├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .swcrc ├── CLAUDE.md ├── LICENSE ├── NOTICE ├── README.md ├── benchmarks ├── consumer.ts ├── murmur2.ts ├── producer.ts └── utils │ ├── definitions.ts │ ├── prepare-consumer-data.ts │ ├── prepare-topics.ts │ └── producer-verifier.ts ├── data ├── jaas │ └── jaas.conf └── ssl │ ├── ca.cert │ ├── ca.key │ ├── client.csr │ ├── client.key │ ├── client.keystore.jks │ ├── client.p12 │ ├── client.pem │ ├── server.csr │ ├── server.keystore.jks │ ├── server.pem │ └── server.truststore.jks ├── docker ├── compose-github-ci.yml ├── compose-multiple.yml ├── compose-single.yml ├── data │ └── jaas │ │ └── jaas.conf └── sasl.yml ├── docs ├── admin.md ├── base.md ├── consumer.md ├── diagnostic.md ├── internals │ ├── apis-status.md │ ├── docker-compose-sasl-plain.md │ ├── docker-compose-sasl-scram-sha.md │ ├── docker-compose-ssl.md │ └── playground.md ├── metrics.md ├── other.md └── producer.md ├── eslint.config.js ├── package.json ├── playground ├── apis │ ├── admin │ │ ├── acl.ts │ │ ├── assignments.ts │ │ ├── client-quotas.ts │ │ ├── config.ts │ │ ├── create-topics.ts │ │ ├── delete-records.ts │ │ ├── delete-topics.ts │ │ ├── group.ts │ │ ├── log-dirs.ts │ │ ├── misc.ts │ │ ├── offset-delete.ts │ │ ├── partitions.ts │ │ └── scram.ts │ ├── consumer │ │ ├── fetch.ts │ │ ├── group.ts │ │ └── list-offsets.ts │ ├── metadata │ │ ├── api-versions.ts │ │ ├── find-coordinator.ts │ │ └── metadata.ts │ ├── producer │ │ ├── init-producer-id.ts │ │ ├── produce-idempotent.ts │ │ ├── produce-with-transaction.ts │ │ └── produce.ts │ └── telemetry │ │ └── telemetry.ts ├── clients │ ├── admin-groups.ts │ ├── admin-topics-multiple.ts │ ├── admin-topics-single.ts │ ├── consumer-hwp.ts │ ├── consumer-rebalance.ts │ ├── consumer.ts │ ├── producer-forever.ts │ ├── producer-idempotent.ts │ ├── producer-metrics.ts │ ├── producer.ts │ └── sasl.ts └── utils.ts ├── pnpm-lock.yaml ├── prettier.config.js ├── scripts ├── bump-version.ts ├── create-api.ts ├── docker.sh ├── generate-apis.ts ├── generate-errors.ts └── utils.ts ├── src ├── apis │ ├── admin │ │ ├── alter-client-quotas-v1.ts │ │ ├── alter-configs-v2.ts │ │ ├── alter-partition-reassignments-v0.ts │ │ ├── alter-partition-v3.ts │ │ ├── alter-replica-log-dirs-v2.ts │ │ ├── alter-user-scram-credentials-v0.ts │ │ ├── consumer-group-describe-v0.ts │ │ ├── create-acls-v3.ts │ │ ├── create-delegation-token-v3.ts │ │ ├── create-partitions-v3.ts │ │ ├── create-topics-v7.ts │ │ ├── delete-acls-v3.ts │ │ ├── delete-groups-v2.ts │ │ ├── delete-records-v2.ts │ │ ├── delete-topics-v6.ts │ │ ├── describe-acls-v3.ts │ │ ├── describe-client-quotas-v0.ts │ │ ├── describe-cluster-v1.ts │ │ ├── describe-configs-v4.ts │ │ ├── describe-delegation-token-v3.ts │ │ ├── describe-groups-v5.ts │ │ ├── describe-log-dirs-v4.ts │ │ ├── describe-producers-v0.ts │ │ ├── describe-quorum-v2.ts │ │ ├── describe-topic-partitions-v0.ts │ │ ├── describe-transactions-v0.ts │ │ ├── describe-user-scram-credentials-v0.ts │ │ ├── envelope-v0.ts │ │ ├── expire-delegation-token-v2.ts │ │ ├── incremental-alter-configs-v1.ts │ │ ├── index.ts │ │ ├── list-groups-v4.ts │ │ ├── list-groups-v5.ts │ │ ├── list-partition-reassignments-v0.ts │ │ ├── list-transactions-v1.ts │ │ ├── offset-delete-v0.ts │ │ ├── renew-delegation-token-v2.ts │ │ ├── unregister-broker-v0.ts │ │ └── update-features-v1.ts │ ├── callbacks.ts │ ├── consumer │ │ ├── consumer-group-heartbeat-v0.ts │ │ ├── fetch-v16.ts │ │ ├── fetch-v17.ts │ │ ├── heartbeat-v4.ts │ │ ├── index.ts │ │ ├── join-group-v9.ts │ │ ├── leave-group-v5.ts │ │ ├── list-offsets-v8.ts │ │ ├── list-offsets-v9.ts │ │ ├── offset-commit-v9.ts │ │ ├── offset-fetch-v9.ts │ │ └── sync-group-v5.ts │ ├── definitions.ts │ ├── enumerations.ts │ ├── index.ts │ ├── metadata │ │ ├── api-versions-v3.ts │ │ ├── api-versions-v4.ts │ │ ├── find-coordinator-v4.ts │ │ ├── find-coordinator-v5.ts │ │ ├── find-coordinator-v6.ts │ │ ├── index.ts │ │ └── metadata-v12.ts │ ├── producer │ │ ├── add-offsets-to-txn-v4.ts │ │ ├── add-partitions-to-txn-v5.ts │ │ ├── end-txn-v4.ts │ │ ├── index.ts │ │ ├── init-producer-id-v4.ts │ │ ├── init-producer-id-v5.ts │ │ ├── produce-v10.ts │ │ ├── produce-v11.ts │ │ └── txn-offset-commit-v4.ts │ ├── security │ │ ├── index.ts │ │ ├── sasl-authenticate-v2.ts │ │ └── sasl-handshake-v1.ts │ └── telemetry │ │ ├── get-telemetry-subscriptions-v0.ts │ │ ├── index.ts │ │ ├── list-client-metrics-resources-v0.ts │ │ └── push-telemetry-v0.ts ├── clients │ ├── admin │ │ ├── admin.ts │ │ ├── index.ts │ │ ├── options.ts │ │ └── types.ts │ ├── base │ │ ├── base.ts │ │ ├── index.ts │ │ ├── options.ts │ │ └── types.ts │ ├── consumer │ │ ├── consumer.ts │ │ ├── index.ts │ │ ├── messages-stream.ts │ │ ├── options.ts │ │ ├── topics-map.ts │ │ └── types.ts │ ├── index.ts │ ├── metrics.ts │ ├── producer │ │ ├── index.ts │ │ ├── options.ts │ │ ├── producer.ts │ │ └── types.ts │ └── serde.ts ├── diagnostic.ts ├── errors.ts ├── global.d.ts ├── index.ts ├── network │ ├── connection-pool.ts │ ├── connection.ts │ └── index.ts ├── protocol │ ├── apis.ts │ ├── compression.ts │ ├── crc32c.ts │ ├── definitions.ts │ ├── dynamic-buffer.ts │ ├── errors.ts │ ├── index.ts │ ├── murmur2.ts │ ├── reader.ts │ ├── records.ts │ ├── sasl │ │ ├── plain.ts │ │ └── scram-sha.ts │ ├── varint.ts │ └── writer.ts ├── symbols.ts └── utils.ts ├── test ├── apis │ ├── admin │ │ ├── alter-client-quotas-v1.test.ts │ │ ├── alter-configs-v2.test.ts │ │ ├── alter-partition-reassignments-v0.test.ts │ │ ├── alter-partition-v3.test.ts │ │ ├── alter-replica-log-dirs-v2.test.ts │ │ ├── alter-user-scram-credentials-v0.test.ts │ │ ├── consumer-group-describe-v0.test.ts │ │ ├── create-acls-v3.test.ts │ │ ├── create-delegation-token-v3.test.ts │ │ ├── create-partitions-v3.test.ts │ │ ├── create-topics-v7.test.ts │ │ ├── delete-acls-v3.test.ts │ │ ├── delete-groups-v2.test.ts │ │ ├── delete-records-v2.test.ts │ │ ├── delete-topics-v6.test.ts │ │ ├── describe-acls-v3.test.ts │ │ ├── describe-client-quotas-v0.test.ts │ │ ├── describe-cluster-v1.test.ts │ │ ├── describe-configs-v4.test.ts │ │ ├── describe-delegation-token-v3.test.ts │ │ ├── describe-groups-v5.test.ts │ │ ├── describe-log-dirs-v4.test.ts │ │ ├── describe-producers-v0.test.ts │ │ ├── describe-quorum-v2.test.ts │ │ ├── describe-topic-partitions-v0.test.ts │ │ ├── describe-transactions-v0.test.ts │ │ ├── describe-user-scram-credentials-v0.test.ts │ │ ├── envelope-v0.test.ts │ │ ├── expire-delegation-token-v2.test.ts │ │ ├── incremental-alter-configs-v1.test.ts │ │ ├── list-groups-v4.test.ts │ │ ├── list-groups-v5.test.ts │ │ ├── list-partition-reassignments-v0.test.ts │ │ ├── list-transactions-v1.test.ts │ │ ├── offset-delete-v0.test.ts │ │ ├── renew-delegation-token-v2.test.ts │ │ ├── unregister-broker-v0.test.ts │ │ └── update-features-v1.test.ts │ ├── consumer │ │ ├── consumer-group-heartbeat-v0.test.ts │ │ ├── fetch-v16.test.ts │ │ ├── fetch-v17.test.ts │ │ ├── heartbeat-v4.test.ts │ │ ├── join-group-v9.test.ts │ │ ├── leave-group-v5.test.ts │ │ ├── list-offsets-v8.test.ts │ │ ├── list-offsets-v9.test.ts │ │ ├── offset-commit-v9.test.ts │ │ ├── offset-fetch-v9.test.ts │ │ └── sync-group-v5.test.ts │ ├── definitions.test.ts │ ├── metadata │ │ ├── api-versions-v3.test.ts │ │ ├── api-versions-v4.test.ts │ │ ├── find-coordinator-v4.test.ts │ │ ├── find-coordinator-v5.test.ts │ │ ├── find-coordinator-v6.test.ts │ │ └── metadata-v12.test.ts │ ├── producer │ │ ├── add-offsets-to-txn-v4.test.ts │ │ ├── add-partitions-to-txn-v5.test.ts │ │ ├── end-txn-v4.test.ts │ │ ├── init-producer-id-v4.test.ts │ │ ├── init-producer-id-v5.test.ts │ │ ├── produce-v10.test.ts │ │ ├── produce-v11.test.ts │ │ └── txn-offset-commit-v4.test.ts │ ├── security │ │ ├── sasl-authenticate-v2.test.ts │ │ └── sasl-handshake-v2.test.ts │ └── telemetry │ │ ├── get-telemetry-subscriptions-v0.test.ts │ │ ├── list-client-metrics-resources-v0.test.ts │ │ └── push-telemetry-v0.test.ts ├── clients │ ├── admin │ │ └── admin.test.ts │ ├── base │ │ ├── base.test.ts │ │ └── sasl.test.ts │ ├── callbacks.test.ts │ ├── consumer │ │ ├── consumer.test.ts │ │ ├── messages-stream.test.ts │ │ └── topics-map.test.ts │ ├── producer │ │ └── producer.test.ts │ └── serde.test.ts ├── config │ ├── c8-ci.json │ ├── c8-local.json │ └── env ├── errors.test.ts ├── helpers.ts ├── network │ ├── connection-pool.test.ts │ └── connection.test.ts ├── protocol │ ├── compression.test.ts │ ├── crc32.test.ts │ ├── dynamic-buffer.test.ts │ ├── murmur2.test.ts │ ├── reader.test.ts │ ├── records.test.ts │ ├── sasl │ │ ├── plain.test.ts │ │ └── scram-sha.test.ts │ ├── varint.test.ts │ └── writer.test.ts └── utils.test.ts ├── tsconfig.base.json └── tsconfig.json /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Run Tests 3 | on: [push, pull_request, workflow_dispatch] 4 | jobs: 5 | ci: 6 | strategy: 7 | matrix: 8 | node-version: [22, 23] 9 | os: [ubuntu-latest] 10 | kafka-version: ['3.7.0', '3.8.0', '3.9.0'] 11 | runs-on: ${{matrix.os}} 12 | timeout-minutes: 10 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | - name: Use supported Node.js Version 17 | uses: actions/setup-node@v4 18 | with: 19 | node-version: ${{ matrix.node-version }} 20 | - name: Restore cached dependencies 21 | uses: actions/cache@v3 22 | with: 23 | path: ~/.pnpm-store 24 | key: node-modules-${{ hashFiles('package.json') }} 25 | - name: Setup pnpm 26 | uses: pnpm/action-setup@v4 27 | with: 28 | version: latest 29 | - name: Install dependencies 30 | run: pnpm install --frozen-lockfile 31 | - name: Start Kafka (${{ matrix.kafka-version }}) Cluster 32 | run: docker compose -f docker/compose-github-ci.yml up -d && pnpx wait-on tcp:9092 tcp:9093 tcp:9094 33 | env: 34 | KAFKA_VERSION: ${{ matrix.kafka-version }} 35 | - name: Run Tests 36 | run: pnpm run ci 37 | env: 38 | KAFKA_VERSION: ${{ matrix.kafka-version }} 39 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version: 7 | description: 'The version number to tag and release' 8 | required: true 9 | type: string 10 | prerelease: 11 | description: 'Release as pre-release' 12 | required: false 13 | type: boolean 14 | default: false 15 | 16 | jobs: 17 | release-npm: 18 | runs-on: ubuntu-latest 19 | environment: main 20 | permissions: 21 | contents: write 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | - name: Setup pnpm 26 | uses: pnpm/action-setup@v4 27 | with: 28 | version: latest 29 | - name: Use supported Node.js Version 30 | uses: actions/setup-node@v4 31 | with: 32 | node-version: 22 33 | registry-url: 'https://registry.npmjs.org' 34 | cache: 'pnpm' 35 | - name: Restore cached dependencies 36 | uses: actions/cache@v3 37 | with: 38 | path: ~/.pnpm-store 39 | key: node-modules-${{ hashFiles('package.json') }} 40 | - name: Install dependencies 41 | run: pnpm install 42 | - name: Bump Version 43 | run: node --experimental-strip-types --disable-warning=ExperimentalWarning scripts/bump-version.ts "${{ inputs.version }}" "${{ github.actor }}" 44 | env: 45 | GITHUB_ACTIONS: 'true' 46 | - name: Push changes 47 | run: git push origin HEAD:${{ github.ref }} 48 | - name: Publish to NPM 49 | run: pnpm publish --access public --tag ${{ inputs.prerelease == true && 'next' || 'latest' }} --publish-branch ${{ github.ref_name }} --no-git-checks 50 | env: 51 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 52 | - name: 'Create release notes' 53 | run: | 54 | pnpx @matteo.collina/release-notes -a ${{ secrets.GH_RELEASE_TOKEN }} -t v${{ inputs.version }} -r kafka -o platformatic ${{ github.event.inputs.prerelease == 'true' && '-p' || '' }} -c ${{ github.ref }} 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | external/ 3 | node_modules 4 | coverage 5 | .eslintcache 6 | -------------------------------------------------------------------------------- /.swcrc: -------------------------------------------------------------------------------- 1 | { 2 | "jsc": { 3 | "target": "esnext", 4 | "parser": { 5 | "syntax": "typescript", 6 | "tsx": false, 7 | "dynamicImport": true 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # @platformatic/kafka Project Guide 2 | 3 | ## Build & Test Commands 4 | 5 | ``` 6 | # Build the project 7 | npm run build 8 | 9 | # Run all tests 10 | npm test 11 | 12 | # Run a single test file 13 | node --env-file=test/config/env --test 'test/path/to/file.test.ts' 14 | 15 | # Run a single test file with coverage 16 | c8 -c test/config/c8-local.json node --test --env-file=test/config/env --test 'test/path/to/file.test.ts' 17 | 18 | # Lint the code 19 | npm run lint 20 | ``` 21 | 22 | ## Code Style Guidelines 23 | 24 | - **TypeScript**: Strict typing with explicit type imports `import type { X }`. Avoid `any` all the times. Ensure types compliance. 25 | - **Formatting**: 2-space indentation, no semicolons, single quotes 26 | - **Imports**: Group related imports, use explicit `.ts` extensions 27 | - **Naming**: camelCase for variables/functions, PascalCase for classes/types 28 | - **Errors**: Extend GenericError class with descriptive error codes prefixed with `PLT_KFK_` 29 | - **Error Handling**: Use try/catch with specific error types. 30 | - **API Design**: Consistent API interface with options objects and promise-based returns. 31 | - **Testing**: Node.js test runner with deep assertions. Use `deepStrictEqual` when appropriate. Never modify the `src` folder. The test file for `src/foo/bar/baz.ts` is `test/foo/bar/baz.test.ts` 32 | 33 | ## Node Requirements 34 | 35 | - Node.js >= 22.14.0 36 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2025 Platformatic 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /benchmarks/murmur2.ts: -------------------------------------------------------------------------------- 1 | import cronometro from 'cronometro' 2 | // @ts-ignore 3 | import kafkaJsMurmur2 from 'kafkajs/src/producer/partitioners/default/murmur2.js' 4 | import { randomBytes } from 'node:crypto' 5 | import { murmur2 } from '../src/index.ts' 6 | 7 | await cronometro({ 8 | kafkajs () { 9 | const value = randomBytes(16) 10 | return kafkaJsMurmur2(value) 11 | }, 12 | '@platformatic/kafka' () { 13 | const value = randomBytes(16) 14 | return murmur2(value) 15 | } 16 | }) 17 | -------------------------------------------------------------------------------- /benchmarks/utils/definitions.ts: -------------------------------------------------------------------------------- 1 | export const topic = 'benchmarks' 2 | export const brokers = ['localhost:9092', 'localhost:9093', 'localhost:9094'] 3 | 4 | // This is needed by KafkaJS 5 | process.env.KAFKAJS_NO_PARTITIONER_WARNING = '1' 6 | -------------------------------------------------------------------------------- /benchmarks/utils/prepare-consumer-data.ts: -------------------------------------------------------------------------------- 1 | import { randomUUID } from 'node:crypto' 2 | import { ProduceAcks, Producer, stringSerializers } from '../../src/index.ts' 3 | import { brokers, topic } from './definitions.ts' 4 | 5 | const producer = new Producer({ 6 | clientId: randomUUID(), 7 | bootstrapBrokers: brokers, 8 | serializers: stringSerializers, 9 | strict: true 10 | }) 11 | 12 | const max = 1e6 13 | 14 | for (let i = 0; i < max; i++) { 15 | await producer.send({ 16 | messages: [ 17 | { 18 | topic, 19 | key: `key-${i}`, 20 | value: `value-${i}`, 21 | headers: new Map([[`header-key-${i}`, `header-value-${i}`]]), 22 | partition: i % brokers.length 23 | } 24 | ], 25 | acks: ProduceAcks.NO_RESPONSE 26 | }) 27 | 28 | if (i % 1000 === 0) { 29 | console.log(`Produced ${i}/${max} messages.`) 30 | } 31 | } 32 | 33 | await producer.close() 34 | -------------------------------------------------------------------------------- /benchmarks/utils/prepare-topics.ts: -------------------------------------------------------------------------------- 1 | import { Admin, debugDump, sleep } from '../../src/index.ts' 2 | import { brokers, topic } from './definitions.ts' 3 | 4 | const admin = new Admin({ clientId: 'id', bootstrapBrokers: brokers, strict: true }) 5 | 6 | try { 7 | await admin.deleteTopics({ topics: [topic] }) 8 | } catch (e) { 9 | // Noop 10 | } 11 | 12 | await admin.createTopics({ topics: [topic], partitions: brokers.length, replicas: 1 }) 13 | await sleep(1000) 14 | debugDump('topic', await admin.metadata({ topics: [topic] })) 15 | 16 | await admin.close() 17 | -------------------------------------------------------------------------------- /benchmarks/utils/producer-verifier.ts: -------------------------------------------------------------------------------- 1 | import { randomUUID } from 'node:crypto' 2 | import { once } from 'node:events' 3 | import { Consumer, debugDump, stringDeserializers } from '../../src/index.ts' 4 | import { brokers, topic } from './definitions.ts' 5 | 6 | const consumer = new Consumer({ 7 | groupId: randomUUID(), 8 | clientId: randomUUID(), 9 | bootstrapBrokers: brokers, 10 | strict: true, 11 | deserializers: stringDeserializers 12 | }) 13 | 14 | const stream = await consumer.consume({ 15 | autocommit: false, 16 | topics: [topic], 17 | sessionTimeout: 10000, 18 | heartbeatInterval: 500, 19 | maxWaitTime: 500 20 | }) 21 | 22 | once(process, 'SIGINT').then(() => consumer.close(true)) 23 | 24 | debugDump('--- Consuming messages ---') 25 | 26 | for await (const message of stream) { 27 | debugDump('messages', message) 28 | } 29 | 30 | debugDump('--- Consuming stopped ---') 31 | 32 | await consumer.close() 33 | -------------------------------------------------------------------------------- /data/jaas/jaas.conf: -------------------------------------------------------------------------------- 1 | KafkaServer { 2 | org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin" user_admin="admin"; 3 | org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin"; 4 | }; 5 | 6 | KafkaClient { 7 | org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; 8 | }; 9 | -------------------------------------------------------------------------------- /data/ssl/ca.cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFAzCCAuugAwIBAgIUKS0c73g9X6AicxUZ3ESyrC8ZDqowDQYJKoZIhvcNAQEL 3 | BQAwETEPMA0GA1UEAwwGZG9ja2VyMB4XDTI1MDQwMTE0NDcyOVoXDTI2MDQwMTE0 4 | NDcyOVowETEPMA0GA1UEAwwGZG9ja2VyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A 5 | MIICCgKCAgEAjosEBi57gB9v/R58TPoSIDk3nxWjZiY3RGwIc1tn1sCmfLkqTQz5 6 | wxicjExtmT7OG31Pf6/fh9jfb7AAbwuNn+83BZR3H6coKo0+7gAZruA+66fFKf6l 7 | gTK6vKQOjrj60Z+C0wWxW7W/I6HgN0NYeBlqTJ+ytph1R2pL46CNGvCIyyO72sln 8 | /XUajeAQGh8OIXUwhtJK4mf+ixubjns0/9QnPfv79M5fRFIBZ0btkrXdrHFCGluD 9 | MP6h9HZRSScI5McGdR+50IqNkPc33Gq9tWms9I0uM0tmHlVIRyniO5H5z4ZZaak/ 10 | vI8Ou3L0z+LZnNah/xTSwAesjKMvwtUg/ULLiqqgMZQIvLlCmQGHbpClYlvpSzjH 11 | O5XIY9FGTxHnqlTl0zcUxUY7MJQ47kJ0oKtDyiQaxKvKRCTICe8va4az6VPYCISt 12 | 5rnbzNxMon0hr3iE5bE9WWvlJgsT5oukfabwqQuduxhSpsYBDJd3wOWeFUQNx7Cx 13 | Rg/IlB923GcOL6qYGRtukdPz/hLqq8iMVSyspWrAdCNrfIE35NBNs+hrgICF7l3F 14 | fqdf/X30A5l12sMVECSZKjjGtYZo404pqsF8MjO6AbxS4ILePYzejJDiOCBEz1ms 15 | 5XUgr2b/oQRW642Wepc7DNcUYiw4Mp1J9C7Cwn6MaF4Y2zQasf4TFEUCAwEAAaNT 16 | MFEwHQYDVR0OBBYEFB16QOQ62Rm1iLE/uU9IUgaJUPopMB8GA1UdIwQYMBaAFB16 17 | QOQ62Rm1iLE/uU9IUgaJUPopMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL 18 | BQADggIBADYajDtRd8lU5UQB1aNhVodkl7soYkwbro1jjA3MbVjc+62T+GhP+7yZ 19 | jF6hGpPOiCVki5AMgR7wtyJzsQ30ym8EG+dQFXLOpJHKYShwkxJgPBVGKCzLz8Py 20 | Qh8OrN7NPCLFRWs92IGEPk9Pt7yu4jMx8kYAMia4qjyN9UGVth2hK+71ANfhx27t 21 | 9VOYcrky8oB8CK+yE9I+KCz9r/9XpE9ngf7WfNaKjaqlyZqbMH9i4weeMdL337JG 22 | FHRutJrYFuTMBEEugj1Y10YS7uvrzWNMTLnlXm7ddFGcvQXpDJXS4N21MZeIkUUf 23 | kr/i7TXlw2nmhy2EXDyegUKxk9Lsau3T7oN2kBXRZAcCrV/BD4RFoFhXdlrXEZvv 24 | enp8oFIkRqwNOA4F7yV2fErg8Vt6hgSecOUceLaYg1jnbUQfCyBZAaeX2SSi0Uoy 25 | 3OtlCgpL60J570dfTd6/ZYDTcMB5A/TDTMgBM55dfuacSqjBGGddWZQTZUJSLKj9 26 | ewkfeJanbvCHJdGoeBkzZuD0OXY//QJKskc+XkdcXoqbflCyZYJTD8KaInfzOmbB 27 | uoUq8lUjSoNXyWg7+4WUyxOwKrOxS0ECdXj6h9HDVyi4m+NXUYWi8bvmRdwVEy4d 28 | d1DA+MGENl1gsSkv7XuERj/0qrfi+6jFx0Msk6qeEjcpbh9ya1la 29 | -----END CERTIFICATE----- 30 | -------------------------------------------------------------------------------- /data/ssl/client.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN NEW CERTIFICATE REQUEST----- 2 | MIIChjCCAW4CAQAwETEPMA0GA1UEAxMGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEF 3 | AAOCAQ8AMIIBCgKCAQEAmIxxGHaUsNXsqFvHixYNvINsnLj3ZOTLmMUi1U9gy/bj 4 | i0nQCArBfkniWPPovyZEvgHFIAMOIVcqfyLxTIMG8LqQzZTiuD4mJ+WBDuQUmzkL 5 | t88+jDIf6eHAFnuVX6teLzc27rbAIQo9chnzyG3cWIsznhMWi2M/Ziwbi5bZFoX1 6 | PV0TVx0Z7fiGVmEpZLLlA4xK5lQpOBJJvWeuSbd9ecChi7HRcLmQAFS1yPxa1DX8 7 | kv2KIyWg3xRssDIMGUnfZtNYNNPng5mbp00kz2Amlcw1ElWoL2D+Z5X6O6oGD4HY 8 | jz+EZ38gyB6Ooh3sk176YYY8kcQDC000BB5hE1KebwIDAQABoDAwLgYJKoZIhvcN 9 | AQkOMSEwHzAdBgNVHQ4EFgQU7swNb4TYQ8UkWd9ewIDndtNE860wDQYJKoZIhvcN 10 | AQELBQADggEBAAXgrommYqOWPD8+F1tK5u2kh3Naj3KEBq2WVuh5TNhUIRg0+BbY 11 | lDKjYeWpha8mrMde6FACk1vfDaPfmnxFEtwLhD7bmJHZnqo2NTVupIRHjo+bGJZk 12 | xuWxwcUWWq4zSine0ZN6fEPz3CyZEfq04kqz6Y0w0K2nlwgE5WWUKLn3it3lMCEE 13 | Gl76jT4MLlCpYVLYYNKjffbbnJDlt2Dcq0lcFfSC5ahtU62ws6YjZ2lE3ZIJGmIv 14 | 04RHnyE6WcbtW/EkPZrginyQtQq42HGoggMrBbzz4Nvi/4Kxde7bZrsDL6s+bCCX 15 | 1Peh0SXWSOYLkymgnV6L5oRc2/d6bIHWY1k= 16 | -----END NEW CERTIFICATE REQUEST----- 17 | -------------------------------------------------------------------------------- /data/ssl/client.key: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: client 3 | localKeyID: 54 69 6D 65 20 31 37 34 33 35 31 39 32 33 33 35 33 37 4 | Key Attributes: 5 | -----BEGIN PRIVATE KEY----- 6 | MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCYjHEYdpSw1eyo 7 | W8eLFg28g2ycuPdk5MuYxSLVT2DL9uOLSdAICsF+SeJY8+i/JkS+AcUgAw4hVyp/ 8 | IvFMgwbwupDNlOK4PiYn5YEO5BSbOQu3zz6MMh/p4cAWe5Vfq14vNzbutsAhCj1y 9 | GfPIbdxYizOeExaLYz9mLBuLltkWhfU9XRNXHRnt+IZWYSlksuUDjErmVCk4Ekm9 10 | Z65Jt315wKGLsdFwuZAAVLXI/FrUNfyS/YojJaDfFGywMgwZSd9m01g00+eDmZun 11 | TSTPYCaVzDUSVagvYP5nlfo7qgYPgdiPP4RnfyDIHo6iHeyTXvphhjyRxAMLTTQE 12 | HmETUp5vAgMBAAECggEAIrcDHbAOkld8bavFMy+nnJjD5L5q/jja5D8oeCaCPqhG 13 | i/IWN15LPjQOIHQRKT7KMfsxIuHPG2M8toX7J5BByhKK1V3mYksyX63ourzIu7I1 14 | qnStJTZa8NyA3Bzvlz/mRRRpmdVYz6wG46poQdpOZI/5BSYO4VWKnfO3qQKBEJC8 15 | dRWLxDZGGGOnbm/6ss0A08ugz/3dP9D8sUxgiFG6l+SKxC+m6u/G7+CfLoWwQ3Iy 16 | GH/FZ5+kLjX6nBIfVphupJNkht/R3pXrXWMizMasHbDp2SNUZ5CMNdQc48md+M5i 17 | 6z0mWNUz+VJSHEAsKSpxdQLBXGDuzKEzwohu83fY4QKBgQDnjyfkEIUzQBIX3/jX 18 | q/2rVaT/sA6FM9i//e3VvV5c90ZAU4eYWFJjqEebFYQz/YaQQZjC4AWD9p2kwfKk 19 | i8wVEUeU5Qece6repW9T6iKO0HDgiNxW+aLuwg6kh6eRIO7vwulYK0sXLDll1z0p 20 | AfjkrzyO1IIpFHahE3/D8pApqQKBgQCopmFjJxymigeHYnjHOGMLmDLycNPiMaO2 21 | aBZcKHor2FpFDmDBNE0v/ScKKynmWNt6APXb+UNoXkhIqvt5qq0Kr5flmltv5QW9 22 | iL22zln39TIBLrgUotyVoXjelEei28E5uIbk9090F5Tnuh65CjsGJ3zBbOKiODPp 23 | N1YeChaGVwKBgFXRv6YN3WAGMe55Sr+b9bLnVfSRaoBBm+ZkFR5vRabM2lOC4bE7 24 | oqVeu9MlYU7nelTCivxGjKTA5OIoI2aKjUONmOr9Cxoa11QXGjCOiOpHJF6DOq5d 25 | bN5yO30M7bSi8QsZQOQ1f2oCMO+mmvs0yOrwa1BAQFE7TVAbavHTV0ypAoGAPRgQ 26 | vXeEtHvaIFw2voyZrLCU/CiNSSqMUN7CAt6S62EzykJcIIFO2OyPAOsEAbL8xuSk 27 | K+zPU3acHt75vGhsySs1DnLtXsqLWs63UwrLoryLQcxn2OnqmLXm8FKs5L1Q8RT8 28 | oONKQBbu1UciLAwdt6wEUJmeO/+6JmjqZHVpFX8CgYBMkgR8h54Kv4YY3tM57aLt 29 | A82OY/aCx4PU2Op/GRMhj0/jqKF6cEGefPiaBmK91LkQaP3G7mM9uifFzPlCXk2L 30 | C78XkiDAvayFdKEkNDBbh1agnQNqP7gJokUSDdjk1zuR7OLJ3Uun3hFjV9F1STR6 31 | JBqt+3WH7Juf6kfKuLCnJg== 32 | -----END PRIVATE KEY----- 33 | -------------------------------------------------------------------------------- /data/ssl/client.keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platformatic/kafka/470a56aec86839fc8579c0f7903397f9be55cadf/data/ssl/client.keystore.jks -------------------------------------------------------------------------------- /data/ssl/client.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platformatic/kafka/470a56aec86839fc8579c0f7903397f9be55cadf/data/ssl/client.p12 -------------------------------------------------------------------------------- /data/ssl/client.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID8jCCAdqgAwIBAgIUaTxqPnKRi3eFLqiVcoBHIxXNTsYwDQYJKoZIhvcNAQEL 3 | BQAwETEPMA0GA1UEAwwGZG9ja2VyMB4XDTI1MDQwMTE0NTM0NVoXDTI2MDQwMTE0 4 | NTM0NVowETEPMA0GA1UEAxMGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A 5 | MIIBCgKCAQEAmIxxGHaUsNXsqFvHixYNvINsnLj3ZOTLmMUi1U9gy/bji0nQCArB 6 | fkniWPPovyZEvgHFIAMOIVcqfyLxTIMG8LqQzZTiuD4mJ+WBDuQUmzkLt88+jDIf 7 | 6eHAFnuVX6teLzc27rbAIQo9chnzyG3cWIsznhMWi2M/Ziwbi5bZFoX1PV0TVx0Z 8 | 7fiGVmEpZLLlA4xK5lQpOBJJvWeuSbd9ecChi7HRcLmQAFS1yPxa1DX8kv2KIyWg 9 | 3xRssDIMGUnfZtNYNNPng5mbp00kz2Amlcw1ElWoL2D+Z5X6O6oGD4HYjz+EZ38g 10 | yB6Ooh3sk176YYY8kcQDC000BB5hE1KebwIDAQABo0IwQDAdBgNVHQ4EFgQU7swN 11 | b4TYQ8UkWd9ewIDndtNE860wHwYDVR0jBBgwFoAUHXpA5DrZGbWIsT+5T0hSBolQ 12 | +ikwDQYJKoZIhvcNAQELBQADggIBAEPyte1JihxDZY0Y3t/W2yIdjmPFCTyHeEKG 13 | wocmUHQw6AcYFeji6KrpKjT+/cVDxYc4XHYzvSll4+VQHxvsrDb/r5UTu5nKKFcJ 14 | amjOaXjfPGEjNKStbFrVVUatgrZgDBrs9+RJvKki/6PIIu9qULBL3+6vCkRQGK5W 15 | X1ZVL+tSQ8sbMpmMhVMObTERDryZDrsWhEC0IEtJdzUyUKIllqiVUTd13AMGz2fA 16 | DGjtLjJ0Pwp7iQy8rwahU2U9Uj9970v7CDdxReUQHuxg4OR7f/t4SUHeWOrgPHvj 17 | PfM+YEZXntZhG62/QCpSmfjeD5yrRuod7EcIgC6apSMK029ds9/pDDK7dPXkMYvc 18 | JmXDIVCIrUtYSK3e06fC56qP0hIYlMunDXFAKqxD1bjc106SPwhRevv1mjHv4o8f 19 | cRETefeayZcxkqai4u9iMxXe9iOBtoS+E7GnI9zpE0ygH0ldjJ/+zbZjiO+yBR/n 20 | JCLWtcYDJrszEU0qQSCGCKntiEf1uKV8gGwClJeRy/HlDsaY9aFh1u9vhhWRq9VG 21 | ffldfeqodi5IcW5fXpWgsl5zeeoskSlVbS6SCBJRRJUD2FsX0DuHSEm7eCS+hxjK 22 | qbYvonF0RQ/dMhd1CE9Ix/QOKGqsDiXRm4XN9DBFpG+UFkU7XRm8DY0uAYHPO/VU 23 | IrO8shnL 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /data/ssl/server.csr: -------------------------------------------------------------------------------- 1 | -----BEGIN NEW CERTIFICATE REQUEST----- 2 | MIIChjCCAW4CAQAwETEPMA0GA1UEAxMGc2VydmVyMIIBIjANBgkqhkiG9w0BAQEF 3 | AAOCAQ8AMIIBCgKCAQEAiIlUpuLQe6waeVd0jQ76U41R81ikYcWOpzxghhAlJe3N 4 | ETbLmZOiuHr3EZVGi7BuHIf4+aVIGO2cjv4XA9TrUJnN1CmzwLSV1qA/UtW2OcIE 5 | sQrmbetU31tDhLN6lHozjq9jHXLYyRVvm5k6tEGfRJ9jyCKWoq5FMzVHd6e/R2Oq 6 | dUG0fL4eWWZVsVQhnNzu1Ks2auIogvGoj6naXGrwU3h+41XAr6E/FMoET4rr42Om 7 | hJOFYc04ZhBo7xuoYElyUW5MJMBXYBw6LyMAe4pr22ppxnpq17RlXnl6rvermAC1 8 | DUxroQ5jeM0jZH7rEC3+GPsRHsHXIflRXFJFBGmapwIDAQABoDAwLgYJKoZIhvcN 9 | AQkOMSEwHzAdBgNVHQ4EFgQUcDj1OmwEC11+08I6TpK25UEFz1cwDQYJKoZIhvcN 10 | AQELBQADggEBACyON/P9TM+KXqVnKPBuBfXGubwjm8B7gAocrzZQ/lY8I2McrNFs 11 | QBp43eWe1rSeIM0LrSaGa1GR17PcVeZkzYlPVUVlpRbyuZqlFGoclUr1C7zyhX9c 12 | dpglKrxti8Lgk/1WVSAAEBrqBd2iPF0wjQVlOxILppa/8NmvYM2ki8GBk6r40DgJ 13 | GUrSkq1/e7yu+5jXW/9XDhxENrtp9xA6ThZCWZYjbmekUeNZMCMpUuP6MHtLl2MB 14 | R5vM2I7nRPlLobw63A6qGWrYdUX5IPLGQAEi7kfAGFWbWqGCJW/eYx6RwvXOblln 15 | O2MG4OYHbLvg4I1ZFLFV+2ErC1Gz9RlQdhg= 16 | -----END NEW CERTIFICATE REQUEST----- 17 | -------------------------------------------------------------------------------- /data/ssl/server.keystore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platformatic/kafka/470a56aec86839fc8579c0f7903397f9be55cadf/data/ssl/server.keystore.jks -------------------------------------------------------------------------------- /data/ssl/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID8jCCAdqgAwIBAgIUCA0crtJhYXmLH/Z94zFqfep3IeAwDQYJKoZIhvcNAQEL 3 | BQAwETEPMA0GA1UEAwwGZG9ja2VyMB4XDTI1MDQwMTE0NTI0MFoXDTI2MDQwMTE0 4 | NTI0MFowETEPMA0GA1UEAxMGc2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A 5 | MIIBCgKCAQEAiIlUpuLQe6waeVd0jQ76U41R81ikYcWOpzxghhAlJe3NETbLmZOi 6 | uHr3EZVGi7BuHIf4+aVIGO2cjv4XA9TrUJnN1CmzwLSV1qA/UtW2OcIEsQrmbetU 7 | 31tDhLN6lHozjq9jHXLYyRVvm5k6tEGfRJ9jyCKWoq5FMzVHd6e/R2OqdUG0fL4e 8 | WWZVsVQhnNzu1Ks2auIogvGoj6naXGrwU3h+41XAr6E/FMoET4rr42OmhJOFYc04 9 | ZhBo7xuoYElyUW5MJMBXYBw6LyMAe4pr22ppxnpq17RlXnl6rvermAC1DUxroQ5j 10 | eM0jZH7rEC3+GPsRHsHXIflRXFJFBGmapwIDAQABo0IwQDAdBgNVHQ4EFgQUcDj1 11 | OmwEC11+08I6TpK25UEFz1cwHwYDVR0jBBgwFoAUHXpA5DrZGbWIsT+5T0hSBolQ 12 | +ikwDQYJKoZIhvcNAQELBQADggIBAAb+CTy0xw/sbB1DcH59IOEC2/J8uEiPZ4EW 13 | 0CI8WEm1GyzKgwRfKh0+TYOxzXmPN10iWeZWToJ9QXWiw8xKHP7ZybdGgjnnqqpV 14 | vyubxpK5t0TC4YUzj6AGwhVM6241HjzcfkigvMIBa2eh3OiM0eo165Ay0qOSyUIl 15 | UcgUyN1gYILZ+aSgWlxn6gE9qu5ruy6k7SZSMJnV590m3BZQT0tf2fEnpyet193t 16 | cyzWTan4d7r5QTg1Hx9jZ3Zv3BB2rGNngGhLj1+wKDvnGijyqVNH9lQ/kRe3KKB8 17 | mZZmCQkE4PjA0EOpoaF9yAa+Z0Tj/gdBz6MWGjDSTT8Qnax/FeUMfhuB4ExXKbCb 18 | DUATRMVzv3oKDakXArynOM3xA6K8sVHfa9uzdp9+7vZsjq6+pYIOotgXqvu+rEnr 19 | 004BT4yA5uJHXBKBXYQLq8XM2dV/tsJp03CkipdzaqAilzndj4vWBBzCrDbPyeB4 20 | OzdWc5xB0TH1gLQDeGWX+rDviVPWQ4k4Y7ChtRAZfUqC3WU/lM5xZZen5rpgn5Kn 21 | e5AZvDHvsl37aZ/HJ248CZSwArJSiGrWX5raaC4LnbYTDm9s0zRBo3cVaCAfIYGg 22 | ICXMtJBX0uwSdvYWb6MLpbkzea1bRdz3Jh4GlnbsKd1v2eC1RSV+KspdEd6GOUta 23 | QDKtZG1f 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /data/ssl/server.truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platformatic/kafka/470a56aec86839fc8579c0f7903397f9be55cadf/data/ssl/server.truststore.jks -------------------------------------------------------------------------------- /docker/data/jaas/jaas.conf: -------------------------------------------------------------------------------- 1 | KafkaServer { 2 | org.apache.kafka.common.security.plain.PlainLoginModule required 3 | username="admin" 4 | password="admin" 5 | user_admin="admin"; 6 | 7 | org.apache.kafka.common.security.scram.ScramLoginModule required 8 | username="admin" 9 | password="admin" 10 | user_admin="admin"; 11 | }; 12 | 13 | KafkaClient { 14 | org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; 15 | }; -------------------------------------------------------------------------------- /docker/sasl.yml: -------------------------------------------------------------------------------- 1 | services: 2 | zookeeper: 3 | image: confluentinc/cp-zookeeper:7.3.2 4 | hostname: zookeeper 5 | container_name: zookeeper 6 | ports: 7 | - '2181:2181' 8 | environment: 9 | ZOOKEEPER_CLIENT_PORT: 2181 10 | ZOOKEEPER_TICK_TIME: 2000 11 | 12 | kafka: 13 | image: confluentinc/cp-kafka:7.3.2 14 | hostname: kafka 15 | container_name: kafka 16 | depends_on: 17 | - zookeeper 18 | ports: 19 | - '9092:9092' 20 | environment: 21 | KAFKA_BROKER_ID: 1 22 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 23 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 24 | KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 25 | KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 26 | KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 27 | KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 28 | KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 29 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT 30 | KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092 31 | KAFKA_LISTENER_NAME_INTERNAL_SASL_ENABLED_MECHANISMS: PLAIN 32 | KAFKA_LISTENER_NAME_EXTERNAL_SASL_ENABLED_MECHANISMS: PLAIN 33 | KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN 34 | KAFKA_LISTENER_NAME_INTERNAL_PLAIN_SASL_JAAS_CONFIG: | 35 | org.apache.kafka.common.security.plain.PlainLoginModule required \ 36 | username="demo" \ 37 | password="demo-password" \ 38 | user_demo="demo-password"; 39 | KAFKA_LISTENER_NAME_EXTERNAL_PLAIN_SASL_JAAS_CONFIG: | 40 | org.apache.kafka.common.security.plain.PlainLoginModule required \ 41 | username="demo" \ 42 | password="demo-password" \ 43 | user_demo="demo-password"; 44 | KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL 45 | volumes: 46 | - type: bind 47 | source: './kafka-client-config' 48 | target: /kafka-client-config 49 | read_only: true 50 | -------------------------------------------------------------------------------- /docs/internals/docker-compose-sasl-plain.md: -------------------------------------------------------------------------------- 1 | # How to enable SASL/PLAIN 2 | 3 | Create a JAAS file, like `data/jaas/jaas.conf` with the following contents: 4 | 5 | ``` 6 | KafkaServer { 7 | org.apache.kafka.common.security.plain.PlainLoginModule required 8 | username="admin" password="admin" 9 | user_admin="admin" 10 | user_client="client"; 11 | }; 12 | 13 | KafkaClient { 14 | org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; 15 | }; 16 | ``` 17 | 18 | (`username/password` are used by the broker to connect to other brokers, while `user_*` defines valid users). 19 | 20 | Ensure the following mapping is enabled in the docker-compose volumes: 21 | 22 | ``` 23 | - ./data/jaas:/var/jaas 24 | ``` 25 | 26 | If you need to run the ACL tools within the docker container, you will need a `admin.conf` structured like this: 27 | 28 | ``` 29 | security.protocol=SASL_PLAINTEXT 30 | sasl.mechanism=PLAIN 31 | ``` 32 | 33 | Permission can be added with a command similar to the following one: 34 | 35 | ``` 36 | /opt/kafka/bin/kafka-acls.sh --bootstrap-server localhost:9092 --add --allow-principal User:client --topic temp --operation all --command-config admin.config 37 | ``` 38 | -------------------------------------------------------------------------------- /docs/internals/docker-compose-sasl-scram-sha.md: -------------------------------------------------------------------------------- 1 | # How to enable SASL/SHA-256 or SASL/SHA-512 2 | 3 | Create a JAAS file, like `data/jaas/sasl.conf` with the following contents: 4 | 5 | ``` 6 | KafkaServer { 7 | org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin" user_admin="admin"; 8 | org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin"; 9 | }; 10 | 11 | KafkaClient { 12 | org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin"; 13 | }; 14 | ``` 15 | 16 | (`username/password` are used by the broker to connect to other brokers, while `user_*` defines valid users). 17 | 18 | Ensure the following mapping is enabled in the docker-compose volumes: 19 | 20 | ``` 21 | - ./data/jaas:/var/jaas 22 | ``` 23 | 24 | If you need to run the ACL tools within the docker container, you will need a `admin.conf` structured like this: 25 | 26 | ``` 27 | security.protocol=SASL_PLAINTEXT 28 | sasl.mechanism=PLAIN 29 | ``` 30 | 31 | Add the user by opening the shell in the Docker container and by doing: 32 | 33 | ``` 34 | /opt/kafka/bin/kafka-configs.sh --bootstrap-server localhost:9092 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=client]' --entity-type users --entity-name client --command-config admin.config 35 | ``` 36 | 37 | Permission can be added with a command similar to the following one: 38 | 39 | ``` 40 | /opt/kafka/bin/kafka-acls.sh --bootstrap-server localhost:9092 --add --allow-principal User:client --topic temp --operation all --command-config admin.config 41 | ``` 42 | -------------------------------------------------------------------------------- /docs/internals/docker-compose-ssl.md: -------------------------------------------------------------------------------- 1 | # Setup SSL 2 | 3 | Prepare the folder: 4 | 5 | `mkdir -p ssl` 6 | 7 | Generate the CA: 8 | 9 | ``` 10 | openssl req -new -newkey rsa:4096 -days 365 -x509 -subj "/CN=docker" -keyout data/ssl/ca.key -out data/ssl/ca.cert -nodes 11 | ``` 12 | 13 | Generate the server keystore: 14 | 15 | ``` 16 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/server.keystore.jks -alias server -genkey -keyalg RSA -validity 365 -dname CN=server -ext SAN=DNS:server 17 | ``` 18 | 19 | Create the trust store: 20 | 21 | ``` 22 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/server.truststore.jks -alias ca -importcert -file data/ssl/ca.cert 23 | ``` 24 | 25 | Create a sign request: 26 | 27 | ``` 28 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/server.keystore.jks -alias server -certreq -file data/ssl/server.csr 29 | ``` 30 | 31 | Sign the certificate with the CA: 32 | 33 | ``` 34 | openssl x509 -req -CA data/ssl/ca.cert -CAkey data/ssl/ca.key -in data/ssl/server.csr -out data/ssl/server.pem -days 365 35 | ``` 36 | 37 | Import the CA and the signed certificate into the keystore: 38 | 39 | ``` 40 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/server.keystore.jks -alias ca -importcert -file data/ssl/ca.cert 41 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/server.keystore.jks -alias server -importcert -file data/ssl/server.pem 42 | ``` 43 | 44 | ## Setup mTLS 45 | 46 | Create the client certificate: 47 | 48 | ``` 49 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/client.keystore.jks -alias client -genkey -keyalg RSA -validity 365 -dname CN=client -ext SAN=DNS:client 50 | ``` 51 | 52 | Create a sign request: 53 | 54 | ``` 55 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/client.keystore.jks -alias client -certreq -file data/ssl/client.csr 56 | ``` 57 | 58 | Sign the certificate with the CA: 59 | 60 | ``` 61 | openssl x509 -req -CA data/ssl/ca.cert -CAkey data/ssl/ca.key -in data/ssl/client.csr -out data/ssl/client.pem -days 365 62 | ``` 63 | 64 | Export the certificate private key: 65 | 66 | ``` 67 | keytool -noprompt -importkeystore -srckeystore data/ssl/client.keystore.jks -srcalias client -srcstorepass 12345678 -deststorepass 12345678 -destkeypass 12345678 -destkeystore data/ssl/client.p12 -deststoretype PKCS12 68 | openssl pkcs12 -nocerts -legacy -in data/ssl/client.p12 -out data/ssl/client.key -passin pass:12345678 -nodes 69 | ``` 70 | 71 | Import the CA and the signed certificate into the keystore: 72 | 73 | ``` 74 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/client.keystore.jks -alias ca -importcert -file data/ssl/ca.cert 75 | keytool -storepass 12345678 -keypass 12345678 -noprompt -keystore data/ssl/client.keystore.jks -alias server -importcert -file data/ssl/client.pem 76 | ``` 77 | -------------------------------------------------------------------------------- /docs/internals/playground.md: -------------------------------------------------------------------------------- 1 | # Playground Files Documentation 2 | 3 | Each file in the `playground` directory is meant to be invoked via shell unless documented differently. 4 | 5 | ``` 6 | node playground/clients/admin-groups.ts 7 | ``` 8 | 9 | ## Admin API 10 | 11 | ### Topic management - Single broker 12 | 13 | The `playground/clients/admin-topics-single.ts` can be used to ensure the presence of `temp1` and `temp2` topics, used in other scripts. 14 | 15 | ### Topic management - Multiple brokers 16 | 17 | The `playground/clients/admin-topics-multiple.ts` can be used to ensure the presence of `temp1` and `temp2` topics, used in other scripts. 18 | 19 | It will try to specifically assign partitions to different brokers (as specified in `docker-compose-multiple.yml`). 20 | 21 | ### Consumer group and rebalancing management 22 | 23 | The `playground/clients/admin-groups.ts` will verify consumer group behaviour by performing the following flow: 24 | 25 | 1. Create a consumer (will create group). 26 | 2. Create a second consumer (will trigger rebalance). 27 | 3. Close second consumer (will trigger rebalance). 28 | 4. List and describe groups. 29 | 5. Delete group (will create new group). 30 | 6. Close first consumer (will trigger rebalance). 31 | 32 | ## Producer API 33 | 34 | All scripts in this API require the `playground/clients/admin-topics-single.ts` or `playground/clients/admin-topics-multiple.ts` to have been run first. 35 | 36 | ### Basic producer 37 | 38 | The `playground/clients/producer.ts` will perform a series of 3 produces with 6 messages each. 39 | 40 | ### Idempotent producer 41 | 42 | The `playground/clients/producer-idempotent.ts` will perform 4 produces of one message each. The produces are started 2 by 2 in parallel, but since Kafka limits the maximum in-flights for idempotent producers to 1, you should see 4 different offsets. 43 | 44 | ### Permanent producer 45 | 46 | The `playground/clients/producer-forever.ts` will perform continuous producing of a message every 100ms. Each message has an increasing value for keys, values, header keys and header values. It can be stopped only via `Ctrl+C`. 47 | 48 | ## Consumer API 49 | 50 | All scripts in this API require the `playground/clients/admin-topics-single.ts` or `playground/clients/admin-topics-multiple.ts` to have been run first. 51 | 52 | ### Basic consumer 53 | 54 | The `playground/clients/consumer.ts` will consume from a topic. Both `on('data')` and `for-await-of` are used. 55 | 56 | ### Consumer rebalance 57 | 58 | The `playground/clients/consumer-rebalance.ts` will verify assignments of topics and partitions by performing the following flow: 59 | 60 | 1. Create a consumer and subscribe to a topic. 61 | 2. Subscribe to a second topic (will trigger rebalance). 62 | 3. Create a second consumer (will trigger rebalance). 63 | 4. Close first consumer (will trigger rebalance). 64 | 5. Close second consumer. 65 | 66 | ## Low level APIs 67 | 68 | The files in the `playground/apis` test low-level APIs directly. 69 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | `@platformatic/kafka` supports integration with [Prometheus](https://prometheus.io/) via [prom-client](https://github.com/siimon/prom-client). 4 | 5 | To ensure maximum compatibility, no `prom-client` version is shipped with the package but you are instead required to provide your own via `metrics.client` option when creating any client. 6 | 7 | If you provide both the `metrics.client` and `metrics.registry` (an instance of `Registry`) options, then `@platformatic/kafka` will register and provide the following metrics: 8 | 9 | | Name | Type | Description | 10 | | ------------------------- | --------- | ----------------------------------------- | 11 | | `kafka_producers` | `Gauge` | Number of active Kafka producers. | 12 | | `kafka_produced_messages` | `Counter` | Number of produced Kafka messages. | 13 | | `kafka_consumers` | `Gauge` | Number of active Kafka consumers. | 14 | | `kafka_consumers_streams` | `Gauge` | Number of active Kafka consumers streams. | 15 | | `kafka_consumers_topics` | `Gauge` | Number of topics being consumed. | 16 | | `kafka_consumed_messages` | `Counter` | Number of consumed Kafka messages. | 17 | 18 | Optionally, you can provide labels with the `metrics.label` option. 19 | 20 | ## Example 21 | 22 | ```javascript 23 | import * as client from 'prom-client' 24 | import { Producer, stringSerializer } from '@platformatic/kafka' 25 | 26 | const registry = new client.Registry() 27 | 28 | // Create a producer with string serialisers 29 | const producer = new Producer({ 30 | clientId: 'my-producer', 31 | bootstrapBrokers: ['localhost:9092'], 32 | serializers: stringSerializers, 33 | metrics: { client, registry } 34 | }) 35 | 36 | // Send messages 37 | await producer.send({ 38 | messages: [ 39 | { 40 | topic: 'events', 41 | key: 'user-123', 42 | value: JSON.stringify({ name: 'John', action: 'login' }), 43 | headers: { source: 'web-app' } 44 | } 45 | ] 46 | }) 47 | 48 | // Close the producer when done 49 | await producer.close() 50 | 51 | console.log(JSON.stringify(await registry.getMetricsAsJSON(), null, 2)) 52 | ``` 53 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import { globalIgnores } from 'eslint/config' 2 | import neostandard from 'neostandard' 3 | 4 | const eslint = [ 5 | ...neostandard({ ts: true }), 6 | globalIgnores(['dist/', 'external/']), 7 | { 8 | files: ['**/*.ts'], 9 | rules: { 10 | '@typescript-eslint/consistent-type-imports': ['error', { fixStyle: 'inline-type-imports' }] 11 | } 12 | } 13 | ] 14 | 15 | export default eslint 16 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@platformatic/kafka", 3 | "version": "1.3.0", 4 | "description": "Modern and performant client for Apache Kafka", 5 | "homepage": "https://github.com/platformatic/kafka", 6 | "author": "Platformatic Inc. (https://platformatic.dev)", 7 | "license": "Apache-2.0", 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/platformatic/kafka.git" 11 | }, 12 | "keywords": [ 13 | "kafka" 14 | ], 15 | "bugs": { 16 | "url": "https://github.com/platformatic/kafka/issues" 17 | }, 18 | "private": false, 19 | "files": [ 20 | "dist", 21 | "LICENSE", 22 | "README.md" 23 | ], 24 | "type": "module", 25 | "exports": "./dist/index.js", 26 | "types": "./dist/index.d.ts", 27 | "scripts": { 28 | "build": "rm -rf dist && tsc -p tsconfig.base.json", 29 | "lint": "eslint --cache", 30 | "typecheck": "tsc -p . --noEmit", 31 | "format": "prettier -w benchmarks playground src test", 32 | "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'", 33 | "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=300000 'test/**/*.test.ts'", 34 | "ci": "npm run build && npm run lint && npm run test:ci", 35 | "prepublishOnly": "npm run build && npm run lint", 36 | "postpublish": "git push origin && git push origin -f --tags", 37 | "generate:apis": "node --experimental-strip-types scripts/generate-apis.ts", 38 | "generate:errors": "node --experimental-strip-types scripts/generate-errors.ts", 39 | "create:api": "node --experimental-strip-types scripts/create-api.ts" 40 | }, 41 | "dependencies": { 42 | "@watchable/unpromise": "^1.0.2", 43 | "ajv": "^8.17.1", 44 | "ajv-errors": "^3.0.0", 45 | "debug": "^4.4.0", 46 | "fastq": "^1.19.1", 47 | "mnemonist": "^0.40.3", 48 | "scule": "^1.3.0" 49 | }, 50 | "optionalDependencies": { 51 | "lz4-napi": "^2.8.0", 52 | "snappy": "^7.2.2" 53 | }, 54 | "devDependencies": { 55 | "@platformatic/rdkafka": "^4.0.0", 56 | "@types/debug": "^4.1.12", 57 | "@types/node": "^22.13.5", 58 | "@types/semver": "^7.7.0", 59 | "c8": "^10.1.3", 60 | "cleaner-spec-reporter": "^0.4.0", 61 | "cronometro": "^5.3.0", 62 | "eslint": "^9.21.0", 63 | "hwp": "^0.4.1", 64 | "json5": "^2.2.3", 65 | "kafkajs": "^2.2.4", 66 | "neostandard": "^0.12.1", 67 | "node-rdkafka": "^3.3.1", 68 | "parse5": "^7.2.1", 69 | "prettier": "^3.5.3", 70 | "prom-client": "^15.1.3", 71 | "semver": "^7.7.1", 72 | "typescript": "^5.7.3" 73 | }, 74 | "engines": { 75 | "node": ">= 22.14.0" 76 | } 77 | } -------------------------------------------------------------------------------- /playground/apis/admin/acl.ts: -------------------------------------------------------------------------------- 1 | import { api as createAclsV3 } from '../../../src/apis/admin/create-acls-v3.ts' 2 | import { api as deleteAclsV3 } from '../../../src/apis/admin/delete-acls-v3.ts' 3 | import { api as describeAclsV3 } from '../../../src/apis/admin/describe-acls-v3.ts' 4 | import { 5 | AclOperations, 6 | AclPermissionTypes, 7 | ResourcePatternTypes, 8 | ResourceTypes 9 | } from '../../../src/apis/enumerations.ts' 10 | import { Connection } from '../../../src/network/connection.ts' 11 | import { performAPICallWithRetry } from '../../utils.ts' 12 | 13 | const connection = new Connection('123') 14 | await connection.connect('localhost', 9092) 15 | 16 | await performAPICallWithRetry('CreateAcls', () => 17 | createAclsV3.async(connection, [ 18 | { 19 | resourceType: ResourceTypes.TOPIC, 20 | resourceName: 'temp', 21 | resourcePatternType: ResourcePatternTypes.LITERAL, 22 | principal: 'abc:cde', 23 | host: '*', 24 | operation: AclOperations.READ, 25 | permissionType: AclPermissionTypes.DENY 26 | } 27 | ]) 28 | ) 29 | 30 | await performAPICallWithRetry('DescribeAcls', () => 31 | describeAclsV3.async( 32 | connection, 33 | ResourceTypes.TOPIC, 34 | 'temp', 35 | ResourcePatternTypes.LITERAL, 36 | null, 37 | null, 38 | AclOperations.READ, 39 | AclPermissionTypes.DENY 40 | ) 41 | ) 42 | 43 | await performAPICallWithRetry('DescribeAcls', () => 44 | describeAclsV3.async( 45 | connection, 46 | ResourceTypes.TOPIC, 47 | 'temp', 48 | ResourcePatternTypes.LITERAL, 49 | null, 50 | null, 51 | AclOperations.READ, 52 | AclPermissionTypes.ALLOW 53 | ) 54 | ) 55 | 56 | await performAPICallWithRetry('DeleteAcls', () => 57 | deleteAclsV3.async(connection, [ 58 | { 59 | resourceTypeFilter: ResourceTypes.TOPIC, 60 | resourceNameFilter: 'temp', 61 | patternTypeFilter: ResourcePatternTypes.LITERAL, 62 | principalFilter: null, 63 | hostFilter: null, 64 | operation: AclOperations.READ, 65 | permissionType: AclPermissionTypes.DENY 66 | } 67 | ]) 68 | ) 69 | 70 | await connection.close() 71 | -------------------------------------------------------------------------------- /playground/apis/admin/assignments.ts: -------------------------------------------------------------------------------- 1 | import { api as alterPartitionReassignmentsV0 } from '../../../src/apis/admin/alter-partition-reassignments-v0.ts' 2 | import { api as listPartitionReassignmentsV0 } from '../../../src/apis/admin/list-partition-reassignments-v0.ts' 3 | import { Connection } from '../../../src/network/connection.ts' 4 | import { performAPICallWithRetry } from '../../utils.ts' 5 | 6 | const connection = new Connection('123') 7 | await connection.connect('localhost', 9092) 8 | 9 | await performAPICallWithRetry('AlterPartitionReassignments', () => 10 | alterPartitionReassignmentsV0.async(connection, 1000, [ 11 | { 12 | name: 'temp', 13 | partitions: [ 14 | { 15 | partitionIndex: 0, 16 | replicas: [1] 17 | } 18 | ] 19 | } 20 | ]) 21 | ) 22 | 23 | await performAPICallWithRetry('ListPartitionReassignments', () => 24 | listPartitionReassignmentsV0.async(connection, 1000, [ 25 | { 26 | name: 'temp', 27 | partitionIndexes: [0] 28 | } 29 | ]) 30 | ) 31 | 32 | await connection.close() 33 | -------------------------------------------------------------------------------- /playground/apis/admin/client-quotas.ts: -------------------------------------------------------------------------------- 1 | import { api as alterClientQuotasV1 } from '../../../src/apis/admin/alter-client-quotas-v1.ts' 2 | import { api as describeClientQuotasV0 } from '../../../src/apis/admin/describe-client-quotas-v0.ts' 3 | import { ClientQuotaMatchTypes } from '../../../src/apis/enumerations.ts' 4 | import { Connection } from '../../../src/network/connection.ts' 5 | import { performAPICallWithRetry } from '../../utils.ts' 6 | 7 | const connection = new Connection('123') 8 | await connection.connect('localhost', 9092) 9 | 10 | await performAPICallWithRetry('DescribeClientQuotas', () => 11 | describeClientQuotasV0.async( 12 | connection, 13 | [ 14 | { 15 | entityType: 'client-id', 16 | matchType: ClientQuotaMatchTypes.EXACT, 17 | match: 'test-id' 18 | } 19 | ], 20 | false 21 | ) 22 | ) 23 | 24 | await performAPICallWithRetry('AlterClientQuotas', () => 25 | alterClientQuotasV1.async( 26 | connection, 27 | [ 28 | { 29 | entities: [ 30 | { 31 | entityType: 'client-id', 32 | entityName: 'test-id' 33 | } 34 | ], 35 | ops: [ 36 | { 37 | key: 'producer_byte_rate', 38 | value: 1000, 39 | remove: false 40 | } 41 | ] 42 | } 43 | ], 44 | false 45 | ) 46 | ) 47 | 48 | await performAPICallWithRetry('DescribeClientQuotas', () => 49 | describeClientQuotasV0.async( 50 | connection, 51 | [ 52 | { 53 | entityType: 'client-id', 54 | matchType: ClientQuotaMatchTypes.EXACT, 55 | match: 'test-id' 56 | } 57 | ], 58 | false 59 | ) 60 | ) 61 | 62 | await connection.close() 63 | -------------------------------------------------------------------------------- /playground/apis/admin/config.ts: -------------------------------------------------------------------------------- 1 | import { api as alterConfigsV2 } from '../../../src/apis/admin/alter-configs-v2.ts' 2 | import { api as describeConfigsV4 } from '../../../src/apis/admin/describe-configs-v4.ts' 3 | import { api as incrementalAlterConfigsV1 } from '../../../src/apis/admin/incremental-alter-configs-v1.ts' 4 | import { IncrementalAlterConfigTypes, ResourceTypes } from '../../../src/apis/enumerations.ts' 5 | import { Connection } from '../../../src/network/connection.ts' 6 | import { performAPICallWithRetry } from '../../utils.ts' 7 | 8 | const connection = new Connection('123') 9 | await connection.connect('localhost', 9092) 10 | 11 | await performAPICallWithRetry('DescribeConfigs', () => 12 | describeConfigsV4.async( 13 | connection, 14 | [ 15 | { 16 | resourceType: ResourceTypes.CLUSTER, 17 | resourceName: '4', 18 | configurationKeys: ['log.retention.ms', 'log.retention.bytes', 'offsets.retention.minutes'] 19 | } 20 | ], 21 | false, 22 | false 23 | ) 24 | ) 25 | 26 | await performAPICallWithRetry('AlterConfig', () => 27 | alterConfigsV2.async( 28 | connection, 29 | [ 30 | { 31 | resourceType: ResourceTypes.CLUSTER, 32 | resourceName: '1', 33 | configs: [ 34 | { 35 | name: 'compression.type', 36 | value: 'gzip' 37 | } 38 | ] 39 | } 40 | ], 41 | false 42 | ) 43 | ) 44 | 45 | await performAPICallWithRetry('IncrementalAlterConfig', () => 46 | incrementalAlterConfigsV1.async( 47 | connection, 48 | [ 49 | { 50 | resourceType: ResourceTypes.CLUSTER, 51 | resourceName: '1', 52 | configs: [ 53 | { 54 | name: 'compression.type', 55 | configOperation: IncrementalAlterConfigTypes.SET, 56 | value: 'gzip' 57 | } 58 | ] 59 | } 60 | ], 61 | false 62 | ) 63 | ) 64 | 65 | await performAPICallWithRetry('DescribeConfigs', () => 66 | describeConfigsV4.async( 67 | connection, 68 | [ 69 | { 70 | resourceType: ResourceTypes.CLUSTER, 71 | resourceName: '1', 72 | configurationKeys: ['compression.type'] 73 | } 74 | ], 75 | false, 76 | false 77 | ) 78 | ) 79 | 80 | await connection.close() 81 | -------------------------------------------------------------------------------- /playground/apis/admin/create-topics.ts: -------------------------------------------------------------------------------- 1 | import { api as createTopicsV7 } from '../../../src/apis/admin/create-topics-v7.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123') 6 | await connection.connect('localhost', 9092) 7 | 8 | await performAPICallWithRetry('CreateTopics', () => 9 | createTopicsV7.async( 10 | connection, 11 | [ 12 | { 13 | name: 'temp', 14 | numPartitions: 1, 15 | replicationFactor: 1, 16 | assignments: [], 17 | configs: [] 18 | } 19 | ], 20 | 1000, 21 | false 22 | ) 23 | ) 24 | 25 | await connection.close() 26 | -------------------------------------------------------------------------------- /playground/apis/admin/delete-records.ts: -------------------------------------------------------------------------------- 1 | import { api as deleteRecordsV2 } from '../../../src/apis/admin/delete-records-v2.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123') 6 | await connection.connect('localhost', 9092) 7 | 8 | await performAPICallWithRetry('DeleteRecords', () => 9 | deleteRecordsV2.async( 10 | connection, 11 | [ 12 | { 13 | name: 'temp', 14 | partitions: [ 15 | { 16 | partitionIndex: 0, 17 | offset: 58n 18 | } 19 | ] 20 | } 21 | ], 22 | 1000 23 | ) 24 | ) 25 | 26 | await connection.close() 27 | -------------------------------------------------------------------------------- /playground/apis/admin/delete-topics.ts: -------------------------------------------------------------------------------- 1 | import { api as deleteTopicsV6 } from '../../../src/apis/admin/delete-topics-v6.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123') 6 | await connection.connect('localhost', 9092) 7 | 8 | await performAPICallWithRetry('DeleteTopics', () => 9 | deleteTopicsV6.async( 10 | connection, 11 | [ 12 | { 13 | name: 'temp' 14 | } 15 | ], 16 | 1000 17 | ) 18 | ) 19 | 20 | await connection.close() 21 | -------------------------------------------------------------------------------- /playground/apis/admin/group.ts: -------------------------------------------------------------------------------- 1 | import { api as deleteGroupsV2 } from '../../../src/apis/admin/delete-groups-v2.ts' 2 | import { api as describeGroupsV5 } from '../../../src/apis/admin/describe-groups-v5.ts' 3 | import { api as listGroupsV5 } from '../../../src/apis/admin/list-groups-v5.ts' 4 | import { FindCoordinatorKeyTypes } from '../../../src/apis/enumerations.ts' 5 | import { api as findCoordinatorV6 } from '../../../src/apis/metadata/find-coordinator-v6.ts' 6 | import { Connection } from '../../../src/network/connection.ts' 7 | import { performAPICallWithRetry } from '../../utils.ts' 8 | 9 | const connection = new Connection('123') 10 | await connection.connect('localhost', 9092) 11 | 12 | const groupId = 'g2' 13 | 14 | await performAPICallWithRetry('FindCoordinator (GROUP)', () => 15 | findCoordinatorV6.async(connection, FindCoordinatorKeyTypes.GROUP, [groupId]) 16 | ) 17 | 18 | await performAPICallWithRetry('DescribeGroups', () => describeGroupsV5.async(connection, [groupId], true)) 19 | 20 | await performAPICallWithRetry('ListGroups', () => listGroupsV5.async(connection, ['STABLE'], [])) 21 | 22 | await performAPICallWithRetry('DeleteGroups', () => deleteGroupsV2.async(connection, [groupId])) 23 | 24 | await connection.close() 25 | -------------------------------------------------------------------------------- /playground/apis/admin/log-dirs.ts: -------------------------------------------------------------------------------- 1 | import { api as alterReplicaLogDirsV2 } from '../../../src/apis/admin/alter-replica-log-dirs-v2.ts' 2 | import { api as describeLogDirsV4 } from '../../../src/apis/admin/describe-log-dirs-v4.ts' 3 | import { Connection } from '../../../src/network/connection.ts' 4 | import { performAPICallWithRetry } from '../../utils.ts' 5 | 6 | const connection = new Connection('123') 7 | await connection.connect('localhost', 9092) 8 | 9 | await performAPICallWithRetry('DescribeLogDirs', () => 10 | describeLogDirsV4.async(connection, [{ name: 'temp', partitions: [0] }]) 11 | ) 12 | 13 | await performAPICallWithRetry('AlterReplicaLogDirs', () => 14 | alterReplicaLogDirsV2.async(connection, [ 15 | { path: '/tmp/kraft-combined-logs', topics: [{ name: 'temp', partitions: [0] }] } 16 | ]) 17 | ) 18 | 19 | await performAPICallWithRetry('DescribeLogDirs', () => 20 | describeLogDirsV4.async(connection, [{ name: 'temp', partitions: [0] }]) 21 | ) 22 | 23 | await connection.close() 24 | -------------------------------------------------------------------------------- /playground/apis/admin/offset-delete.ts: -------------------------------------------------------------------------------- 1 | import { api as offsetDeleteV0 } from '../../../src/apis/admin/offset-delete-v0.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123') 6 | await connection.connect('localhost', 9092) 7 | 8 | await performAPICallWithRetry('OffsetDelete', () => 9 | offsetDeleteV0.async(connection, 'g2', [ 10 | { 11 | name: 'temp', 12 | partitions: [ 13 | { 14 | partitionIndex: 0 15 | } 16 | ] 17 | } 18 | ]) 19 | ) 20 | 21 | await connection.close() 22 | -------------------------------------------------------------------------------- /playground/apis/admin/partitions.ts: -------------------------------------------------------------------------------- 1 | import { api as alterPartitionV3 } from '../../../src/apis/admin/alter-partition-v3.ts' 2 | import { api as createPartitionsV3 } from '../../../src/apis/admin/create-partitions-v3.ts' 3 | import { Connection } from '../../../src/network/connection.ts' 4 | import { performAPICallWithRetry } from '../../utils.ts' 5 | 6 | const performCreatePartitions = false 7 | const performAlterPartition = false 8 | 9 | const connection = new Connection('123') 10 | await connection.connect('localhost', 9092) 11 | 12 | if (performCreatePartitions) { 13 | await performAPICallWithRetry('CreatePartitions', () => 14 | createPartitionsV3.async( 15 | connection, 16 | [ 17 | { 18 | name: 'temp', 19 | count: 2, 20 | assignments: [ 21 | { 22 | brokerIds: [1] 23 | } 24 | ] 25 | } 26 | ], 27 | 1000, 28 | true 29 | ) 30 | ) 31 | } 32 | 33 | if (performAlterPartition) { 34 | await performAPICallWithRetry('AlterPartition', () => 35 | alterPartitionV3.async(connection, 1, -1n, [ 36 | { 37 | topicId: '8d5e2533-cd5c-4498-9ec2-a4d7f741a044', 38 | partitions: [ 39 | { 40 | partitionIndex: 0, 41 | leaderEpoch: 0, 42 | newIsrWithEpochs: [{ brokerId: 1, brokerEpoch: -1n }], 43 | leaderRecoveryState: 0, 44 | partitionEpoch: 0 45 | } 46 | ] 47 | } 48 | ]) 49 | ) 50 | } 51 | 52 | await connection.close() 53 | -------------------------------------------------------------------------------- /playground/apis/admin/scram.ts: -------------------------------------------------------------------------------- 1 | import { api as alterUserScramCredentialsV0 } from '../../../src/apis/admin/alter-user-scram-credentials-v0.ts' 2 | import { api as describeUserScramCredentialsV0 } from '../../../src/apis/admin/describe-user-scram-credentials-v0.ts' 3 | import { ScramMechanisms } from '../../../src/apis/enumerations.ts' 4 | import { Connection } from '../../../src/network/connection.ts' 5 | import { performAPICallWithRetry } from '../../utils.ts' 6 | 7 | const connection = new Connection('123') 8 | await connection.connect('localhost', 9092) 9 | 10 | await performAPICallWithRetry('AlterUserScramCredentials', () => 11 | alterUserScramCredentialsV0.async( 12 | connection, 13 | [], 14 | [ 15 | { 16 | name: 'user', 17 | mechanism: ScramMechanisms.SCRAM_SHA_256, 18 | iterations: 16384, 19 | salt: Buffer.alloc(10), 20 | saltedPassword: Buffer.alloc(20) 21 | } 22 | ] 23 | ) 24 | ) 25 | 26 | await performAPICallWithRetry('DescribeUserScramCredentials', () => 27 | describeUserScramCredentialsV0.async(connection, [{ name: 'user' }]) 28 | ) 29 | 30 | await connection.close() 31 | -------------------------------------------------------------------------------- /playground/apis/consumer/group.ts: -------------------------------------------------------------------------------- 1 | import { api as consumerGroupHeartbeatV0 } from '../../../src/apis/consumer/consumer-group-heartbeat-v0.ts' 2 | import { api as heartbeatV4 } from '../../../src/apis/consumer/heartbeat-v4.ts' 3 | import { api as leaveGroupV5 } from '../../../src/apis/consumer/leave-group-v5.ts' 4 | import { api as syncGroupV5 } from '../../../src/apis/consumer/sync-group-v5.ts' 5 | import { FindCoordinatorKeyTypes } from '../../../src/apis/enumerations.ts' 6 | import { api as findCoordinatorV6 } from '../../../src/apis/metadata/find-coordinator-v6.ts' 7 | import { Connection } from '../../../src/network/connection.ts' 8 | import { joinGroup, performAPICallWithRetry } from '../../utils.ts' 9 | 10 | const performConsumerHeartbeat = false 11 | 12 | const connection = new Connection('123') 13 | await connection.connect('localhost', 9092) 14 | 15 | const topicName = 'temp' 16 | const groupId = 'g2' 17 | 18 | await performAPICallWithRetry('FindCoordinator (GROUP)', () => 19 | findCoordinatorV6.async(connection, FindCoordinatorKeyTypes.GROUP, [groupId]) 20 | ) 21 | 22 | const joinGroupResponse = await joinGroup(connection, groupId) 23 | const memberId = joinGroupResponse.memberId! 24 | 25 | await performAPICallWithRetry('Heartbeat', () => 26 | heartbeatV4.async(connection, groupId, joinGroupResponse.generationId, memberId) 27 | ) 28 | 29 | if (performConsumerHeartbeat) { 30 | await performAPICallWithRetry('ConsumerHeartbeat', () => 31 | consumerGroupHeartbeatV0.async(connection, groupId, memberId, 0, null, null, 60000, [topicName], null, []) 32 | ) 33 | } 34 | 35 | await performAPICallWithRetry('SyncGroup', () => 36 | syncGroupV5.async(connection, groupId, joinGroupResponse.generationId, memberId, null, 'whatever', 'whatever', []) 37 | ) 38 | 39 | await performAPICallWithRetry('LeaveGroup', () => leaveGroupV5.async(connection, groupId, [{ memberId }])) 40 | 41 | await connection.close() 42 | -------------------------------------------------------------------------------- /playground/apis/consumer/list-offsets.ts: -------------------------------------------------------------------------------- 1 | import { api as listOffsetsV9 } from '../../../src/apis/consumer/list-offsets-v9.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123') 6 | await connection.connect('localhost', 9092) 7 | 8 | await performAPICallWithRetry('ListOffsets', () => 9 | listOffsetsV9.async(connection, -1, 0, [ 10 | { name: 'temp', partitions: [{ partitionIndex: 0, currentLeaderEpoch: -1, timestamp: -1n }] } 11 | ]) 12 | ) 13 | 14 | await connection.close() 15 | -------------------------------------------------------------------------------- /playground/apis/metadata/api-versions.ts: -------------------------------------------------------------------------------- 1 | import { api as apiVersionsV3 } from '../../../src/apis/metadata/api-versions-v3.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123', { 6 | // tls: { 7 | // rejectUnauthorized: false, 8 | // cert: await readFile(resolve(import.meta.dirname, '../../ssl/client.pem')), 9 | // key: await readFile(resolve(import.meta.dirname, '../../ssl/client.key')) 10 | // } 11 | }) 12 | await connection.connect('localhost', 9092) 13 | 14 | await performAPICallWithRetry('ApiVersions', () => apiVersionsV3.async(connection, 'kafka', '1.0.0')) 15 | 16 | await connection.close() 17 | -------------------------------------------------------------------------------- /playground/apis/metadata/find-coordinator.ts: -------------------------------------------------------------------------------- 1 | import { FindCoordinatorKeyTypes } from '../../../src/apis/enumerations.ts' 2 | import { api as findCoordinatorV6 } from '../../../src/apis/metadata/find-coordinator-v6.ts' 3 | import { Connection } from '../../../src/network/connection.ts' 4 | import { performAPICallWithRetry } from '../../utils.ts' 5 | 6 | const connection = new Connection('foo') 7 | await connection.connect('localhost', 9092) 8 | 9 | await performAPICallWithRetry('FindCoordinator (GROUP)', () => 10 | findCoordinatorV6.async(connection, FindCoordinatorKeyTypes.GROUP, ['f1']) 11 | ) 12 | 13 | await performAPICallWithRetry('FindCoordinator (TRANSACTION)', () => 14 | findCoordinatorV6.async(connection, FindCoordinatorKeyTypes.TRANSACTION, ['f1']) 15 | ) 16 | 17 | await connection.close() 18 | -------------------------------------------------------------------------------- /playground/apis/metadata/metadata.ts: -------------------------------------------------------------------------------- 1 | import { api as metadataV12 } from '../../../src/apis/metadata/metadata-v12.ts' 2 | import { api as saslAuthenticateV2 } from '../../../src/apis/security/sasl-authenticate-v2.ts' 3 | import { api as saslHandshakeV1 } from '../../../src/apis/security/sasl-handshake-v1.ts' 4 | import { Connection } from '../../../src/network/connection.ts' 5 | import { authenticate } from '../../../src/protocol/sasl/scram-sha.ts' 6 | import { performAPICallWithRetry } from '../../utils.ts' 7 | 8 | const sasl = false 9 | const connection = new Connection('123') 10 | await connection.connect('localhost', 9092) 11 | 12 | if (sasl) { 13 | // await performAPICallWithRetry('SaslHandshake', () => saslHandshakeV1.async(connection, 'PLAIN')) 14 | 15 | // await performAPICallWithRetry('SaslAuthenticate', () => 16 | // saslAuthenticateV2.async(connection, createAuthenticationRequest('client', 'client')) 17 | // ) 18 | 19 | await performAPICallWithRetry('SaslHandshake', () => saslHandshakeV1.async(connection, 'SCRAM-SHA-256')) 20 | 21 | await performAPICallWithRetry('SaslAuthenticate', () => 22 | authenticate(saslAuthenticateV2, connection, 'SHA-256', 'client', 'client') 23 | ) 24 | } 25 | 26 | await performAPICallWithRetry('Metadata', () => metadataV12.async(connection, null, false, true)) 27 | 28 | await connection.close() 29 | -------------------------------------------------------------------------------- /playground/apis/producer/init-producer-id.ts: -------------------------------------------------------------------------------- 1 | import { FindCoordinatorKeyTypes } from '../../../src/apis/enumerations.ts' 2 | import { api as findCoordinatorV6 } from '../../../src/apis/metadata/find-coordinator-v6.ts' 3 | import { api as initProducerIdV5 } from '../../../src/apis/producer/init-producer-id-v5.ts' 4 | import { Connection } from '../../../src/network/connection.ts' 5 | import { performAPICallWithRetry } from '../../utils.ts' 6 | 7 | const transactionalId = 'eeeee' 8 | 9 | const connection = new Connection('111') 10 | await connection.connect('localhost', 9092) 11 | 12 | await performAPICallWithRetry('FindCoordinator (TRANSACTION)', () => 13 | findCoordinatorV6.async(connection, FindCoordinatorKeyTypes.TRANSACTION, [transactionalId]) 14 | ) 15 | 16 | await performAPICallWithRetry('InitProducerId', () => 17 | initProducerIdV5.async(connection, transactionalId, 60000, -1n, -1) 18 | ) 19 | 20 | await connection.close() 21 | -------------------------------------------------------------------------------- /playground/apis/producer/produce-idempotent.ts: -------------------------------------------------------------------------------- 1 | import { api as initProducerIdV5 } from '../../../src/apis/producer/init-producer-id-v5.ts' 2 | import { api as produceV11 } from '../../../src/apis/producer/produce-v11.ts' 3 | import { Connection } from '../../../src/network/connection.ts' 4 | import { performAPICallWithRetry } from '../../utils.ts' 5 | 6 | const connection = new Connection('my-client') 7 | await connection.connect('localhost', 9092) 8 | 9 | const { producerId, producerEpoch } = await initProducerIdV5.async(connection, null, 1000, 0n, 0) 10 | 11 | // The following two calls are identical. Since the same producerId and producerEpoch are used, 12 | // the second call will be discarded from the server 13 | await performAPICallWithRetry('Produce', () => 14 | produceV11.async( 15 | connection, 16 | 1, 17 | 0, 18 | [ 19 | { 20 | topic: 'temp1', 21 | partition: 0, 22 | key: Buffer.from('111'), 23 | value: Buffer.from('222'), 24 | headers: new Map([ 25 | [Buffer.from('a'), Buffer.from('123')], 26 | [Buffer.from('b'), Buffer.from([97, 98, 99])] 27 | ]) 28 | }, 29 | { topic: 'temp1', partition: 0, key: Buffer.from('333'), value: Buffer.from('444'), timestamp: 12345678n } 30 | ], 31 | { compression: 'zstd', producerId, producerEpoch } 32 | ) 33 | ) 34 | 35 | await performAPICallWithRetry('Produce', () => 36 | produceV11.async( 37 | connection, 38 | 1, 39 | 0, 40 | [ 41 | { 42 | topic: 'temp1', 43 | partition: 0, 44 | key: Buffer.from('111'), 45 | value: Buffer.from('222'), 46 | headers: new Map([ 47 | [Buffer.from('a'), Buffer.from('123')], 48 | [Buffer.from('b'), Buffer.from([97, 98, 99])] 49 | ]) 50 | }, 51 | { topic: 'temp1', partition: 0, key: Buffer.from('333'), value: Buffer.from('444'), timestamp: 12345678n } 52 | ], 53 | { compression: 'zstd', producerId, producerEpoch } 54 | ) 55 | ) 56 | 57 | await connection.close() 58 | -------------------------------------------------------------------------------- /playground/apis/producer/produce.ts: -------------------------------------------------------------------------------- 1 | import { api as produceV11 } from '../../../src/apis/producer/produce-v11.ts' 2 | import { Connection } from '../../../src/network/connection.ts' 3 | import { performAPICallWithRetry } from '../../utils.ts' 4 | 5 | const connection = new Connection('123') 6 | await connection.connect('localhost', 9092) 7 | 8 | const NUM_RECORDS = 1e4 9 | const prefix = Date.now().toString() 10 | 11 | for (let i = 0; i < NUM_RECORDS; i++) { 12 | const key = `${prefix}-${i.toString().padStart(3, '0')}` 13 | 14 | await performAPICallWithRetry('Produce', () => 15 | produceV11.async(connection, 1, 0, [ 16 | // { 17 | // topic: 'temp', 18 | // partition: 0, 19 | // key: Buffer.from('aaa'), 20 | // value: Buffer.from('bbb'), 21 | // headers: new Map([[Buffer.from('ccc'), Buffer.from('ddd')]]) 22 | // } 23 | { 24 | topic: 'temp1', 25 | partition: 0, 26 | key: Buffer.from(key + '-1'), 27 | value: Buffer.from(Math.floor(Math.random() * 1e5).toString()), 28 | headers: new Map([[Buffer.from('key'), Buffer.from('value')]]) 29 | }, 30 | { 31 | topic: 'temp2', 32 | partition: 0, 33 | key: Buffer.from(key + '-2'), 34 | value: Buffer.from(Math.floor(Math.random() * 1e5).toString()), 35 | headers: new Map([[Buffer.from('key'), Buffer.from('value')]]) 36 | } 37 | ]) 38 | ) 39 | } 40 | 41 | await connection.close() 42 | -------------------------------------------------------------------------------- /playground/apis/telemetry/telemetry.ts: -------------------------------------------------------------------------------- 1 | import { CompressionTypes } from 'kafkajs' 2 | import { api as getTelemetrySubscriptionsV0 } from '../../../src/apis/telemetry/get-telemetry-subscriptions-v0.ts' 3 | import { api as listClientMetricsResourcesV0 } from '../../../src/apis/telemetry/list-client-metrics-resources-v0.ts' 4 | import { api as pushTelemetryV0 } from '../../../src/apis/telemetry/push-telemetry-v0.ts' 5 | import { Connection } from '../../../src/network/connection.ts' 6 | import { performAPICallWithRetry } from '../../utils.ts' 7 | 8 | const connection = new Connection('123') 9 | await connection.connect('localhost', 9092) 10 | 11 | const { clientInstanceId, subscriptionId } = await performAPICallWithRetry('GetTelemetrySubscriptions', () => 12 | getTelemetrySubscriptionsV0.async(connection) 13 | ) 14 | 15 | await performAPICallWithRetry('ListClientMetricsResources', () => listClientMetricsResourcesV0.async(connection)) 16 | 17 | await performAPICallWithRetry('PushTelemetry', () => 18 | pushTelemetryV0.async( 19 | connection, 20 | clientInstanceId, 21 | subscriptionId, 22 | false, 23 | CompressionTypes.None, 24 | Buffer.from('metrics') 25 | ) 26 | ) 27 | await connection.close() 28 | -------------------------------------------------------------------------------- /playground/clients/admin-groups.ts: -------------------------------------------------------------------------------- 1 | import { Admin, Consumer, debugDump } from '../../src/index.ts' 2 | 3 | const consumer1 = new Consumer({ 4 | groupId: 'id1', 5 | clientId: 'id', 6 | bootstrapBrokers: ['localhost:9092'], 7 | strict: true 8 | }) 9 | 10 | const consumer2 = new Consumer({ 11 | groupId: 'id2', 12 | clientId: 'id', 13 | bootstrapBrokers: ['localhost:9092'], 14 | strict: true 15 | }) 16 | 17 | consumer1.topics.track('temp1') 18 | await consumer1.joinGroup({ sessionTimeout: 10000, heartbeatInterval: 500, rebalanceTimeout: 15000 }) 19 | await consumer2.joinGroup({ sessionTimeout: 10000, heartbeatInterval: 500, rebalanceTimeout: 15000 }) 20 | 21 | const admin = new Admin({ clientId: 'id', bootstrapBrokers: ['localhost:9092'], strict: true }) 22 | 23 | debugDump('listGroups', await admin.listGroups()) 24 | debugDump('describeGroups', await admin.describeGroups({ groups: ['id1', 'id2'] })) 25 | debugDump('describeGroups', await admin.describeGroups({ groups: ['id1'] })) 26 | debugDump('describeGroups', await admin.describeGroups({ groups: ['id2'] })) 27 | 28 | await consumer2.close() 29 | 30 | await admin.deleteGroups({ groups: ['id2'] }) 31 | 32 | await consumer1.close() 33 | 34 | await admin.close() 35 | -------------------------------------------------------------------------------- /playground/clients/admin-topics-multiple.ts: -------------------------------------------------------------------------------- 1 | import { Admin, debugDump, sleep } from '../../src/index.ts' 2 | 3 | const retries = 0 4 | const admin = new Admin({ clientId: 'id', bootstrapBrokers: ['localhost:9092'], retries, strict: true }) 5 | const metadataDelay = retries === 0 ? 500 : 0 6 | 7 | try { 8 | await admin.deleteTopics({ topics: ['temp1', 'temp2'] }) 9 | } catch (e) { 10 | // Noop 11 | } 12 | 13 | await admin.createTopics({ topics: ['temp1'], partitions: 3, replicas: 1 }) 14 | await admin.createTopics({ 15 | topics: ['temp2'], 16 | partitions: -1, 17 | replicas: -1, 18 | assignments: [ 19 | { partition: 0, brokers: [6] }, 20 | { partition: 1, brokers: [4] }, 21 | { partition: 2, brokers: [5] } 22 | ] 23 | }) 24 | 25 | await sleep(metadataDelay) 26 | 27 | debugDump('metadata', await admin.metadata({ topics: ['temp1', 'temp2'] })) 28 | 29 | await admin.close() 30 | -------------------------------------------------------------------------------- /playground/clients/admin-topics-single.ts: -------------------------------------------------------------------------------- 1 | import { Admin, debugDump, sleep } from '../../src/index.ts' 2 | 3 | const retries = 0 4 | const admin = new Admin({ clientId: 'id', bootstrapBrokers: ['localhost:9092'], retries, strict: true }) 5 | const metadataDelay = retries === 0 ? 500 : 0 6 | 7 | try { 8 | await admin.deleteTopics({ topics: ['temp1', 'temp2'] }) 9 | } catch (e) { 10 | // Noop 11 | } 12 | 13 | await admin.createTopics({ topics: ['temp1', 'temp2'], partitions: 3, replicas: 1 }) 14 | await sleep(metadataDelay) 15 | debugDump('metadata', await admin.metadata({ topics: ['temp1', 'temp2'] })) 16 | 17 | await admin.close() 18 | -------------------------------------------------------------------------------- /playground/clients/consumer-hwp.ts: -------------------------------------------------------------------------------- 1 | import { forEach } from 'hwp' 2 | import { once } from 'node:events' 3 | import { setTimeout as sleep } from 'node:timers/promises' 4 | import { Consumer, debugDump, stringDeserializers } from '../../src/index.ts' 5 | 6 | // const consumer = new Consumer({ groupId: 'id7', clientId: 'id', bootstrapBrokers: ['localhost:9092'], strict: true }) 7 | const consumer = new Consumer({ 8 | groupId: 'id9', 9 | clientId: 'id', 10 | bootstrapBrokers: ['localhost:9092'], 11 | strict: true, 12 | deserializers: stringDeserializers 13 | }) 14 | 15 | const stream = await consumer.consume({ 16 | autocommit: false, 17 | topics: ['temp1', 'temp2'], 18 | sessionTimeout: 10000, 19 | heartbeatInterval: 500, 20 | maxWaitTime: 500, 21 | mode: 'earliest' 22 | }) 23 | 24 | // This is purposely not catched to show the error handling if we remove the force parameter 25 | once(process, 'SIGINT').then(() => consumer.close(true)) 26 | 27 | debugDump('start') 28 | 29 | await forEach( 30 | stream, 31 | async message => { 32 | console.log('data', message.partition, message.offset, message.key, message.value, message.headers) 33 | await sleep(1000) 34 | console.log('done', message.partition, message.offset, message.key, message.value, message.headers) 35 | }, 36 | 16 37 | ) 38 | 39 | debugDump('end') 40 | await consumer.close() 41 | -------------------------------------------------------------------------------- /playground/clients/consumer-rebalance.ts: -------------------------------------------------------------------------------- 1 | import { once } from 'node:events' 2 | import { Consumer, debugDump, stringDeserializers } from '../../src/index.ts' 3 | 4 | const consumer1 = new Consumer({ 5 | groupId: 'id9', 6 | clientId: 'id', 7 | bootstrapBrokers: ['localhost:9092'], 8 | strict: true, 9 | deserializers: stringDeserializers 10 | }) 11 | 12 | const consumer2 = new Consumer({ 13 | groupId: 'id9', 14 | clientId: 'id', 15 | bootstrapBrokers: ['localhost:9092'], 16 | strict: true, 17 | deserializers: stringDeserializers 18 | }) 19 | 20 | consumer1.topics.trackAll('temp1') 21 | await consumer1.joinGroup({ sessionTimeout: 10000, heartbeatInterval: 500, rebalanceTimeout: 15000 }) 22 | debugDump({ id: 1, memberId: consumer1.memberId, assignments: consumer1.assignments }) 23 | 24 | consumer1.topics.trackAll('temp2') 25 | await consumer1.joinGroup({ sessionTimeout: 10000, heartbeatInterval: 500, rebalanceTimeout: 15000 }) 26 | debugDump({ id: 1, memberId: consumer1.memberId, assignments: consumer1.assignments }) 27 | 28 | consumer2.topics.trackAll('temp1', 'temp2') 29 | await consumer2.joinGroup({ sessionTimeout: 10000, heartbeatInterval: 500, rebalanceTimeout: 15000 }) 30 | debugDump({ id: 1, memberId: consumer1.memberId, assignments: consumer1.assignments }) 31 | debugDump({ id: 2, memberId: consumer2.memberId, assignments: consumer2.assignments }) 32 | 33 | await consumer1.close() 34 | 35 | await once(consumer2, 'consumer:group:join') 36 | debugDump({ id: 2, memberId: consumer2.memberId, assignments: consumer2.assignments }) 37 | 38 | await consumer2.close() 39 | -------------------------------------------------------------------------------- /playground/clients/consumer.ts: -------------------------------------------------------------------------------- 1 | import { once } from 'node:events' 2 | import { Consumer, debugDump, stringDeserializers } from '../../src/index.ts' 3 | 4 | // const consumer = new Consumer({ groupId: 'id7', clientId: 'id', bootstrapBrokers: ['localhost:9092'], strict: true }) 5 | const consumer = new Consumer({ 6 | groupId: 'id9', 7 | clientId: 'id', 8 | bootstrapBrokers: ['localhost:9092'], 9 | strict: true, 10 | deserializers: stringDeserializers 11 | }) 12 | 13 | const stream = await consumer.consume({ 14 | autocommit: false, 15 | topics: ['temp1', 'temp2'], 16 | sessionTimeout: 10000, 17 | heartbeatInterval: 500, 18 | maxWaitTime: 500 19 | }) 20 | 21 | // This is purposely not catched to show the error handling if we remove the force parameter 22 | once(process, 'SIGINT').then(() => consumer.close(true)) 23 | 24 | debugDump('start') 25 | 26 | stream.on('data', message => { 27 | console.log('data', message.partition, message.offset, message.key, message.value, message.headers) 28 | }) 29 | 30 | for await (const message of stream) { 31 | console.log('data', message.partition, message.offset, message.key, message.value, message.headers) 32 | } 33 | 34 | debugDump('end') 35 | await consumer.close() 36 | -------------------------------------------------------------------------------- /playground/clients/producer-forever.ts: -------------------------------------------------------------------------------- 1 | import { Producer, debugDump, sleep, stringSerializers } from '../../src/index.ts' 2 | 3 | const producer = new Producer({ 4 | clientId: 'id', 5 | bootstrapBrokers: ['localhost:9092'], 6 | serializers: stringSerializers, 7 | strict: true 8 | }) 9 | 10 | let i = 0 11 | while (true) { 12 | i++ 13 | 14 | try { 15 | debugDump( 16 | 'produce', 17 | await producer.send({ 18 | messages: [ 19 | { 20 | topic: 'temp1', 21 | key: `key-${i}`, 22 | value: `value-${i}`, 23 | headers: new Map([[`header-key-${i}`, `header-value-${i}`]]), 24 | partition: i % 2 25 | } 26 | ] 27 | }) 28 | ) 29 | await sleep(100) 30 | } catch (e) { 31 | debugDump(e) 32 | break 33 | } 34 | } 35 | 36 | await producer.close() 37 | -------------------------------------------------------------------------------- /playground/clients/producer-idempotent.ts: -------------------------------------------------------------------------------- 1 | import { ProduceAcks, Producer, debugDump, stringSerializers } from '../../src/index.ts' 2 | 3 | const producer = new Producer({ 4 | clientId: 'id', 5 | bootstrapBrokers: ['localhost:9092'], 6 | idempotent: true, 7 | autocreateTopics: true, 8 | serializers: stringSerializers, 9 | strict: true 10 | }) 11 | 12 | function callbackProduce (cb?: Function): void { 13 | producer.send( 14 | { 15 | messages: [ 16 | { 17 | topic: 'temp1', 18 | key: 'key-idempotent', 19 | value: 'value-idempotent', 20 | headers: new Map([['header-idempotent', 'header-value-idempotent']]), 21 | partition: 0 22 | } 23 | ], 24 | acks: ProduceAcks.ALL 25 | }, 26 | (error, result) => { 27 | if (error) { 28 | console.error('ERROR', error) 29 | return 30 | } 31 | 32 | debugDump('produce', result) 33 | cb?.() 34 | } 35 | ) 36 | } 37 | 38 | callbackProduce(() => { 39 | callbackProduce() 40 | }) 41 | 42 | callbackProduce(() => { 43 | callbackProduce(() => { 44 | producer.close() 45 | }) 46 | }) 47 | -------------------------------------------------------------------------------- /playground/clients/producer-metrics.ts: -------------------------------------------------------------------------------- 1 | import * as client from 'prom-client' 2 | import { Producer, stringSerializers } from '../../src/index.ts' 3 | 4 | const registry = new client.Registry() 5 | 6 | // Create a producer with string serialisers 7 | const producer1 = new Producer({ 8 | clientId: 'my-producer', 9 | bootstrapBrokers: ['localhost:9092'], 10 | serializers: stringSerializers, 11 | metrics: { client, registry, labels: { a: 1, b: 2 } }, 12 | autocreateTopics: true 13 | }) 14 | 15 | // Send messages 16 | await producer1.send({ 17 | messages: [ 18 | { 19 | topic: 'events', 20 | key: 'user-123', 21 | value: JSON.stringify({ name: 'John', action: 'login' }), 22 | headers: { source: 'web-app' } 23 | } 24 | ] 25 | }) 26 | 27 | // Close the producer when done 28 | await producer1.close() 29 | 30 | // Create another producer with string serialisers but different labels 31 | const producer2 = new Producer({ 32 | clientId: 'my-producer', 33 | bootstrapBrokers: ['localhost:9092'], 34 | serializers: stringSerializers, 35 | metrics: { client, registry, labels: { b: 3, c: 4 } }, 36 | autocreateTopics: true 37 | }) 38 | 39 | // Send messages 40 | await producer2.send({ 41 | messages: [ 42 | { 43 | topic: 'events', 44 | key: 'user-123', 45 | value: JSON.stringify({ name: 'John', action: 'login' }), 46 | headers: { source: 'web-app' } 47 | } 48 | ] 49 | }) 50 | 51 | // Close the producer when done 52 | await producer2.close() 53 | 54 | console.log(JSON.stringify(await registry.getMetricsAsJSON(), null, 2)) 55 | -------------------------------------------------------------------------------- /playground/clients/producer.ts: -------------------------------------------------------------------------------- 1 | import { ProduceAcks, Producer, debugDump, stringSerializers } from '../../src/index.ts' 2 | 3 | const producer = new Producer({ 4 | clientId: 'id', 5 | bootstrapBrokers: ['localhost:9092'], 6 | serializers: stringSerializers, 7 | strict: true 8 | }) 9 | 10 | debugDump( 11 | 'produce', 12 | await Promise.all([ 13 | producer.send({ 14 | messages: [ 15 | { 16 | topic: 'temp1', 17 | partition: 0, 18 | key: 'key1', 19 | value: 'value1', 20 | headers: new Map([['headerKey1', 'headerValue1']]) 21 | }, 22 | { topic: 'temp1', partition: 0, key: 'key2', value: 'value2' }, 23 | { 24 | topic: 'temp1', 25 | partition: 0, 26 | key: 'key3', 27 | value: 'value3', 28 | headers: new Map([['headerKey3', 'headerValue3']]) 29 | }, 30 | { topic: 'temp1', partition: 0, key: 'key4', value: 'value4' }, 31 | { 32 | topic: 'temp1', 33 | partition: 0, 34 | key: 'key5', 35 | value: 'value5', 36 | headers: { headerKey5: 'headerValue5' } 37 | }, 38 | { topic: 'temp1', partition: 0, key: 'key6', value: 'value6' } 39 | ], 40 | acks: ProduceAcks.LEADER 41 | }), 42 | 43 | producer.send({ 44 | messages: [ 45 | { 46 | topic: 'temp1', 47 | partition: 0, 48 | key: 'key1', 49 | value: 'value1', 50 | headers: new Map([['headerKey1', 'headerValue1']]) 51 | }, 52 | { topic: 'temp1', partition: 0, key: 'key2', value: 'value2' }, 53 | { 54 | topic: 'temp1', 55 | partition: 0, 56 | key: 'key3', 57 | value: 'value3', 58 | headers: new Map([['headerKey3', 'headerValue3']]) 59 | }, 60 | { topic: 'temp1', partition: 0, key: 'key4', value: 'value4' }, 61 | { 62 | topic: 'temp1', 63 | partition: 0, 64 | key: 'key5', 65 | value: 'value5', 66 | headers: { headerKey5: 'headerValue5' } 67 | }, 68 | { topic: 'temp1', partition: 0, key: 'key6', value: 'value6' } 69 | ], 70 | acks: ProduceAcks.LEADER 71 | }), 72 | 73 | producer.send({ 74 | messages: [ 75 | { 76 | topic: 'temp1', 77 | partition: 0, 78 | key: 'key1', 79 | value: 'value1', 80 | headers: new Map([['headerKey1', 'headerValue1']]) 81 | }, 82 | { topic: 'temp1', partition: 0, key: 'key2', value: 'value2' }, 83 | { 84 | topic: 'temp1', 85 | partition: 0, 86 | key: 'key3', 87 | value: 'value3', 88 | headers: new Map([['headerKey3', 'headerValue3']]) 89 | }, 90 | { topic: 'temp1', partition: 0, key: 'key4', value: 'value4' }, 91 | { 92 | topic: 'temp1', 93 | partition: 0, 94 | key: 'key5', 95 | value: 'value5', 96 | headers: { headerKey5: 'headerValue5' } 97 | }, 98 | { topic: 'temp1', partition: 0, key: 'key6', value: 'value6' } 99 | ], 100 | acks: ProduceAcks.LEADER 101 | }) 102 | ]) 103 | ) 104 | 105 | await producer.close() 106 | -------------------------------------------------------------------------------- /playground/clients/sasl.ts: -------------------------------------------------------------------------------- 1 | import { Consumer } from '../../src/index.ts' 2 | 3 | async function main () { 4 | const consumer = new Consumer({ 5 | clientId: 'clientId', 6 | groupId: 'groupId', 7 | bootstrapBrokers: ['localhost:3012'], 8 | sasl: { 9 | mechanism: 'SCRAM-SHA-256', 10 | username: 'admin', 11 | password: 'admin' 12 | }, 13 | retries: 0 14 | }) 15 | 16 | console.log(await consumer.metadata({ topics: [] })) 17 | } 18 | 19 | await main() 20 | -------------------------------------------------------------------------------- /prettier.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | printWidth: 120, 3 | semi: false, 4 | singleQuote: true, 5 | bracketSpacing: true, 6 | trailingComma: 'none', 7 | arrowParens: 'avoid' 8 | } 9 | -------------------------------------------------------------------------------- /scripts/bump-version.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S node --experimental-strip-types --disable-warning=ExperimentalWarning 2 | 3 | import { execSync } from 'node:child_process' 4 | import { readFile, writeFile } from 'node:fs/promises' 5 | import { inc, type ReleaseType } from 'semver' 6 | 7 | type UserInfo = [string, string] 8 | 9 | function getUserInfo (): UserInfo { 10 | const username = process.argv[3] ?? process.env.GITHUB_ACTOR 11 | const defaultUser = 'mcollina' 12 | 13 | const users: Record = { 14 | mcollina: ['Matteo Collina', 'hello@matteocollina.com'], 15 | ShogunPanda: ['Paolo Insogna', 'paolo@cowtech.it'] 16 | } 17 | 18 | let userInfo = users[username] 19 | 20 | if (!userInfo) { 21 | userInfo = users[defaultUser] 22 | } 23 | 24 | return userInfo 25 | } 26 | 27 | async function getVersion (): Promise { 28 | const version = process.argv[2].replace(/^v/, '') 29 | 30 | if (['minor', 'major', 'patch'].includes(process.argv[2])) { 31 | const packageJson = JSON.parse(await readFile('package.json', 'utf8')) 32 | return inc(packageJson.version, version as ReleaseType)! 33 | } 34 | 35 | return version 36 | } 37 | 38 | async function updatePackageJson (version: string): Promise { 39 | const packageJson = JSON.parse(await readFile('package.json', 'utf8')) 40 | packageJson.version = version 41 | await writeFile('package.json', JSON.stringify(packageJson, null, 2)) 42 | } 43 | 44 | const userInfo = getUserInfo() 45 | const version = await getVersion() 46 | 47 | await updatePackageJson(version) 48 | 49 | if (process.env.GITHUB_ACTIONS === 'true') { 50 | execSync(`git config --global user.name "${userInfo[0]}"`) 51 | execSync(`git config --global user.email "${userInfo[1]}"`) 52 | } 53 | 54 | execSync(`git commit -a -m "chore: Bumped v${version}." -m "Signed-off-by: ${userInfo[0]} <${userInfo[1]}>"`) 55 | -------------------------------------------------------------------------------- /scripts/create-api.ts: -------------------------------------------------------------------------------- 1 | import { writeFile } from 'node:fs/promises' 2 | import { resolve } from 'node:path' 3 | import { parseArgs } from 'node:util' 4 | import { camelCase, kebabCase, pascalCase } from 'scule' 5 | import { formatOutput } from './utils.ts' 6 | 7 | const template = ` 8 | import { Reader } from '../../protocol/reader.ts' 9 | import { Writer } from '../../protocol/writer.ts' 10 | import { createAPI, type ResponseErrorWithLocation } from '../index.ts' 11 | import { ResponseError } from '../../errors.ts' 12 | 13 | export type #{type}Request = Parameters 14 | 15 | export interface #{type}Response {} 16 | 17 | /* 18 | 19 | */ 20 | function createRequest (): Writer { 21 | return Writer.create().appendTaggedFields() 22 | } 23 | 24 | /* 25 | 26 | */ 27 | function parseResponse (_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): #{type}Response { 28 | const errors: ResponseErrorWithLocation[] = [] 29 | 30 | const response: #{type}Response = {} 31 | 32 | if (errors.length) { 33 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 34 | } 35 | 36 | return response 37 | } 38 | 39 | export const #{api} = createAPI<#{type}Request, #{type}Response>(#{code}, #{version}, createRequest, parseResponse) 40 | ` 41 | 42 | async function main () { 43 | let values: Record = {} 44 | 45 | try { 46 | values = parseArgs({ 47 | args: process.argv.slice(2), 48 | options: { 49 | name: { type: 'string', required: true }, 50 | section: { type: 'string', required: true }, 51 | code: { type: 'string', required: true }, 52 | version: { type: 'string', required: true } 53 | } 54 | }).values 55 | 56 | if (!values.section || !values.name || !values.code || !values.version) { 57 | throw new Error('Missing argument.') 58 | } 59 | } catch (e) { 60 | console.error('Please provide the --section, --name, --code and --version options.') 61 | process.exit(1) 62 | } 63 | 64 | const { section, name, code, version } = values 65 | 66 | const typeName = pascalCase(name!) 67 | const fileName = kebabCase(typeName) 68 | 69 | const context: Record = { 70 | type: typeName, 71 | api: camelCase(typeName + 'V' + version), 72 | code, 73 | version 74 | } 75 | 76 | const output = template.replace(/#\{([a-z]+)\}/g, (_, key) => context[key]!) 77 | const destination = resolve(import.meta.dirname, `../src/apis/${section}/${fileName}.ts`) 78 | 79 | await writeFile(destination, await formatOutput(output, destination), 'utf-8') 80 | } 81 | 82 | await main() 83 | -------------------------------------------------------------------------------- /scripts/docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x -e 4 | [ -z "$KAFKA_CONFIGURATION" ] && KAFKA_CONFIGURATION=multiple 5 | docker-compose -f docker/compose-$KAFKA_CONFIGURATION.yml $@ -------------------------------------------------------------------------------- /scripts/generate-apis.ts: -------------------------------------------------------------------------------- 1 | import JSON5 from 'json5' 2 | import { glob, readFile, writeFile } from 'node:fs/promises' 3 | import { basename, resolve } from 'node:path' 4 | import { formatOutput } from './utils.ts' 5 | 6 | async function loadAPIs (root: string): Promise<[Record, Record]> { 7 | const apiByName: Record = {} 8 | const apiById: Record = {} 9 | 10 | for await (const file of glob(resolve(root, 'clients/src/main/resources/common/message/*Request.json'))) { 11 | const spec = JSON5.parse(await readFile(file, 'utf-8')) 12 | const key = spec.apiKey as number 13 | const name = basename(file, 'Request.json') 14 | 15 | apiByName[name] = key 16 | apiById[key] = name 17 | } 18 | 19 | return [apiByName, apiById] 20 | } 21 | 22 | async function main () { 23 | // Load Kafka API names and IDs 24 | const kafkaSource = process.argv[2] 25 | 26 | if (!kafkaSource) { 27 | console.error('Please provide the path to the Kafka source code') 28 | process.exit(1) 29 | } 30 | 31 | const [apiByName, apiById] = await loadAPIs(resolve(process.cwd(), kafkaSource)) 32 | 33 | const output = ` 34 | // This is autogenerated from the generate:apis script, do not edit manually. 35 | 36 | export const protocolAPIsByName: Record = Object.freeze(${JSON.stringify(apiByName, null, 2)}) 37 | 38 | export const protocolAPIsById: Record = Object.freeze(${JSON.stringify(apiById, null, 2)}) 39 | ` 40 | 41 | const destination = resolve(import.meta.dirname, '../src/protocol/apis.ts') 42 | await writeFile(destination, await formatOutput(output, destination), 'utf-8') 43 | } 44 | 45 | await main() 46 | -------------------------------------------------------------------------------- /scripts/generate-errors.ts: -------------------------------------------------------------------------------- 1 | import { readFile, writeFile } from 'node:fs/promises' 2 | import { resolve } from 'node:path' 3 | import { Parser } from 'parse5' 4 | import { formatOutput } from './utils.ts' 5 | 6 | interface ProtocolError { 7 | id: string 8 | code: number 9 | canRetry: boolean 10 | message: string 11 | } 12 | 13 | async function loadErrors (root: string): Promise { 14 | // Open the main protocol.html file to the folder used 15 | const protocolIndex = await readFile(resolve(root, 'protocol.html'), 'utf-8') 16 | 17 | const [, folder] = protocolIndex.match(/#include virtual="(.+?)\/protocol.html"/)! 18 | 19 | // Now open the actual protocol.html file 20 | const errorsDefinitions = await readFile(resolve(root, folder, 'generated/protocol_errors.html'), 'utf-8') 21 | 22 | const contents = Parser.parse(errorsDefinitions)! 23 | // html -> body -> table -> tbody 24 | 25 | return ( 26 | // @ts-ignore 27 | contents.childNodes[0].childNodes[1].childNodes[0].childNodes[0].childNodes 28 | // @ts-ignore 29 | .filter(n => n.nodeName === 'tr') 30 | .slice(1) // Discard the header row 31 | // @ts-ignore 32 | .map(row => { 33 | return { 34 | id: row.childNodes[0].childNodes[0].value, 35 | code: parseInt(row.childNodes[1].childNodes[0].value), 36 | canRetry: row.childNodes[2].childNodes[0].value === 'True', 37 | message: row.childNodes[3].childNodes[0]?.value ?? '' 38 | } 39 | }) 40 | ) 41 | } 42 | 43 | async function main () { 44 | // Load Kafka API names and IDs 45 | const kafkaSource = process.argv[2] 46 | 47 | if (!kafkaSource) { 48 | console.error('Please provide the path to the Kafka Website source code') 49 | process.exit(1) 50 | } 51 | 52 | const errors = await loadErrors(resolve(process.cwd(), kafkaSource)) 53 | 54 | const output = ` 55 | // This is autogenerated from the generate:errors script, do not edit manually. 56 | 57 | export interface ProtocolError { 58 | id: string 59 | code: number 60 | canRetry: boolean 61 | message: string 62 | } 63 | 64 | export const protocolErrorsCodesById: Record = ${JSON.stringify( 65 | Object.fromEntries(errors.map(e => [e.code, e.id])), 66 | null, 67 | 2 68 | )} 69 | 70 | export const protocolErrors: Record = ${JSON.stringify( 71 | Object.fromEntries(errors.map(e => [e.id, e])), 72 | null, 73 | 2 74 | )} 75 | ` 76 | 77 | const destination = resolve(import.meta.dirname, '../src/protocol/errors.ts') 78 | await writeFile(destination, await formatOutput(output, destination), 'utf-8') 79 | } 80 | 81 | await main() 82 | -------------------------------------------------------------------------------- /scripts/utils.ts: -------------------------------------------------------------------------------- 1 | import { ESLint } from 'eslint' 2 | import { format } from 'prettier' 3 | 4 | export async function formatOutput (output: string, filePath: string): Promise { 5 | // Format with prettier 6 | output = await format(output, { 7 | parser: 'typescript', 8 | printWidth: 120, 9 | semi: false, 10 | singleQuote: true, 11 | bracketSpacing: true, 12 | trailingComma: 'none', 13 | arrowParens: 'avoid' 14 | }) 15 | 16 | // Lint with eslint 17 | const eslint = new ESLint({ fix: true }) 18 | const [result] = await eslint.lintText(output, { filePath }) 19 | 20 | return result.output ?? result.source! ?? output 21 | } 22 | -------------------------------------------------------------------------------- /src/apis/admin/alter-configs-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export interface AlterConfigsRequestConfig { 8 | name: string 9 | value?: NullableString 10 | } 11 | 12 | export interface AlterConfigsRequestResource { 13 | resourceType: number 14 | resourceName: string 15 | configs: AlterConfigsRequestConfig[] 16 | } 17 | 18 | export type AlterConfigsRequest = Parameters 19 | 20 | export interface AlterConfigsResponseResult { 21 | errorCode: number 22 | errorMessage: NullableString 23 | resourceType: number 24 | resourceName: string 25 | } 26 | 27 | export interface AlterConfigsResponse { 28 | throttleTimeMs: number 29 | responses: AlterConfigsResponseResult[] 30 | } 31 | 32 | /* 33 | AlterConfigs Request (Version: 2) => [resources] validate_only TAG_BUFFER 34 | resources => resource_type resource_name [configs] TAG_BUFFER 35 | resource_type => INT8 36 | resource_name => COMPACT_STRING 37 | configs => name value TAG_BUFFER 38 | name => COMPACT_STRING 39 | value => COMPACT_NULLABLE_STRING 40 | validate_only => BOOLEAN 41 | */ 42 | export function createRequest (resources: AlterConfigsRequestResource[], validateOnly: boolean): Writer { 43 | return Writer.create() 44 | .appendArray(resources, (w, r) => { 45 | w.appendInt8(r.resourceType) 46 | .appendString(r.resourceName) 47 | .appendArray(r.configs, (w, r) => { 48 | w.appendString(r.name).appendString(r.value) 49 | }) 50 | }) 51 | .appendBoolean(validateOnly) 52 | .appendTaggedFields() 53 | } 54 | 55 | /* 56 | AlterConfigs Response (Version: 2) => throttle_time_ms [responses] TAG_BUFFER 57 | throttle_time_ms => INT32 58 | responses => error_code error_message resource_type resource_name TAG_BUFFER 59 | error_code => INT16 60 | error_message => COMPACT_NULLABLE_STRING 61 | resource_type => INT8 62 | resource_name => COMPACT_STRING 63 | */ 64 | export function parseResponse ( 65 | _correlationId: number, 66 | apiKey: number, 67 | apiVersion: number, 68 | reader: Reader 69 | ): AlterConfigsResponse { 70 | const errors: ResponseErrorWithLocation[] = [] 71 | 72 | const response: AlterConfigsResponse = { 73 | throttleTimeMs: reader.readInt32(), 74 | responses: reader.readArray((r, i) => { 75 | const errorCode = r.readInt16() 76 | 77 | if (errorCode !== 0) { 78 | errors.push([`/responses/${i}`, errorCode]) 79 | } 80 | 81 | return { 82 | errorCode, 83 | errorMessage: r.readNullableString(), 84 | resourceType: r.readInt8(), 85 | resourceName: r.readString() 86 | } 87 | }) 88 | } 89 | 90 | if (errors.length) { 91 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 92 | } 93 | 94 | return response 95 | } 96 | 97 | export const api = createAPI(33, 2, createRequest, parseResponse) 98 | -------------------------------------------------------------------------------- /src/apis/admin/alter-replica-log-dirs-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 5 | 6 | export interface AlterReplicaLogDirsRequestTopic { 7 | name: string 8 | partitions: number[] 9 | } 10 | 11 | export interface AlterReplicaLogDirsRequestDir { 12 | path: string 13 | topics: AlterReplicaLogDirsRequestTopic[] 14 | } 15 | export type AlterReplicaLogDirsRequest = Parameters 16 | 17 | export interface AlterReplicaLogDirsResponsePartition { 18 | partitionIndex: number 19 | errorCode: number 20 | } 21 | 22 | export interface AlterReplicaLogDirsResponseResult { 23 | topicName: string 24 | partitions: AlterReplicaLogDirsResponsePartition[] 25 | } 26 | 27 | export interface AlterReplicaLogDirsResponse { 28 | throttleTimeMs?: number 29 | results: AlterReplicaLogDirsResponseResult[] 30 | } 31 | 32 | /* 33 | AlterReplicaLogDirs Request (Version: 2) => [dirs] TAG_BUFFER 34 | dirs => path [topics] TAG_BUFFER 35 | path => COMPACT_STRING 36 | topics => name [partitions] TAG_BUFFER 37 | name => COMPACT_STRING 38 | partitions => INT32 39 | */ 40 | export function createRequest (dirs: AlterReplicaLogDirsRequestDir[]): Writer { 41 | return Writer.create() 42 | .appendArray(dirs, (w, d) => { 43 | w.appendString(d.path).appendArray(d.topics, (w, t) => { 44 | w.appendString(t.name).appendArray(t.partitions, (w, p) => w.appendInt32(p), true, false) 45 | }) 46 | }) 47 | .appendTaggedFields() 48 | } 49 | 50 | /* 51 | AlterReplicaLogDirs Response (Version: 2) => throttle_time_ms [results] TAG_BUFFER 52 | throttle_time_ms => INT32 53 | results => topic_name [partitions] TAG_BUFFER 54 | topic_name => COMPACT_STRING 55 | partitions => partition_index error_code TAG_BUFFER 56 | partition_index => INT32 57 | error_code => INT16 58 | */ 59 | export function parseResponse ( 60 | _correlationId: number, 61 | apiKey: number, 62 | apiVersion: number, 63 | reader: Reader 64 | ): AlterReplicaLogDirsResponse { 65 | const errors: ResponseErrorWithLocation[] = [] 66 | 67 | const response: AlterReplicaLogDirsResponse = { 68 | throttleTimeMs: reader.readInt32(), 69 | results: reader.readArray((r, i) => { 70 | return { 71 | topicName: r.readString(), 72 | partitions: r.readArray((r, j) => { 73 | const partition = { 74 | partitionIndex: r.readInt32(), 75 | errorCode: r.readInt16() 76 | } 77 | 78 | if (partition.errorCode !== 0) { 79 | errors.push([`/results/${i}/partitions/${j}`, partition.errorCode]) 80 | } 81 | 82 | return partition 83 | }) 84 | } 85 | }) 86 | } 87 | 88 | if (errors.length) { 89 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 90 | } 91 | 92 | return response 93 | } 94 | 95 | export const api = createAPI( 96 | 34, 97 | 2, 98 | createRequest, 99 | parseResponse 100 | ) 101 | -------------------------------------------------------------------------------- /src/apis/admin/create-acls-v3.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export interface CreateAclsRequestCreation { 8 | resourceType: number 9 | resourceName: string 10 | resourcePatternType: number 11 | principal: string 12 | host: string 13 | operation: number 14 | permissionType: number 15 | } 16 | 17 | export type CreateAclsRequest = Parameters 18 | 19 | export interface CreateAclsResponseResult { 20 | errorCode: number 21 | errorMessage: NullableString 22 | } 23 | 24 | export interface CreateAclsResponse { 25 | throttleTimeMs: number 26 | results: CreateAclsResponseResult[] 27 | } 28 | 29 | /* 30 | CreateAcls Request (Version: 3) => [creations] TAG_BUFFER 31 | creations => resource_type resource_name resource_pattern_type principal host operation permission_type TAG_BUFFER 32 | resource_type => INT8 33 | resource_name => COMPACT_STRING 34 | resource_pattern_type => INT8 35 | principal => COMPACT_STRING 36 | host => COMPACT_STRING 37 | operation => INT8 38 | permission_type => INT8 39 | */ 40 | export function createRequest (creations: CreateAclsRequestCreation[]): Writer { 41 | return Writer.create() 42 | .appendArray(creations, (w, c) => { 43 | w.appendInt8(c.resourceType) 44 | .appendString(c.resourceName) 45 | .appendInt8(c.resourcePatternType) 46 | .appendString(c.principal) 47 | .appendString(c.host) 48 | .appendInt8(c.operation) 49 | .appendInt8(c.permissionType) 50 | }) 51 | .appendTaggedFields() 52 | } 53 | 54 | /* 55 | CreateAcls Response (Version: 3) => throttle_time_ms [results] TAG_BUFFER 56 | throttle_time_ms => INT32 57 | results => error_code error_message TAG_BUFFER 58 | error_code => INT16 59 | error_message => COMPACT_NULLABLE_STRING 60 | */ 61 | export function parseResponse ( 62 | _correlationId: number, 63 | apiKey: number, 64 | apiVersion: number, 65 | reader: Reader 66 | ): CreateAclsResponse { 67 | const errors: ResponseErrorWithLocation[] = [] 68 | 69 | const response: CreateAclsResponse = { 70 | throttleTimeMs: reader.readInt32(), 71 | results: reader.readArray((r, i) => { 72 | const result = { 73 | errorCode: r.readInt16(), 74 | errorMessage: r.readNullableString() 75 | } 76 | 77 | if (result.errorCode !== 0) { 78 | errors.push([`/results/${i}`, result.errorCode]) 79 | } 80 | 81 | return result 82 | }) 83 | } 84 | 85 | if (errors.length) { 86 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 87 | } 88 | 89 | return response 90 | } 91 | 92 | export const api = createAPI(30, 3, createRequest, parseResponse) 93 | -------------------------------------------------------------------------------- /src/apis/admin/create-partitions-v3.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export interface CreatePartitionsRequestAssignment { 8 | brokerIds: number[] 9 | } 10 | 11 | export interface CreatePartitionsRequestTopic { 12 | name: string 13 | count: number 14 | assignments: CreatePartitionsRequestAssignment[] 15 | } 16 | 17 | export type CreatePartitionsRequest = Parameters 18 | 19 | export interface CreatePartitionsResponseResult { 20 | name: string 21 | errorCode: number 22 | errorMessage: NullableString 23 | } 24 | 25 | export interface CreatePartitionsResponse { 26 | throttleTimeMs: number 27 | results: CreatePartitionsResponseResult[] 28 | } 29 | 30 | /* 31 | CreatePartitions Request (Version: 3) => [topics] timeout_ms validate_only TAG_BUFFER 32 | topics => name count [assignments] TAG_BUFFER 33 | name => COMPACT_STRING 34 | count => INT32 35 | assignments => [broker_ids] TAG_BUFFER 36 | broker_ids => INT32 37 | timeout_ms => INT32 38 | validate_only => BOOLEAN 39 | */ 40 | export function createRequest ( 41 | topics: CreatePartitionsRequestTopic[], 42 | timeoutMs: number, 43 | validateOnly: boolean 44 | ): Writer { 45 | return Writer.create() 46 | .appendArray(topics, (w, t) => { 47 | w.appendString(t.name) 48 | .appendInt32(t.count) 49 | .appendArray(t.assignments, (w, a) => w.appendArray(a.brokerIds, (w, b) => w.appendInt32(b), true, false)) 50 | }) 51 | .appendInt32(timeoutMs) 52 | .appendBoolean(validateOnly) 53 | .appendTaggedFields() 54 | } 55 | 56 | /* 57 | CreatePartitions Response (Version: 3) => throttle_time_ms [results] TAG_BUFFER 58 | throttle_time_ms => INT32 59 | results => name error_code error_message TAG_BUFFER 60 | name => COMPACT_STRING 61 | error_code => INT16 62 | error_message => COMPACT_NULLABLE_STRING 63 | */ 64 | export function parseResponse ( 65 | _correlationId: number, 66 | apiKey: number, 67 | apiVersion: number, 68 | reader: Reader 69 | ): CreatePartitionsResponse { 70 | const errors: ResponseErrorWithLocation[] = [] 71 | 72 | const response: CreatePartitionsResponse = { 73 | throttleTimeMs: reader.readInt32(), 74 | results: reader.readArray((r, i) => { 75 | const result = { 76 | name: r.readString(), 77 | errorCode: r.readInt16(), 78 | errorMessage: r.readNullableString() 79 | } 80 | 81 | if (result.errorCode !== 0) { 82 | errors.push([`/results/${i}`, result.errorCode]) 83 | } 84 | 85 | return result 86 | }) 87 | } 88 | 89 | if (errors.length) { 90 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 91 | } 92 | 93 | return response 94 | } 95 | 96 | export const api = createAPI(37, 3, createRequest, parseResponse) 97 | -------------------------------------------------------------------------------- /src/apis/admin/delete-groups-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 5 | 6 | export type DeleteGroupsRequest = Parameters 7 | 8 | export interface DeleteGroupsResponseGroup { 9 | groupId: string 10 | errorCode: number 11 | } 12 | 13 | export interface DeleteGroupsResponse { 14 | throttleTimeMs: number 15 | results: DeleteGroupsResponseGroup[] 16 | } 17 | 18 | /* 19 | DeleteGroups Request (Version: 2) => [groups_names] TAG_BUFFER 20 | groups_names => COMPACT_STRING 21 | */ 22 | export function createRequest (groupsNames: string[]): Writer { 23 | return Writer.create() 24 | .appendArray(groupsNames, (w, r) => w.appendString(r), true, false) 25 | .appendTaggedFields() 26 | } 27 | 28 | /* 29 | DeleteGroups Response (Version: 2) => throttle_time_ms [results] TAG_BUFFER 30 | throttle_time_ms => INT32 31 | results => group_id error_code TAG_BUFFER 32 | group_id => COMPACT_STRING 33 | error_code => INT16 34 | */ 35 | export function parseResponse ( 36 | _correlationId: number, 37 | apiKey: number, 38 | apiVersion: number, 39 | reader: Reader 40 | ): DeleteGroupsResponse { 41 | const errors: ResponseErrorWithLocation[] = [] 42 | 43 | const response: DeleteGroupsResponse = { 44 | throttleTimeMs: reader.readInt32(), 45 | results: reader.readArray((r, i) => { 46 | const group = { 47 | groupId: r.readString(), 48 | errorCode: r.readInt16() 49 | } 50 | 51 | if (group.errorCode !== 0) { 52 | errors.push([`/results/${i}`, group.errorCode]) 53 | } 54 | 55 | return group 56 | }) 57 | } 58 | 59 | if (errors.length) { 60 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 61 | } 62 | 63 | return response 64 | } 65 | 66 | export const api = createAPI(42, 2, createRequest, parseResponse) 67 | -------------------------------------------------------------------------------- /src/apis/admin/delete-records-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 5 | 6 | export interface DeleteRecordsRequestPartitions { 7 | partitionIndex: number 8 | offset: bigint 9 | } 10 | 11 | export interface DeleteRecordsRequestTopics { 12 | name: string 13 | partitions: DeleteRecordsRequestPartitions[] 14 | } 15 | 16 | export type DeleteRecordsRequest = Parameters 17 | 18 | export interface DeleteRecordsResponsePartition { 19 | partitionIndex: number 20 | lowWatermark: bigint 21 | errorCode: number 22 | } 23 | 24 | export interface DeleteRecordsResponseTopic { 25 | name: string 26 | partitions: DeleteRecordsResponsePartition[] 27 | } 28 | 29 | export interface DeleteRecordsResponse { 30 | throttleTimeMs: number 31 | topics: DeleteRecordsResponseTopic[] 32 | } 33 | 34 | /* 35 | DeleteRecords Request (Version: 2) => [topics] timeout_ms TAG_BUFFER 36 | topics => name [partitions] TAG_BUFFER 37 | name => COMPACT_STRING 38 | partitions => partition_index offset TAG_BUFFER 39 | partition_index => INT32 40 | offset => INT64 41 | timeout_ms => INT32 42 | */ 43 | export function createRequest (topics: DeleteRecordsRequestTopics[], timeoutMs: number): Writer { 44 | return Writer.create() 45 | .appendArray(topics, (w, t) => { 46 | w.appendString(t.name).appendArray(t.partitions, (w, p) => { 47 | w.appendInt32(p.partitionIndex).appendInt64(p.offset) 48 | }) 49 | }) 50 | .appendInt32(timeoutMs) 51 | .appendTaggedFields() 52 | } 53 | 54 | /* 55 | DeleteRecords Response (Version: 2) => throttle_time_ms [topics] TAG_BUFFER 56 | throttle_time_ms => INT32 57 | topics => name [partitions] TAG_BUFFER 58 | name => COMPACT_STRING 59 | partitions => partition_index low_watermark error_code TAG_BUFFER 60 | partition_index => INT32 61 | low_watermark => INT64 62 | error_code => INT16 63 | */ 64 | export function parseResponse ( 65 | _correlationId: number, 66 | apiKey: number, 67 | apiVersion: number, 68 | reader: Reader 69 | ): DeleteRecordsResponse { 70 | const errors: ResponseErrorWithLocation[] = [] 71 | 72 | const response: DeleteRecordsResponse = { 73 | throttleTimeMs: reader.readInt32(), 74 | topics: reader.readArray((r, i) => { 75 | return { 76 | name: r.readString(), 77 | partitions: r.readArray((r, j) => { 78 | const partition = { 79 | partitionIndex: r.readInt32(), 80 | lowWatermark: r.readInt64(), 81 | errorCode: r.readInt16() 82 | } 83 | 84 | if (partition.errorCode !== 0) { 85 | errors.push([`topics[${i}].partitions[${j}]`, partition.errorCode]) 86 | } 87 | 88 | return partition 89 | }) 90 | } 91 | }) 92 | } 93 | 94 | if (errors.length) { 95 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 96 | } 97 | 98 | return response 99 | } 100 | 101 | export const api = createAPI(21, 2, createRequest, parseResponse) 102 | -------------------------------------------------------------------------------- /src/apis/admin/delete-topics-v6.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export interface DeleteTopicsRequestTopic { 8 | name: string 9 | topicId?: NullableString 10 | } 11 | 12 | export type DeleteTopicsRequest = Parameters 13 | 14 | export interface DeleteTopicsResponseResponse { 15 | name: NullableString 16 | topicId: string 17 | errorCode: number 18 | errorMessage: NullableString 19 | } 20 | 21 | export interface DeleteTopicsResponse { 22 | throttleTimeMs: number 23 | responses: DeleteTopicsResponseResponse[] 24 | } 25 | 26 | /* 27 | DeleteTopics Request (Version: 6) => [topics] timeout_ms TAG_BUFFER 28 | topics => name topic_id TAG_BUFFER 29 | name => COMPACT_NULLABLE_STRING 30 | topic_id => UUID 31 | timeout_ms => INT32 32 | */ 33 | export function createRequest (topics: DeleteTopicsRequestTopic[], timeoutMs: number): Writer { 34 | return Writer.create() 35 | .appendArray(topics, (w, topic) => { 36 | w.appendString(topic.name).appendUUID(topic.topicId) 37 | }) 38 | .appendInt32(timeoutMs) 39 | .appendTaggedFields() 40 | } 41 | 42 | /* 43 | DeleteTopics Response (Version: 6) => throttle_time_ms [responses] TAG_BUFFER 44 | throttle_time_ms => INT32 45 | responses => name topic_id error_code error_message TAG_BUFFER 46 | name => COMPACT_NULLABLE_STRING 47 | topic_id => UUID 48 | error_code => INT16 49 | error_message => COMPACT_NULLABLE_STRING 50 | */ 51 | export function parseResponse ( 52 | _correlationId: number, 53 | apiKey: number, 54 | apiVersion: number, 55 | reader: Reader 56 | ): DeleteTopicsResponse { 57 | const errors: ResponseErrorWithLocation[] = [] 58 | 59 | const response: DeleteTopicsResponse = { 60 | throttleTimeMs: reader.readInt32(), 61 | responses: reader.readArray((r, i) => { 62 | const topicResponse = { 63 | name: r.readNullableString(), 64 | topicId: r.readUUID(), 65 | errorCode: r.readInt16(), 66 | errorMessage: r.readNullableString() 67 | } 68 | 69 | if (topicResponse.errorCode !== 0) { 70 | errors.push([`/responses/${i}`, topicResponse.errorCode]) 71 | } 72 | 73 | return topicResponse 74 | }) 75 | } 76 | 77 | if (errors.length) { 78 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 79 | } 80 | 81 | return response 82 | } 83 | 84 | export const api = createAPI(20, 6, createRequest, parseResponse) 85 | -------------------------------------------------------------------------------- /src/apis/admin/describe-cluster-v1.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type DescribeClusterRequest = Parameters 8 | 9 | export interface DescribeClusterResponseBroker { 10 | brokerId: number 11 | host: string 12 | port: number 13 | rack: NullableString 14 | } 15 | 16 | export interface DescribeClusterResponse { 17 | throttleTimeMs: number 18 | errorCode: number 19 | errorMessage: NullableString 20 | endpointType: number 21 | clusterId: string 22 | controllerId: number 23 | brokers: DescribeClusterResponseBroker[] 24 | clusterAuthorizedOperations: number 25 | } 26 | 27 | /* 28 | DescribeCluster Request (Version: 1) => include_cluster_authorized_operations endpoint_type TAG_BUFFER 29 | include_cluster_authorized_operations => BOOLEAN 30 | endpoint_type => INT8 31 | */ 32 | export function createRequest (includeClusterAuthorizedOperations: boolean, endpointType: number): Writer { 33 | return Writer.create().appendBoolean(includeClusterAuthorizedOperations).appendInt8(endpointType).appendTaggedFields() 34 | } 35 | 36 | /* 37 | DescribeCluster Response (Version: 1) => throttle_time_ms error_code error_message endpoint_type cluster_id controller_id [brokers] cluster_authorized_operations TAG_BUFFER 38 | throttle_time_ms => INT32 39 | error_code => INT16 40 | error_message => COMPACT_NULLABLE_STRING 41 | endpoint_type => INT8 42 | cluster_id => COMPACT_STRING 43 | controller_id => INT32 44 | brokers => broker_id host port rack TAG_BUFFER 45 | broker_id => INT32 46 | host => COMPACT_STRING 47 | port => INT32 48 | rack => COMPACT_NULLABLE_STRING 49 | cluster_authorized_operations => INT32 50 | */ 51 | export function parseResponse ( 52 | _correlationId: number, 53 | apiKey: number, 54 | apiVersion: number, 55 | reader: Reader 56 | ): DescribeClusterResponse { 57 | const response: DescribeClusterResponse = { 58 | throttleTimeMs: reader.readInt32(), 59 | errorCode: reader.readInt16(), 60 | errorMessage: reader.readNullableString(), 61 | endpointType: reader.readInt8(), 62 | clusterId: reader.readString(), 63 | controllerId: reader.readInt32(), 64 | brokers: reader.readArray(r => { 65 | return { 66 | brokerId: r.readInt32(), 67 | host: r.readString(), 68 | port: r.readInt32(), 69 | rack: r.readNullableString() 70 | } 71 | }), 72 | clusterAuthorizedOperations: reader.readInt32() 73 | } 74 | 75 | if (response.errorCode !== 0) { 76 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 77 | } 78 | 79 | return response 80 | } 81 | 82 | export const api = createAPI(60, 1, createRequest, parseResponse) 83 | -------------------------------------------------------------------------------- /src/apis/admin/envelope-v0.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type EnvelopeRequest = Parameters 7 | 8 | export interface EnvelopeResponse { 9 | responseData: Buffer | null 10 | errorCode: number 11 | } 12 | 13 | /* 14 | Envelope Request (Version: 0) => request_data request_principal client_host_address TAG_BUFFER 15 | request_data => COMPACT_BYTES 16 | request_principal => COMPACT_NULLABLE_BYTES 17 | client_host_address => COMPACT_BYTES 18 | */ 19 | export function createRequest ( 20 | requestData: Buffer, 21 | requestPrincipal: Buffer | undefined | null, 22 | clientHostAddress: Buffer 23 | ): Writer { 24 | return Writer.create() 25 | .appendBytes(requestData) 26 | .appendBytes(requestPrincipal) 27 | .appendBytes(clientHostAddress) 28 | .appendTaggedFields() 29 | } 30 | 31 | /* 32 | Envelope Response (Version: 0) => response_data error_code TAG_BUFFER 33 | response_data => COMPACT_NULLABLE_BYTES 34 | error_code => INT16 35 | */ 36 | export function parseResponse ( 37 | _correlationId: number, 38 | apiKey: number, 39 | apiVersion: number, 40 | reader: Reader 41 | ): EnvelopeResponse { 42 | const response: EnvelopeResponse = { 43 | responseData: reader.readNullableBytes(), 44 | errorCode: reader.readInt16() 45 | } 46 | 47 | if (response.errorCode) { 48 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 49 | } 50 | 51 | return response 52 | } 53 | 54 | export const api = createAPI(58, 0, createRequest, parseResponse) 55 | -------------------------------------------------------------------------------- /src/apis/admin/expire-delegation-token-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type ExpireDelegationTokenRequest = Parameters 7 | 8 | export interface ExpireDelegationTokenResponse { 9 | errorCode: number 10 | expiryTimestampMs: bigint 11 | throttleTimeMs: number 12 | } 13 | 14 | /* 15 | ExpireDelegationToken Request (Version: 2) => hmac expiry_time_period_ms TAG_BUFFER 16 | hmac => COMPACT_BYTES 17 | expiry_time_period_ms => INT64 18 | */ 19 | export function createRequest (hmac: Buffer, expiryTimePeriodMs: bigint): Writer { 20 | return Writer.create().appendBytes(hmac).appendInt64(expiryTimePeriodMs).appendTaggedFields() 21 | } 22 | 23 | /* 24 | ExpireDelegationToken Response (Version: 2) => error_code expiry_timestamp_ms throttle_time_ms TAG_BUFFER 25 | error_code => INT16 26 | expiry_timestamp_ms => INT64 27 | throttle_time_ms => INT32 28 | */ 29 | export function parseResponse ( 30 | _correlationId: number, 31 | apiKey: number, 32 | apiVersion: number, 33 | reader: Reader 34 | ): ExpireDelegationTokenResponse { 35 | const response: ExpireDelegationTokenResponse = { 36 | errorCode: reader.readInt16(), 37 | expiryTimestampMs: reader.readInt64(), 38 | throttleTimeMs: reader.readInt32() 39 | } 40 | 41 | if (response.errorCode !== 0) { 42 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 43 | } 44 | 45 | return response 46 | } 47 | 48 | export const api = createAPI( 49 | 40, 50 | 2, 51 | createRequest, 52 | parseResponse 53 | ) 54 | -------------------------------------------------------------------------------- /src/apis/admin/index.ts: -------------------------------------------------------------------------------- 1 | export * as alterClientQuotasV1 from './alter-client-quotas-v1.ts' 2 | export * as alterConfigsV2 from './alter-configs-v2.ts' 3 | export * as alterPartitionReassignmentsV0 from './alter-partition-reassignments-v0.ts' 4 | export * as alterPartitionV3 from './alter-partition-v3.ts' 5 | export * as alterReplicaLogDirsV2 from './alter-replica-log-dirs-v2.ts' 6 | export * as alterUserScramCredentialsV0 from './alter-user-scram-credentials-v0.ts' 7 | export * as consumerGroupDescribeV0 from './consumer-group-describe-v0.ts' 8 | export * as createAclsV3 from './create-acls-v3.ts' 9 | export * as createDelegationTokenV3 from './create-delegation-token-v3.ts' 10 | export * as createPartitionsV3 from './create-partitions-v3.ts' 11 | export * as createTopicsV7 from './create-topics-v7.ts' 12 | export * as deleteAclsV3 from './delete-acls-v3.ts' 13 | export * as deleteGroupsV2 from './delete-groups-v2.ts' 14 | export * as deleteRecordsV2 from './delete-records-v2.ts' 15 | export * as deleteTopicsV6 from './delete-topics-v6.ts' 16 | export * as describeAclsV3 from './describe-acls-v3.ts' 17 | export * as describeClientQuotasV0 from './describe-client-quotas-v0.ts' 18 | export * as describeClusterV1 from './describe-cluster-v1.ts' 19 | export * as describeConfigsV4 from './describe-configs-v4.ts' 20 | export * as describeDelegationTokenV3 from './describe-delegation-token-v3.ts' 21 | export * as describeGroupsV5 from './describe-groups-v5.ts' 22 | export * as describeLogDirsV4 from './describe-log-dirs-v4.ts' 23 | export * as describeProducersV0 from './describe-producers-v0.ts' 24 | export * as describeQuorumV2 from './describe-quorum-v2.ts' 25 | export * as describeTopicPartitionsV0 from './describe-topic-partitions-v0.ts' 26 | export * as describeTransactionsV0 from './describe-transactions-v0.ts' 27 | export * as describeUserScramCredentialsV0 from './describe-user-scram-credentials-v0.ts' 28 | export * as envelopeV0 from './envelope-v0.ts' 29 | export * as expireDelegationTokenV2 from './expire-delegation-token-v2.ts' 30 | export * as incrementalAlterConfigsV1 from './incremental-alter-configs-v1.ts' 31 | export * as listGroupsV4 from './list-groups-v4.ts' 32 | export * as listGroupsV5 from './list-groups-v5.ts' 33 | export * as listPartitionReassignmentsV0 from './list-partition-reassignments-v0.ts' 34 | export * as listTransactionsV1 from './list-transactions-v1.ts' 35 | export * as offsetDeleteV0 from './offset-delete-v0.ts' 36 | export * as renewDelegationTokenV2 from './renew-delegation-token-v2.ts' 37 | export * as unregisterBrokerV0 from './unregister-broker-v0.ts' 38 | export * as updateFeaturesV1 from './update-features-v1.ts' 39 | -------------------------------------------------------------------------------- /src/apis/admin/list-groups-v4.ts: -------------------------------------------------------------------------------- 1 | import { pascalCase } from 'scule' 2 | import { ResponseError } from '../../errors.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | import { type ConsumerGroupState } from '../enumerations.ts' 7 | 8 | export type ListGroupsRequest = Parameters 9 | 10 | export interface ListGroupsResponseGroup { 11 | groupId: string 12 | protocolType: string 13 | groupState: string 14 | } 15 | 16 | export interface ListGroupsResponse { 17 | throttleTimeMs: number 18 | errorCode: number 19 | groups: ListGroupsResponseGroup[] 20 | } 21 | 22 | /* 23 | ListGroups Request (Version: 4) => [states_filter] TAG_BUFFER 24 | states_filter => COMPACT_STRING 25 | */ 26 | export function createRequest (statesFilter: ConsumerGroupState[]): Writer { 27 | return Writer.create() 28 | .appendArray(statesFilter, (w, s) => w.appendString(pascalCase(s, { normalize: true })), true, false) 29 | .appendTaggedFields() 30 | } 31 | 32 | /* 33 | ListGroups Response (Version: 4) => throttle_time_ms error_code [groups] TAG_BUFFER 34 | throttle_time_ms => INT32 35 | error_code => INT16 36 | groups => group_id protocol_type group_state group_type TAG_BUFFER 37 | group_id => COMPACT_STRING 38 | protocol_type => COMPACT_STRING 39 | group_state => COMPACT_STRING 40 | */ 41 | export function parseResponse ( 42 | _correlationId: number, 43 | apiKey: number, 44 | apiVersion: number, 45 | reader: Reader 46 | ): ListGroupsResponse { 47 | const response: ListGroupsResponse = { 48 | throttleTimeMs: reader.readInt32(), 49 | errorCode: reader.readInt16(), 50 | groups: reader.readArray(r => { 51 | return { 52 | groupId: r.readNullableString(), 53 | protocolType: r.readString(), 54 | groupState: r.readString() 55 | } as ListGroupsResponseGroup 56 | }) 57 | } 58 | 59 | if (response.errorCode !== 0) { 60 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 61 | } 62 | 63 | return response 64 | } 65 | 66 | export const api = createAPI(16, 4, createRequest, parseResponse) 67 | -------------------------------------------------------------------------------- /src/apis/admin/list-groups-v5.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | import { type ConsumerGroupState } from '../enumerations.ts' 6 | 7 | export type ListGroupsRequest = Parameters 8 | 9 | export interface ListGroupsResponseGroup { 10 | groupId: string 11 | protocolType: string 12 | groupState: string 13 | groupType: string 14 | } 15 | 16 | export interface ListGroupsResponse { 17 | throttleTimeMs: number 18 | errorCode: number 19 | groups: ListGroupsResponseGroup[] 20 | } 21 | 22 | /* 23 | ListGroups Request (Version: 5) => [states_filter] [types_filter] TAG_BUFFER 24 | states_filter => COMPACT_STRING 25 | types_filter => COMPACT_STRING 26 | */ 27 | export function createRequest (statesFilter: ConsumerGroupState[], typesFilter: string[]): Writer { 28 | return Writer.create() 29 | .appendArray(statesFilter, (w, s) => w.appendString(s as string), true, false) 30 | .appendArray(typesFilter, (w, t) => w.appendString(t), true, false) 31 | .appendTaggedFields() 32 | } 33 | 34 | /* 35 | ListGroups Response (Version: 5) => throttle_time_ms error_code [groups] TAG_BUFFER 36 | throttle_time_ms => INT32 37 | error_code => INT16 38 | groups => group_id protocol_type group_state group_type TAG_BUFFER 39 | group_id => COMPACT_STRING 40 | protocol_type => COMPACT_STRING 41 | group_state => COMPACT_STRING 42 | group_type => COMPACT_STRING 43 | */ 44 | export function parseResponse ( 45 | _correlationId: number, 46 | apiKey: number, 47 | apiVersion: number, 48 | reader: Reader 49 | ): ListGroupsResponse { 50 | const response: ListGroupsResponse = { 51 | throttleTimeMs: reader.readInt32(), 52 | errorCode: reader.readInt16(), 53 | groups: reader.readArray(r => { 54 | return { 55 | groupId: r.readNullableString(), 56 | protocolType: r.readString(), 57 | groupState: r.readString(), 58 | groupType: r.readString() 59 | } as ListGroupsResponseGroup 60 | }) 61 | } 62 | 63 | if (response.errorCode !== 0) { 64 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 65 | } 66 | 67 | return response 68 | } 69 | 70 | export const api = createAPI(16, 5, createRequest, parseResponse) 71 | -------------------------------------------------------------------------------- /src/apis/admin/list-transactions-v1.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | import { type TransactionState } from '../enumerations.ts' 6 | 7 | export type ListTransactionsRequest = Parameters 8 | 9 | export interface ListTransactionsResponseTransactionState { 10 | transactionalId: string 11 | producerId: bigint 12 | transactionState: string 13 | } 14 | 15 | export interface ListTransactionsResponse { 16 | throttleTimeMs: number 17 | errorCode: number 18 | unknownStateFilters: string[] 19 | transactionStates: ListTransactionsResponseTransactionState[] 20 | } 21 | 22 | /* 23 | ListTransactions Request (Version: 1) => [state_filters] [producer_id_filters] duration_filter TAG_BUFFER 24 | state_filters => COMPACT_STRING 25 | producer_id_filters => INT64 26 | duration_filter => INT64 27 | */ 28 | export function createRequest ( 29 | stateFilters: TransactionState[], 30 | producerIdFilters: bigint[], 31 | durationFilter: bigint 32 | ): Writer { 33 | return Writer.create() 34 | .appendArray(stateFilters, (w, t) => w.appendString(t), true, false) 35 | .appendArray(producerIdFilters, (w, p) => w.appendInt64(p), true, false) 36 | .appendInt64(durationFilter) 37 | .appendTaggedFields() 38 | } 39 | 40 | /* 41 | ListTransactions Response (Version: 1) => throttle_time_ms error_code [unknown_state_filters] [transaction_states] TAG_BUFFER 42 | throttle_time_ms => INT32 43 | error_code => INT16 44 | unknown_state_filters => COMPACT_STRING 45 | transaction_states => transactional_id producer_id transaction_state TAG_BUFFER 46 | transactional_id => COMPACT_STRING 47 | producer_id => INT64 48 | transaction_state => COMPACT_STRING 49 | */ 50 | export function parseResponse ( 51 | _correlationId: number, 52 | apiKey: number, 53 | apiVersion: number, 54 | reader: Reader 55 | ): ListTransactionsResponse { 56 | const response: ListTransactionsResponse = { 57 | throttleTimeMs: reader.readInt32(), 58 | errorCode: reader.readInt16(), 59 | unknownStateFilters: reader.readArray(r => r.readString(), true, false)!, 60 | transactionStates: reader.readArray(r => { 61 | return { 62 | transactionalId: r.readString(), 63 | producerId: r.readInt64(), 64 | transactionState: r.readString() 65 | } 66 | }) 67 | } 68 | 69 | if (response.errorCode !== 0) { 70 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 71 | } 72 | 73 | return response 74 | } 75 | 76 | export const api = createAPI(66, 1, createRequest, parseResponse) 77 | -------------------------------------------------------------------------------- /src/apis/admin/offset-delete-v0.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 5 | 6 | export interface OffsetDeleteRequestPartition { 7 | partitionIndex: number 8 | } 9 | 10 | export interface OffsetDeleteRequestTopic { 11 | name: string 12 | partitions: OffsetDeleteRequestPartition[] 13 | } 14 | 15 | export type OffsetDeleteRequest = Parameters 16 | 17 | export interface OffsetDeleteResponsePartition { 18 | partitionIndex: number 19 | errorCode: number 20 | } 21 | 22 | export interface OffsetDeleteResponseTopic { 23 | name: string 24 | partitions: OffsetDeleteResponsePartition[] 25 | } 26 | 27 | export interface OffsetDeleteResponse { 28 | errorCode: number 29 | throttleTimeMs: number 30 | topics: OffsetDeleteResponseTopic[] 31 | } 32 | 33 | /* 34 | OffsetDelete Request (Version: 0) => group_id [topics] 35 | group_id => STRING 36 | topics => name [partitions] 37 | name => STRING 38 | partitions => partition_index 39 | partition_index => INT32 40 | */ 41 | export function createRequest (groupId: string, topics: OffsetDeleteRequestTopic[]): Writer { 42 | return Writer.create() 43 | .appendString(groupId, false) 44 | .appendArray( 45 | topics, 46 | (w, t) => { 47 | w.appendString(t.name, false).appendArray(t.partitions, (w, p) => w.appendInt32(p.partitionIndex), false, false) 48 | }, 49 | false, 50 | false 51 | ) 52 | } 53 | 54 | /* 55 | OffsetDelete Response (Version: 0) => error_code throttle_time_ms [topics] 56 | error_code => INT16 57 | throttle_time_ms => INT32 58 | topics => name [partitions] 59 | name => STRING 60 | partitions => partition_index error_code 61 | partition_index => INT32 62 | error_code => INT16 63 | */ 64 | export function parseResponse ( 65 | _correlationId: number, 66 | apiKey: number, 67 | apiVersion: number, 68 | reader: Reader 69 | ): OffsetDeleteResponse { 70 | const errors: ResponseErrorWithLocation[] = [] 71 | 72 | const errorCode = reader.readInt16() 73 | 74 | if (errorCode !== 0) { 75 | errors.push(['', errorCode]) 76 | } 77 | 78 | const response: OffsetDeleteResponse = { 79 | errorCode, 80 | throttleTimeMs: reader.readInt32(), 81 | topics: reader.readArray((r, i) => { 82 | return { 83 | name: r.readString(), 84 | partitions: r.readArray((r, j) => { 85 | const partition = { 86 | partitionIndex: r.readInt32(), 87 | errorCode: r.readInt16() 88 | } 89 | 90 | if (partition.errorCode !== 0) { 91 | errors.push([`/topics/${i}/partitions/${j}`, partition.errorCode]) 92 | } 93 | 94 | return partition 95 | }) 96 | } 97 | }) 98 | } 99 | 100 | if (errors.length) { 101 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 102 | } 103 | 104 | return response 105 | } 106 | 107 | export const api = createAPI( 108 | 47, 109 | 0, 110 | createRequest, 111 | parseResponse, 112 | false, 113 | false 114 | ) 115 | -------------------------------------------------------------------------------- /src/apis/admin/renew-delegation-token-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type RenewDelegationTokenRequest = Parameters 7 | 8 | export interface RenewDelegationTokenResponse { 9 | errorCode: number 10 | expiryTimestampMs: bigint 11 | throttleTimeMs: number 12 | } 13 | 14 | /* 15 | RenewDelegationToken Request (Version: 2) => hmac renew_period_ms TAG_BUFFER 16 | hmac => COMPACT_BYTES 17 | renew_period_ms => INT64 18 | */ 19 | export function createRequest (hmac: Buffer, renewPeriodMs: bigint): Writer { 20 | return Writer.create().appendBytes(hmac).appendInt64(renewPeriodMs).appendTaggedFields() 21 | } 22 | 23 | /* 24 | RenewDelegationToken Response (Version: 2) => error_code expiry_timestamp_ms throttle_time_ms TAG_BUFFER 25 | error_code => INT16 26 | expiry_timestamp_ms => INT64 27 | throttle_time_ms => INT32 28 | */ 29 | export function parseResponse ( 30 | _correlationId: number, 31 | apiKey: number, 32 | apiVersion: number, 33 | reader: Reader 34 | ): RenewDelegationTokenResponse { 35 | const response: RenewDelegationTokenResponse = { 36 | errorCode: reader.readInt16(), 37 | expiryTimestampMs: reader.readInt64(), 38 | throttleTimeMs: reader.readInt32() 39 | } 40 | 41 | if (response.errorCode !== 0) { 42 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 43 | } 44 | 45 | return response 46 | } 47 | 48 | export const api = createAPI( 49 | 39, 50 | 2, 51 | createRequest, 52 | parseResponse 53 | ) 54 | -------------------------------------------------------------------------------- /src/apis/admin/unregister-broker-v0.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type UnregisterBrokerRequest = Parameters 8 | 9 | export interface UnregisterBrokerResponse { 10 | throttleTimeMs: number 11 | errorCode: number 12 | errorMessage: NullableString 13 | } 14 | 15 | /* 16 | UnregisterBroker Request (Version: 0) => broker_id TAG_BUFFER 17 | broker_id => INT32 18 | */ 19 | export function createRequest (brokerId: number): Writer { 20 | return Writer.create().appendInt32(brokerId).appendTaggedFields() 21 | } 22 | 23 | /* 24 | UnregisterBroker Response (Version: 0) => throttle_time_ms error_code error_message TAG_BUFFER 25 | throttle_time_ms => INT32 26 | error_code => INT16 27 | error_message => COMPACT_NULLABLE_STRING 28 | */ 29 | export function parseResponse ( 30 | _correlationId: number, 31 | apiKey: number, 32 | apiVersion: number, 33 | reader: Reader 34 | ): UnregisterBrokerResponse { 35 | const response: UnregisterBrokerResponse = { 36 | throttleTimeMs: reader.readInt32(), 37 | errorCode: reader.readInt16(), 38 | errorMessage: reader.readNullableString() 39 | } 40 | 41 | if (response.errorCode !== 0) { 42 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 43 | } 44 | 45 | return response 46 | } 47 | 48 | export const api = createAPI(64, 0, createRequest, parseResponse) 49 | -------------------------------------------------------------------------------- /src/apis/callbacks.ts: -------------------------------------------------------------------------------- 1 | import { MultipleErrors } from '../errors.ts' 2 | import { type Callback } from './definitions.ts' 3 | 4 | export const kCallbackPromise = Symbol('plt.kafka.callbackPromise') 5 | 6 | // This is only meaningful for testing 7 | export const kNoopCallbackReturnValue = Symbol('plt.kafka.noopCallbackReturnValue') 8 | 9 | export const noopCallback: CallbackWithPromise = () => { 10 | return Promise.resolve(kNoopCallbackReturnValue) 11 | } 12 | 13 | export type CallbackWithPromise = Callback & { [kCallbackPromise]?: Promise } 14 | 15 | export function createPromisifiedCallback (): CallbackWithPromise { 16 | const { promise, resolve, reject } = Promise.withResolvers() 17 | 18 | function callback (error?: Error | null, payload?: ReturnType): void { 19 | if (error) { 20 | reject(error) 21 | } else { 22 | resolve(payload!) 23 | } 24 | } 25 | 26 | callback[kCallbackPromise] = promise 27 | 28 | return callback 29 | } 30 | 31 | export function runConcurrentCallbacks ( 32 | errorMessage: string, 33 | collection: unknown[] | Set | Map, 34 | operation: (item: any, cb: Callback) => void, 35 | callback: Callback 36 | ): void { 37 | let remaining = Array.isArray(collection) ? collection.length : collection.size 38 | let hasErrors = false 39 | const errors: Error[] = Array.from(Array(remaining)) 40 | const results: ReturnType[] = Array.from(Array(remaining)) 41 | 42 | let i = 0 43 | 44 | function operationCallback (index: number, e: Error | null, result: ReturnType): void { 45 | if (e) { 46 | hasErrors = true 47 | errors[index] = e 48 | } else { 49 | results[index] = result 50 | } 51 | 52 | remaining-- 53 | 54 | if (remaining === 0) { 55 | callback(hasErrors ? new MultipleErrors(errorMessage, errors) : null, results) 56 | } 57 | } 58 | 59 | for (const item of collection) { 60 | operation(item, operationCallback.bind(null, i++)) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/apis/consumer/heartbeat-v4.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type HeartbeatRequest = Parameters 8 | 9 | export interface HeartbeatResponse { 10 | throttleTimeMs: number 11 | errorCode: number 12 | } 13 | 14 | /* 15 | Heartbeat Request (Version: 4) => group_id generation_id member_id group_instance_id TAG_BUFFER 16 | group_id => COMPACT_STRING 17 | generation_id => INT32 18 | member_id => COMPACT_STRING 19 | group_instance_id => COMPACT_NULLABLE_STRING 20 | */ 21 | export function createRequest ( 22 | groupId: string, 23 | generationId: number, 24 | memberId: string, 25 | groupInstanceId?: NullableString 26 | ): Writer { 27 | return Writer.create() 28 | .appendString(groupId) 29 | .appendInt32(generationId) 30 | .appendString(memberId) 31 | .appendString(groupInstanceId) 32 | .appendTaggedFields() 33 | } 34 | 35 | /* 36 | Heartbeat Response (Version: 4) => throttle_time_ms error_code TAG_BUFFER 37 | throttle_time_ms => INT32 38 | error_code => INT16 39 | */ 40 | export function parseResponse ( 41 | _correlationId: number, 42 | apiKey: number, 43 | apiVersion: number, 44 | reader: Reader 45 | ): HeartbeatResponse { 46 | const response: HeartbeatResponse = { 47 | throttleTimeMs: reader.readInt32(), 48 | errorCode: reader.readInt16() 49 | } 50 | 51 | if (response.errorCode !== 0) { 52 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 53 | } 54 | 55 | return response 56 | } 57 | 58 | export const api = createAPI(12, 4, createRequest, parseResponse) 59 | -------------------------------------------------------------------------------- /src/apis/consumer/index.ts: -------------------------------------------------------------------------------- 1 | export * as consumerGroupHeartbeatV0 from './consumer-group-heartbeat-v0.ts' 2 | export * as fetchV16 from './fetch-v16.ts' 3 | export * as fetchV17 from './fetch-v17.ts' 4 | export * as heartbeatV4 from './heartbeat-v4.ts' 5 | export * as joinGroupV9 from './join-group-v9.ts' 6 | export * as leaveGroupV5 from './leave-group-v5.ts' 7 | export * as listOffsetsV8 from './list-offsets-v8.ts' 8 | export * as listOffsetsV9 from './list-offsets-v9.ts' 9 | export * as offsetCommitV9 from './offset-commit-v9.ts' 10 | export * as offsetFetchV9 from './offset-fetch-v9.ts' 11 | export * as syncGroupV5 from './sync-group-v5.ts' 12 | -------------------------------------------------------------------------------- /src/apis/consumer/leave-group-v5.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export interface LeaveGroupRequestMember { 8 | memberId: string 9 | groupInstanceId?: NullableString 10 | reason?: NullableString 11 | } 12 | 13 | export type LeaveGroupRequest = Parameters 14 | 15 | export interface LeaveGroupResponseMember { 16 | memberId: NullableString 17 | groupInstanceId: NullableString 18 | errorCode: number 19 | } 20 | 21 | export interface LeaveGroupResponse { 22 | throttleTimeMs: number 23 | errorCode: number 24 | members: LeaveGroupResponseMember[] 25 | } 26 | 27 | /* 28 | LeaveGroup Request (Version: 5) => group_id [members] TAG_BUFFER 29 | group_id => COMPACT_STRING 30 | members => member_id group_instance_id reason TAG_BUFFER 31 | member_id => COMPACT_STRING 32 | group_instance_id => COMPACT_NULLABLE_STRING 33 | reason => COMPACT_NULLABLE_STRING 34 | */ 35 | export function createRequest (groupId: string, members: LeaveGroupRequestMember[]): Writer { 36 | return Writer.create() 37 | .appendString(groupId) 38 | .appendArray(members, (w, m) => { 39 | w.appendString(m.memberId).appendString(m.groupInstanceId).appendString(m.reason) 40 | }) 41 | .appendTaggedFields() 42 | } 43 | 44 | /* 45 | LeaveGroup Response (Version: 5) => throttle_time_ms error_code [members] TAG_BUFFER 46 | throttle_time_ms => INT32 47 | error_code => INT16 48 | members => member_id group_instance_id error_code TAG_BUFFER 49 | member_id => COMPACT_STRING 50 | group_instance_id => COMPACT_NULLABLE_STRING 51 | error_code => INT16 52 | 53 | */ 54 | export function parseResponse ( 55 | _correlationId: number, 56 | apiKey: number, 57 | apiVersion: number, 58 | reader: Reader 59 | ): LeaveGroupResponse { 60 | const errors: ResponseErrorWithLocation[] = [] 61 | 62 | const throttleTimeMs = reader.readInt32() 63 | const errorCode = reader.readInt16() 64 | 65 | if (errorCode !== 0) { 66 | errors.push(['', errorCode]) 67 | } 68 | 69 | const response: LeaveGroupResponse = { 70 | throttleTimeMs, 71 | errorCode, 72 | members: reader.readArray((r, i) => { 73 | const member: LeaveGroupResponseMember = { 74 | memberId: r.readNullableString(), 75 | groupInstanceId: r.readNullableString(), 76 | errorCode: r.readInt16() 77 | } 78 | 79 | if (member.errorCode !== 0) { 80 | errors.push([`/members/${i}`, member.errorCode]) 81 | } 82 | 83 | return member 84 | }) 85 | } 86 | 87 | if (errors.length) { 88 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 89 | } 90 | 91 | return response 92 | } 93 | 94 | export const api = createAPI(13, 5, createRequest, parseResponse) 95 | -------------------------------------------------------------------------------- /src/apis/consumer/sync-group-v5.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export interface SyncGroupRequestAssignment { 8 | memberId: string 9 | assignment: Buffer 10 | } 11 | 12 | export type SyncGroupRequest = Parameters 13 | 14 | export interface SyncGroupResponse { 15 | throttleTimeMs: number 16 | errorCode: number 17 | protocolType: NullableString 18 | protocolName: NullableString 19 | assignment: Buffer 20 | } 21 | 22 | /* 23 | SyncGroup Request (Version: 5) => group_id generation_id member_id group_instance_id protocol_type protocol_name [assignments] TAG_BUFFER 24 | group_id => COMPACT_STRING 25 | generation_id => INT32 26 | member_id => COMPACT_STRING 27 | group_instance_id => COMPACT_NULLABLE_STRING 28 | protocol_type => COMPACT_NULLABLE_STRING 29 | protocol_name => COMPACT_NULLABLE_STRING 30 | assignments => member_id assignment TAG_BUFFER 31 | member_id => COMPACT_STRING 32 | assignment => COMPACT_BYTES 33 | 34 | */ 35 | export function createRequest ( 36 | groupId: string, 37 | generationId: number, 38 | memberId: string, 39 | groupInstanceId: NullableString, 40 | protocolType: NullableString, 41 | protocolName: NullableString, 42 | assignments: SyncGroupRequestAssignment[] 43 | ): Writer { 44 | return Writer.create() 45 | .appendString(groupId) 46 | .appendInt32(generationId) 47 | .appendString(memberId) 48 | .appendString(groupInstanceId) 49 | .appendString(protocolType) 50 | .appendString(protocolName) 51 | .appendArray(assignments, (w, a) => w.appendString(a.memberId).appendBytes(a.assignment)) 52 | .appendTaggedFields() 53 | } 54 | 55 | /* 56 | SyncGroup Response (Version: 5) => throttle_time_ms error_code protocol_type protocol_name assignment TAG_BUFFER 57 | throttle_time_ms => INT32 58 | error_code => INT16 59 | protocol_type => COMPACT_NULLABLE_STRING 60 | protocol_name => COMPACT_NULLABLE_STRING 61 | assignment => COMPACT_BYTES 62 | */ 63 | export function parseResponse ( 64 | _correlationId: number, 65 | apiKey: number, 66 | apiVersion: number, 67 | reader: Reader 68 | ): SyncGroupResponse { 69 | const response: SyncGroupResponse = { 70 | throttleTimeMs: reader.readInt32(), 71 | errorCode: reader.readInt16(), 72 | protocolType: reader.readNullableString(), 73 | protocolName: reader.readNullableString(), 74 | assignment: reader.readBytes() 75 | } 76 | 77 | if (response.errorCode !== 0) { 78 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 79 | } 80 | 81 | return response 82 | } 83 | 84 | export const api = createAPI(14, 5, createRequest, parseResponse) 85 | -------------------------------------------------------------------------------- /src/apis/definitions.ts: -------------------------------------------------------------------------------- 1 | import { promisify } from 'node:util' 2 | import { type Connection } from '../network/connection.ts' 3 | import { type Reader } from '../protocol/reader.ts' 4 | import { type Writer } from '../protocol/writer.ts' 5 | 6 | export type Callback = (error: Error | null, payload: ReturnType) => void 7 | 8 | export type CallbackArguments = [cb: Callback] 9 | 10 | export type RequestCreator = (...args: any[]) => Writer 11 | 12 | export type ResponseParser = ( 13 | correlationId: number, 14 | apiKey: number, 15 | apiVersion: number, 16 | reader: Reader 17 | ) => ReturnType | Promise 18 | 19 | export type ResponseErrorWithLocation = [string, number] 20 | 21 | export type APIWithCallback, ResponseType> = ( 22 | connection: Connection, 23 | ...args: [...RequestArguments, ...Partial>] 24 | ) => void 25 | 26 | export type APIWithPromise, ResponseType> = ( 27 | connection: Connection, 28 | ...args: RequestArguments 29 | ) => Promise 30 | 31 | export type API, ResponseType> = APIWithCallback & { 32 | async: APIWithPromise 33 | key: number 34 | version: number 35 | } 36 | 37 | export function createAPI, ResponseType> ( 38 | apiKey: number, 39 | apiVersion: number, 40 | createRequest: RequestCreator, 41 | parseResponse: ResponseParser, 42 | hasRequestHeaderTaggedFields: boolean = true, 43 | hasResponseHeaderTaggedFields: boolean = true 44 | ): API { 45 | const api = function api ( 46 | connection: Connection, 47 | ...args: [...RequestArguments, ...Partial>] 48 | ): void { 49 | const cb = typeof args[args.length - 1] === 'function' ? (args.pop() as Callback) : () => {} 50 | 51 | connection.send( 52 | apiKey, 53 | apiVersion, 54 | () => createRequest(...args), 55 | parseResponse, 56 | hasRequestHeaderTaggedFields, 57 | hasResponseHeaderTaggedFields, 58 | cb 59 | ) 60 | } 61 | 62 | api.async = promisify(api) as APIWithPromise 63 | api.key = apiKey 64 | api.version = apiVersion 65 | 66 | return api 67 | } 68 | -------------------------------------------------------------------------------- /src/apis/index.ts: -------------------------------------------------------------------------------- 1 | // Generics 2 | export * from './callbacks.ts' 3 | export * from './definitions.ts' 4 | export * from './enumerations.ts' 5 | 6 | // Low-level APIs 7 | export * from './admin/index.ts' 8 | export * from './consumer/index.ts' 9 | export * from './metadata/index.ts' 10 | export * from './producer/index.ts' 11 | export * from './security/index.ts' 12 | export * from './telemetry/index.ts' 13 | -------------------------------------------------------------------------------- /src/apis/metadata/api-versions-v3.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { protocolAPIsById } from '../../protocol/apis.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type ApiVersionsRequest = Parameters 8 | 9 | export interface ApiVersionsResponseApi { 10 | apiKey: number 11 | name: string 12 | minVersion: number 13 | maxVersion: number 14 | } 15 | 16 | export type ApiVersionsResponse = { 17 | errorCode: number 18 | apiKeys: ApiVersionsResponseApi[] 19 | throttleTimeMs: number 20 | } 21 | 22 | /* 23 | ApiVersions Request (Version: 3) => client_software_name client_software_version TAG_BUFFER 24 | client_software_name => COMPACT_STRING 25 | client_software_version => COMPACT_STRING 26 | */ 27 | export function createRequest (clientSoftwareName: string, clientSoftwareVersion: string): Writer { 28 | return Writer.create().appendString(clientSoftwareName).appendString(clientSoftwareVersion).appendTaggedFields() 29 | } 30 | 31 | /* 32 | ApiVersions Response (Version: 3) => error_code [api_keys] throttle_time_ms TAG_BUFFER 33 | error_code => INT16 34 | api_keys => api_key min_version max_version TAG_BUFFER 35 | api_key => INT16 36 | min_version => INT16 37 | max_version => INT16 38 | throttle_time_ms => INT32 39 | */ 40 | export function parseResponse ( 41 | _correlationId: number, 42 | apiKey: number, 43 | apiVersion: number, 44 | reader: Reader 45 | ): ApiVersionsResponse { 46 | const response: ApiVersionsResponse = { 47 | errorCode: reader.readInt16(), 48 | apiKeys: reader.readArray(r => { 49 | const apiKey = r.readInt16() 50 | 51 | return { 52 | apiKey, 53 | name: protocolAPIsById[apiKey], 54 | minVersion: r.readInt16(), 55 | maxVersion: r.readInt16() 56 | } 57 | }), 58 | throttleTimeMs: reader.readInt32() 59 | } 60 | 61 | if (response.errorCode !== 0) { 62 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 63 | } 64 | 65 | return response 66 | } 67 | 68 | export const api = createAPI(18, 3, createRequest, parseResponse, true, false) 69 | -------------------------------------------------------------------------------- /src/apis/metadata/api-versions-v4.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { protocolAPIsById } from '../../protocol/apis.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type ApiVersionsRequest = Parameters 8 | 9 | export interface ApiVersionsResponseApi { 10 | apiKey: number 11 | name: string 12 | minVersion: number 13 | maxVersion: number 14 | } 15 | 16 | export type ApiVersionsResponse = { 17 | errorCode: number 18 | apiKeys: ApiVersionsResponseApi[] 19 | throttleTimeMs: number 20 | } 21 | 22 | /* 23 | ApiVersions Request (Version: 4) => client_software_name client_software_version TAG_BUFFER 24 | client_software_name => COMPACT_STRING 25 | client_software_version => COMPACT_STRING 26 | */ 27 | export function createRequest (clientSoftwareName: string, clientSoftwareVersion: string): Writer { 28 | return Writer.create().appendString(clientSoftwareName).appendString(clientSoftwareVersion).appendTaggedFields() 29 | } 30 | 31 | /* 32 | ApiVersions Response (Version: 4) => error_code [api_keys] throttle_time_ms TAG_BUFFER 33 | error_code => INT16 34 | api_keys => api_key min_version max_version TAG_BUFFER 35 | api_key => INT16 36 | min_version => INT16 37 | max_version => INT16 38 | throttle_time_ms => INT32 39 | */ 40 | export function parseResponse ( 41 | _correlationId: number, 42 | apiKey: number, 43 | apiVersion: number, 44 | reader: Reader 45 | ): ApiVersionsResponse { 46 | const response: ApiVersionsResponse = { 47 | errorCode: reader.readInt16(), 48 | apiKeys: reader.readArray(r => { 49 | const apiKey = r.readInt16() 50 | 51 | return { 52 | apiKey, 53 | name: protocolAPIsById[apiKey], 54 | minVersion: r.readInt16(), 55 | maxVersion: r.readInt16() 56 | } 57 | }), 58 | throttleTimeMs: reader.readInt32() 59 | } 60 | 61 | if (response.errorCode !== 0) { 62 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 63 | } 64 | 65 | return response 66 | } 67 | 68 | export const api = createAPI(18, 4, createRequest, parseResponse, true, false) 69 | -------------------------------------------------------------------------------- /src/apis/metadata/find-coordinator-v4.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export type FindCoordinatorRequest = Parameters 8 | 9 | export interface FindCoordinatorResponseCoordinator { 10 | key: string 11 | nodeId: number 12 | host: string 13 | port: number 14 | errorCode: number 15 | errorMessage: NullableString 16 | } 17 | 18 | export interface FindCoordinatorResponse { 19 | throttleTimeMs: number 20 | coordinators: FindCoordinatorResponseCoordinator[] 21 | } 22 | 23 | /* 24 | FindCoordinator Request (Version: 4) => key_type [coordinator_keys] TAG_BUFFER 25 | key_type => INT8 26 | coordinator_keys => COMPACT_STRING 27 | */ 28 | export function createRequest (keyType: number, coordinatorKeys: string[]): Writer { 29 | return Writer.create() 30 | .appendInt8(keyType) 31 | .appendArray(coordinatorKeys, (w, k) => w.appendString(k), true, false) 32 | .appendTaggedFields() 33 | } 34 | 35 | /* 36 | FindCoordinator Response (Version: 4) => throttle_time_ms [coordinators] TAG_BUFFER 37 | throttle_time_ms => INT32 38 | coordinators => key node_id host port error_code error_message TAG_BUFFER 39 | key => COMPACT_STRING 40 | node_id => INT32 41 | host => COMPACT_STRING 42 | port => INT32 43 | error_code => INT16 44 | error_message => COMPACT_NULLABLE_STRING 45 | */ 46 | export function parseResponse ( 47 | _correlationId: number, 48 | apiKey: number, 49 | apiVersion: number, 50 | reader: Reader 51 | ): FindCoordinatorResponse { 52 | const errors: ResponseErrorWithLocation[] = [] 53 | 54 | const response: FindCoordinatorResponse = { 55 | throttleTimeMs: reader.readInt32(), 56 | coordinators: reader.readArray((r, i) => { 57 | const coordinator = { 58 | key: r.readString(), 59 | nodeId: r.readInt32(), 60 | host: r.readString(), 61 | port: r.readInt32(), 62 | errorCode: r.readInt16(), 63 | errorMessage: r.readNullableString() 64 | } 65 | 66 | if (coordinator.errorCode !== 0) { 67 | errors.push([`/coordinators/${i}`, coordinator.errorCode]) 68 | } 69 | 70 | return coordinator 71 | }) 72 | } 73 | 74 | if (errors.length) { 75 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 76 | } 77 | 78 | return response 79 | } 80 | 81 | export const api = createAPI(10, 4, createRequest, parseResponse) 82 | -------------------------------------------------------------------------------- /src/apis/metadata/find-coordinator-v5.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export type FindCoordinatorRequest = Parameters 8 | 9 | export interface FindCoordinatorResponseCoordinator { 10 | key: string 11 | nodeId: number 12 | host: string 13 | port: number 14 | errorCode: number 15 | errorMessage: NullableString 16 | } 17 | 18 | export interface FindCoordinatorResponse { 19 | throttleTimeMs: number 20 | coordinators: FindCoordinatorResponseCoordinator[] 21 | } 22 | 23 | /* 24 | FindCoordinator Request (Version: 5) => key_type [coordinator_keys] TAG_BUFFER 25 | key_type => INT8 26 | coordinator_keys => COMPACT_STRING 27 | */ 28 | export function createRequest (keyType: number, coordinatorKeys: string[]): Writer { 29 | return Writer.create() 30 | .appendInt8(keyType) 31 | .appendArray(coordinatorKeys, (w, k) => w.appendString(k), true, false) 32 | .appendTaggedFields() 33 | } 34 | 35 | /* 36 | FindCoordinator Response (Version: 5) => throttle_time_ms [coordinators] TAG_BUFFER 37 | throttle_time_ms => INT32 38 | coordinators => key node_id host port error_code error_message TAG_BUFFER 39 | key => COMPACT_STRING 40 | node_id => INT32 41 | host => COMPACT_STRING 42 | port => INT32 43 | error_code => INT16 44 | error_message => COMPACT_NULLABLE_STRING 45 | */ 46 | export function parseResponse ( 47 | _correlationId: number, 48 | apiKey: number, 49 | apiVersion: number, 50 | reader: Reader 51 | ): FindCoordinatorResponse { 52 | const errors: ResponseErrorWithLocation[] = [] 53 | 54 | const response: FindCoordinatorResponse = { 55 | throttleTimeMs: reader.readInt32(), 56 | coordinators: reader.readArray((r, i) => { 57 | const coordinator = { 58 | key: r.readString(), 59 | nodeId: r.readInt32(), 60 | host: r.readString(), 61 | port: r.readInt32(), 62 | errorCode: r.readInt16(), 63 | errorMessage: r.readNullableString() 64 | } 65 | 66 | if (coordinator.errorCode !== 0) { 67 | errors.push([`/coordinators/${i}`, coordinator.errorCode]) 68 | } 69 | 70 | return coordinator 71 | }) 72 | } 73 | 74 | if (errors.length) { 75 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 76 | } 77 | 78 | return response 79 | } 80 | 81 | export const api = createAPI(10, 5, createRequest, parseResponse) 82 | -------------------------------------------------------------------------------- /src/apis/metadata/find-coordinator-v6.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export type FindCoordinatorRequest = Parameters 8 | 9 | export interface FindCoordinatorResponseCoordinator { 10 | key: string 11 | nodeId: number 12 | host: string 13 | port: number 14 | errorCode: number 15 | errorMessage: NullableString 16 | } 17 | 18 | export interface FindCoordinatorResponse { 19 | throttleTimeMs: number 20 | coordinators: FindCoordinatorResponseCoordinator[] 21 | } 22 | 23 | /* 24 | FindCoordinator Request (Version: 6) => key_type [coordinator_keys] TAG_BUFFER 25 | key_type => INT8 26 | coordinator_keys => COMPACT_STRING 27 | */ 28 | export function createRequest (keyType: number, coordinatorKeys: string[]): Writer { 29 | return Writer.create() 30 | .appendInt8(keyType) 31 | .appendArray(coordinatorKeys, (w, k) => w.appendString(k), true, false) 32 | .appendTaggedFields() 33 | } 34 | 35 | /* 36 | FindCoordinator Response (Version: 6) => throttle_time_ms [coordinators] TAG_BUFFER 37 | throttle_time_ms => INT32 38 | coordinators => key node_id host port error_code error_message TAG_BUFFER 39 | key => COMPACT_STRING 40 | node_id => INT32 41 | host => COMPACT_STRING 42 | port => INT32 43 | error_code => INT16 44 | error_message => COMPACT_NULLABLE_STRING 45 | */ 46 | export function parseResponse ( 47 | _correlationId: number, 48 | apiKey: number, 49 | apiVersion: number, 50 | reader: Reader 51 | ): FindCoordinatorResponse { 52 | const errors: ResponseErrorWithLocation[] = [] 53 | 54 | const response: FindCoordinatorResponse = { 55 | throttleTimeMs: reader.readInt32(), 56 | coordinators: reader.readArray((r, i) => { 57 | const coordinator = { 58 | key: r.readString(), 59 | nodeId: r.readInt32(), 60 | host: r.readString(), 61 | port: r.readInt32(), 62 | errorCode: r.readInt16(), 63 | errorMessage: r.readNullableString() 64 | } 65 | 66 | if (coordinator.errorCode !== 0) { 67 | errors.push([`/coordinators/${i}`, coordinator.errorCode]) 68 | } 69 | 70 | return coordinator 71 | }) 72 | } 73 | 74 | if (errors.length) { 75 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 76 | } 77 | 78 | return response 79 | } 80 | 81 | export const api = createAPI(10, 6, createRequest, parseResponse) 82 | -------------------------------------------------------------------------------- /src/apis/metadata/index.ts: -------------------------------------------------------------------------------- 1 | export * as apiVersionsV3 from './api-versions-v3.ts' 2 | export * as apiVersionsV4 from './api-versions-v4.ts' 3 | export * as findCoordinatorV4 from './find-coordinator-v4.ts' 4 | export * as findCoordinatorV5 from './find-coordinator-v5.ts' 5 | export * as findCoordinatorV6 from './find-coordinator-v6.ts' 6 | export * as metadataV12 from './metadata-v12.ts' 7 | -------------------------------------------------------------------------------- /src/apis/producer/add-offsets-to-txn-v4.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type AddOffsetsToTxnRequest = Parameters 7 | 8 | export interface AddOffsetsToTxnResponse { 9 | throttleTimeMs: number 10 | errorCode: number 11 | } 12 | 13 | /* 14 | AddOffsetsToTxn Request (Version: 4) => transactional_id producer_id producer_epoch group_id TAG_BUFFER 15 | transactional_id => COMPACT_STRING 16 | producer_id => INT64 17 | producer_epoch => INT16 18 | group_id => COMPACT_STRING 19 | */ 20 | export function createRequest ( 21 | transactionalId: string, 22 | producerId: bigint, 23 | producerEpoch: number, 24 | groupId: string 25 | ): Writer { 26 | return Writer.create() 27 | .appendString(transactionalId, true) 28 | .appendInt64(producerId) 29 | .appendInt16(producerEpoch) 30 | .appendString(groupId, true) 31 | .appendTaggedFields() 32 | } 33 | 34 | /* 35 | AddOffsetsToTxn Response (Version: 4) => throttle_time_ms error_code TAG_BUFFER 36 | throttle_time_ms => INT32 37 | error_code => INT16 38 | */ 39 | export function parseResponse ( 40 | _correlationId: number, 41 | apiKey: number, 42 | apiVersion: number, 43 | reader: Reader 44 | ): AddOffsetsToTxnResponse { 45 | const response = { 46 | throttleTimeMs: reader.readInt32(), 47 | errorCode: reader.readInt16() 48 | } 49 | 50 | if (response.errorCode !== 0) { 51 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 52 | } 53 | 54 | return response 55 | } 56 | 57 | export const api = createAPI(25, 4, createRequest, parseResponse) 58 | -------------------------------------------------------------------------------- /src/apis/producer/end-txn-v4.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type EndTxnRequest = Parameters 7 | 8 | export interface EndTxnResponse { 9 | throttleTimeMs: number 10 | errorCode: number 11 | } 12 | 13 | /* 14 | EndTxn Request (Version: 4) => transactional_id producer_id producer_epoch committed TAG_BUFFER 15 | transactional_id => COMPACT_STRING 16 | producer_id => INT64 17 | producer_epoch => INT16 18 | committed => BOOLEAN 19 | */ 20 | export function createRequest ( 21 | transactionalId: string, 22 | producerId: bigint, 23 | producerEpoch: number, 24 | committed: boolean 25 | ): Writer { 26 | return Writer.create() 27 | .appendString(transactionalId, true) 28 | .appendInt64(producerId) 29 | .appendInt16(producerEpoch) 30 | .appendBoolean(committed) 31 | .appendTaggedFields() 32 | } 33 | 34 | /* 35 | EndTxn Response (Version: 4) => throttle_time_ms error_code TAG_BUFFER 36 | throttle_time_ms => INT32 37 | error_code => INT16 38 | */ 39 | export function parseResponse ( 40 | _correlationId: number, 41 | apiKey: number, 42 | apiVersion: number, 43 | reader: Reader 44 | ): EndTxnResponse { 45 | const response = { 46 | throttleTimeMs: reader.readInt32(), 47 | errorCode: reader.readInt16() 48 | } 49 | 50 | if (response.errorCode !== 0) { 51 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 52 | } 53 | 54 | return response 55 | } 56 | 57 | export const api = createAPI(26, 4, createRequest, parseResponse) 58 | -------------------------------------------------------------------------------- /src/apis/producer/index.ts: -------------------------------------------------------------------------------- 1 | export * as addOffsetsToTxnV4 from './add-offsets-to-txn-v4.ts' 2 | export * as addPartitionsToTxnV5 from './add-partitions-to-txn-v5.ts' 3 | export * as endTxnV4 from './end-txn-v4.ts' 4 | export * as initProducerIdV4 from './init-producer-id-v4.ts' 5 | export * as initProducerIdV5 from './init-producer-id-v5.ts' 6 | export * as produceV10 from './produce-v10.ts' 7 | export * as produceV11 from './produce-v11.ts' 8 | export * as txnOffsetCommitV4 from './txn-offset-commit-v4.ts' 9 | -------------------------------------------------------------------------------- /src/apis/producer/init-producer-id-v4.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type InitProducerIdRequest = Parameters 8 | 9 | export interface InitProducerIdResponseCoordinator { 10 | key: string 11 | nodeId: number 12 | host: string 13 | port: number 14 | errorCode: number 15 | errorMessage: NullableString 16 | } 17 | 18 | export interface InitProducerIdResponse { 19 | throttleTimeMs: number 20 | errorCode: number 21 | producerId: bigint 22 | producerEpoch: number 23 | } 24 | 25 | /* 26 | InitProducerId Request (Version: 4) => transactional_id transaction_timeout_ms producer_id producer_epoch TAG_BUFFER 27 | transactional_id => COMPACT_NULLABLE_STRING 28 | transaction_timeout_ms => INT32 29 | producer_id => INT64 30 | producer_epoch => INT16 31 | */ 32 | export function createRequest ( 33 | transactionalId: NullableString, 34 | transactionTimeoutMs: number, 35 | producerId: bigint, 36 | producerEpoch: number 37 | ): Writer { 38 | return Writer.create() 39 | .appendString(transactionalId, true) 40 | .appendInt32(transactionTimeoutMs) 41 | .appendInt64(producerId) 42 | .appendInt16(producerEpoch) 43 | .appendTaggedFields() 44 | } 45 | 46 | /* 47 | InitProducerId Response (Version: 4) => throttle_time_ms error_code producer_id producer_epoch TAG_BUFFER 48 | throttle_time_ms => INT32 49 | error_code => INT16 50 | producer_id => INT64 51 | producer_epoch => INT16 52 | */ 53 | export function parseResponse ( 54 | _correlationId: number, 55 | apiKey: number, 56 | apiVersion: number, 57 | reader: Reader 58 | ): InitProducerIdResponse { 59 | const response = { 60 | throttleTimeMs: reader.readInt32(), 61 | errorCode: reader.readInt16(), 62 | producerId: reader.readInt64(), 63 | producerEpoch: reader.readInt16() 64 | } 65 | 66 | if (response.errorCode !== 0) { 67 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 68 | } 69 | 70 | return response 71 | } 72 | 73 | export const api = createAPI(22, 4, createRequest, parseResponse) 74 | -------------------------------------------------------------------------------- /src/apis/producer/init-producer-id-v5.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI } from '../definitions.ts' 6 | 7 | export type InitProducerIdRequest = Parameters 8 | 9 | export interface InitProducerIdResponseCoordinator { 10 | key: string 11 | nodeId: number 12 | host: string 13 | port: number 14 | errorCode: number 15 | errorMessage: NullableString 16 | } 17 | 18 | export interface InitProducerIdResponse { 19 | throttleTimeMs: number 20 | errorCode: number 21 | producerId: bigint 22 | producerEpoch: number 23 | } 24 | 25 | /* 26 | InitProducerId Request (Version: 5) => transactional_id transaction_timeout_ms producer_id producer_epoch TAG_BUFFER 27 | transactional_id => COMPACT_NULLABLE_STRING 28 | transaction_timeout_ms => INT32 29 | producer_id => INT64 30 | producer_epoch => INT16 31 | */ 32 | export function createRequest ( 33 | transactionalId: NullableString, 34 | transactionTimeoutMs: number, 35 | producerId: bigint, 36 | producerEpoch: number 37 | ): Writer { 38 | return Writer.create() 39 | .appendString(transactionalId, true) 40 | .appendInt32(transactionTimeoutMs) 41 | .appendInt64(producerId) 42 | .appendInt16(producerEpoch) 43 | .appendTaggedFields() 44 | } 45 | 46 | /* 47 | InitProducerId Response (Version: 5) => throttle_time_ms error_code producer_id producer_epoch TAG_BUFFER 48 | throttle_time_ms => INT32 49 | error_code => INT16 50 | producer_id => INT64 51 | producer_epoch => INT16 52 | */ 53 | export function parseResponse ( 54 | _correlationId: number, 55 | apiKey: number, 56 | apiVersion: number, 57 | reader: Reader 58 | ): InitProducerIdResponse { 59 | const response = { 60 | throttleTimeMs: reader.readInt32(), 61 | errorCode: reader.readInt16(), 62 | producerId: reader.readInt64(), 63 | producerEpoch: reader.readInt16() 64 | } 65 | 66 | if (response.errorCode !== 0) { 67 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 68 | } 69 | 70 | return response 71 | } 72 | 73 | export const api = createAPI(22, 5, createRequest, parseResponse) 74 | -------------------------------------------------------------------------------- /src/apis/security/index.ts: -------------------------------------------------------------------------------- 1 | export * as saslAuthenticateV2 from './sasl-authenticate-v2.ts' 2 | export * as saslHandshakeV1 from './sasl-handshake-v1.ts' 3 | -------------------------------------------------------------------------------- /src/apis/security/sasl-authenticate-v2.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { type API, createAPI } from '../definitions.ts' 6 | 7 | export type SaslAuthenticateRequest = Parameters 8 | 9 | export interface SaslAuthenticateResponse { 10 | errorCode: number 11 | errorMessage: NullableString 12 | authBytes: Buffer 13 | sessionLifetimeMs: bigint 14 | } 15 | 16 | export type SASLAuthenticationAPI = API<[Buffer], SaslAuthenticateResponse> 17 | 18 | /* 19 | SaslAuthenticate Request (Version: 2) => auth_bytes TAG_BUFFER 20 | auth_bytes => COMPACT_BYTES 21 | */ 22 | export function createRequest (authBytes: Buffer): Writer { 23 | return Writer.create().appendBytes(authBytes).appendTaggedFields() 24 | } 25 | 26 | /* 27 | SaslAuthenticate Response (Version: 2) => error_code error_message auth_bytes session_lifetime_ms TAG_BUFFER 28 | error_code => INT16 29 | error_message => COMPACT_NULLABLE_STRING 30 | auth_bytes => COMPACT_BYTES 31 | session_lifetime_ms => INT64 32 | */ 33 | export function parseResponse ( 34 | _correlationId: number, 35 | apiKey: number, 36 | apiVersion: number, 37 | reader: Reader 38 | ): SaslAuthenticateResponse { 39 | const response: SaslAuthenticateResponse = { 40 | errorCode: reader.readInt16(), 41 | errorMessage: reader.readNullableString(), 42 | authBytes: reader.readBytes(), 43 | sessionLifetimeMs: reader.readInt64() 44 | } 45 | 46 | if (response.errorCode !== 0) { 47 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 48 | } 49 | 50 | return response 51 | } 52 | 53 | export const api = createAPI(36, 2, createRequest, parseResponse) 54 | -------------------------------------------------------------------------------- /src/apis/security/sasl-handshake-v1.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type SaslHandshakeRequest = Parameters 7 | 8 | export interface SaslHandshakeResponse { 9 | errorCode?: number 10 | mechanisms?: string[] 11 | } 12 | 13 | /* 14 | SaslHandshake Request (Version: 1) => mechanism 15 | mechanism => STRING 16 | */ 17 | export function createRequest (mechanism: string): Writer { 18 | return Writer.create().appendString(mechanism, false) 19 | } 20 | 21 | /* 22 | SaslHandshake Response (Version: 1) => error_code [mechanisms] 23 | error_code => INT16 24 | mechanisms => STRING 25 | */ 26 | export function parseResponse ( 27 | _correlationId: number, 28 | apiKey: number, 29 | apiVersion: number, 30 | reader: Reader 31 | ): SaslHandshakeResponse { 32 | const response: SaslHandshakeResponse = { 33 | errorCode: reader.readInt16(), 34 | mechanisms: reader.readArray( 35 | r => { 36 | return r.readString(false)! 37 | }, 38 | false, 39 | false 40 | )! 41 | } 42 | 43 | if (response.errorCode !== 0) { 44 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode! }, response) 45 | } 46 | 47 | return response 48 | } 49 | 50 | export const api = createAPI( 51 | 17, 52 | 1, 53 | createRequest, 54 | parseResponse, 55 | false, 56 | false 57 | ) 58 | -------------------------------------------------------------------------------- /src/apis/telemetry/get-telemetry-subscriptions-v0.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type Reader } from '../../protocol/reader.ts' 4 | import { Writer } from '../../protocol/writer.ts' 5 | import { createAPI, type ResponseErrorWithLocation } from '../definitions.ts' 6 | 7 | export type GetTelemetrySubscriptionsRequest = Parameters 8 | 9 | export interface GetTelemetrySubscriptionsResponse { 10 | throttleTimeMs: number 11 | errorCode: number 12 | clientInstanceId: string 13 | subscriptionId: number 14 | acceptedCompressionTypes: number[] 15 | pushIntervalMs: number 16 | telemetryMaxBytes: number 17 | deltaTemporality: boolean 18 | requestedMetrics: string[] 19 | } 20 | 21 | /* 22 | GetTelemetrySubscriptions Request (Version: 0) => client_instance_id TAG_BUFFER 23 | client_instance_id => UUID 24 | */ 25 | export function createRequest (clientInstanceId?: NullableString): Writer { 26 | return Writer.create().appendUUID(clientInstanceId).appendTaggedFields() 27 | } 28 | 29 | /* 30 | GetTelemetrySubscriptions Response (Version: 0) => throttle_time_ms error_code client_instance_id subscription_id [accepted_compression_types] push_interval_ms telemetry_max_bytes delta_temporality [requested_metrics] TAG_BUFFER 31 | throttle_time_ms => INT32 32 | error_code => INT16 33 | client_instance_id => UUID 34 | subscription_id => INT32 35 | accepted_compression_types => INT8 36 | push_interval_ms => INT32 37 | telemetry_max_bytes => INT32 38 | delta_temporality => BOOLEAN 39 | requested_metrics => COMPACT_STRING 40 | */ 41 | export function parseResponse ( 42 | _correlationId: number, 43 | apiKey: number, 44 | apiVersion: number, 45 | reader: Reader 46 | ): GetTelemetrySubscriptionsResponse { 47 | const errors: ResponseErrorWithLocation[] = [] 48 | 49 | const throttleTimeMs = reader.readInt32() 50 | const errorCode = reader.readInt16() 51 | 52 | if (errorCode !== 0) { 53 | errors.push(['', errorCode]) 54 | } 55 | 56 | const response: GetTelemetrySubscriptionsResponse = { 57 | throttleTimeMs, 58 | errorCode, 59 | clientInstanceId: reader.readUUID(), 60 | subscriptionId: reader.readInt32(), 61 | acceptedCompressionTypes: reader.readArray(r => r.readInt8(), true, false)!, 62 | pushIntervalMs: reader.readInt32(), 63 | telemetryMaxBytes: reader.readInt32(), 64 | deltaTemporality: reader.readBoolean(), 65 | requestedMetrics: reader.readArray(r => r.readString(), true, false)! 66 | } 67 | 68 | if (errors.length) { 69 | throw new ResponseError(apiKey, apiVersion, Object.fromEntries(errors), response) 70 | } 71 | 72 | return response 73 | } 74 | 75 | export const api = createAPI( 76 | 71, 77 | 0, 78 | createRequest, 79 | parseResponse 80 | ) 81 | -------------------------------------------------------------------------------- /src/apis/telemetry/index.ts: -------------------------------------------------------------------------------- 1 | export * as getTelemetrySubscriptionsV0 from './get-telemetry-subscriptions-v0.ts' 2 | export * as listClientMetricsResourcesV0 from './list-client-metrics-resources-v0.ts' 3 | export * as pushTelemetryV0 from './push-telemetry-v0.ts' 4 | -------------------------------------------------------------------------------- /src/apis/telemetry/list-client-metrics-resources-v0.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type ListClientMetricsResourcesRequest = Parameters 7 | 8 | export interface ListClientMetricsResourcesResource { 9 | name: string 10 | } 11 | 12 | export interface ListClientMetricsResourcesResponse { 13 | throttleTimeMs: number 14 | errorCode: number 15 | clientMetricsResources: ListClientMetricsResourcesResource[] 16 | } 17 | 18 | /* 19 | ListClientMetricsResources Request (Version: 0) => TAG_BUFFER 20 | */ 21 | export function createRequest (): Writer { 22 | return Writer.create().appendTaggedFields() 23 | } 24 | 25 | /* 26 | ListClientMetricsResources Response (Version: 0) => throttle_time_ms error_code [client_metrics_resources] TAG_BUFFER 27 | throttle_time_ms => INT32 28 | error_code => INT16 29 | client_metrics_resources => name TAG_BUFFER 30 | name => COMPACT_STRING 31 | */ 32 | export function parseResponse ( 33 | _correlationId: number, 34 | apiKey: number, 35 | apiVersion: number, 36 | reader: Reader 37 | ): ListClientMetricsResourcesResponse { 38 | const response: ListClientMetricsResourcesResponse = { 39 | throttleTimeMs: reader.readInt32(), 40 | errorCode: reader.readInt16(), 41 | clientMetricsResources: reader.readArray(r => { 42 | return { 43 | name: r.readString() 44 | } 45 | }) 46 | } 47 | 48 | if (response.errorCode !== 0) { 49 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 50 | } 51 | 52 | return response 53 | } 54 | 55 | export const api = createAPI( 56 | 74, 57 | 0, 58 | createRequest, 59 | parseResponse 60 | ) 61 | -------------------------------------------------------------------------------- /src/apis/telemetry/push-telemetry-v0.ts: -------------------------------------------------------------------------------- 1 | import { ResponseError } from '../../errors.ts' 2 | import { type Reader } from '../../protocol/reader.ts' 3 | import { Writer } from '../../protocol/writer.ts' 4 | import { createAPI } from '../definitions.ts' 5 | 6 | export type PushTelemetryRequest = Parameters 7 | 8 | export interface PushTelemetryResponse { 9 | throttleTimeMs: number 10 | errorCode: number 11 | } 12 | 13 | /* 14 | PushTelemetry Request (Version: 0) => client_instance_id subscription_id terminating compression_type metrics TAG_BUFFER 15 | client_instance_id => UUID 16 | subscription_id => INT32 17 | terminating => BOOLEAN 18 | compression_type => INT8 19 | metrics => COMPACT_BYTES 20 | */ 21 | export function createRequest ( 22 | clientInstanceId: string, 23 | subscriptionId: number, 24 | terminating: boolean, 25 | compressionType: number, 26 | metrics: Buffer 27 | ): Writer { 28 | return Writer.create() 29 | .appendUUID(clientInstanceId) 30 | .appendInt32(subscriptionId) 31 | .appendBoolean(terminating) 32 | .appendInt8(compressionType) 33 | .appendBytes(metrics) 34 | .appendTaggedFields() 35 | } 36 | 37 | /* 38 | PushTelemetry Response (Version: 0) => throttle_time_ms error_code TAG_BUFFER 39 | throttle_time_ms => INT32 40 | error_code => INT16 41 | */ 42 | export function parseResponse ( 43 | _correlationId: number, 44 | apiKey: number, 45 | apiVersion: number, 46 | reader: Reader 47 | ): PushTelemetryResponse { 48 | const response: PushTelemetryResponse = { 49 | throttleTimeMs: reader.readInt32(), 50 | errorCode: reader.readInt16() 51 | } 52 | 53 | if (response.errorCode !== 0) { 54 | throw new ResponseError(apiKey, apiVersion, { '': response.errorCode }, response) 55 | } 56 | 57 | return response 58 | } 59 | 60 | export const api = createAPI(72, 0, createRequest, parseResponse) 61 | -------------------------------------------------------------------------------- /src/clients/admin/index.ts: -------------------------------------------------------------------------------- 1 | export * from './admin.ts' 2 | export * from './options.ts' 3 | export * from './types.ts' 4 | -------------------------------------------------------------------------------- /src/clients/admin/options.ts: -------------------------------------------------------------------------------- 1 | import { ConsumerGroupStates } from '../../apis/enumerations.ts' 2 | import { ajv, listErrorMessage } from '../../utils.ts' 3 | import { idProperty } from '../base/options.ts' 4 | 5 | export const groupsProperties = { 6 | groups: { 7 | type: 'array', 8 | items: idProperty, 9 | minItems: 1 10 | } 11 | } 12 | 13 | export const createTopicOptionsSchema = { 14 | type: 'object', 15 | properties: { 16 | topics: { type: 'array', items: idProperty }, 17 | partitions: { type: 'number' }, 18 | replicas: { type: 'number' }, 19 | assignments: { 20 | type: 'array', 21 | items: { 22 | type: 'object', 23 | properties: { 24 | partition: { type: 'number', minimum: 0 }, 25 | brokers: { type: 'array', items: { type: 'number' }, minItems: 1 } 26 | }, 27 | required: ['partition', 'brokers'], 28 | additionalProperties: false 29 | }, 30 | minItems: 1 31 | } 32 | }, 33 | required: ['topics'], 34 | additionalProperties: false 35 | } 36 | 37 | export const deleteTopicOptionsSchema = { 38 | type: 'object', 39 | properties: { 40 | topics: { type: 'array', items: idProperty } 41 | }, 42 | required: ['topics'], 43 | additionalProperties: false 44 | } 45 | 46 | export const listGroupsOptionsSchema = { 47 | type: 'object', 48 | properties: { 49 | states: { 50 | type: 'array', 51 | items: { 52 | type: 'string', 53 | enum: ConsumerGroupStates, 54 | errorMessage: listErrorMessage(ConsumerGroupStates as unknown as string[]) 55 | }, 56 | minItems: 0 57 | }, 58 | types: { 59 | type: 'array', 60 | items: idProperty, 61 | minItems: 0 62 | } 63 | }, 64 | additionalProperties: false 65 | } 66 | 67 | export const describeGroupsOptionsSchema = { 68 | type: 'object', 69 | properties: { 70 | ...groupsProperties, 71 | includeAuthorizedOperations: { type: 'boolean' } 72 | }, 73 | required: ['groups'], 74 | additionalProperties: false 75 | } 76 | 77 | export const deleteGroupsOptionsSchema = { 78 | type: 'object', 79 | properties: groupsProperties, 80 | required: ['groups'], 81 | additionalProperties: false 82 | } 83 | 84 | export const createTopicsOptionsValidator = ajv.compile(createTopicOptionsSchema) 85 | export const deleteTopicsOptionsValidator = ajv.compile(deleteTopicOptionsSchema) 86 | export const listGroupsOptionsValidator = ajv.compile(listGroupsOptionsSchema) 87 | export const describeGroupsOptionsValidator = ajv.compile(describeGroupsOptionsSchema) 88 | export const deleteGroupsOptionsValidator = ajv.compile(deleteGroupsOptionsSchema) 89 | -------------------------------------------------------------------------------- /src/clients/admin/types.ts: -------------------------------------------------------------------------------- 1 | import { type ConsumerGroupState } from '../../apis/enumerations.ts' 2 | import { type NullableString } from '../../protocol/definitions.ts' 3 | import { type BaseOptions } from '../base/types.ts' 4 | import { type ExtendedGroupProtocolSubscription, type GroupAssignment } from '../consumer/types.ts' 5 | 6 | export interface BrokerAssignment { 7 | partition: number 8 | brokers: number[] 9 | } 10 | 11 | export interface CreatedTopic { 12 | id: string 13 | name: string 14 | partitions: number 15 | replicas: number 16 | configuration: Record 17 | } 18 | 19 | export interface GroupMember { 20 | id: string 21 | groupInstanceId: NullableString 22 | clientId: string 23 | clientHost: string 24 | metadata: Omit 25 | assignments: Map 26 | } 27 | 28 | export interface GroupBase { 29 | id: string 30 | state: ConsumerGroupState 31 | groupType: string 32 | protocolType: string 33 | } 34 | 35 | export interface Group extends Omit { 36 | protocol: string 37 | members: Map 38 | authorizedOperations: number 39 | } 40 | 41 | // Currently empty but reserved for future use 42 | export interface AdminOptions extends BaseOptions {} 43 | 44 | export interface CreateTopicsOptions { 45 | topics: string[] 46 | partitions?: number 47 | replicas?: number 48 | assignments?: BrokerAssignment[] 49 | } 50 | 51 | export interface DeleteTopicsOptions { 52 | topics: string[] 53 | } 54 | 55 | export interface ListGroupsOptions { 56 | states?: ConsumerGroupState[] 57 | types?: string[] 58 | } 59 | 60 | export interface DescribeGroupsOptions { 61 | groups: string[] 62 | includeAuthorizedOperations?: boolean 63 | } 64 | 65 | export interface DeleteGroupsOptions { 66 | groups: string[] 67 | } 68 | -------------------------------------------------------------------------------- /src/clients/base/index.ts: -------------------------------------------------------------------------------- 1 | export { Base } from './base.ts' 2 | export * from './options.ts' 3 | export * from './types.ts' 4 | -------------------------------------------------------------------------------- /src/clients/base/options.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from 'node:fs' 2 | import { SASLMechanisms } from '../../apis/enumerations.ts' 3 | import { ajv } from '../../utils.ts' 4 | import { type BaseOptions } from './types.ts' 5 | 6 | const packageJson = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf-8')) 7 | 8 | // Note: clientSoftwareName can only contain alphanumeric characters, hyphens and dots 9 | export const clientSoftwareName = 'platformatic-kafka' 10 | export const clientSoftwareVersion = packageJson.version 11 | 12 | export const idProperty = { type: 'string', pattern: '^\\S+$' } 13 | 14 | export const topicWithPartitionAndOffsetProperties = { 15 | topic: idProperty, 16 | partition: { type: 'number', minimum: 0 }, 17 | offset: { bigint: true } 18 | } 19 | 20 | export const baseOptionsSchema = { 21 | type: 'object', 22 | properties: { 23 | clientId: idProperty, 24 | bootstrapBrokers: { 25 | oneOf: [ 26 | { type: 'array', items: { type: 'string' } }, 27 | { 28 | type: 'array', 29 | items: { 30 | type: 'object', 31 | properties: { host: { type: 'string' }, port: { type: 'number', minimum: 0, maximum: 65535 } } 32 | } 33 | } 34 | ] 35 | }, 36 | timeout: { type: 'number', minimum: 0 }, 37 | connectTimeout: { type: 'number', minimum: 0 }, 38 | retries: { type: 'number', minimum: 0 }, 39 | retryDelay: { type: 'number', minimum: 0 }, 40 | maxInflights: { type: 'number', minimum: 0 }, 41 | tls: { type: 'object', additionalProperties: true }, // No validation as they come from Node.js 42 | sasl: { 43 | type: 'object', 44 | properties: { 45 | mechanism: { type: 'string', enum: SASLMechanisms }, 46 | username: { type: 'string' }, 47 | password: { type: 'string' } 48 | }, 49 | required: ['mechanism', 'username', 'password'], 50 | additionalProperties: false 51 | }, 52 | metadataMaxAge: { type: 'number', minimum: 0 }, 53 | autocreateTopics: { type: 'boolean' }, 54 | strict: { type: 'boolean' }, 55 | metrics: { type: 'object', additionalProperties: true } 56 | }, 57 | required: ['clientId', 'bootstrapBrokers'], 58 | additionalProperties: true 59 | } 60 | 61 | export const metadataOptionsSchema = { 62 | type: 'object', 63 | properties: { 64 | topics: { type: 'array', items: idProperty }, 65 | autocreateTopics: { type: 'boolean' }, 66 | forceUpdate: { type: 'boolean' }, 67 | metadataMaxAge: { type: 'number', minimum: 0 } 68 | }, 69 | required: ['topics'], 70 | additionalProperties: false 71 | } 72 | 73 | export const baseOptionsValidator = ajv.compile(baseOptionsSchema) 74 | 75 | export const metadataOptionsValidator = ajv.compile(metadataOptionsSchema) 76 | 77 | export const defaultPort = 9092 78 | export const defaultBaseOptions: Partial = { 79 | connectTimeout: 5000, 80 | maxInflights: 5, 81 | timeout: 5000, 82 | retries: 3, 83 | retryDelay: 1000, 84 | metadataMaxAge: 5000, // 5 minutes 85 | autocreateTopics: false, 86 | strict: false 87 | } 88 | -------------------------------------------------------------------------------- /src/clients/base/types.ts: -------------------------------------------------------------------------------- 1 | import { type Broker, type ConnectionOptions } from '../../network/connection.ts' 2 | import { type Metrics } from '../metrics.ts' 3 | 4 | export interface TopicWithPartitionAndOffset { 5 | topic: string 6 | partition: number 7 | offset: bigint 8 | } 9 | 10 | export interface ClusterPartitionMetadata { 11 | leader: number 12 | leaderEpoch: number 13 | replicas: number[] 14 | } 15 | 16 | export interface ClusterTopicMetadata { 17 | id: string 18 | partitions: ClusterPartitionMetadata[] 19 | partitionsCount: number 20 | } 21 | 22 | export interface ClusterMetadata { 23 | id: string 24 | brokers: Map 25 | topics: Map 26 | lastUpdate: number 27 | } 28 | 29 | export interface BaseOptions extends ConnectionOptions { 30 | clientId: string 31 | bootstrapBrokers: Broker[] | string[] 32 | timeout?: number 33 | retries?: number 34 | retryDelay?: number 35 | metadataMaxAge?: number 36 | autocreateTopics?: boolean 37 | strict?: boolean 38 | metrics?: Metrics 39 | } 40 | 41 | export interface MetadataOptions { 42 | topics: string[] 43 | autocreateTopics?: boolean 44 | forceUpdate?: boolean 45 | metadataMaxAge?: number 46 | } 47 | -------------------------------------------------------------------------------- /src/clients/consumer/index.ts: -------------------------------------------------------------------------------- 1 | export * from './consumer.ts' 2 | export * from './messages-stream.ts' 3 | export * from './options.ts' 4 | export * from './topics-map.ts' 5 | export * from './types.ts' 6 | -------------------------------------------------------------------------------- /src/clients/consumer/topics-map.ts: -------------------------------------------------------------------------------- 1 | import { type Gauge } from '../metrics.ts' 2 | 3 | export class TopicsMap extends Map { 4 | #current: string[] = [] 5 | #metric: Gauge | undefined 6 | 7 | get current (): string[] { 8 | return this.#current 9 | } 10 | 11 | clear () { 12 | for (const k of this.keys()) { 13 | this.untrack(k) 14 | } 15 | 16 | super.clear() 17 | } 18 | 19 | track (topic: string): boolean { 20 | let updated = false 21 | 22 | let existing = this.get(topic) 23 | if (typeof existing === 'undefined') { 24 | existing = 0 25 | updated = true 26 | } 27 | 28 | this.set(topic, existing + 1) 29 | 30 | if (existing === 0) { 31 | this.#metric?.inc() 32 | } 33 | 34 | if (updated) { 35 | this.#updateCurrentList() 36 | } 37 | 38 | return updated 39 | } 40 | 41 | trackAll (...topics: string[]): boolean[] { 42 | const updated = [] 43 | for (const topic of topics.flat()) { 44 | updated.push(this.track(topic)) 45 | } 46 | 47 | return updated 48 | } 49 | 50 | untrack (topic: string): boolean { 51 | const existing = this.get(topic) 52 | 53 | if (existing === 1) { 54 | this.delete(topic) 55 | this.#updateCurrentList() 56 | this.#metric?.dec() 57 | return true 58 | } else if (typeof existing === 'number') { 59 | this.set(topic, existing - 1) 60 | } 61 | 62 | return false 63 | } 64 | 65 | untrackAll (...topics: string[]): boolean[] { 66 | const updated = [] 67 | for (const topic of topics.flat()) { 68 | updated.push(this.untrack(topic)) 69 | } 70 | 71 | return updated 72 | } 73 | 74 | setMetric (metric: Gauge): void { 75 | this.#metric = metric 76 | } 77 | 78 | #updateCurrentList (): void { 79 | this.#current = Array.from(this.keys()) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/clients/index.ts: -------------------------------------------------------------------------------- 1 | export * from './serde.ts' 2 | 3 | export * from './admin/index.ts' 4 | export * from './base/index.ts' 5 | export * from './consumer/index.ts' 6 | export * from './producer/index.ts' 7 | -------------------------------------------------------------------------------- /src/clients/metrics.ts: -------------------------------------------------------------------------------- 1 | // Interfaces to make the package compatible with prom-client 2 | 3 | // Unused in this package 4 | type RegistryContentType = 5 | | 'application/openmetrics-text; version=1.0.0; charset=utf-8' 6 | | 'text/plain; version=0.0.4; charset=utf-8' 7 | 8 | // Unused in this package 9 | export interface Metric { 10 | name?: string 11 | get(): Promise 12 | reset: () => void 13 | labels(labels: any): any 14 | } 15 | 16 | export interface Counter extends Metric { 17 | inc: (value?: number) => void 18 | } 19 | 20 | export interface Gauge extends Metric { 21 | inc: (value?: number) => void 22 | dec: (value?: number) => void 23 | } 24 | 25 | export interface Histogram {} 26 | 27 | export interface Summary {} 28 | 29 | export interface Registry { 30 | getSingleMetric: (name: string) => Counter | Gauge | any 31 | 32 | // Unused in this package 33 | metrics(): Promise 34 | clear(): void 35 | resetMetrics(): void 36 | registerMetric(metric: Metric): void 37 | getMetricsAsJSON(): Promise 38 | getMetricsAsArray(): any[] 39 | removeSingleMetric(name: string): void 40 | setDefaultLabels(labels: object): void 41 | getSingleMetricAsString(name: string): Promise 42 | readonly contentType: RegistryContentType 43 | setContentType(contentType: RegistryContentType): void 44 | } 45 | 46 | export interface Prometheus { 47 | Counter: new (options: { name: string; help: string; registers: Registry[]; labelNames?: string[] }) => Counter 48 | Gauge: new (options: { name: string; help: string; registers: Registry[]; labelNames?: string[] }) => Gauge 49 | Registry: new (contentType?: string) => Registry 50 | } 51 | 52 | export interface Metrics { 53 | registry: Registry 54 | client: Prometheus 55 | labels?: Record 56 | } 57 | 58 | export function ensureMetric ( 59 | metrics: Metrics, 60 | type: 'Gauge' | 'Counter', 61 | name: string, 62 | help: string 63 | ): MetricType { 64 | let metric = metrics.registry.getSingleMetric(name) as MetricType 65 | const labels = Object.keys(metrics.labels ?? {}) 66 | 67 | if (!metric) { 68 | metric = new metrics.client[type]({ 69 | name, 70 | help, 71 | registers: [metrics.registry], 72 | labelNames: labels 73 | }) as unknown as MetricType 74 | } else { 75 | // @ts-expect-error Overriding internal API 76 | metric.labelNames = metric.sortedLabelNames = Array.from(new Set([...metric.labelNames, ...labels])).sort() 77 | } 78 | 79 | return metric.labels(metrics.labels ?? {}) as MetricType 80 | } 81 | -------------------------------------------------------------------------------- /src/clients/producer/index.ts: -------------------------------------------------------------------------------- 1 | export * from './options.ts' 2 | export * from './producer.ts' 3 | export * from './types.ts' 4 | -------------------------------------------------------------------------------- /src/clients/producer/options.ts: -------------------------------------------------------------------------------- 1 | import { ProduceAcks } from '../../apis/enumerations.ts' 2 | import { compressionsAlgorithms } from '../../protocol/compression.ts' 3 | import { messageSchema } from '../../protocol/records.ts' 4 | import { ajv, enumErrorMessage } from '../../utils.ts' 5 | import { serdeProperties } from '../serde.ts' 6 | 7 | export const produceOptionsProperties = { 8 | producerId: { bigint: true }, 9 | producerEpoch: { type: 'number' }, 10 | idempotent: { type: 'boolean' }, 11 | acks: { 12 | type: 'number', 13 | enum: Object.values(ProduceAcks), 14 | errorMessage: enumErrorMessage(ProduceAcks) 15 | }, 16 | compression: { 17 | type: 'string', 18 | enum: Object.keys(compressionsAlgorithms), 19 | errorMessage: enumErrorMessage(compressionsAlgorithms, true) 20 | }, 21 | partitioner: { function: true }, 22 | autocreateTopics: { type: 'boolean' }, 23 | repeatOnStaleMetadata: { type: 'boolean' } 24 | } 25 | 26 | export const produceOptionsSchema = { 27 | type: 'object', 28 | properties: produceOptionsProperties, 29 | additionalProperties: false 30 | } 31 | 32 | export const produceOptionsValidator = ajv.compile(produceOptionsSchema) 33 | export const producerOptionsValidator = ajv.compile({ 34 | type: 'object', 35 | properties: { 36 | ...produceOptionsProperties, 37 | serializers: serdeProperties 38 | }, 39 | additionalProperties: true 40 | }) 41 | 42 | export const sendOptionsSchema = { 43 | type: 'object', 44 | properties: { 45 | messages: { type: 'array', items: messageSchema }, 46 | ...produceOptionsProperties 47 | }, 48 | required: ['messages'], 49 | additionalProperties: false 50 | } 51 | 52 | export const sendOptionsValidator = ajv.compile(sendOptionsSchema) 53 | -------------------------------------------------------------------------------- /src/clients/producer/types.ts: -------------------------------------------------------------------------------- 1 | import { type CompressionAlgorithms } from '../../protocol/compression.ts' 2 | import { type MessageToProduce } from '../../protocol/records.ts' 3 | import { type BaseOptions, type TopicWithPartitionAndOffset } from '../base/types.ts' 4 | import { type Serializers } from '../serde.ts' 5 | 6 | export interface ProducerInfo { 7 | producerId: bigint 8 | producerEpoch: number 9 | } 10 | 11 | export interface ProduceResult { 12 | // This is only defined when ack is not NO_RESPONSE 13 | offsets?: TopicWithPartitionAndOffset[] 14 | // This is only defined when ack is NO_RESPONSE 15 | unwritableNodes?: number[] 16 | } 17 | 18 | export type Partitioner = ( 19 | message: MessageToProduce 20 | ) => number 21 | 22 | export interface ProduceOptions { 23 | producerId?: bigint 24 | producerEpoch?: number 25 | idempotent?: boolean 26 | acks?: number 27 | compression?: CompressionAlgorithms 28 | partitioner?: Partitioner 29 | autocreateTopics?: boolean 30 | repeatOnStaleMetadata?: boolean 31 | } 32 | 33 | export type ProducerOptions = BaseOptions & 34 | ProduceOptions & { 35 | serializers?: Partial> 36 | } 37 | 38 | export type SendOptions = { 39 | messages: MessageToProduce[] 40 | } & ProduceOptions 41 | -------------------------------------------------------------------------------- /src/clients/serde.ts: -------------------------------------------------------------------------------- 1 | export type Serializer = (data?: InputType) => Buffer | undefined 2 | export type Deserializer = (data?: Buffer) => OutputType | undefined 3 | 4 | export type SerializerWithHeaders = ( 5 | data?: InputType, 6 | headers?: Map 7 | ) => Buffer | undefined 8 | export type DeserializerWithHeaders = ( 9 | data?: Buffer, 10 | headers?: Map 11 | ) => OutputType | undefined 12 | 13 | export interface Serializers { 14 | key: SerializerWithHeaders 15 | value: SerializerWithHeaders 16 | headerKey: Serializer 17 | headerValue: Serializer 18 | } 19 | 20 | export interface Deserializers { 21 | key: DeserializerWithHeaders 22 | value: DeserializerWithHeaders 23 | headerKey: Deserializer 24 | headerValue: Deserializer 25 | } 26 | 27 | export function stringSerializer (data?: string): Buffer | undefined { 28 | if (typeof data !== 'string') { 29 | return undefined 30 | } 31 | 32 | return Buffer.from(data, 'utf-8') 33 | } 34 | 35 | export function stringDeserializer (data?: string | Buffer): string | undefined { 36 | if (!Buffer.isBuffer(data)) { 37 | return undefined 38 | } 39 | 40 | return data.toString('utf-8') 41 | } 42 | 43 | export function jsonSerializer> (data?: T): Buffer | undefined { 44 | return Buffer.from(JSON.stringify(data), 'utf-8') 45 | } 46 | 47 | export function jsonDeserializer> (data?: string | Buffer): T | undefined { 48 | if (!Buffer.isBuffer(data)) { 49 | return undefined 50 | } 51 | 52 | return JSON.parse(data.toString('utf-8')) as T 53 | } 54 | 55 | export function serializersFrom (serializer: Serializer): Serializers { 56 | return { 57 | key: serializer, 58 | value: serializer, 59 | headerKey: serializer, 60 | headerValue: serializer 61 | } 62 | } 63 | 64 | export function deserializersFrom (deserializer: Deserializer): Deserializers { 65 | return { 66 | key: deserializer, 67 | value: deserializer, 68 | headerKey: deserializer, 69 | headerValue: deserializer 70 | } 71 | } 72 | 73 | export const serdeProperties = { 74 | type: 'object', 75 | properties: { 76 | key: { function: true }, 77 | value: { function: true }, 78 | headerKey: { function: true }, 79 | headerValue: { function: true } 80 | }, 81 | additionalProperties: false 82 | } 83 | 84 | export const stringSerializers = serializersFrom(stringSerializer) 85 | export const stringDeserializers = deserializersFrom(stringDeserializer) 86 | -------------------------------------------------------------------------------- /src/global.d.ts: -------------------------------------------------------------------------------- 1 | declare namespace NodeJS { 2 | export interface Process { 3 | _rawDebug: (...args: any[]) => void 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | // General 2 | export * from './diagnostic.ts' 3 | export * from './errors.ts' 4 | export * from './utils.ts' 5 | 6 | // Networking 7 | export * from './network/index.ts' 8 | export * from './protocol/index.ts' 9 | 10 | // APIs 11 | export * from './apis/index.ts' 12 | 13 | // Clients 14 | export * from './clients/index.ts' 15 | -------------------------------------------------------------------------------- /src/network/index.ts: -------------------------------------------------------------------------------- 1 | export * from './connection-pool.ts' 2 | export * from './connection.ts' 3 | -------------------------------------------------------------------------------- /src/protocol/definitions.ts: -------------------------------------------------------------------------------- 1 | import { type DynamicBuffer } from './dynamic-buffer.ts' 2 | 3 | export const INT8_SIZE = 1 4 | export const INT16_SIZE = 2 5 | export const INT32_SIZE = 4 6 | export const INT64_SIZE = 8 7 | export const UUID_SIZE = 16 8 | 9 | export const EMPTY_BUFFER = Buffer.alloc(0) 10 | export const EMPTY_UUID = Buffer.alloc(UUID_SIZE) 11 | 12 | // Since it is serialized at either 0 (for nullable) or 1 (since length is stored as length + 1), it always uses a single byte 13 | export const EMPTY_OR_SINGLE_COMPACT_LENGTH_SIZE = INT8_SIZE 14 | 15 | // TODO(ShogunPanda): Tagged fields are not supported yet 16 | export const EMPTY_TAGGED_FIELDS_BUFFER = Buffer.from([0]) 17 | 18 | export type Collection = string | Buffer | DynamicBuffer | Array | Map | Set 19 | 20 | export type NullableString = string | undefined | null 21 | -------------------------------------------------------------------------------- /src/protocol/index.ts: -------------------------------------------------------------------------------- 1 | export * from './apis.ts' 2 | export * from './compression.ts' 3 | export * from './crc32c.ts' 4 | export * from './definitions.ts' 5 | export * from './dynamic-buffer.ts' 6 | export * from './errors.ts' 7 | export * from './index.ts' 8 | export * from './murmur2.ts' 9 | export * from './reader.ts' 10 | export * from './records.ts' 11 | export * as saslPlain from './sasl/plain.ts' 12 | export * as saslScramSha from './sasl/scram-sha.ts' 13 | export * from './varint.ts' 14 | export * from './writer.ts' 15 | -------------------------------------------------------------------------------- /src/protocol/murmur2.ts: -------------------------------------------------------------------------------- 1 | import { LRUCache } from 'mnemonist' 2 | 3 | // This is a Javascript port of 4 | // https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L492 5 | 6 | // 'm' and 'r' are mixing constants generated offline. They're not really 'magic', they just happen to work well. 7 | const seed = 0x9747b28cn 8 | const m = 0x5bd1e995n 9 | const r = 24 10 | 11 | const cache = new LRUCache(1000) 12 | const tail = Buffer.alloc(4) 13 | 14 | function tripleRightShift (num: bigint, bits: number) { 15 | return BigInt(Number(BigInt.asIntN(32, num)) >>> bits) 16 | } 17 | 18 | export function murmur2 (data: string | Buffer, useCache: boolean = false): number { 19 | let key: string | undefined 20 | 21 | if (typeof data === 'string') { 22 | if (useCache) { 23 | key = data 24 | const existing = key ? cache.get(key) : undefined 25 | 26 | if (existing) { 27 | return existing 28 | } 29 | } 30 | 31 | data = Buffer.from(data) 32 | } 33 | 34 | const length = data.length 35 | 36 | // Initialize the hash to a random value 37 | let h: number | bigint = seed ^ BigInt(length) 38 | 39 | let i = 0 40 | while (i < length - 3) { 41 | let k = BigInt(data.readInt32LE(i)) 42 | i += 4 43 | 44 | k *= m 45 | k ^= tripleRightShift(k, r) 46 | k *= m 47 | 48 | h *= m 49 | h ^= k 50 | } 51 | 52 | // Read the last few bytes of the input array 53 | // If this becomes an hot-path, consider to optimize it further and avoid copying 54 | if (length % 4 > 0) { 55 | data.copy(tail, 0, length - (length % 4)) 56 | h ^= BigInt(tail.readInt32LE(0)) 57 | tail.fill(0) 58 | h *= m 59 | } 60 | 61 | // Perform the final mixing 62 | h ^= tripleRightShift(h, 13) 63 | h *= m 64 | h ^= tripleRightShift(h, 15) 65 | 66 | const hash = Number(BigInt.asIntN(32, h)) 67 | 68 | if (key !== undefined) { 69 | cache.set(key, hash) 70 | } 71 | 72 | return hash 73 | } 74 | -------------------------------------------------------------------------------- /src/protocol/sasl/plain.ts: -------------------------------------------------------------------------------- 1 | import { createPromisifiedCallback, kCallbackPromise, type CallbackWithPromise } from '../../apis/callbacks.ts' 2 | import { type SASLAuthenticationAPI, type SaslAuthenticateResponse } from '../../apis/security/sasl-authenticate-v2.ts' 3 | import { type Connection } from '../../network/connection.ts' 4 | 5 | export function authenticate ( 6 | authenticateAPI: SASLAuthenticationAPI, 7 | connection: Connection, 8 | username: string, 9 | password: string, 10 | callback: CallbackWithPromise 11 | ): void 12 | export function authenticate ( 13 | authenticateAPI: SASLAuthenticationAPI, 14 | connection: Connection, 15 | username: string, 16 | password: string 17 | ): Promise 18 | export function authenticate ( 19 | authenticateAPI: SASLAuthenticationAPI, 20 | connection: Connection, 21 | username: string, 22 | password: string, 23 | callback?: CallbackWithPromise 24 | ): void | Promise { 25 | if (!callback) { 26 | callback = createPromisifiedCallback() 27 | } 28 | 29 | authenticateAPI(connection, Buffer.from(['', username, password].join('\0')), callback) 30 | 31 | return callback[kCallbackPromise] 32 | } 33 | -------------------------------------------------------------------------------- /src/protocol/varint.ts: -------------------------------------------------------------------------------- 1 | export const MOST_SIGNIFICANT_BIT_FLAG = 0x80 // 128 or 1000 0000 2 | export const MOST_SIGNIFICANT_BIT_FLAG_64 = 0x80n // 128 or 1000 0000 3 | export const LEAST_SIGNIFICANT_7_BITS = 0x7f // 127 or 0111 1111 4 | export const LEAST_SIGNIFICANT_7_BITS_64 = 0x7fn // 127 or 0111 1111 5 | 6 | // This is used in varint to check if there are any other bits set after the first 7 bits, 7 | // which means it still needs more than a byte to represent the number in varint encoding 8 | export const BITS_8PLUS_MASK = 0xffffffff - 0x7f 9 | export const BITS_8PLUS_MASK_64 = 0xffffffffn - 0x7fn 10 | 11 | export function intZigZagEncode (value: number): number { 12 | return (value << 1) ^ (value >> 31) 13 | } 14 | 15 | export function intZigZagDecode (value: number): number { 16 | return (value >> 1) ^ -(value & 1) 17 | } 18 | 19 | export function int64ZigZagEncode (value: bigint): bigint { 20 | return (value << 1n) ^ (value >> 31n) 21 | } 22 | 23 | export function int64ZigZagDecode (value: bigint): bigint { 24 | return (value >> 1n) ^ -(value & 1n) 25 | } 26 | 27 | export function sizeOfUnsignedVarInt (value: number): number { 28 | let bytes = 1 29 | 30 | while ((value & BITS_8PLUS_MASK) !== 0) { 31 | bytes++ 32 | value >>>= 7 33 | } 34 | 35 | return bytes 36 | } 37 | 38 | export function sizeOfUnsignedVarInt64 (value: bigint): number { 39 | let bytes = 1 40 | 41 | while ((value & BITS_8PLUS_MASK_64) !== 0n) { 42 | bytes++ 43 | value >>= 7n 44 | } 45 | 46 | return bytes 47 | } 48 | -------------------------------------------------------------------------------- /src/symbols.ts: -------------------------------------------------------------------------------- 1 | // IMPORTANT: Never export this file in index.ts - The symbols are meant to be private 2 | 3 | export const kInstance = Symbol('plt.kafka.base.instance') 4 | -------------------------------------------------------------------------------- /test/apis/definitions.test.ts: -------------------------------------------------------------------------------- 1 | // This file uses internal APIs to check the general low-level API developer experience. 2 | 3 | import { deepStrictEqual } from 'assert' 4 | import test from 'node:test' 5 | // Technically V4 is the latest version, but we use V3 in the tests so that it is also compatible with older brokers (2.4.0+) 6 | import { api } from '../../src/apis/metadata/api-versions-v3.ts' 7 | import { Connection } from '../../src/index.ts' 8 | import { kafkaBootstrapServers } from '../helpers.ts' 9 | 10 | test('any API should work in promise mode or callback mode', async t => { 11 | const connection = new Connection('clientId') 12 | t.after(() => connection.close()) 13 | 14 | const [host, port] = kafkaBootstrapServers[0].split(':') 15 | await connection.connect(host, Number(port)) 16 | 17 | const promiseResponse = await api.async(connection, 'test-client', '1.0.0') 18 | 19 | const callbackResponse = await new Promise((resolve, reject) => { 20 | api(connection, 'test-client', '1.0.0', (error, response) => { 21 | if (error) { 22 | reject(error) 23 | } else { 24 | resolve(response) 25 | } 26 | }) 27 | }) 28 | 29 | // This call has no callback but it will not fail 30 | api(connection, 'test-client', '1.0.0') 31 | 32 | deepStrictEqual(promiseResponse, callbackResponse) 33 | deepStrictEqual(promiseResponse.apiKeys[0].name, 'Produce') 34 | }) 35 | -------------------------------------------------------------------------------- /test/apis/security/sasl-handshake-v2.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ok, throws } from 'node:assert' 2 | import test from 'node:test' 3 | import { Reader, ResponseError, saslHandshakeV1, Writer } from '../../../src/index.ts' 4 | 5 | const { createRequest, parseResponse } = saslHandshakeV1 6 | 7 | test('createRequest serializes mechanism correctly', () => { 8 | const mechanism = 'SCRAM-SHA-256' 9 | const writer = createRequest(mechanism) 10 | 11 | // Verify it returns a Writer 12 | ok(writer instanceof Writer) 13 | 14 | // Read the serialized data to verify correctness 15 | const reader = Reader.from(writer) 16 | 17 | // Note: SaslHandshake V1 uses non-compact strings (false flag in readString) 18 | // Verify the complete request structure with deepStrictEqual 19 | deepStrictEqual( 20 | { 21 | mechanism: reader.readString(false) // non-compact string 22 | }, 23 | { 24 | mechanism 25 | }, 26 | 'Serialized request should match expected structure' 27 | ) 28 | }) 29 | 30 | test('createRequest handles PLAIN mechanism', () => { 31 | const mechanism = 'PLAIN' 32 | const writer = createRequest(mechanism) 33 | 34 | // Verify it returns a Writer 35 | ok(writer instanceof Writer) 36 | 37 | // Read the serialized data to verify correctness 38 | const reader = Reader.from(writer) 39 | 40 | // Verify the complete request structure with deepStrictEqual 41 | deepStrictEqual( 42 | { 43 | mechanism: reader.readString(false) // non-compact string 44 | }, 45 | { 46 | mechanism 47 | }, 48 | 'Serialized request with PLAIN mechanism should match expected structure' 49 | ) 50 | }) 51 | 52 | test('parseResponse correctly processes a successful response', () => { 53 | // Create a successful response 54 | const writer = Writer.create() 55 | .appendInt16(0) // errorCode (success) 56 | // Mechanisms array (this is a non-compact array without tagged fields) 57 | .appendArray( 58 | ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512'], 59 | (w, mechanism) => w.appendString(mechanism, false), 60 | false, 61 | false 62 | ) // non-compact array without tagged fields 63 | 64 | const response = parseResponse(1, 17, 1, Reader.from(writer)) 65 | 66 | // Verify structure 67 | deepStrictEqual(response, { 68 | errorCode: 0, 69 | mechanisms: ['PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512'] 70 | }) 71 | }) 72 | 73 | test('parseResponse throws error on non-zero error code', () => { 74 | // Create a response with error 75 | const writer = Writer.create() 76 | .appendInt16(58) // errorCode (UNSUPPORTED_SASL_MECHANISM) 77 | // Mechanisms array (empty because the requested mechanism is not supported) 78 | .appendArray([], () => {}, false, false) // empty non-compact array 79 | 80 | throws( 81 | () => { 82 | parseResponse(1, 17, 1, Reader.from(writer)) 83 | }, 84 | (err: any) => { 85 | ok(err instanceof ResponseError) 86 | ok(err.message.includes('Received response with error while executing API')) 87 | 88 | // Verify the error response details 89 | deepStrictEqual(err.response, { 90 | errorCode: 58, 91 | mechanisms: [] 92 | }) 93 | 94 | return true 95 | } 96 | ) 97 | }) 98 | -------------------------------------------------------------------------------- /test/clients/base/sasl.test.ts: -------------------------------------------------------------------------------- 1 | import { deepStrictEqual, ok, rejects } from 'node:assert' 2 | import { randomBytes } from 'node:crypto' 3 | import { test } from 'node:test' 4 | import { alterUserScramCredentialsV0, Base, Connection, MultipleErrors, ScramMechanisms } from '../../../src/index.ts' 5 | import { hi, ScramAlgorithms } from '../../../src/protocol/sasl/scram-sha.ts' 6 | 7 | test('should not connect to SASL protected broker by default', async t => { 8 | const base = new Base({ clientId: 'clientId', bootstrapBrokers: ['localhost:3012'], strict: true, retries: 0 }) 9 | t.after(() => base.close()) 10 | 11 | await rejects(() => base.metadata({ topics: [] })) 12 | }) 13 | 14 | test('should connect to SASL protected broker using SASL/PLAIN', async t => { 15 | const base = new Base({ 16 | clientId: 'clientId', 17 | bootstrapBrokers: ['localhost:3012'], 18 | strict: true, 19 | retries: 0, 20 | sasl: { mechanism: 'PLAIN', username: 'admin', password: 'admin' } 21 | }) 22 | 23 | t.after(() => base.close()) 24 | 25 | const metadata = await base.metadata({ topics: [] }) 26 | 27 | deepStrictEqual(metadata.brokers.get(1), { host: 'localhost', port: 9092 }) 28 | }) 29 | 30 | test('should connect to SASL protected broker using SASL/SCRAM-SHA-256', async t => { 31 | // To use SCRAM-SHA-256, we need to create the user via the alterUserScramCredentialsV0 API 32 | { 33 | const connection = new Connection('clientId', { 34 | sasl: { mechanism: 'PLAIN', username: 'admin', password: 'admin' } 35 | }) 36 | 37 | const iterations = ScramAlgorithms['SHA-256'].minIterations 38 | const salt = randomBytes(10) 39 | const saltedPassword = hi(ScramAlgorithms['SHA-256'], 'admin', salt, iterations) 40 | await connection.connect('localhost', 3012) 41 | 42 | await alterUserScramCredentialsV0.api.async( 43 | connection, 44 | [], 45 | [ 46 | { 47 | name: 'admin', 48 | mechanism: ScramMechanisms.SCRAM_SHA_256, 49 | iterations: ScramAlgorithms['SHA-256'].minIterations, 50 | salt, 51 | saltedPassword 52 | } 53 | ] 54 | ) 55 | 56 | await connection.close() 57 | } 58 | 59 | const base = new Base({ 60 | clientId: 'clientId', 61 | bootstrapBrokers: ['localhost:3012'], 62 | strict: true, 63 | retries: 0, 64 | sasl: { mechanism: 'SCRAM-SHA-256', username: 'admin', password: 'admin' } 65 | }) 66 | 67 | t.after(() => base.close()) 68 | 69 | const metadata = await base.metadata({ topics: [] }) 70 | 71 | deepStrictEqual(metadata.brokers.get(1), { host: 'localhost', port: 9092 }) 72 | }) 73 | 74 | test('should handle authentication errors', async t => { 75 | const base = new Base({ 76 | clientId: 'clientId', 77 | bootstrapBrokers: ['localhost:3012'], 78 | retries: 0, 79 | sasl: { mechanism: 'PLAIN', username: 'admin', password: 'invalid' } 80 | }) 81 | 82 | t.after(() => base.close()) 83 | 84 | try { 85 | await base.metadata({ topics: [] }) 86 | throw new Error('Expected error not thrown') 87 | } catch (error) { 88 | ok(error instanceof MultipleErrors) 89 | deepStrictEqual(error.errors[0].cause.message, 'SASL authentication failed.') 90 | } 91 | }) 92 | -------------------------------------------------------------------------------- /test/config/c8-ci.json: -------------------------------------------------------------------------------- 1 | { 2 | "check-coverage": true, 3 | "reporter": ["text", "json"], 4 | "branches": 90, 5 | "functions": 90, 6 | "lines": 90, 7 | "statements": 90 8 | } 9 | -------------------------------------------------------------------------------- /test/config/c8-local.json: -------------------------------------------------------------------------------- 1 | { 2 | "reporter": ["text", "html"] 3 | } 4 | -------------------------------------------------------------------------------- /test/config/env: -------------------------------------------------------------------------------- 1 | NODE_OPTIONS="--experimental-strip-types --test-reporter=cleaner-spec-reporter --disable-warning=ExperimentalWarning" 2 | -------------------------------------------------------------------------------- /test/protocol/varint.test.ts: -------------------------------------------------------------------------------- 1 | import { strictEqual } from 'node:assert' 2 | import test from 'node:test' 3 | import { 4 | int64ZigZagDecode, 5 | int64ZigZagEncode, 6 | intZigZagDecode, 7 | intZigZagEncode, 8 | sizeOfUnsignedVarInt, 9 | sizeOfUnsignedVarInt64 10 | } from '../../src/index.ts' 11 | 12 | test('zigzag encoding (32-bit)', () => { 13 | strictEqual(intZigZagEncode(0), 0) 14 | strictEqual(intZigZagEncode(-1), 1) 15 | strictEqual(intZigZagEncode(1), 2) 16 | strictEqual(intZigZagEncode(-2), 3) 17 | strictEqual(intZigZagEncode(2), 4) 18 | }) 19 | 20 | test('zigzag decoding (32-bit)', () => { 21 | strictEqual(intZigZagDecode(0), 0) 22 | strictEqual(intZigZagDecode(1), -1) 23 | strictEqual(intZigZagDecode(2), 1) 24 | strictEqual(intZigZagDecode(3), -2) 25 | strictEqual(intZigZagDecode(4), 2) 26 | }) 27 | 28 | test('size of unsigned varint (32-bit)', () => { 29 | strictEqual(sizeOfUnsignedVarInt(0), 1) 30 | strictEqual(sizeOfUnsignedVarInt(127), 1) 31 | 32 | strictEqual(sizeOfUnsignedVarInt(128), 2) 33 | strictEqual(sizeOfUnsignedVarInt(16383), 2) 34 | 35 | strictEqual(sizeOfUnsignedVarInt(16384), 3) 36 | strictEqual(sizeOfUnsignedVarInt(2097151), 3) 37 | 38 | strictEqual(sizeOfUnsignedVarInt(2097152), 4) 39 | strictEqual(sizeOfUnsignedVarInt(268435455), 4) 40 | 41 | strictEqual(sizeOfUnsignedVarInt(268435456), 5) 42 | strictEqual(sizeOfUnsignedVarInt(2147483647), 5) 43 | }) 44 | 45 | test('zigzag encoding (64-bit)', () => { 46 | strictEqual(int64ZigZagEncode(0n), 0n) 47 | strictEqual(int64ZigZagEncode(-1n), 1n) 48 | strictEqual(int64ZigZagEncode(1n), 2n) 49 | strictEqual(int64ZigZagEncode(-2n), 3n) 50 | strictEqual(int64ZigZagEncode(2n), 4n) 51 | }) 52 | 53 | test('zigzag decoding (64-bit)', () => { 54 | strictEqual(int64ZigZagDecode(0n), 0n) 55 | strictEqual(int64ZigZagDecode(1n), -1n) 56 | strictEqual(int64ZigZagDecode(2n), 1n) 57 | strictEqual(int64ZigZagDecode(3n), -2n) 58 | strictEqual(int64ZigZagDecode(4n), 2n) 59 | }) 60 | 61 | test('size of unsigned varint (64-bit)', () => { 62 | strictEqual(sizeOfUnsignedVarInt64(0n), 1) 63 | strictEqual(sizeOfUnsignedVarInt64(127n), 1) 64 | 65 | strictEqual(sizeOfUnsignedVarInt64(128n), 2) 66 | strictEqual(sizeOfUnsignedVarInt64(16383n), 2) 67 | 68 | strictEqual(sizeOfUnsignedVarInt64(2147483647n), 5) 69 | strictEqual(sizeOfUnsignedVarInt64(9223372036854775807n), 9) 70 | }) 71 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "NodeNext", 5 | "moduleResolution": "NodeNext", 6 | "lib": ["ESNext", "ESNext.Promise"], 7 | "jsx": "preserve", 8 | "declaration": true, 9 | "outDir": "dist", 10 | "allowJs": false, 11 | "allowSyntheticDefaultImports": true, 12 | "esModuleInterop": true, 13 | "strict": true, 14 | "skipLibCheck": true, 15 | "noImplicitAny": true, 16 | "noUnusedLocals": true, 17 | "noUnusedParameters": true, 18 | "strictNullChecks": true, 19 | "useUnknownInCatchVariables": false, 20 | "allowImportingTsExtensions": true, 21 | "rewriteRelativeImportExtensions": true 22 | }, 23 | "include": ["src/*.ts", "src/**/*.ts"] 24 | } 25 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "include": [ 4 | "src/*.ts", 5 | "src/**/*.ts", 6 | "test/*.ts", 7 | "test/**/*.ts", 8 | "scripts/*.ts", 9 | "benchmarks/**/*.ts", 10 | "playground/*.ts", 11 | "playground/**/*.ts" 12 | ] 13 | } 14 | --------------------------------------------------------------------------------