├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .dockerignore ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── README.md ├── about.hbs ├── about.toml ├── compose.yaml ├── data └── .gitignore ├── docs ├── sarama.md └── schema-registry.md ├── duckdb-init.sql ├── etc ├── data │ ├── employees.json │ ├── grades.json │ ├── observations.json │ ├── persons.json │ └── trips.json ├── grafana │ ├── dashboards │ │ └── home.json │ └── provisioning │ │ ├── dashboards │ │ └── tansu.yaml │ │ └── datasources │ │ └── prometheus.yaml ├── initdb.d │ └── 010-schema.sql ├── lakekeeper │ └── create-default-warehouse.json ├── prometheus.yaml └── schema │ ├── employee.proto │ ├── grade.json │ ├── observation.avsc │ ├── people.proto │ ├── person.json │ ├── search.proto │ └── taxi.proto ├── example.env ├── justfile ├── logs ├── tansu-kafka-sans-io │ └── .gitignore ├── tansu-schema-registry │ └── .gitignore ├── tansu-server │ └── .gitignore └── tansu-storage │ └── .gitignore ├── rust-toolchain.toml ├── rustfmt.toml ├── tansu-cat ├── Cargo.toml └── src │ ├── consume.rs │ ├── lib.rs │ └── produce.rs ├── tansu-cli ├── Cargo.toml └── src │ ├── bin │ └── tansu.rs │ ├── cli.rs │ ├── cli │ ├── broker.rs │ ├── cat.rs │ ├── proxy.rs │ └── topic.rs │ └── lib.rs ├── tansu-kafka-model ├── .gitignore ├── Cargo.toml └── src │ ├── error.rs │ ├── lib.rs │ └── wv.rs ├── tansu-kafka-sans-io ├── Cargo.toml ├── benches │ └── decode_bench.rs ├── build.rs ├── message │ ├── AddOffsetsToTxnRequest.json │ ├── AddOffsetsToTxnResponse.json │ ├── AddPartitionsToTxnRequest.json │ ├── AddPartitionsToTxnResponse.json │ ├── AddRaftVoterRequest.json │ ├── AddRaftVoterResponse.json │ ├── AllocateProducerIdsRequest.json │ ├── AllocateProducerIdsResponse.json │ ├── AlterClientQuotasRequest.json │ ├── AlterClientQuotasResponse.json │ ├── AlterConfigsRequest.json │ ├── AlterConfigsResponse.json │ ├── AlterPartitionReassignmentsRequest.json │ ├── AlterPartitionReassignmentsResponse.json │ ├── AlterPartitionRequest.json │ ├── AlterPartitionResponse.json │ ├── AlterReplicaLogDirsRequest.json │ ├── AlterReplicaLogDirsResponse.json │ ├── AlterUserScramCredentialsRequest.json │ ├── AlterUserScramCredentialsResponse.json │ ├── ApiVersionsRequest.json │ ├── ApiVersionsResponse.json │ ├── AssignReplicasToDirsRequest.json │ ├── AssignReplicasToDirsResponse.json │ ├── BeginQuorumEpochRequest.json │ ├── BeginQuorumEpochResponse.json │ ├── BrokerHeartbeatRequest.json │ ├── BrokerHeartbeatResponse.json │ ├── BrokerRegistrationRequest.json │ ├── BrokerRegistrationResponse.json │ ├── ConsumerGroupDescribeRequest.json │ ├── ConsumerGroupDescribeResponse.json │ ├── ConsumerGroupHeartbeatRequest.json │ ├── ConsumerGroupHeartbeatResponse.json │ ├── ConsumerProtocolAssignment.json │ ├── ConsumerProtocolSubscription.json │ ├── ControlledShutdownRequest.json │ ├── ControlledShutdownResponse.json │ ├── ControllerRegistrationRequest.json │ ├── ControllerRegistrationResponse.json │ ├── CreateAclsRequest.json │ ├── CreateAclsResponse.json │ ├── CreateDelegationTokenRequest.json │ ├── CreateDelegationTokenResponse.json │ ├── CreatePartitionsRequest.json │ ├── CreatePartitionsResponse.json │ ├── CreateTopicsRequest.json │ ├── CreateTopicsResponse.json │ ├── DefaultPrincipalData.json │ ├── DeleteAclsRequest.json │ ├── DeleteAclsResponse.json │ ├── DeleteGroupsRequest.json │ ├── DeleteGroupsResponse.json │ ├── DeleteRecordsRequest.json │ ├── DeleteRecordsResponse.json │ ├── DeleteShareGroupStateRequest.json │ ├── DeleteShareGroupStateResponse.json │ ├── DeleteTopicsRequest.json │ ├── DeleteTopicsResponse.json │ ├── DescribeAclsRequest.json │ ├── DescribeAclsResponse.json │ ├── DescribeClientQuotasRequest.json │ ├── DescribeClientQuotasResponse.json │ ├── DescribeClusterRequest.json │ ├── DescribeClusterResponse.json │ ├── DescribeConfigsRequest.json │ ├── DescribeConfigsResponse.json │ ├── DescribeDelegationTokenRequest.json │ ├── DescribeDelegationTokenResponse.json │ ├── DescribeGroupsRequest.json │ ├── DescribeGroupsResponse.json │ ├── DescribeLogDirsRequest.json │ ├── DescribeLogDirsResponse.json │ ├── DescribeProducersRequest.json │ ├── DescribeProducersResponse.json │ ├── DescribeQuorumRequest.json │ ├── DescribeQuorumResponse.json │ ├── DescribeTopicPartitionsRequest.json │ ├── DescribeTopicPartitionsResponse.json │ ├── DescribeTransactionsRequest.json │ ├── DescribeTransactionsResponse.json │ ├── DescribeUserScramCredentialsRequest.json │ ├── DescribeUserScramCredentialsResponse.json │ ├── ElectLeadersRequest.json │ ├── ElectLeadersResponse.json │ ├── EndQuorumEpochRequest.json │ ├── EndQuorumEpochResponse.json │ ├── EndTxnRequest.json │ ├── EndTxnResponse.json │ ├── EnvelopeRequest.json │ ├── EnvelopeResponse.json │ ├── ExpireDelegationTokenRequest.json │ ├── ExpireDelegationTokenResponse.json │ ├── FetchRequest.json │ ├── FetchResponse.json │ ├── FetchSnapshotRequest.json │ ├── FetchSnapshotResponse.json │ ├── FindCoordinatorRequest.json │ ├── FindCoordinatorResponse.json │ ├── GetTelemetrySubscriptionsRequest.json │ ├── GetTelemetrySubscriptionsResponse.json │ ├── HeartbeatRequest.json │ ├── HeartbeatResponse.json │ ├── IncrementalAlterConfigsRequest.json │ ├── IncrementalAlterConfigsResponse.json │ ├── InitProducerIdRequest.json │ ├── InitProducerIdResponse.json │ ├── InitializeShareGroupStateRequest.json │ ├── InitializeShareGroupStateResponse.json │ ├── JoinGroupRequest.json │ ├── JoinGroupResponse.json │ ├── LeaderAndIsrRequest.json │ ├── LeaderAndIsrResponse.json │ ├── LeaderChangeMessage.json │ ├── LeaveGroupRequest.json │ ├── LeaveGroupResponse.json │ ├── ListClientMetricsResourcesRequest.json │ ├── ListClientMetricsResourcesResponse.json │ ├── ListGroupsRequest.json │ ├── ListGroupsResponse.json │ ├── ListOffsetsRequest.json │ ├── ListOffsetsResponse.json │ ├── ListPartitionReassignmentsRequest.json │ ├── ListPartitionReassignmentsResponse.json │ ├── ListTransactionsRequest.json │ ├── ListTransactionsResponse.json │ ├── MetadataRequest.json │ ├── MetadataResponse.json │ ├── OffsetCommitRequest.json │ ├── OffsetCommitResponse.json │ ├── OffsetDeleteRequest.json │ ├── OffsetDeleteResponse.json │ ├── OffsetFetchRequest.json │ ├── OffsetFetchResponse.json │ ├── OffsetForLeaderEpochRequest.json │ ├── OffsetForLeaderEpochResponse.json │ ├── ProduceRequest.json │ ├── ProduceResponse.json │ ├── PushTelemetryRequest.json │ ├── PushTelemetryResponse.json │ ├── README.md │ ├── ReadShareGroupStateRequest.json │ ├── ReadShareGroupStateResponse.json │ ├── ReadShareGroupStateSummaryRequest.json │ ├── ReadShareGroupStateSummaryResponse.json │ ├── RemoveRaftVoterRequest.json │ ├── RemoveRaftVoterResponse.json │ ├── RenewDelegationTokenRequest.json │ ├── RenewDelegationTokenResponse.json │ ├── RequestHeader.json │ ├── ResponseHeader.json │ ├── SaslAuthenticateRequest.json │ ├── SaslAuthenticateResponse.json │ ├── SaslHandshakeRequest.json │ ├── SaslHandshakeResponse.json │ ├── ShareAcknowledgeRequest.json │ ├── ShareAcknowledgeResponse.json │ ├── ShareFetchRequest.json │ ├── ShareFetchResponse.json │ ├── ShareGroupDescribeRequest.json │ ├── ShareGroupDescribeResponse.json │ ├── ShareGroupHeartbeatRequest.json │ ├── ShareGroupHeartbeatResponse.json │ ├── SnapshotFooterRecord.json │ ├── SnapshotHeaderRecord.json │ ├── StopReplicaRequest.json │ ├── StopReplicaResponse.json │ ├── SyncGroupRequest.json │ ├── SyncGroupResponse.json │ ├── TxnOffsetCommitRequest.json │ ├── TxnOffsetCommitResponse.json │ ├── UnregisterBrokerRequest.json │ ├── UnregisterBrokerResponse.json │ ├── UpdateFeaturesRequest.json │ ├── UpdateFeaturesResponse.json │ ├── UpdateMetadataRequest.json │ ├── UpdateMetadataResponse.json │ ├── UpdateRaftVoterRequest.json │ ├── UpdateRaftVoterResponse.json │ ├── VoteRequest.json │ ├── VoteResponse.json │ ├── WriteShareGroupStateRequest.json │ ├── WriteShareGroupStateResponse.json │ ├── WriteTxnMarkersRequest.json │ └── WriteTxnMarkersResponse.json ├── src │ ├── de.rs │ ├── lib.rs │ ├── primitive.rs │ ├── primitive │ │ ├── tagged.rs │ │ ├── tagged │ │ │ ├── de.rs │ │ │ └── ser.rs │ │ ├── uuid.rs │ │ └── varint.rs │ ├── record.rs │ ├── record │ │ ├── codec.rs │ │ ├── deflated.rs │ │ ├── header.rs │ │ └── inflated.rs │ └── ser.rs └── tests │ ├── .gitignore │ ├── add_partitions_to_txn.rs │ ├── codec.rs │ ├── common │ └── mod.rs │ ├── create_topics.rs │ ├── decode.rs │ ├── describe_configs.rs │ ├── describe_topic_partitions.rs │ ├── encode.rs │ ├── fetch.rs │ ├── fetch_response.rs │ ├── find_coordinator.rs │ └── version.rs ├── tansu-proxy ├── Cargo.toml └── src │ ├── lib.rs │ └── main.rs ├── tansu-schema-registry ├── .gitignore ├── Cargo.toml ├── src │ ├── avro.rs │ ├── json.rs │ ├── lake.rs │ ├── lake │ │ ├── berg.rs │ │ ├── delta.rs │ │ ├── quet.rs │ │ └── tansu.rs │ ├── lib.rs │ ├── meta.avsc │ ├── meta.json │ ├── meta.proto │ ├── proto.rs │ └── sql.rs └── tests │ ├── berg.rs │ └── common │ └── mod.rs ├── tansu-server ├── .gitignore ├── Cargo.toml ├── src │ ├── broker.rs │ ├── broker │ │ ├── api_versions.rs │ │ ├── create_topic.rs │ │ ├── delete_records.rs │ │ ├── delete_topics.rs │ │ ├── describe_cluster.rs │ │ ├── describe_configs.rs │ │ ├── fetch.rs │ │ ├── find_coordinator.rs │ │ ├── group.rs │ │ ├── group │ │ │ ├── heartbeat.rs │ │ │ ├── join.rs │ │ │ ├── leave.rs │ │ │ ├── offset_commit.rs │ │ │ ├── offset_fetch.rs │ │ │ └── sync.rs │ │ ├── init_producer_id.rs │ │ ├── list_offsets.rs │ │ ├── list_partition_reassignments.rs │ │ ├── metadata.rs │ │ ├── produce.rs │ │ ├── telemetry.rs │ │ ├── txn.rs │ │ └── txn │ │ │ ├── add_offsets.rs │ │ │ ├── add_partitions.rs │ │ │ └── offset_commit.rs │ ├── coordinator.rs │ ├── coordinator │ │ ├── group.rs │ │ └── group │ │ │ ├── administrator.rs │ │ │ └── consumer.rs │ ├── lib.rs │ ├── otel.rs │ └── otel │ │ ├── prom.rs │ │ └── tracing.rs └── tests │ ├── cg.rs │ ├── cg_dynamic.rs │ ├── cg_static.rs │ ├── common │ └── mod.rs │ ├── describe_cluster.rs │ ├── describe_configs.rs │ ├── fetch.rs │ ├── idempotent.rs │ ├── list_offsets.rs │ ├── metadata.rs │ ├── person.rs │ ├── pg_init_producer.rs │ ├── pg_txn.rs │ ├── produce_fetch.rs │ ├── topic.rs │ └── txn.rs ├── tansu-storage ├── .gitignore ├── Cargo.toml ├── benches │ └── segment_bench.rs ├── src │ ├── dynostore.rs │ ├── dynostore │ │ ├── metadata.rs │ │ └── opticon.rs │ ├── index.rs │ ├── index │ │ ├── offset.rs │ │ └── time.rs │ ├── lib.rs │ ├── os.rs │ ├── pg.rs │ ├── pg │ │ ├── broker_metadata_select.sql │ │ ├── consumer_group_delete.sql │ │ ├── consumer_group_detail.sql │ │ ├── consumer_group_detail_delete_by_cg.sql │ │ ├── consumer_group_detail_insert.sql │ │ ├── consumer_group_insert.sql │ │ ├── consumer_group_select.sql │ │ ├── consumer_group_select_by_name.sql │ │ ├── consumer_offset_delete_by_cg.sql │ │ ├── consumer_offset_delete_by_topic.sql │ │ ├── consumer_offset_insert.sql │ │ ├── consumer_offset_insert_from_txn.sql │ │ ├── consumer_offset_select.sql │ │ ├── consumer_offset_select_by_group.sql │ │ ├── header_delete_by_topic.sql │ │ ├── header_fetch.sql │ │ ├── header_insert.sql │ │ ├── init_txn_producer.sql │ │ ├── list_earliest_offset.sql │ │ ├── list_latest_offset_committed.sql │ │ ├── list_latest_offset_timestamp.sql │ │ ├── list_latest_offset_uncommitted.sql │ │ ├── producer_detail_delete_by_topic.sql │ │ ├── producer_detail_insert.sql │ │ ├── producer_epoch_current_for_producer.sql │ │ ├── producer_epoch_for_current_txn.sql │ │ ├── producer_epoch_insert.sql │ │ ├── producer_insert.sql │ │ ├── producer_select_for_update.sql │ │ ├── producer_update_epoch_with_txn.sql │ │ ├── producer_update_sequence.sql │ │ ├── record_delete_by_topic.sql │ │ ├── record_fetch.sql │ │ ├── record_insert.sql │ │ ├── topic_by_cluster.sql │ │ ├── topic_by_uuid.sql │ │ ├── topic_configuration_delete.sql │ │ ├── topic_configuration_delete_by_topic.sql │ │ ├── topic_configuration_upsert.sql │ │ ├── topic_delete_by.sql │ │ ├── topic_insert.sql │ │ ├── topic_select_name.sql │ │ ├── topic_select_uuid.sql │ │ ├── topition_delete_by_topic.sql │ │ ├── topition_insert.sql │ │ ├── topition_select.sql │ │ ├── txn_detail_insert.sql │ │ ├── txn_detail_select.sql │ │ ├── txn_detail_select_current.sql │ │ ├── txn_detail_select_for_update.sql │ │ ├── txn_detail_update_sequence.sql │ │ ├── txn_detail_update_started_at.sql │ │ ├── txn_insert.sql │ │ ├── txn_offset_commit_delete_by_txn.sql │ │ ├── txn_offset_commit_insert.sql │ │ ├── txn_offset_commit_tp_delete_by_topic.sql │ │ ├── txn_offset_commit_tp_delete_by_txn.sql │ │ ├── txn_offset_commit_tp_insert.sql │ │ ├── txn_produce_offset_delete_by_topic.sql │ │ ├── txn_produce_offset_delete_by_txn.sql │ │ ├── txn_produce_offset_insert.sql │ │ ├── txn_produce_offset_select_offset_range.sql │ │ ├── txn_produce_offset_select_overlapping_txn.sql │ │ ├── txn_select_name.sql │ │ ├── txn_select_produced_topitions.sql │ │ ├── txn_select_producer_epoch.sql │ │ ├── txn_status_update.sql │ │ ├── txn_topition_delete_by_topic.sql │ │ ├── txn_topition_delete_by_txn.sql │ │ ├── txn_topition_insert.sql │ │ ├── txn_topition_select.sql │ │ ├── txn_topition_select_txns.sql │ │ ├── watermark_delete_by_topic.sql │ │ ├── watermark_insert.sql │ │ ├── watermark_insert_from_txn.sql │ │ ├── watermark_select.sql │ │ ├── watermark_select_for_update.sql │ │ ├── watermark_select_stable.sql │ │ └── watermark_update.sql │ └── segment.rs └── tests │ ├── pg_list_offsets.rs │ ├── pg_producer.rs │ ├── pg_register_broker.rs │ └── pg_topic_lifecycle.rs ├── tansu-topic ├── Cargo.toml └── src │ ├── create.rs │ ├── delete.rs │ └── lib.rs └── typos.toml /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright ⓒ 2024 Peter Morgan 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU Affero General Public License as 5 | # published by the Free Software Foundation, either version 3 of the 6 | # License, or (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU Affero General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU Affero General Public License 14 | # along with this program. If not, see . 15 | 16 | FROM rust:1.85 17 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tansu", 3 | 4 | "dockerFile": "Dockerfile", 5 | 6 | "hostRequirements": { 7 | "cpus": 2, 8 | "memory": "4gb", 9 | "storage": "32gb" 10 | }, 11 | 12 | "forwardPorts": [3000, 5432, 9090, 9092], 13 | 14 | "portsAttributes": { 15 | "3000": { "label": "grafana" }, 16 | "5432": { "label": "postgresql" }, 17 | "9090": { "label": "prom" }, 18 | "9092": { "label": "tansu" } 19 | }, 20 | 21 | "features": { 22 | "ghcr.io/devcontainers/features/sshd:1": {}, 23 | 24 | "ghcr.io/devcontainers/features/common-utils:2": {}, 25 | 26 | "ghcr.io/devcontainers/features/docker-in-docker:2": {}, 27 | 28 | "ghcr.io/devcontainers/features/github-cli:1": {} 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /target 2 | /work-dir 3 | /logs 4 | /.git 5 | perf.data 6 | *.log 7 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | Cargo.lock -diff 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | target-branch: "main" 8 | - package-ecosystem: "cargo" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | target-branch: "main" 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | debug 2 | target 3 | Cargo.lock 4 | **/*.rs.bk 5 | *~ 6 | *# 7 | .#* 8 | .*DS_Store 9 | .DS_Store 10 | *.log 11 | .env 12 | flamegraph.svg 13 | perf.data 14 | .vscode 15 | license.html 16 | *.duckdb 17 | .zed 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright ⓒ 2024-2025 Peter Morgan 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU Affero General Public License as 5 | # published by the Free Software Foundation, either version 3 of the 6 | # License, or (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU Affero General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU Affero General Public License 14 | # along with this program. If not, see . 15 | 16 | FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx 17 | 18 | FROM --platform=$BUILDPLATFORM rust:1.87-alpine AS builder 19 | COPY --from=xx / / 20 | RUN apk add clang cmake lld 21 | RUN rustup target add $(xx-cargo --print-target-triple) 22 | 23 | WORKDIR /usr/src 24 | ADD / /usr/src/ 25 | 26 | ARG TARGETPLATFORM 27 | RUN xx-apk add --no-cache musl-dev zlib-dev gcc 28 | RUN xx-cargo build --bin tansu --release --target-dir ./build 29 | RUN xx-verify --static ./build/$(xx-cargo --print-target-triple)/release/tansu 30 | 31 | RUN < 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use dotenv::dotenv; 17 | use tansu_cli::{Cli, Result}; 18 | use tansu_kafka_sans_io::ErrorCode; 19 | use tansu_server::{TracingFormat, otel}; 20 | use tracing::{debug, error}; 21 | 22 | #[tokio::main] 23 | async fn main() -> Result { 24 | dotenv().ok(); 25 | 26 | let _guard = otel::init(TracingFormat::Text)?; 27 | 28 | Cli::main() 29 | .await 30 | .inspect(|error_code| debug!(%error_code)) 31 | .inspect_err(|err| error!(%err)) 32 | } 33 | -------------------------------------------------------------------------------- /tansu-cli/src/cli/proxy.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use crate::EnvVarExp; 17 | 18 | use super::DEFAULT_BROKER; 19 | use clap::Args; 20 | use url::Url; 21 | 22 | #[derive(Args, Clone, Debug)] 23 | pub(super) struct Arg { 24 | #[arg(long, env = "LISTENER_URL", default_value = "tcp://[::]:9092")] 25 | pub(super) listener_url: EnvVarExp, 26 | 27 | #[arg(long, default_value = DEFAULT_BROKER)] 28 | pub(super) origin_url: EnvVarExp, 29 | } 30 | -------------------------------------------------------------------------------- /tansu-kafka-model/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /tansu-kafka-model/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-kafka-model" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | # [lints] 9 | # workspace = true 10 | 11 | [dependencies] 12 | convert_case.workspace = true 13 | lazy_static.workspace = true 14 | proc-macro2.workspace = true 15 | quote.workspace = true 16 | regex.workspace = true 17 | serde_json.workspace = true 18 | serde.workspace = true 19 | syn.workspace = true 20 | tracing.workspace = true 21 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-kafka-sans-io" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | # [lints.clippy] 9 | # all = "warn" 10 | 11 | # [lints.rust] 12 | # deprecated_in_future = "warn" 13 | # elided_lifetimes_in_paths = "deny" 14 | # missing_copy_implementations = "allow" 15 | # missing_debug_implementations = "allow" 16 | # non_ascii_idents = "forbid" 17 | # nonstandard_style = "deny" 18 | # rust_2018_idioms = "deny" 19 | # unreachable_pub = "allow" 20 | # unsafe_code = "forbid" 21 | # unused_import_braces = "warn" 22 | # unused_labels = "warn" 23 | # unused_lifetimes = "warn" 24 | # unused_qualifications = "allow" 25 | # unused_results = "warn" 26 | # unused_variables = "allow" 27 | 28 | # [lints.rustdoc] 29 | # broken_intra_doc_links = "deny" 30 | # private_intra_doc_links = "deny" 31 | 32 | [dependencies] 33 | bytes.workspace = true 34 | crc.workspace = true 35 | flate2.workspace = true 36 | lz4.workspace = true 37 | serde.workspace = true 38 | snap.workspace = true 39 | tansu-kafka-model = { path = "../tansu-kafka-model" } 40 | thiserror.workspace = true 41 | tracing-subscriber.workspace = true 42 | tracing.workspace = true 43 | zstd.workspace = true 44 | 45 | [build-dependencies] 46 | convert_case.workspace = true 47 | glob.workspace = true 48 | prettyplease.workspace = true 49 | proc-macro2.workspace = true 50 | quote.workspace = true 51 | serde_json.workspace = true 52 | syn.workspace = true 53 | tansu-kafka-model = { path = "../tansu-kafka-model" } 54 | 55 | [dev-dependencies] 56 | criterion.workspace = true 57 | pretty_assertions.workspace = true 58 | 59 | [features] 60 | default = [] 61 | nightly-features = [] 62 | diagnostics = [] 63 | 64 | 65 | [[bench]] 66 | name = "decode_bench" 67 | harness = false 68 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/benches/decode_bench.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024-2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use criterion::{Criterion, black_box, criterion_group, criterion_main}; 17 | use tansu_kafka_sans_io::Frame; 18 | 19 | fn api_versions_request_v3_000(c: &mut Criterion) { 20 | _ = c.bench_function("api_versions_request_v3_000", |b| { 21 | b.iter(|| { 22 | Frame::request_from_bytes(black_box(&[ 23 | 0, 0, 0, 52, 0, 18, 0, 3, 0, 0, 0, 3, 0, 16, 99, 111, 110, 115, 111, 108, 101, 45, 24 | 112, 114, 111, 100, 117, 99, 101, 114, 0, 18, 97, 112, 97, 99, 104, 101, 45, 107, 25 | 97, 102, 107, 97, 45, 106, 97, 118, 97, 6, 51, 46, 54, 46, 49, 0, 26 | ])) 27 | }) 28 | }); 29 | } 30 | 31 | criterion_group!(benches, api_versions_request_v3_000); 32 | criterion_main!(benches); 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/AddOffsetsToTxnResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 25, 18 | "type": "response", 19 | "name": "AddOffsetsToTxnResponse", 20 | // Starting in version 1, on quota violation brokers send out responses before throttling. 21 | // 22 | // Version 2 adds the support for new error code PRODUCER_FENCED. 23 | // 24 | // Version 3 enables flexible versions. 25 | // 26 | // Version 4 adds support for new error code TRANSACTION_ABORTABLE (KIP-890). 27 | "validVersions": "0-4", 28 | "flexibleVersions": "3+", 29 | "fields": [ 30 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 31 | "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 32 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 33 | "about": "The response error code, or 0 if there was no error." } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/AddRaftVoterResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 80, 18 | "type": "response", 19 | "name": "AddRaftVoterResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 24 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The error code, or 0 if there was no error" }, 27 | { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", "ignorable": true, 28 | "about": "The error message, or null if there was no error." } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/AllocateProducerIdsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implie 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 67, 18 | "type": "request", 19 | "listeners": ["zkBroker", "controller"], 20 | "name": "AllocateProducerIdsRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId", 25 | "about": "The ID of the requesting broker" }, 26 | { "name": "BrokerEpoch", "type": "int64", "versions": "0+", "default": "-1", 27 | "about": "The epoch of the requesting broker" } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/AllocateProducerIdsResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 67, 18 | "type": "response", 19 | "name": "AllocateProducerIdsResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 24 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The top level response error code" }, 27 | { "name": "ProducerIdStart", "type": "int64", "versions": "0+", "entityType": "producerId", 28 | "about": "The first producer ID in this range, inclusive"}, 29 | { "name": "ProducerIdLen", "type": "int32", "versions": "0+", 30 | "about": "The number of producer IDs in this range"} 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ApiVersionsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 18, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "ApiVersionsRequest", 21 | // Versions 0 through 2 of ApiVersionsRequest are the same. 22 | // 23 | // Version 3 is the first flexible version and adds ClientSoftwareName and ClientSoftwareVersion. 24 | // 25 | // Version 4 fixes KAFKA-17011, which blocked SupportedFeatures.MinVersion in the response from being 0. 26 | "validVersions": "0-4", 27 | "flexibleVersions": "3+", 28 | "fields": [ 29 | { "name": "ClientSoftwareName", "type": "string", "versions": "3+", 30 | "ignorable": true, "about": "The name of the client." }, 31 | { "name": "ClientSoftwareVersion", "type": "string", "versions": "3+", 32 | "ignorable": true, "about": "The version of the client." } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/BrokerRegistrationResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // Version 1 adds Zk broker epoch to the request if the broker is migrating from Zk mode to KRaft mode. 17 | // 18 | // Version 2 adds the PreviousBrokerEpoch to the request for the KIP-966 19 | { 20 | "apiKey": 62, 21 | "type": "response", 22 | "name": "BrokerRegistrationResponse", 23 | "validVersions": "0-4", 24 | "flexibleVersions": "0+", 25 | "fields": [ 26 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 27 | "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 28 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 29 | "about": "The error code, or 0 if there was no error." }, 30 | { "name": "BrokerEpoch", "type": "int64", "versions": "0+", "default": "-1", 31 | "about": "The broker's assigned epoch, or -1 if none was assigned." } 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ConsumerGroupDescribeRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 69, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker"], 20 | "name": "ConsumerGroupDescribeRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "GroupIds", "type": "[]string", "versions": "0+", "entityType": "groupId", 25 | "about": "The ids of the groups to describe" }, 26 | { "name": "IncludeAuthorizedOperations", "type": "bool", "versions": "0+", 27 | "about": "Whether to include authorized operations." } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ControlledShutdownRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 7, 18 | "type": "request", 19 | "listeners": ["zkBroker", "controller"], 20 | "name": "ControlledShutdownRequest", 21 | // Version 0 of ControlledShutdownRequest has a non-standard request header 22 | // which does not include clientId. Version 1 and later use the standard 23 | // request header. 24 | // 25 | // Version 1 is the same as version 0. 26 | // 27 | // Version 2 adds BrokerEpoch. 28 | "validVersions": "0-3", 29 | "flexibleVersions": "3+", 30 | "fields": [ 31 | { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId", 32 | "about": "The id of the broker for which controlled shutdown has been requested." }, 33 | { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "default": "-1", "ignorable": true, 34 | "about": "The broker epoch." } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ControlledShutdownResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 7, 18 | "type": "response", 19 | "name": "ControlledShutdownResponse", 20 | // Versions 1 and 2 are the same as version 0. 21 | "validVersions": "0-3", 22 | "flexibleVersions": "3+", 23 | "fields": [ 24 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 25 | "about": "The top-level error code." }, 26 | { "name": "RemainingPartitions", "type": "[]RemainingPartition", "versions": "0+", 27 | "about": "The partitions that the broker still leads.", "fields": [ 28 | { "name": "TopicName", "type": "string", "versions": "0+", "mapKey": true, "entityType": "topicName", 29 | "about": "The name of the topic." }, 30 | { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true, 31 | "about": "The index of the partition." } 32 | ]} 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ControllerRegistrationResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 70, 18 | "type": "response", 19 | "name": "ControllerRegistrationResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 24 | "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The response error code." }, 27 | { "name": "ErrorMessage", "type": "string", "nullableVersions": "0+", "versions": "0+", 28 | "about": "The response error message, or null if there was no error." } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DefaultPrincipalData.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "type": "data", 18 | "name": "DefaultPrincipalData", 19 | // The encoding format for default Kafka principal in 20 | // org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder. 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | {"name": "Type", "type": "string", "versions": "0+", 25 | "about": "The principal type"}, 26 | {"name": "Name", "type": "string", "versions": "0+", 27 | "about": "The principal name"}, 28 | {"name": "TokenAuthenticated", "type": "bool", "versions": "0+", 29 | "about": "Whether the principal was authenticated by a delegation token on the forwarding broker."} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DeleteGroupsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 42, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker"], 20 | "name": "DeleteGroupsRequest", 21 | // Version 1 is the same as version 0. 22 | // 23 | // Version 2 is the first flexible version. 24 | "validVersions": "0-2", 25 | "flexibleVersions": "2+", 26 | "fields": [ 27 | { "name": "GroupsNames", "type": "[]string", "versions": "0+", "entityType": "groupId", 28 | "about": "The group names to delete." } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DeleteShareGroupStateRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 86, 18 | "type": "request", 19 | "listeners": ["broker"], 20 | "name": "DeleteShareGroupStateRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "latestVersionUnstable": true, 24 | "fields": [ 25 | { "name": "GroupId", "type": "string", "versions": "0+", 26 | "about":"The group identifier." }, 27 | { "name": "Topics", "type": "[]DeleteStateData", "versions": "0+", 28 | "about": "The data for the topics.", "fields": [ 29 | { "name": "TopicId", "type": "uuid", "versions": "0+", 30 | "about": "The topic identifier." }, 31 | { "name": "Partitions", "type": "[]PartitionData", "versions": "0+", 32 | "about": "The data for the partitions.", "fields": [ 33 | { "name": "Partition", "type": "int32", "versions": "0+", 34 | "about": "The partition index." } 35 | ]} 36 | ]} 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DescribeClusterRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 60, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "DescribeClusterRequest", 21 | // 22 | // Version 1 adds EndpointType for KIP-919 support. 23 | // 24 | "validVersions": "0-1", 25 | "flexibleVersions": "0+", 26 | "fields": [ 27 | { "name": "IncludeClusterAuthorizedOperations", "type": "bool", "versions": "0+", 28 | "about": "Whether to include cluster authorized operations." }, 29 | { "name": "EndpointType", "type": "int8", "versions": "1+", "default": "1", 30 | "about": "The endpoint type to describe. 1=brokers, 2=controllers." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DescribeGroupsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 15, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker"], 20 | "name": "DescribeGroupsRequest", 21 | // Versions 1 and 2 are the same as version 0. 22 | // 23 | // Starting in version 3, authorized operations can be requested. 24 | // 25 | // Starting in version 4, the response will include group.instance.id info for members. 26 | // 27 | // Version 5 is the first flexible version. 28 | "validVersions": "0-5", 29 | "flexibleVersions": "5+", 30 | "fields": [ 31 | { "name": "Groups", "type": "[]string", "versions": "0+", "entityType": "groupId", 32 | "about": "The names of the groups to describe" }, 33 | { "name": "IncludeAuthorizedOperations", "type": "bool", "versions": "3+", 34 | "about": "Whether to include authorized operations." } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DescribeProducersRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 61, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker"], 20 | "name": "DescribeProducersRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "Topics", "type": "[]TopicRequest", "versions": "0+", "fields": [ 25 | { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName", 26 | "about": "The topic name." }, 27 | { "name": "PartitionIndexes", "type": "[]int32", "versions": "0+", 28 | "about": "The indexes of the partitions to list producers for." } 29 | ]} 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DescribeTransactionsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 65, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker"], 20 | "name": "DescribeTransactionsRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "TransactionalIds", "entityType": "transactionalId", "type": "[]string", "versions": "0+", 25 | "about": "Array of transactionalIds to include in describe results. If empty, then no results will be returned." } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/DescribeUserScramCredentialsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 50, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "DescribeUserScramCredentialsRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "Users", "type": "[]UserName", "versions": "0+", "nullableVersions": "0+", 25 | "about": "The users to describe, or null/empty to describe all users.", "fields": [ 26 | { "name": "Name", "type": "string", "versions": "0+", 27 | "about": "The user name." } 28 | ]} 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/EndTxnResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 26, 18 | "type": "response", 19 | "name": "EndTxnResponse", 20 | // Starting in version 1, on quota violation, brokers send out responses before throttling. 21 | // 22 | // Version 2 adds the support for new error code PRODUCER_FENCED. 23 | // 24 | // Version 3 enables flexible versions. 25 | // 26 | // Version 4 adds support for new error code TRANSACTION_ABORTABLE (KIP-890). 27 | "validVersions": "0-4", 28 | "flexibleVersions": "3+", 29 | "fields": [ 30 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 31 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 32 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 33 | "about": "The error code, or 0 if there was no error." } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/EnvelopeRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 58, 18 | "type": "request", 19 | "listeners": ["controller", "zkBroker"], 20 | "name": "EnvelopeRequest", 21 | // Request struct for forwarding. 22 | "validVersions": "0", 23 | "flexibleVersions": "0+", 24 | "fields": [ 25 | { "name": "RequestData", "type": "bytes", "versions": "0+", "zeroCopy": true, 26 | "about": "The embedded request header and data."}, 27 | { "name": "RequestPrincipal", "type": "bytes", "versions": "0+", "nullableVersions": "0+", 28 | "about": "Value of the initial client principal when the request is redirected by a broker." }, 29 | { "name": "ClientHostAddress", "type": "bytes", "versions": "0+", 30 | "about": "The original client's address in bytes." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/EnvelopeResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 58, 18 | "type": "response", 19 | "name": "EnvelopeResponse", 20 | // Response struct for forwarding. 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "ResponseData", "type": "bytes", "versions": "0+", "nullableVersions": "0+", 25 | "zeroCopy": true, "default": "null", 26 | "about": "The embedded response header and data."}, 27 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 28 | "about": "The error code, or 0 if there was no error." } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ExpireDelegationTokenRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 40, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "ExpireDelegationTokenRequest", 21 | // Version 1 is the same as version 0. 22 | // Version 2 adds flexible version support 23 | "validVersions": "0-2", 24 | "deprecatedVersions": "0", 25 | "flexibleVersions": "2+", 26 | "fields": [ 27 | { "name": "Hmac", "type": "bytes", "versions": "0+", 28 | "about": "The HMAC of the delegation token to be expired." }, 29 | { "name": "ExpiryTimePeriodMs", "type": "int64", "versions": "0+", 30 | "about": "The expiry time period in milliseconds." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ExpireDelegationTokenResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 40, 18 | "type": "response", 19 | "name": "ExpireDelegationTokenResponse", 20 | // Starting in version 1, on quota violation, brokers send out responses before throttling. 21 | // Version 2 adds flexible version support 22 | "validVersions": "0-2", 23 | "flexibleVersions": "2+", 24 | "fields": [ 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The error code, or 0 if there was no error." }, 27 | { "name": "ExpiryTimestampMs", "type": "int64", "versions": "0+", 28 | "about": "The timestamp in milliseconds at which this token expires." }, 29 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 30 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/GetTelemetrySubscriptionsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 71, 18 | "type": "request", 19 | "listeners": ["broker"], 20 | "name": "GetTelemetrySubscriptionsRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { 25 | "name": "ClientInstanceId", "type": "uuid", "versions": "0+", 26 | "about": "Unique id for this client instance, must be set to 0 on the first request." 27 | } 28 | ] 29 | } -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/LeaderChangeMessage.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "type": "data", 18 | "name": "LeaderChangeMessage", 19 | "validVersions": "0", 20 | "flexibleVersions": "0+", 21 | "fields": [ 22 | {"name": "Version", "type": "int16", "versions": "0+", 23 | "about": "The version of the leader change message"}, 24 | {"name": "LeaderId", "type": "int32", "versions": "0+", "entityType": "brokerId", 25 | "about": "The ID of the newly elected leader"}, 26 | {"name": "Voters", "type": "[]Voter", "versions": "0+", 27 | "about": "The set of voters in the quorum for this epoch"}, 28 | {"name": "GrantingVoters", "type": "[]Voter", "versions": "0+", 29 | "about": "The voters who voted for the leader at the time of election"} 30 | ], 31 | "commonStructs": [ 32 | { "name": "Voter", "versions": "0+", "fields": [ 33 | {"name": "VoterId", "type": "int32", "versions": "0+"} 34 | ]} 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ListClientMetricsResourcesRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 74, 18 | "type": "request", 19 | "listeners": ["broker"], 20 | "name": "ListClientMetricsResourcesRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ListClientMetricsResourcesResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 74, 18 | "type": "response", 19 | "name": "ListClientMetricsResourcesResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 24 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The error code, or 0 if there was no error." }, 27 | { "name": "ClientMetricsResources", "type": "[]ClientMetricsResource", "versions": "0+", 28 | "about": "Each client metrics resource in the response.", "fields": [ 29 | { "name": "Name", "type": "string", "versions": "0+", 30 | "about": "The resource name." } 31 | ]} 32 | ] 33 | } 34 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ListGroupsRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 16, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker"], 20 | "name": "ListGroupsRequest", 21 | // Version 1 and 2 are the same as version 0. 22 | // 23 | // Version 3 is the first flexible version. 24 | // 25 | // Version 4 adds the StatesFilter field (KIP-518). 26 | // 27 | // Version 5 adds the TypesFilter field (KIP-848). 28 | "validVersions": "0-5", 29 | "flexibleVersions": "3+", 30 | "fields": [ 31 | { "name": "StatesFilter", "type": "[]string", "versions": "4+", 32 | "about": "The states of the groups we want to list. If empty, all groups are returned with their state." }, 33 | { "name": "TypesFilter", "type": "[]string", "versions": "5+", 34 | "about": "The types of the groups we want to list. If empty, all groups are returned with their type." } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/PushTelemetryResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 72, 18 | "type": "response", 19 | "name": "PushTelemetryResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { 24 | "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 25 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." 26 | }, 27 | { 28 | "name": "ErrorCode", "type": "int16", "versions": "0+", 29 | "about": "The error code, or 0 if there was no error." 30 | } 31 | ] 32 | } -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/RemoveRaftVoterRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 81, 18 | "type": "request", 19 | "listeners": ["controller", "broker"], 20 | "name": "RemoveRaftVoterRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "ClusterId", "type": "string", "versions": "0+", "nullableVersions": "0+" }, 25 | { "name": "VoterId", "type": "int32", "versions": "0+", 26 | "about": "The replica id of the voter getting removed from the topic partition" }, 27 | { "name": "VoterDirectoryId", "type": "uuid", "versions": "0+", 28 | "about": "The directory id of the voter getting removed from the topic partition" } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/RemoveRaftVoterResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 81, 18 | "type": "response", 19 | "name": "RemoveRaftVoterResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 24 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The error code, or 0 if there was no error" }, 27 | { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", "ignorable": true, 28 | "about": "The error message, or null if there was no error." } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/RenewDelegationTokenRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 39, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "RenewDelegationTokenRequest", 21 | // Version 1 is the same as version 0. 22 | // Version 2 adds flexible version support 23 | "validVersions": "0-2", 24 | "deprecatedVersions": "0", 25 | "flexibleVersions": "2+", 26 | "fields": [ 27 | { "name": "Hmac", "type": "bytes", "versions": "0+", 28 | "about": "The HMAC of the delegation token to be renewed." }, 29 | { "name": "RenewPeriodMs", "type": "int64", "versions": "0+", 30 | "about": "The renewal time period in milliseconds." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/RenewDelegationTokenResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 39, 18 | "type": "response", 19 | "name": "RenewDelegationTokenResponse", 20 | // Starting in version 1, on quota violation, brokers send out responses before throttling. 21 | // Version 2 adds flexible version support 22 | "validVersions": "0-2", 23 | "flexibleVersions": "2+", 24 | "fields": [ 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The error code, or 0 if there was no error." }, 27 | { "name": "ExpiryTimestampMs", "type": "int64", "versions": "0+", 28 | "about": "The timestamp in milliseconds at which this token expires." }, 29 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 30 | "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ResponseHeader.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "type": "header", 18 | "name": "ResponseHeader", 19 | // Version 1 is the first flexible version. 20 | "validVersions": "0-1", 21 | "flexibleVersions": "1+", 22 | "fields": [ 23 | { "name": "CorrelationId", "type": "int32", "versions": "0+", 24 | "about": "The correlation ID of this response." } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/SaslAuthenticateRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 36, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "SaslAuthenticateRequest", 21 | // Version 1 is the same as version 0. 22 | // Version 2 adds flexible version support 23 | "validVersions": "0-2", 24 | "flexibleVersions": "2+", 25 | "fields": [ 26 | { "name": "AuthBytes", "type": "bytes", "versions": "0+", 27 | "about": "The SASL authentication bytes from the client, as defined by the SASL mechanism." } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/SaslHandshakeRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 17, 18 | "type": "request", 19 | "listeners": ["zkBroker", "broker", "controller"], 20 | "name": "SaslHandshakeRequest", 21 | // Version 1 supports SASL_AUTHENTICATE. 22 | // NOTE: Version cannot be easily bumped due to incorrect 23 | // client negotiation for clients <= 2.4. 24 | // See https://issues.apache.org/jira/browse/KAFKA-9577 25 | "validVersions": "0-1", 26 | "flexibleVersions": "none", 27 | "fields": [ 28 | { "name": "Mechanism", "type": "string", "versions": "0+", 29 | "about": "The SASL mechanism chosen by the client." } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/SaslHandshakeResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 17, 18 | "type": "response", 19 | "name": "SaslHandshakeResponse", 20 | // Version 1 is the same as version 0. 21 | // NOTE: Version cannot be easily bumped due to incorrect 22 | // client negotiation for clients <= 2.4. 23 | // See https://issues.apache.org/jira/browse/KAFKA-9577 24 | "validVersions": "0-1", 25 | "flexibleVersions": "none", 26 | "fields": [ 27 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 28 | "about": "The error code, or 0 if there was no error." }, 29 | { "name": "Mechanisms", "type": "[]string", "versions": "0+", 30 | "about": "The mechanisms enabled in the server." } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/ShareGroupDescribeRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 77, 18 | "type": "request", 19 | "listeners": ["broker"], 20 | "name": "ShareGroupDescribeRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | // The ShareGroupDescribeRequest API is added as part of KIP-932 and is still under 24 | // development. Hence, the API is not exposed by default by brokers unless 25 | // explicitly enabled. 26 | "latestVersionUnstable": true, 27 | "fields": [ 28 | { "name": "GroupIds", "type": "[]string", "versions": "0+", "entityType": "groupId", 29 | "about": "The ids of the groups to describe" }, 30 | { "name": "IncludeAuthorizedOperations", "type": "bool", "versions": "0+", 31 | "about": "Whether to include authorized operations." } 32 | ] 33 | } -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/SnapshotFooterRecord.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "type": "data", 18 | "name": "SnapshotFooterRecord", 19 | "validVersions": "0", 20 | "flexibleVersions": "0+", 21 | "fields": [ 22 | {"name": "Version", "type": "int16", "versions": "0+", 23 | "about": "The version of the snapshot footer record"} 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/SnapshotHeaderRecord.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "type": "data", 18 | "name": "SnapshotHeaderRecord", 19 | "validVersions": "0", 20 | "flexibleVersions": "0+", 21 | "fields": [ 22 | {"name": "Version", "type": "int16", "versions": "0+", 23 | "about": "The version of the snapshot header record"}, 24 | {"name": "LastContainedLogTimestamp", "type": "int64", "versions": "0+", 25 | "about": "The append time of the last record from the log contained in this snapshot"} 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/UnregisterBrokerRequest.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 64, 18 | "type": "request", 19 | "listeners": ["broker", "controller"], 20 | "name": "UnregisterBrokerRequest", 21 | "validVersions": "0", 22 | "flexibleVersions": "0+", 23 | "fields": [ 24 | { "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId", 25 | "about": "The broker ID to unregister." } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/UnregisterBrokerResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 64, 18 | "type": "response", 19 | "name": "UnregisterBrokerResponse", 20 | "validVersions": "0", 21 | "flexibleVersions": "0+", 22 | "fields": [ 23 | { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", 24 | "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, 25 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 26 | "about": "The error code, or 0 if there was no error." }, 27 | { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", 28 | "about": "The top-level error message, or `null` if there was no top-level error." } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/message/UpdateMetadataResponse.json: -------------------------------------------------------------------------------- 1 | // Licensed to the Apache Software Foundation (ASF) under one or more 2 | // contributor license agreements. See the NOTICE file distributed with 3 | // this work for additional information regarding copyright ownership. 4 | // The ASF licenses this file to You under the Apache License, Version 2.0 5 | // (the "License"); you may not use this file except in compliance with 6 | // the License. You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | { 17 | "apiKey": 6, 18 | "type": "response", 19 | "name": "UpdateMetadataResponse", 20 | // Versions 1, 2, 3, 4, and 5 are the same as version 0 21 | "validVersions": "0-8", 22 | "flexibleVersions": "6+", 23 | "fields": [ 24 | { "name": "ErrorCode", "type": "int16", "versions": "0+", 25 | "about": "The error code, or 0 if there was no error." } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/src/primitive.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use crate::Result; 17 | 18 | pub trait ByteSize { 19 | fn size_in_bytes(&self) -> Result; 20 | } 21 | 22 | pub mod tagged; 23 | pub mod uuid; 24 | pub mod varint; 25 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/src/primitive/uuid.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use serde::{Deserialize, Serialize}; 17 | 18 | use super::ByteSize; 19 | use crate::Result; 20 | 21 | #[derive( 22 | Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, 23 | )] 24 | pub struct Uuid(pub [u8; 16]); 25 | 26 | impl From<[u8; 16]> for Uuid { 27 | fn from(value: [u8; 16]) -> Self { 28 | Self(value) 29 | } 30 | } 31 | 32 | impl ByteSize for Uuid { 33 | fn size_in_bytes(&self) -> Result { 34 | Ok(size_of_val(&self.0)) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tansu-kafka-sans-io/tests/.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | -------------------------------------------------------------------------------- /tansu-proxy/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-proxy" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | # [lints] 9 | # workspace = true 10 | 11 | [dependencies] 12 | clap.workspace = true 13 | tansu-kafka-sans-io = { path = "../tansu-kafka-sans-io" } 14 | thiserror.workspace = true 15 | tokio.workspace = true 16 | tracing-subscriber.workspace = true 17 | tracing.workspace = true 18 | url.workspace = true 19 | -------------------------------------------------------------------------------- /tansu-schema-registry/.gitignore: -------------------------------------------------------------------------------- 1 | *.parquet 2 | -------------------------------------------------------------------------------- /tansu-schema-registry/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-schema-registry" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | [dependencies] 9 | anyhow.workspace = true 10 | apache-avro.workspace = true 11 | arrow.workspace = true 12 | async-trait.workspace = true 13 | bytes.workspace = true 14 | chrono.workspace = true 15 | datafusion.workspace = true 16 | delta_kernel.workspace = true 17 | deltalake.workspace = true 18 | dotenv.workspace = true 19 | futures.workspace = true 20 | iceberg-catalog-memory.workspace = true 21 | iceberg-catalog-rest.workspace = true 22 | iceberg-datafusion.workspace = true 23 | iceberg.workspace = true 24 | jsonschema.workspace = true 25 | num-bigint.workspace = true 26 | object_store.workspace = true 27 | opentelemetry-semantic-conventions.workspace = true 28 | opentelemetry.workspace = true 29 | ordered-float.workspace = true 30 | parquet.workspace = true 31 | protobuf-json-mapping.workspace = true 32 | protobuf-parse.workspace = true 33 | protobuf.workspace = true 34 | serde.workspace = true 35 | serde_json.workspace = true 36 | tansu-kafka-sans-io = { path = "../tansu-kafka-sans-io" } 37 | tempfile.workspace = true 38 | thiserror.workspace = true 39 | tokio.workspace = true 40 | tracing-subscriber.workspace = true 41 | tracing.workspace = true 42 | url.workspace = true 43 | uuid.workspace = true 44 | 45 | [dev-dependencies] 46 | pretty_assertions.workspace = true 47 | rand.workspace = true 48 | 49 | # [lints] 50 | # workspace = true 51 | -------------------------------------------------------------------------------- /tansu-schema-registry/src/lake/tansu.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use dotenv::dotenv; 17 | use tansu_cli::{Cli, Result}; 18 | use tansu_kafka_sans_io::ErrorCode; 19 | use tansu_server::{TracingFormat, otel}; 20 | use tracing::{debug, error}; 21 | 22 | #[tokio::main] 23 | async fn main() -> Result { 24 | dotenv().ok(); 25 | 26 | let _guard = otel::init(TracingFormat::Text)?; 27 | 28 | Cli::main() 29 | .await 30 | .inspect(|error_code| debug!(%error_code)) 31 | .inspect_err(|err| error!(%err)) 32 | } 33 | -------------------------------------------------------------------------------- /tansu-schema-registry/src/meta.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type": "record", 3 | "name": "metadata", 4 | "fields": [ 5 | { "name": "partition", "type": "int" }, 6 | { "name": "timestamp", "type": "long", "logicalType": "timestamp-micros" }, 7 | { "name": "year", "type": "int" }, 8 | { "name": "month", "type": "int" }, 9 | { "name": "day", "type": "int" } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /tansu-schema-registry/src/meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "meta", 3 | "type": "object", 4 | 5 | "properties": { 6 | "partition": { "type": "integer" }, 7 | "timestamp": { "type": "string", "format": "date-time" }, 8 | "year": { "type": "integer" }, 9 | "month": { "type": "integer" }, 10 | "day": { "type": "integer" } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /tansu-schema-registry/src/meta.proto: -------------------------------------------------------------------------------- 1 | syntax = 'proto3'; 2 | 3 | import "google/protobuf/timestamp.proto"; 4 | 5 | message Meta { 6 | int32 partition = 1; 7 | google.protobuf.Timestamp timestamp = 2; 8 | int32 year = 3; 9 | int32 month = 4; 10 | int32 day = 5; 11 | } 12 | -------------------------------------------------------------------------------- /tansu-server/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /tansu-server/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-server" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | # [lints] 9 | # workspace = true 10 | 11 | [dependencies] 12 | async-trait.workspace = true 13 | bytes.workspace = true 14 | clap.workspace = true 15 | deadpool-postgres.workspace = true 16 | futures.workspace = true 17 | http-body-util.workspace = true 18 | hyper-util.workspace = true 19 | hyper.workspace = true 20 | jsonschema.workspace = true 21 | object_store.workspace = true 22 | opentelemetry-jaeger.workspace = true 23 | opentelemetry-otlp.workspace = true 24 | opentelemetry-prometheus.workspace = true 25 | opentelemetry-semantic-conventions.workspace = true 26 | opentelemetry.workspace = true 27 | opentelemetry_sdk.workspace = true 28 | prometheus.workspace = true 29 | rand.workspace = true 30 | regex.workspace = true 31 | serde.workspace = true 32 | serde_json.workspace = true 33 | tansu-kafka-model = { path = "../tansu-kafka-model" } 34 | tansu-kafka-sans-io = { path = "../tansu-kafka-sans-io" } 35 | tansu-schema-registry = { path = "../tansu-schema-registry" } 36 | tansu-storage = { path = "../tansu-storage" } 37 | thiserror.workspace = true 38 | tokio-postgres.workspace = true 39 | tokio.workspace = true 40 | tracing-opentelemetry.workspace = true 41 | tracing-subscriber.workspace = true 42 | tracing.workspace = true 43 | url.workspace = true 44 | uuid.workspace = true 45 | 46 | [dev-dependencies] 47 | pretty_assertions.workspace = true 48 | 49 | [features] 50 | default = [] 51 | nightly-features = [] 52 | -------------------------------------------------------------------------------- /tansu-server/src/broker/delete_records.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024-2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use crate::Result; 17 | use tansu_kafka_sans_io::{Body, delete_records_request::DeleteRecordsTopic}; 18 | use tansu_storage::Storage; 19 | use tracing::debug; 20 | 21 | #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] 22 | pub struct DeleteRecordsRequest { 23 | storage: S, 24 | } 25 | 26 | impl DeleteRecordsRequest 27 | where 28 | S: Storage, 29 | { 30 | pub fn with_storage(storage: S) -> Self { 31 | Self { storage } 32 | } 33 | 34 | pub async fn request(&mut self, topics: &[DeleteRecordsTopic]) -> Result { 35 | let topics = self 36 | .storage 37 | .delete_records(topics) 38 | .await 39 | .inspect_err(|err| debug!(?err, ?topics)) 40 | .map(Some)?; 41 | 42 | Ok(Body::DeleteRecordsResponse { 43 | throttle_time_ms: 0, 44 | topics, 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tansu-server/src/broker/group.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | pub mod heartbeat; 17 | pub mod join; 18 | pub mod leave; 19 | pub mod offset_commit; 20 | pub mod offset_fetch; 21 | pub mod sync; 22 | -------------------------------------------------------------------------------- /tansu-server/src/broker/group/heartbeat.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024-2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use tansu_kafka_sans_io::Body; 17 | 18 | use crate::{Result, coordinator::group::Coordinator}; 19 | 20 | #[derive(Debug)] 21 | pub struct HeartbeatRequest { 22 | coordinator: C, 23 | } 24 | 25 | impl HeartbeatRequest { 26 | pub fn with_coordinator(coordinator: C) -> Self 27 | where 28 | C: Coordinator, 29 | { 30 | Self { coordinator } 31 | } 32 | } 33 | 34 | impl HeartbeatRequest 35 | where 36 | C: Coordinator, 37 | { 38 | pub async fn response( 39 | &mut self, 40 | group_id: &str, 41 | generation_id: i32, 42 | member_id: &str, 43 | group_instance_id: Option<&str>, 44 | ) -> Result { 45 | self.coordinator 46 | .heartbeat(group_id, generation_id, member_id, group_instance_id) 47 | .await 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tansu-server/src/broker/group/leave.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024-2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use tansu_kafka_sans_io::{Body, leave_group_request::MemberIdentity}; 17 | 18 | use crate::{Result, coordinator::group::Coordinator}; 19 | 20 | #[derive(Debug)] 21 | pub struct LeaveRequest { 22 | coordinator: C, 23 | } 24 | 25 | impl LeaveRequest 26 | where 27 | C: Coordinator, 28 | { 29 | pub async fn response( 30 | &mut self, 31 | group_id: &str, 32 | member_id: Option<&str>, 33 | members: Option<&[MemberIdentity]>, 34 | ) -> Result { 35 | self.coordinator.leave(group_id, member_id, members).await 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /tansu-server/src/broker/group/offset_fetch.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024-2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use tansu_kafka_sans_io::{ 17 | Body, 18 | offset_fetch_request::{OffsetFetchRequestGroup, OffsetFetchRequestTopic}, 19 | }; 20 | 21 | use crate::{Result, coordinator::group::Coordinator}; 22 | 23 | #[derive(Debug)] 24 | pub struct OffsetFetchRequest { 25 | coordinator: C, 26 | } 27 | 28 | impl OffsetFetchRequest 29 | where 30 | C: Coordinator, 31 | { 32 | pub fn with_coordinator(coordinator: C) -> Self { 33 | Self { coordinator } 34 | } 35 | 36 | pub async fn response( 37 | &mut self, 38 | group_id: Option<&str>, 39 | topics: Option<&[OffsetFetchRequestTopic]>, 40 | groups: Option<&[OffsetFetchRequestGroup]>, 41 | require_stable: Option, 42 | ) -> Result { 43 | self.coordinator 44 | .offset_fetch(group_id, topics, groups, require_stable) 45 | .await 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tansu-server/src/broker/telemetry.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use tansu_kafka_sans_io::{Body, ErrorCode}; 17 | use uuid::Uuid; 18 | 19 | #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] 20 | pub struct GetTelemetrySubscriptionsRequest; 21 | 22 | impl GetTelemetrySubscriptionsRequest { 23 | pub fn response(&self, client_instance_id: [u8; 16]) -> Body { 24 | let _ = client_instance_id; 25 | 26 | let client_instance_id = *Uuid::new_v4().as_bytes(); 27 | 28 | Body::GetTelemetrySubscriptionsResponse { 29 | throttle_time_ms: 0, 30 | error_code: ErrorCode::None.into(), 31 | client_instance_id, 32 | subscription_id: 0, 33 | accepted_compression_types: Some([0].into()), 34 | push_interval_ms: 5_000, 35 | telemetry_max_bytes: 1_024, 36 | delta_temporality: false, 37 | requested_metrics: Some([].into()), 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tansu-server/src/broker/txn.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | pub mod add_offsets; 17 | pub mod add_partitions; 18 | pub mod offset_commit; 19 | -------------------------------------------------------------------------------- /tansu-server/src/coordinator.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | pub mod group; 17 | -------------------------------------------------------------------------------- /tansu-server/src/coordinator/group/consumer.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use std::collections::HashMap; 17 | 18 | use bytes::Bytes; 19 | 20 | #[allow(dead_code)] 21 | struct Group { 22 | id: String, 23 | session_timeout_ms: i32, 24 | rebalance_timeout_ms: i32, 25 | members: HashMap, 26 | } 27 | 28 | #[allow(dead_code)] 29 | struct Member { 30 | metadata: Bytes, 31 | } 32 | 33 | #[allow(dead_code)] 34 | struct MemberMetadata { 35 | version: i16, 36 | topics: Vec, 37 | user_data: Option, 38 | owned_partitions: Vec, 39 | generation_id: Option, 40 | rack_id: Option, 41 | } 42 | 43 | #[allow(dead_code)] 44 | struct TopicPartition { 45 | topic: String, 46 | partitions: Vec, 47 | } 48 | -------------------------------------------------------------------------------- /tansu-server/src/otel.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024-2025 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use crate::{Result, TracingFormat}; 17 | 18 | pub mod prom; 19 | mod tracing; 20 | 21 | #[derive(Debug)] 22 | pub struct Guard { 23 | #[allow(dead_code)] 24 | tracer: tracing::Guard, 25 | } 26 | 27 | pub fn init(tracing_format: TracingFormat) -> Result { 28 | tracing::init_tracing_subscriber(tracing_format).map(|tracer| Guard { tracer }) 29 | } 30 | -------------------------------------------------------------------------------- /tansu-storage/.gitignore: -------------------------------------------------------------------------------- 1 | test-*.log 2 | -------------------------------------------------------------------------------- /tansu-storage/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-storage" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | # [lints] 9 | # workspace = true 10 | 11 | [dependencies] 12 | async-trait.workspace = true 13 | bytes.workspace = true 14 | deadpool-postgres.workspace = true 15 | deadpool.workspace = true 16 | futures-core.workspace = true 17 | futures-util.workspace = true 18 | futures.workspace = true 19 | glob.workspace = true 20 | object_store.workspace = true 21 | opentelemetry-semantic-conventions.workspace = true 22 | opentelemetry.workspace = true 23 | protobuf.workspace = true 24 | rand.workspace = true 25 | regex.workspace = true 26 | serde.workspace = true 27 | serde_json.workspace = true 28 | tansu-kafka-sans-io = { path = "../tansu-kafka-sans-io" } 29 | tansu-schema-registry = { path = "../tansu-schema-registry" } 30 | thiserror.workspace = true 31 | tokio-postgres.workspace = true 32 | tokio.workspace = true 33 | tracing-subscriber.workspace = true 34 | tracing.workspace = true 35 | url.workspace = true 36 | uuid.workspace = true 37 | 38 | [dev-dependencies] 39 | criterion.workspace = true 40 | tempfile.workspace = true 41 | time.workspace = true 42 | tracing-subscriber.workspace = true 43 | 44 | [[bench]] 45 | name = "segment_bench" 46 | harness = false 47 | -------------------------------------------------------------------------------- /tansu-storage/src/os.rs: -------------------------------------------------------------------------------- 1 | // Copyright ⓒ 2024 Peter Morgan 2 | // 3 | // This program is free software: you can redistribute it and/or modify 4 | // it under the terms of the GNU Affero General Public License as 5 | // published by the Free Software Foundation, either version 3 of the 6 | // License, or (at your option) any later version. 7 | // 8 | // This program is distributed in the hope that it will be useful, 9 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | // GNU Affero General Public License for more details. 12 | // 13 | // You should have received a copy of the GNU Affero General Public License 14 | // along with this program. If not, see . 15 | 16 | use object_store::{ObjectMeta, PutResult, UpdateVersion}; 17 | 18 | use crate::Version; 19 | 20 | impl From for UpdateVersion { 21 | fn from(value: Version) -> Self { 22 | Self { 23 | e_tag: value.e_tag, 24 | version: value.version, 25 | } 26 | } 27 | } 28 | 29 | impl From for Version { 30 | fn from(value: ObjectMeta) -> Self { 31 | Self { 32 | e_tag: value.e_tag, 33 | version: value.version, 34 | } 35 | } 36 | } 37 | 38 | impl From for Version { 39 | fn from(value: UpdateVersion) -> Self { 40 | Self { 41 | e_tag: value.e_tag, 42 | version: value.version, 43 | } 44 | } 45 | } 46 | 47 | impl From for Version { 48 | fn from(value: PutResult) -> Self { 49 | Self { 50 | e_tag: value.e_tag, 51 | version: value.version, 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/broker_metadata_select.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select b.node, l.host, l.port, b.rack 18 | 19 | from 20 | 21 | cluster c 22 | join broker b on b.cluster = c.id 23 | join listener l on l.broker = b.id 24 | 25 | where 26 | 27 | c.name = $1 28 | and l.name = 'broker'; 29 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_delete.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from consumer_group 18 | using cluster c 19 | where c.name = $1 20 | and consumer_group.name = $2 21 | and consumer_group.cluster = c.id; 22 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_detail.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select cgd.e_tag, cgd.detail 18 | 19 | from 20 | 21 | cluster c 22 | join consumer_group cg on cg.cluster = c.id 23 | join consumer_group_detail cgd on cgd.consumer_group = cg.id 24 | 25 | where cg.name = $1 26 | and c.name = $2; 27 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_detail_delete_by_cg.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from consumer_group_detail 18 | using cluster c, consumer_group cg 19 | where c.name = $1 20 | and cg.name = $2 21 | and cg.cluster = c.id; 22 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_detail_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare cg_update (text, text, uuid, uuid, json) as 18 | insert into consumer_group_detail 19 | (consumer_group, e_tag, detail) 20 | 21 | select cg.id, $4, $5 22 | 23 | from cluster c 24 | join consumer_group cg on cg.cluster = c.id 25 | 26 | where c.name = $1 27 | and cg.name = $2 28 | 29 | on conflict (consumer_group) 30 | 31 | do update set 32 | 33 | detail = excluded.detail, 34 | last_updated = excluded.last_updated, 35 | e_tag = $4 36 | 37 | where consumer_group_detail.e_tag = $3 38 | 39 | returning $2, $1, e_tag, detail; 40 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare cg_update (text, text, uuid, uuid, json) as 18 | insert into consumer_group 19 | (cluster, name) 20 | 21 | select c.id, $2 22 | from cluster c 23 | where c.name = $1 24 | 25 | on conflict (cluster, name) 26 | do update set 27 | last_updated = excluded.last_updated; 28 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_select.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select cg.name 18 | 19 | from 20 | 21 | cluster c 22 | join consumer_group cg on cg.cluster = c.id 23 | 24 | where c.name = $1; 25 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_group_select_by_name.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select cg.name, cgd.detail 18 | 19 | from 20 | 21 | cluster c 22 | join consumer_group cg on cg.cluster = c.id 23 | left join consumer_group_detail cgd on cgd.consumer_group = cg.id 24 | 25 | where c.name = $1 26 | and cg.name = $2; 27 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_offset_delete_by_cg.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from consumer_offset 18 | using cluster c, consumer_group cg 19 | where c.name = $1 20 | and cg.name = $2 21 | and cg.cluster = c.id 22 | and consumer_offset.consumer_group = cg.id; 23 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_offset_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from consumer_offset 18 | using cluster c,consumer_group cg, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and cg.cluster = c.id 22 | and tp.topic = t.id 23 | and consumer_offset.topition = tp.id 24 | and consumer_offset.consumer_group = cg.id; 25 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_offset_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into consumer_offset 18 | (consumer_group, topition, committed_offset, leader_epoch, timestamp, metadata) 19 | 20 | select cg.id, tp.id, $5, $6, $7, $8 21 | 22 | from cluster c 23 | join topic t on t.cluster = c.id 24 | join topition tp on tp.topic = t.id 25 | join consumer_group cg on cg.cluster = c.id 26 | 27 | where c.name = $1 28 | and t.name = $2 29 | and tp.partition = $3 30 | and cg.name = $4 31 | 32 | on conflict (consumer_group, topition) 33 | do update set 34 | committed_offset = excluded.committed_offset, 35 | leader_epoch = excluded.leader_epoch, 36 | timestamp = excluded.timestamp, 37 | metadata = excluded.metadata; 38 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_offset_select.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare consumer_offset_select(text, text, text, integer) as 18 | select co.committed_offset 19 | 20 | from cluster c 21 | join consumer_group cg on cg.cluster = c.id 22 | join topic t on t.cluster = c.id 23 | join topition tp on tp.topic = t.id 24 | join consumer_offset co on co.consumer_group = cg.id and co.topition = tp.id 25 | 26 | where c.name = $1 27 | and cg.name = $2 28 | and t.name = $3 29 | and tp.partition = $4; 30 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/consumer_offset_select_by_group.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare consumer_offset_select_by_group (text, text) as 18 | 19 | select t.name, tp.partition, co.committed_offset 20 | 21 | from cluster c 22 | join consumer_group cg on cg.cluster = c.id 23 | join topic t on t.cluster = c.id 24 | join topition tp on tp.topic = t.id 25 | join consumer_offset co on co.consumer_group = cg.id and co.topition = tp.id 26 | 27 | where c.name = $1 28 | and cg.name = $2; 29 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/header_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024-2025 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from header 18 | using cluster c, record r, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and r.topition = tp.id 24 | and header.topition = r.topition 25 | and header.offset_id = r.offset_id; 26 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/header_fetch.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024-2025 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select h.k, h.v 18 | from header h 19 | join record r on h.topition = r.topition and h.offset_id = r.offset_id 20 | join topition tp on tp.id = r.topition 21 | join topic t on tp.topic = t.id 22 | join cluster c on t.cluster = c.id 23 | where c.name = $1 24 | and t.name = $2 25 | and tp.partition = $3 26 | and r.offset_id = $4; 27 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/header_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024-2025 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into header 18 | (topition, offset_id, k, v) 19 | 20 | select 21 | 22 | r.topition, r.offset_id, $5, $6 23 | 24 | from 25 | 26 | cluster c 27 | join topic t on t.cluster = c.id 28 | join topition tp on tp.topic = t.id 29 | join record r on r.topition = tp.id 30 | 31 | where 32 | 33 | c.name = $1 34 | and t.name = $2 35 | and tp.partition = $3 36 | and r.offset_id = $4; 37 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/init_txn_producer.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into producer (id, epoch, cluster) 18 | select $2, $3, cluster.id 19 | from cluster where cluster.name = $1 20 | on conflict (id, epoch) 21 | do update set 22 | epoch = excluded.epoch + 1, 23 | last_updated = excluded.last_updated 24 | returning id, epoch; 25 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/list_earliest_offset.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare list_earliest_offset (text, text, integer) as 18 | select 19 | 20 | r.offset_id, 21 | r.timestamp 22 | 23 | from 24 | 25 | cluster c 26 | join topic t on t.cluster = c.id 27 | join topition tp on tp.topic = t.id 28 | join watermark w on w.topition = tp.id 29 | join record r on r.topition = tp.id and r.offset_id = w.low 30 | 31 | where 32 | 33 | c.name = $1 34 | and t.name = $2 35 | and tp.partition = $3; 36 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/list_latest_offset_timestamp.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select 18 | id as offset, timestamp 19 | from 20 | record 21 | join ( 22 | select 23 | coalesce(min(record.id), (select last_value from record_id_seq)) as offset 24 | from record, topic, cluster 25 | where 26 | topic.cluster = cluster.id 27 | and cluster.name = $1 28 | and topic.name = $2 29 | and record.partition = $3 30 | and record.timestamp >= $4 31 | and record.topic = topic.id) as minimum 32 | on record.id = minimum.offset; 33 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/list_latest_offset_uncommitted.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare list_latest_offset (text, text, integer) as 18 | select 19 | 20 | r.offset_id + 1, r.timestamp 21 | 22 | from 23 | 24 | cluster c 25 | join topic t on t.cluster = c.id 26 | join topition tp on tp.topic = t.id 27 | join watermark w on w.topition = tp.id 28 | join record r on r.topition = tp.id and r.offset_id = w.high - 1 29 | 30 | where 31 | 32 | c.name = $1 33 | and t.name = $2 34 | and tp.partition = $3; 35 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_detail_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from producer_detail 18 | using cluster c, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and producer_detail.topition = tp.id; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_detail_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into producer_detail (producer_epoch, topition, sequence) 18 | 19 | select pe.id, tp.id, $6 20 | 21 | from 22 | 23 | cluster c 24 | join producer p on p.cluster = c.id 25 | join topic t on t.cluster = c.id 26 | join topition tp on tp.topic = t.id 27 | join producer_epoch pe on pe.producer = p.id 28 | 29 | where 30 | 31 | c.name = $1 32 | and t.name = $2 33 | and tp.partition = $3 34 | and p.id = $4 35 | and pe.epoch = $5 36 | 37 | on conflict (producer_epoch, topition) 38 | 39 | do update set 40 | 41 | sequence = producer_detail.sequence + $6, 42 | last_updated = excluded.last_updated 43 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_epoch_current_for_producer.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare producer_epoch_current_for_producer (text, integer) as 18 | 19 | select 20 | 21 | pe.epoch 22 | 23 | from 24 | 25 | cluster c 26 | join producer p on p.cluster = c.id 27 | join producer_epoch pe on pe.producer = p.id 28 | 29 | where 30 | 31 | c.name = $1 32 | and p.id = $2 33 | 34 | order by pe.epoch desc 35 | 36 | limit 1; 37 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_epoch_for_current_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare producer_epoch_for_current_txn (text, text) as 18 | 19 | select 20 | 21 | p.id as producer, 22 | pe.epoch as epoch, 23 | txn_d.status as status 24 | 25 | from 26 | 27 | cluster c 28 | join producer p on p.cluster = c.id 29 | join producer_epoch pe on pe.producer = p.id 30 | join txn on txn.producer = p.id 31 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 32 | 33 | where 34 | 35 | c.name = $1 36 | and txn.name = $2 37 | 38 | order by p.id, pe.epoch desc 39 | 40 | limit 1; 41 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_epoch_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare producer_epoch_insert(text, integer) as 18 | 19 | with curr as ( 20 | 21 | select 22 | 23 | p.id, 24 | coalesce(max(pe.epoch) + 1, 0) as epoch 25 | 26 | from 27 | 28 | cluster c 29 | join producer p on p.cluster = c.id 30 | left join producer_epoch pe on pe.producer = p.id 31 | 32 | where 33 | 34 | c.name = $1 35 | and p.id = $2 36 | 37 | group by p.id 38 | 39 | ) 40 | 41 | insert into producer_epoch (producer, epoch) 42 | select * from curr 43 | returning epoch; 44 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into producer (cluster) 18 | select cluster.id 19 | from cluster where cluster.name = $1 20 | returning id; 21 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_select_for_update.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select 18 | 19 | coalesce(pd.sequence, 0) 20 | 21 | from 22 | 23 | cluster c 24 | join producer p on p.cluster = c.id 25 | join topic t on t.cluster = c.id 26 | join producer_epoch pe on pe.producer = p.id 27 | join topition tp on tp.topic = t.id 28 | left join producer_detail pd on pd.producer_epoch = pe.id and pd.topition = tp.id 29 | 30 | where 31 | 32 | c.name = $1 33 | and t.name = $2 34 | and tp.partition = $3 35 | and p.id = $4 36 | and pe.epoch = $5; 37 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_update_epoch_with_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | update producer 18 | 19 | set 20 | 21 | epoch = epoch + 1, 22 | sequence = 0 23 | 24 | from 25 | 26 | cluster c, 27 | txn 28 | 29 | where c.name = $1 30 | and txn.name = $2 31 | and txn.cluster = c.id 32 | and txn.producer = producer.id 33 | and producer.cluster = c.id 34 | 35 | returning producer.id, producer.epoch; 36 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/producer_update_sequence.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | update producer 18 | 19 | set 20 | 21 | sequence = sequence + $4 22 | 23 | from 24 | 25 | cluster c 26 | 27 | where 28 | 29 | c.name = $1 30 | and producer.id = $2 31 | and producer.epoch = $3 32 | and producer.cluster = c.id; 33 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/record_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from record 18 | using cluster c, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and record.topition = tp.id; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/record_fetch.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare record_fetch (text, text, integer, integer, integer, integer) as 18 | with sized as ( 19 | select 20 | 21 | r.offset_id, 22 | r.attributes, 23 | r.timestamp, 24 | r.k, 25 | r.v, 26 | sum(coalesce(length(r.k), 0) + coalesce(length(r.v), 0)) over (order by r.offset_id) as bytes, 27 | r.producer_id, 28 | r.producer_epoch 29 | 30 | from 31 | 32 | cluster c 33 | join topic t on t.cluster = c.id 34 | join topition tp on tp.topic = t.id 35 | join record r on r.topition = tp.id 36 | 37 | where 38 | 39 | c.name = $1 40 | and t.name = $2 41 | and tp.partition = $3 42 | and r.offset_id >= $4 43 | and r.offset_id < $6) 44 | 45 | select * from sized where bytes < $5; 46 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/record_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into record 18 | (topition, offset_id, attributes, producer_id, producer_epoch, timestamp, k, v) 19 | 20 | select 21 | 22 | tp.id, $4, $5, $6, $7, $8, $9, $10 23 | 24 | from 25 | 26 | cluster c 27 | join topic t on t.cluster = c.id 28 | join topition tp on tp.topic = t.id 29 | 30 | where 31 | 32 | c.name = $1 33 | and t.name = $2 34 | and tp.partition = $3; 35 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_by_cluster.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select t.uuid, t.name, is_internal, partitions, replication_factor 18 | 19 | from 20 | 21 | cluster c 22 | join topic t on t.cluster = c.id 23 | 24 | where c.name = $1; 25 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_by_uuid.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select t.uuid, t.name, is_internal, partitions, replication_factor 18 | from topic t, cluster c 19 | where c.name = $1 20 | and t.uuid = $2 21 | and t.cluster = c.id; 22 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_configuration_delete.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2025 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from topic_configuration 18 | using cluster c, topic t 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and topic_configuration.topic = t.id 23 | and topic_configuration.name = $3; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_configuration_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2025 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from topic_configuration 18 | using cluster c, topic t 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and topic_configuration.topic = t.id; 23 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_configuration_upsert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024-2025 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into topic_configuration (topic, name, value) 18 | 19 | select t.id, $3, $4 20 | 21 | from cluster c 22 | join topic t on t.cluster = c.id 23 | 24 | where c.name = $1 25 | and t.name = $2 26 | 27 | on conflict (topic, name) 28 | 29 | do update set 30 | 31 | value = excluded.value, 32 | last_updated = excluded.last_updated 33 | 34 | returning topic_configuration.id; 35 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_delete_by.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from topic 18 | using cluster c 19 | where c.name = $1 20 | and topic.name = $2 21 | and topic.cluster = c.id 22 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into topic 18 | (cluster, name, partitions, replication_factor, is_internal) 19 | select c.id, $2, $3, $4, false 20 | from cluster c 21 | where c.name = $1 22 | returning topic.uuid; 23 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_select_name.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select t.uuid, t.name, is_internal, partitions, replication_factor 18 | 19 | from 20 | 21 | cluster c 22 | join topic t on t.cluster = c.id 23 | 24 | where c.name = $1 25 | and t.name = $2; 26 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topic_select_uuid.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select t.uuid, t.name, is_internal, partitions, replication_factor 18 | 19 | from 20 | 21 | cluster c 22 | join topic t on t.cluster = c.id 23 | 24 | where 25 | 26 | c.name = $1 27 | and t.uuid = $2; 28 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topition_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from topition 18 | using cluster c, topic t 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and topition.topic = t.id; 23 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topition_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into topition 18 | (topic, partition) 19 | select t.id, $3 20 | from cluster c, topic t 21 | where c.name = $1 22 | and t.name = $2 23 | and t.cluster = c.id 24 | returning topition.id; 25 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/topition_select.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select t.uuid, t.name, is_internal, partitions, replication_factor 18 | 19 | from 20 | 21 | cluster c 22 | join topic t on t.cluster = c.id 23 | join topition tp on tp.topic = t.id 24 | 25 | where c.name = $1 26 | and t.name = $2 27 | and tp.partition = $3; 28 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_detail_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into txn_detail (transaction, producer_epoch, transaction_timeout_ms) 18 | 19 | select txn.id, pe.id, $5 20 | 21 | from 22 | 23 | cluster c 24 | join txn on txn.cluster = c.id 25 | join producer p on p.cluster = c.id 26 | join producer_epoch pe on pe.producer = p.id 27 | 28 | where 29 | 30 | c.name = $1 31 | and txn.name = $2 32 | and p.id = $3 33 | and pe.epoch = $4; 34 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_detail_select.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select 18 | 19 | txn_detail.epoch, txn_detail.sequence 20 | 21 | from 22 | 23 | cluster c 24 | join producer p on p.cluster = c.id 25 | join txn on txn.cluster = c.id and txn.producer = p.id 26 | join txn_detail on txn_detail.transaction = txn.id 27 | 28 | where 29 | 30 | c.name = $1 31 | and p.id = $2 32 | and p.epoch = $3 33 | 34 | for no key update; 35 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_detail_select_current.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare txn_detail_select_current(text, text) as 18 | 19 | select 20 | 21 | p.id, max(pe.epoch), txn_d.status 22 | 23 | from 24 | 25 | cluster c 26 | join producer p on p.cluster = c.id 27 | join producer_epoch pe on pe.producer = p.id 28 | join txn on txn.cluster = c.id and txn.producer = p.id 29 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 30 | 31 | where 32 | 33 | c.name = $1 34 | and txn.name = $2 35 | 36 | group by p.id; 37 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_detail_select_for_update.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select 18 | 19 | txn_detail.epoch, txn_detail.sequence, txn_detail.status 20 | 21 | from 22 | 23 | cluster c 24 | join producer p on p.cluster = c.id 25 | join txn on txn.cluster = c.id and txn.producer = p.id 26 | join txn_detail on txn_detail.transaction = txn.id 27 | 28 | where 29 | 30 | c.name = $1 31 | and txn.name = $2 32 | and p.id = $3 33 | and txn_detail.epoch = $4 34 | 35 | for no key update; 36 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_detail_update_sequence.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | 18 | -- prepare txn_detail_update_sequence (text, integer, integer) as 19 | 20 | update txn_detail 21 | 22 | set 23 | 24 | sequence = txn_detail.sequence + $5 25 | 26 | from 27 | 28 | cluster c, 29 | producer p, 30 | txn 31 | 32 | where 33 | 34 | c.name = $1 35 | and txn.name = $2 36 | and p.id = $3 37 | and txn_detail.epoch = $4 38 | 39 | and p.cluster = c.id 40 | and txn.cluster = c.id 41 | and txn.producer = p.id 42 | and txn_detail.transaction = txn.id; 43 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_detail_update_started_at.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | update txn_detail 18 | 19 | set 20 | 21 | started_at = current_timestamp, 22 | status = 'BEGIN' 23 | 24 | from 25 | 26 | cluster c 27 | join producer p on p.cluster = c.id 28 | join producer_epoch pe on pe.producer = p.id 29 | join txn on txn.cluster = c.id and txn.producer = p.id 30 | 31 | where 32 | 33 | c.name = $1 34 | and txn.name = $2 35 | and p.id = $3 36 | and pe.epoch = $4 37 | and txn_detail.transaction = txn.id 38 | and txn_detail.producer_epoch = pe.id 39 | and txn_detail.started_at is null 40 | and txn_detail.status is null; 41 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into txn (cluster, name, producer) 18 | 19 | select c.id, $2, $3 20 | 21 | from cluster c 22 | 23 | where c.name = $1; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_offset_commit_delete_by_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_offset_commit 18 | 19 | using cluster c, producer p, producer_epoch pe, txn, txn_detail txn_d 20 | 21 | where c.name = $1 22 | and txn.name = $2 23 | and p.id = $3 24 | and pe.epoch = $4 25 | 26 | and p.cluster = c.id 27 | and pe.producer = p.id 28 | and txn.cluster = c.id 29 | and txn.producer = p.id 30 | and txn_d.transaction = txn.id 31 | and txn_d.producer_epoch = pe.id 32 | and txn_offset_commit.txn_detail = txn_d.id; 33 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_offset_commit_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into txn_offset_commit 18 | (txn_detail, consumer_group, generation_id, member_id) 19 | 20 | select 21 | txn_d.id, 22 | cg.id, 23 | $6, 24 | $7 25 | 26 | from 27 | 28 | cluster c 29 | join consumer_group cg on cg.cluster = c.id 30 | join producer p on p.cluster = c.id 31 | join producer_epoch pe on pe.producer = p.id 32 | join txn on txn.cluster = c.id and txn.producer = p.id 33 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 34 | 35 | where 36 | 37 | c.name = $1 38 | and txn.name = $2 39 | and cg.name = $3 40 | and p.id = $4 41 | and pe.epoch = $5; 42 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_offset_commit_tp_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_offset_commit_tp 18 | using cluster c, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and txn_offset_commit_tp.topition = tp.id; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_offset_commit_tp_delete_by_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_offset_commit_tp 18 | using cluster c, producer p, producer_epoch pe, txn, txn_detail txn_d, txn_offset_commit txn_oc 19 | 20 | where c.name = $1 21 | and txn.name = $2 22 | and p.id = $3 23 | and pe.epoch = $4 24 | 25 | and p.cluster = c.id 26 | and pe.producer = p.id 27 | and txn.cluster = c.id 28 | and txn.producer = p.id 29 | and txn_d.transaction = txn.id 30 | and txn_d.producer_epoch = pe.id 31 | and txn_oc.txn_detail = txn_d.id 32 | and txn_offset_commit_tp.offset_commit = txn_oc.id; 33 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_offset_commit_tp_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare txn_offset_commit_tp (text, text, text, integer, integer, text, integer) as 18 | insert into txn_offset_commit_tp 19 | (offset_commit, topition, committed_offset, leader_epoch, metadata) 20 | 21 | select 22 | 23 | oc.id, 24 | tp.id, 25 | $8, 26 | $9, 27 | $10 28 | 29 | from 30 | 31 | cluster c 32 | join consumer_group cg on cg.cluster = c.id 33 | join producer p on p.cluster = c.id 34 | join producer_epoch pe on pe.producer = p.id 35 | join topic t on t.cluster = c.id 36 | join topition tp on tp.topic = t.id 37 | join txn on txn.cluster = c.id and txn.producer = p.id 38 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 39 | join txn_offset_commit oc on oc.txn_detail = txn_d.id and oc.consumer_group = cg.id 40 | 41 | where 42 | 43 | c.name = $1 44 | and txn.name = $2 45 | and cg.name = $3 46 | and p.id = $4 47 | and pe.epoch = $5 48 | and t.name = $6 49 | and tp.partition = $7; 50 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_produce_offset_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_produce_offset 18 | using cluster c, topic t, topition tp, txn_topition txn_tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and txn_tp.topition = tp.id 24 | and txn_produce_offset.txn_topition = txn_tp.id; 25 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_produce_offset_delete_by_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_produce_offset 18 | 19 | using cluster c, producer p, producer_epoch pe, txn, txn_detail txn_d, txn_topition txn_tp 20 | 21 | where 22 | 23 | c.name = $1 24 | and txn.name = $2 25 | and p.id = $3 26 | and pe.epoch = $4 27 | 28 | and p.cluster = c.id 29 | and pe.producer = p.id 30 | and txn.cluster = c.id 31 | and txn.producer = p.id 32 | and txn_d.transaction = txn.id 33 | and txn_tp.txn_detail = txn_d.id 34 | and txn_produce_offset.txn_topition = txn_tp.id; 35 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_produce_offset_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into txn_produce_offset 18 | (txn_topition, offset_start, offset_end) 19 | 20 | select txn_tp.id, $7, $8 21 | 22 | from 23 | 24 | cluster c 25 | join producer p on p.cluster = c.id 26 | join producer_epoch pe on pe.producer = p.id 27 | join txn on txn.cluster = c.id and txn.producer = p.id 28 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 29 | join topic t on t.cluster = c.id 30 | join topition tp on tp.topic = t.id 31 | join txn_topition txn_tp on txn_tp.txn_detail = txn_d.id and txn_tp.topition = tp.id 32 | 33 | where 34 | 35 | c.name = $1 36 | and txn.name = $2 37 | and p.id = $3 38 | and pe.epoch = $4 39 | and t.name = $5 40 | and tp.partition = $6 41 | 42 | on conflict (txn_topition) 43 | 44 | do update 45 | 46 | set offset_end = excluded.offset_end, 47 | last_updated = excluded.last_updated; 48 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_produce_offset_select_offset_range.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare txn_produce_offset_select_offset_range (text, text, integer, integer, text, integer) as 18 | 19 | select 20 | 21 | txn_po.offset_start, 22 | txn_po.offset_end 23 | 24 | from 25 | 26 | cluster c 27 | join producer p on p.cluster = c.id 28 | join producer_epoch pe on pe.producer = p.id 29 | join txn on txn.cluster = c.id and txn.producer = p.id 30 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 31 | join topic t on t.cluster = c.id 32 | join topition tp on tp.topic = t.id 33 | join txn_topition txn_tp on txn_tp.txn_detail = txn_d.id and txn_tp.topition = tp.id 34 | join txn_produce_offset txn_po on txn_po.txn_topition = txn_tp.id 35 | 36 | where 37 | 38 | c.name = $1 39 | and txn.name = $2 40 | and p.id = $3 41 | and pe.epoch = $4 42 | and t.name = $5 43 | and tp.partition = $6; 44 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_select_name.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select 18 | 19 | p.id 20 | 21 | from 22 | 23 | cluster c 24 | join producer p on p.cluster = c.id 25 | join txn on txn.cluster = c.id and txn.producer = p.id 26 | 27 | where 28 | 29 | c.name = $1 30 | and txn.name = $2; 31 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_select_produced_topitions.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare txn_produced_topitions (text, text, integer, integer) as 18 | 19 | select 20 | 21 | t.name as topic, 22 | tp.partition as partition 23 | 24 | from 25 | 26 | cluster c 27 | join producer p on p.cluster = c.id 28 | join producer_epoch pe on pe.producer = p.id 29 | join topic t on t.cluster = c.id 30 | join txn on txn.cluster = c.id and txn.producer = p.id 31 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 32 | join txn_topition txn_tp on txn_tp.txn_detail = txn_d.id 33 | join txn_produce_offset txn_po on txn_po.txn_topition = txn_tp.id 34 | join topition tp on tp.topic = t.id and txn_tp.topition = tp.id 35 | 36 | where 37 | 38 | c.name = $1 39 | and txn.name = $2 40 | and p.id = $3 41 | and pe.epoch = $4; 42 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_select_producer_epoch.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- prepare txn_select_producer_epoch (text, text) as 18 | 19 | select p.id, p.epoch 20 | 21 | from 22 | 23 | cluster c 24 | join producer p on p.cluster = c.id 25 | join txn on txn.cluster = c.id and txn.producer = p.id 26 | 27 | where c.name = $1 28 | and txn.name = $2; 29 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_status_update.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | update txn_detail 18 | 19 | set 20 | 21 | status = $5, 22 | last_updated = current_timestamp 23 | 24 | from cluster c, producer p, producer_epoch pe, txn 25 | 26 | where c.name = $1 27 | and txn.name = $2 28 | and p.id = $3 29 | and pe.epoch = $4 30 | and p.cluster = c.id 31 | and pe.producer = p.id 32 | and txn.cluster = c.id 33 | and txn.producer = p.id 34 | and txn_detail.producer_epoch = pe.id 35 | and txn_detail.transaction = txn.id; 36 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_topition_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_topition 18 | using cluster c, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and txn_topition.topition = tp.id; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_topition_delete_by_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from txn_topition 18 | 19 | using cluster c, producer p, producer_epoch pe, txn, txn_detail txn_d 20 | 21 | where 22 | 23 | c.name = $1 24 | and txn.name = $2 25 | and p.id = $3 26 | and pe.epoch = $4 27 | 28 | and p.cluster = c.id 29 | and pe.producer = p.id 30 | and txn.cluster = c.id 31 | and txn.producer = p.id 32 | and txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 33 | and txn_topition.txn_detail = txn_d.id; 34 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_topition_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into txn_topition 18 | (txn_detail, topition) 19 | 20 | select txn_d.id, tp.id 21 | 22 | from 23 | 24 | cluster c 25 | join producer p on p.cluster = c.id 26 | join topic t on t.cluster = c.id 27 | join topition tp on tp.topic = t.id 28 | join txn on txn.cluster = c.id and txn.producer = p.id 29 | join producer_epoch pe on pe.producer = p.id 30 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 31 | 32 | where 33 | 34 | c.name = $1 35 | and t.name = $2 36 | and tp.partition = $3 37 | and txn.name = $4 38 | and p.id = $5 39 | and pe.epoch = $6; 40 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_topition_select.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select t.name, tp.partition, txnp.offset_start, txnp.offset_end 18 | from cluster c, 19 | producer p, 20 | topic t, 21 | topition tp, 22 | txn, 23 | txn_topition txn_tp 24 | where c.name = $1 25 | and p.id = $2 26 | and p.epoch = $3 27 | and t.name = $4 28 | and tp.partition = $5 29 | and txn.cluster = c.id 30 | and txn.producer = p.id 31 | and p.cluster = c.id 32 | and t.cluster = c.id 33 | and tp.topic = t.id 34 | and txn_tp.transaction = txn.id 35 | and txn_tp.topition = tp.id; 36 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/txn_topition_select_txns.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select count(distinct txn.name) 18 | 19 | from 20 | 21 | cluster c 22 | join topic t on t.cluster = c.id 23 | join topition tp on tp.topic = t.id 24 | join txn on txn.cluster = c.id 25 | join txn_detail on txn_detail.transaction = txn.id 26 | join txn_topition txn_tp on txn_tp.txn_detail = txn_detail.id and txn_tp.topition = tp.id 27 | 28 | where 29 | 30 | c.name = $1 31 | and t.name = $2 32 | and tp.partition = $3; 33 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/watermark_delete_by_topic.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | delete from watermark 18 | using cluster c, topic t, topition tp 19 | where c.name = $1 20 | and t.name = $2 21 | and t.cluster = c.id 22 | and tp.topic = t.id 23 | and watermark.topition = tp.id; 24 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/watermark_insert.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into watermark (topition) 18 | select tp.id 19 | from cluster c, topic t, topition tp 20 | where c.name = $1 21 | and t.name = $2 22 | and tp.partition = $3 23 | and t.cluster = c.id 24 | and tp.topic = t.id 25 | returning watermark.id; 26 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/watermark_insert_from_txn.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | insert into watermark 18 | (topition, stable) 19 | 20 | -- prepare watermark_from_txn (text, text, integer, integer) as 21 | select tp.id, txn_po.offset_end 22 | 23 | from 24 | 25 | cluster c 26 | join producer p on p.cluster = c.id 27 | join producer_epoch pe on pe.producer = p.id 28 | join topic t on t.cluster = c.id 29 | join txn on txn.cluster = c.id and txn.producer = p.id 30 | join txn_detail txn_d on txn_d.transaction = txn.id and txn_d.producer_epoch = pe.id 31 | join txn_topition txn_tp on txn_tp.txn_detail = txn_d.id 32 | join txn_produce_offset txn_po on txn_po.txn_topition = txn_tp.id 33 | join topition tp on tp.topic = t.id and txn_tp.topition = tp.id 34 | 35 | where 36 | 37 | c.name = $1 38 | and txn.name = $2 39 | and p.id = $3 40 | and pe.epoch = $4 41 | 42 | on conflict (topition) 43 | do update 44 | set 45 | stable = excluded.stable; 46 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/watermark_select_for_update.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | select 18 | 19 | w.low, w.high 20 | 21 | from 22 | 23 | cluster c, 24 | topic t, 25 | topition tp, 26 | watermark w 27 | 28 | where 29 | 30 | c.name = $1 31 | and t.name = $2 32 | and tp.partition = $3 33 | and t.cluster = c.id 34 | and tp.topic = t.id 35 | and w.topition = tp.id 36 | 37 | for no key update; 38 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/watermark_select_stable.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | -- Last Stable Offset Tracking: To do this, the broker must maintain in 18 | -- memory the set of active transactions along with their initial 19 | -- offsets. The last stable offset is always equal to the minimum 20 | -- of the initial offsets across all active transactions. 21 | -- 22 | -- prepare watermark (text, text, integer) as 23 | select 24 | 25 | min(txn_po.offset_start) as stable 26 | 27 | from cluster c 28 | 29 | join topic t on t.cluster = c.id 30 | join txn on txn.cluster = c.id 31 | join txn_topition txn_tp on txn_tp.transaction = txn.id 32 | join txn_produce_offset txn_po on txn_po.txn_topition = txn_tp.id 33 | join topition tp on tp.topic = t.id and txn_tp.topition = tp.id 34 | join watermark w on w.topition = tp.id 35 | 36 | where 37 | 38 | c.name = $1 39 | and t.name = $2 40 | and tp.partition = $3; 41 | -------------------------------------------------------------------------------- /tansu-storage/src/pg/watermark_update.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres; -*- 2 | -- Copyright ⓒ 2024 Peter Morgan 3 | -- 4 | -- This program is free software: you can redistribute it and/or modify 5 | -- it under the terms of the GNU Affero General Public License as 6 | -- published by the Free Software Foundation, either version 3 of the 7 | -- License, or (at your option) any later version. 8 | -- 9 | -- This program is distributed in the hope that it will be useful, 10 | -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | -- GNU Affero General Public License for more details. 13 | -- 14 | -- You should have received a copy of the GNU Affero General Public License 15 | -- along with this program. If not, see . 16 | 17 | update watermark w 18 | 19 | set 20 | 21 | low = $4, 22 | high = $5 23 | 24 | from 25 | 26 | cluster c 27 | join topic t on t.cluster = c.id 28 | join topition tp on tp.topic = t.id 29 | 30 | where 31 | 32 | c.name = $1 33 | and t.name = $2 34 | and tp.partition = $3 35 | and t.cluster = c.id 36 | and tp.topic = t.id 37 | and w.topition = tp.id; 38 | -------------------------------------------------------------------------------- /tansu-topic/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tansu-topic" 3 | edition.workspace = true 4 | version.workspace = true 5 | license.workspace = true 6 | publish = false 7 | 8 | [dependencies] 9 | futures-core.workspace = true 10 | futures-util.workspace = true 11 | futures.workspace = true 12 | tansu-kafka-sans-io = { path = "../tansu-kafka-sans-io" } 13 | thiserror.workspace = true 14 | tokio-util.workspace = true 15 | tokio.workspace = true 16 | tracing-subscriber.workspace = true 17 | tracing.workspace = true 18 | url.workspace = true 19 | -------------------------------------------------------------------------------- /typos.toml: -------------------------------------------------------------------------------- 1 | [files] 2 | ignore-files = true 3 | ignore-hidden = true 4 | extend-exclude = [ 5 | # not our typos 6 | "license.html", 7 | "tansu-kafka-sans-io/message", 8 | ] 9 | --------------------------------------------------------------------------------