├── VERSION ├── rust-toolchain ├── .dockerignore ├── rhio ├── src │ ├── http │ │ └── mod.rs │ ├── utils │ │ ├── mod.rs │ │ ├── retry │ │ │ ├── mod.rs │ │ │ └── types.rs │ │ └── nats │ │ │ ├── mod.rs │ │ │ └── factory.rs │ ├── node │ │ └── mod.rs │ ├── nats │ │ ├── client │ │ │ ├── fake │ │ │ │ ├── mod.rs │ │ │ │ └── blocking.rs │ │ │ ├── mod.rs │ │ │ └── types.rs │ │ └── mod.rs │ ├── tests │ │ ├── mod.rs │ │ ├── utils.rs │ │ ├── fake_rhio_server.rs │ │ ├── http_api.rs │ │ └── service_configuration.rs │ ├── lib.rs │ ├── metrics.rs │ ├── tracing.rs │ ├── main.rs │ ├── network │ │ ├── mod.rs │ │ ├── membership.rs │ │ └── actor.rs │ └── blobs │ │ ├── proxy.rs │ │ └── mod.rs ├── build.rs └── Cargo.toml ├── rhio-config ├── src │ └── lib.rs └── Cargo.toml ├── rhio-operator ├── src │ ├── operations │ │ ├── mod.rs │ │ ├── graceful_shutdown.rs │ │ └── pdb.rs │ ├── rhio │ │ ├── mod.rs │ │ ├── private_key.rs │ │ └── fixtures │ │ │ ├── mod.rs │ │ │ ├── service.yaml │ │ │ ├── rhio.yaml │ │ │ └── statefulset.yaml │ ├── configuration │ │ ├── mod.rs │ │ ├── fixtures │ │ │ ├── full │ │ │ │ ├── ros.yaml │ │ │ │ ├── rhio_private_key_secret.yaml │ │ │ │ ├── rms.yaml │ │ │ │ ├── rhio_nats_credentials_secret.yaml │ │ │ │ ├── rhio_s3_credentials_secret.yaml │ │ │ │ ├── rmss.yaml │ │ │ │ ├── ross.yaml │ │ │ │ └── rhio.yaml │ │ │ ├── minimal │ │ │ │ └── rhio.yaml │ │ │ └── mod.rs │ │ └── error.rs │ ├── api │ │ ├── mod.rs │ │ ├── object_store.rs │ │ ├── message_stream.rs │ │ ├── message_stream_subscription.rs │ │ ├── role.rs │ │ └── object_store_subscription.rs │ ├── lib.rs │ ├── cli.rs │ └── main.rs ├── build.rs ├── config-spec │ └── properties.yaml ├── examples │ ├── ros.yaml │ ├── rms.yaml │ ├── rmss.yaml │ ├── ross.yaml │ └── rhio-service.yaml └── Cargo.toml ├── .rusty-hook.toml ├── env └── dev │ ├── overlays │ ├── kind-cluster1 │ │ ├── pb.txt │ │ ├── pk.txt │ │ ├── kustomization.yaml │ │ ├── minio │ │ │ └── kustomization.yaml │ │ ├── rhio │ │ │ ├── rmss.cluster2.yaml │ │ │ ├── rmss.cluster3.yaml │ │ │ ├── ross.cluster2.yaml │ │ │ ├── ross.cluster3.yaml │ │ │ └── kustomization.yaml │ │ └── nats │ │ │ └── kustomization.yaml │ ├── kind-cluster2 │ │ ├── pb.txt │ │ ├── pk.txt │ │ ├── kustomization.yaml │ │ ├── minio │ │ │ └── kustomization.yaml │ │ ├── rhio │ │ │ ├── rmss.cluster1.yaml │ │ │ ├── rmss.cluster3.yaml │ │ │ ├── ross.cluster1.yaml │ │ │ ├── ross.cluster3.yaml │ │ │ └── kustomization.yaml │ │ └── nats │ │ │ └── kustomization.yaml │ └── kind-cluster3 │ │ ├── pb.txt │ │ ├── pk.txt │ │ ├── kustomization.yaml │ │ ├── minio │ │ └── kustomization.yaml │ │ ├── rhio │ │ ├── rmss.cluster1.yaml │ │ ├── rmss.cluster2.yaml │ │ ├── ross.cluster1.yaml │ │ ├── ross.cluster2.yaml │ │ └── kustomization.yaml │ │ └── nats │ │ └── kustomization.yaml │ ├── apps │ ├── minio │ │ ├── kustomization.yaml │ │ ├── minio-tenant.balancer.yaml │ │ ├── minio.yaml │ │ └── minio-tenant.yaml │ ├── rhio │ │ ├── minio-credentials-secret.yaml │ │ ├── kustomization.yaml │ │ ├── ros.yaml │ │ ├── rms.yaml │ │ ├── rhio-operator.yaml │ │ └── rhio-service.yaml │ └── nats │ │ ├── kustomization.yaml │ │ ├── nats.stream.yaml │ │ ├── nats.balancer.yaml │ │ ├── nats.nack.yaml │ │ └── nats.nats.yaml │ ├── README.md │ ├── argocd.install.sh │ └── applications.sh ├── charts ├── rhio-operator │ ├── configs │ │ └── properties.yaml │ ├── templates │ │ ├── configmap.yaml │ │ ├── serviceaccount.yaml │ │ ├── _helpers.tpl │ │ ├── deployment.yaml │ │ └── roles.yaml │ ├── Chart.yaml │ ├── .helmignore │ ├── README.md │ └── values.yaml └── rhio │ ├── templates │ ├── secret.yaml │ ├── cm.yaml │ ├── serviceaccount.yaml │ ├── service.yaml │ ├── _helpers.tpl │ └── deployment.yaml │ ├── Chart.yaml │ └── .helmignore ├── rhio-http-api ├── src │ ├── lib.rs │ ├── api.rs │ ├── blocking.rs │ ├── client.rs │ ├── server.rs │ └── status.rs └── Cargo.toml ├── .gitignore ├── rhio-core ├── src │ ├── lib.rs │ ├── private_key.rs │ ├── nats.rs │ └── subject.rs └── Cargo.toml ├── .cargo └── audit.toml ├── Dockerfile ├── s3-server └── Cargo.toml ├── rhio-blobs ├── Cargo.toml └── src │ ├── paths.rs │ ├── utils.rs │ └── lib.rs ├── osv-scanner.toml ├── .github └── workflows │ ├── daily-security.yaml │ └── rust.yaml ├── LICENSE ├── CHANGELOG.md ├── Cargo.toml └── config.example.yaml /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.1 -------------------------------------------------------------------------------- /rust-toolchain: -------------------------------------------------------------------------------- 1 | 1.91 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | ./target -------------------------------------------------------------------------------- /rhio/src/http/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | -------------------------------------------------------------------------------- /rhio-config/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod configuration; 2 | -------------------------------------------------------------------------------- /rhio/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod nats; 2 | pub mod retry; 3 | -------------------------------------------------------------------------------- /rhio/src/utils/retry/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod stream; 2 | pub mod types; 3 | -------------------------------------------------------------------------------- /rhio/src/node/mod.rs: -------------------------------------------------------------------------------- 1 | mod actor; 2 | pub mod config; 3 | pub mod rhio; 4 | -------------------------------------------------------------------------------- /rhio-operator/src/operations/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod graceful_shutdown; 2 | pub mod pdb; 3 | -------------------------------------------------------------------------------- /rhio/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().unwrap(); 3 | } 4 | -------------------------------------------------------------------------------- /rhio-operator/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().unwrap(); 3 | } 4 | -------------------------------------------------------------------------------- /rhio/src/utils/nats/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod factory; 3 | pub mod stream; 4 | -------------------------------------------------------------------------------- /rhio/src/nats/client/fake/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod blocking; 2 | pub mod client; 3 | pub mod server; 4 | -------------------------------------------------------------------------------- /.rusty-hook.toml: -------------------------------------------------------------------------------- 1 | [hooks] 2 | pre-push = "cargo fmt -- --check" 3 | 4 | [logging] 5 | verbose = true -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/pb.txt: -------------------------------------------------------------------------------- 1 | 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/pk.txt: -------------------------------------------------------------------------------- 1 | f19ec4f81213f7bd0711ae4fe76f4b7c35a0b9b72b270dbf4ac0972c474206ca -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/pb.txt: -------------------------------------------------------------------------------- 1 | b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/pk.txt: -------------------------------------------------------------------------------- 1 | 239dab4b4034ef72f1965b5e40b4737d7b315f4722a02a4c09ed947879777dcf -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/pb.txt: -------------------------------------------------------------------------------- 1 | 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/pk.txt: -------------------------------------------------------------------------------- 1 | 712b540d7c994cde1123937efb22571cbbee8dabe60747398119df2fb003edec -------------------------------------------------------------------------------- /rhio/src/nats/client/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod nats; 2 | pub mod types; 3 | 4 | #[cfg(test)] 5 | pub mod fake; 6 | -------------------------------------------------------------------------------- /charts/rhio-operator/configs/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | 5 | properties: [] 6 | -------------------------------------------------------------------------------- /rhio-operator/config-spec/properties.yaml: -------------------------------------------------------------------------------- 1 | version: 0.1.0 2 | spec: 3 | units: [] 4 | 5 | properties: [] 6 | -------------------------------------------------------------------------------- /rhio-http-api/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod api; 2 | pub mod blocking; 3 | pub mod client; 4 | pub mod server; 5 | pub mod status; 6 | -------------------------------------------------------------------------------- /rhio-operator/src/rhio/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod builders; 2 | pub mod controller; 3 | pub mod error; 4 | #[cfg(test)] 5 | pub mod fixtures; 6 | pub mod private_key; 7 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod configmap; 2 | pub mod controllers; 3 | pub mod error; 4 | #[cfg(test)] 5 | pub mod fixtures; 6 | pub mod secret; 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | target-docker 3 | ./blobs 4 | config*.yaml 5 | !config.example.yaml 6 | !configmap.yaml 7 | private-key* 8 | .DS_Store 9 | tmp 10 | /env/dev/nats 11 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ./minio 5 | - ./nats 6 | - ./rhio 7 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ./minio 5 | - ./nats 6 | - ./rhio 7 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ./minio 5 | - ./nats 6 | - ./rhio 7 | -------------------------------------------------------------------------------- /rhio-operator/examples/ros.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStore 3 | metadata: 4 | name: test-object-store 5 | spec: 6 | buckets: 7 | - source 8 | -------------------------------------------------------------------------------- /charts/rhio/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ .Release.Name }}-key 5 | data: 6 | secretKey: {{ .Values.secretKey | b64enc }} 7 | 8 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/ros.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStore 3 | metadata: 4 | name: test-store 5 | spec: 6 | buckets: 7 | - source 8 | -------------------------------------------------------------------------------- /rhio-operator/src/api/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod message_stream; 2 | pub mod message_stream_subscription; 3 | pub mod object_store; 4 | pub mod object_store_subscription; 5 | pub mod role; 6 | pub mod service; 7 | -------------------------------------------------------------------------------- /env/dev/apps/minio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - minio-tenant.yaml 5 | - minio-tenant.balancer.yaml 6 | - minio.yaml 7 | -------------------------------------------------------------------------------- /env/dev/apps/rhio/minio-credentials-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: s3-credentials 5 | namespace: rhio 6 | stringData: 7 | access_key: minio 8 | secret_key: minio123 -------------------------------------------------------------------------------- /rhio-operator/examples/rms.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStream 3 | metadata: 4 | name: test-stream 5 | spec: 6 | streamName: test-stream 7 | subjects: 8 | - test.subject 9 | -------------------------------------------------------------------------------- /rhio/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod blob_replication; 2 | pub mod configuration; 3 | pub mod fake_rhio_server; 4 | pub mod http_api; 5 | pub mod message_replication; 6 | pub mod service_configuration; 7 | pub mod utils; 8 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/rhio_private_key_secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | immutable: true 4 | metadata: 5 | name: rhio_private_key_secret 6 | namespace: ns 7 | data: 8 | secretKey: key -------------------------------------------------------------------------------- /env/dev/apps/nats/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - nats.crds.yml 5 | - nats.nack.yaml 6 | - nats.nats.yaml 7 | - nats.balancer.yaml 8 | - nats.stream.yaml -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/rms.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStream 3 | metadata: 4 | name: test-stream 5 | spec: 6 | streamName: test-stream 7 | subjects: 8 | - test.subject 9 | -------------------------------------------------------------------------------- /env/dev/apps/rhio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - rhio.crds.yaml 5 | - rhio-operator.yaml 6 | - minio-credentials-secret.yaml 7 | - rhio-service.yaml 8 | - rms.yaml 9 | - ros.yaml -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/rhio_nats_credentials_secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | immutable: true 4 | metadata: 5 | name: rhio_nats_credentials_secret 6 | namespace: ns 7 | data: 8 | password: cGFzc3dvcmQ= 9 | username: dXNlcm5hbWU= -------------------------------------------------------------------------------- /charts/rhio/templates/cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "app.fullname" . }}-config 5 | labels: 6 | {{- include "app.labels" . | nindent 4 }} 7 | data: 8 | config.yaml: |- 9 | {{- .Values.configuration | toYaml | nindent 4 }} -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/rhio_s3_credentials_secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | immutable: true 4 | metadata: 5 | name: rhio_s3_credentials_secret 6 | namespace: ns 7 | data: 8 | access_key: YWNjZXNzLWtleQ== 9 | secret_key: c2VjcmV0LWtleQ== -------------------------------------------------------------------------------- /rhio-operator/src/rhio/private_key.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[derive(Clone, Deserialize, Debug, Eq, PartialEq, Serialize)] 4 | #[serde(rename_all = "camelCase")] 5 | pub struct PrivateKey { 6 | pub secret_key: String, 7 | pub public_key: String, 8 | } 9 | -------------------------------------------------------------------------------- /charts/rhio-operator/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | data: 4 | {{ (.Files.Glob "configs/*").AsConfig | indent 2 }} 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ include "operator.fullname" . }}-configmap 8 | labels: 9 | {{- include "operator.labels" . | nindent 4 }} 10 | -------------------------------------------------------------------------------- /rhio-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod message; 2 | pub mod nats; 3 | mod private_key; 4 | mod subject; 5 | 6 | pub use message::{NetworkMessage, NetworkPayload}; 7 | pub use nats::{NATS_RHIO_PUBLIC_KEY, NATS_RHIO_SIGNATURE}; 8 | pub use private_key::load_private_key_from_file; 9 | pub use subject::{Subject, subjects_to_str}; 10 | -------------------------------------------------------------------------------- /rhio-operator/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::result_large_err)] 2 | 3 | pub mod api; 4 | pub mod cli; 5 | pub mod configuration; 6 | pub mod operations; 7 | pub mod rhio; 8 | 9 | pub mod built_info { 10 | // The file has been placed there by the build script. 11 | include!(concat!(env!("OUT_DIR"), "/built.rs")); 12 | } 13 | -------------------------------------------------------------------------------- /env/dev/apps/rhio/ros.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStore 3 | metadata: 4 | name: test-object-store 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | buckets: 11 | - source 12 | -------------------------------------------------------------------------------- /rhio-operator/examples/rmss.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: test-stream-subscription 5 | spec: 6 | publicKey: b2030d8df6c0a8bc53513e1c1746446ff00424e39f0ba25441f76b3d68752b8c 7 | subscriptions: 8 | - subject: test.subject 9 | stream: test-stream 10 | -------------------------------------------------------------------------------- /.cargo/audit.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | # advisory IDs to ignore e.g. ["RUSTSEC-2019-0001", ...] 3 | ignore = [ 4 | "RUSTSEC-2023-0071" # no fixes available 5 | ] 6 | informational_warnings = ["unmaintained"] # warn for categories of informational advisories 7 | severity_threshold = "medium" # CVSS severity ("none", "low", "medium", "high", "critical") -------------------------------------------------------------------------------- /charts/rhio-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v2 3 | name: rhio-operator 4 | version: "0.0.0" 5 | appVersion: "0.0.0" 6 | description: The Stackable Operator for Rhio Service 7 | home: docker pull ghcr.io/hiro-microdatacenters-bv/rhio/rhio-operator 8 | maintainers: 9 | - name: Hiro 10 | url: https://hiro-microdatacenters.nl/ 11 | -------------------------------------------------------------------------------- /rhio-operator/examples/ross.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: test-object-store-subscription 5 | spec: 6 | publicKey: b2030d8df6c0a8bc53513e1c1746446ff00424e39f0ba25441f76b3d68752b8c 7 | buckets: 8 | - remoteBucket: source 9 | localBucket: target 10 | 11 | -------------------------------------------------------------------------------- /charts/rhio/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: rhio 3 | description: peer-to-peer message stream and blob storage solution 4 | type: application 5 | version: "0.0.0" 6 | appVersion: "0.0.0" 7 | home: https://github.com/HIRO-MicroDataCenters-BV/rhio 8 | maintainers: 9 | - name: HIRO-Microdatacenters 10 | url: https://hiro-microdatacenters.nl 11 | -------------------------------------------------------------------------------- /env/dev/apps/rhio/rms.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStream 3 | metadata: 4 | name: test-stream 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | streamName: stream 11 | subjects: 12 | - cluster1 13 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/rmss.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: test-stream-subscription 5 | spec: 6 | publicKey: b2030d8df6c0a8bc53513e1c1746446ff00424e39f0ba25441f76b3d68752b8c 7 | subscriptions: 8 | - subject: test.subject 9 | stream: test-stream 10 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/ross.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: test-store-subscription 5 | spec: 6 | publicKey: b2030d8df6c0a8bc53513e1c1746446ff00424e39f0ba25441f76b3d68752b8c 7 | buckets: 8 | - remoteBucket: source 9 | localBucket: target 10 | -------------------------------------------------------------------------------- /env/dev/apps/minio/minio-tenant.balancer.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: minio-tenant-console-service-external 6 | namespace: minio 7 | spec: 8 | type: LoadBalancer 9 | externalTrafficPolicy: Local 10 | selector: 11 | v1.min.io/tenant: minio-tenant 12 | ports: 13 | - name: http-console 14 | port: 9090 15 | protocol: TCP 16 | targetPort: 9090 17 | -------------------------------------------------------------------------------- /env/dev/apps/nats/nats.stream.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: jetstream.nats.io/v1beta2 3 | kind: Stream 4 | metadata: 5 | name: nats-stream 6 | namespace: nats 7 | annotations: 8 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 9 | argocd.argoproj.io/sync-wave: "10" 10 | spec: 11 | name: stream 12 | subjects: ["cluster1", "cluster2", "cluster3"] 13 | storage: file 14 | replicas: 1 -------------------------------------------------------------------------------- /charts/rhio/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/rhio/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "app.serviceAccountName" . }} 6 | labels: 7 | {{- include "app.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | automountServiceAccountToken: {{ .Values.serviceAccount.automount }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /rhio-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhio-core" 3 | version = "0.1.1" 4 | edition = "2024" 5 | publish = false 6 | 7 | [lints] 8 | workspace = true 9 | 10 | [dependencies] 11 | anyhow.workspace = true 12 | async-nats.workspace = true 13 | bytes.workspace = true 14 | ciborium.workspace = true 15 | hex.workspace = true 16 | p2panda-core.workspace = true 17 | rhio-blobs.workspace = true 18 | serde.workspace = true 19 | 20 | [dev-dependencies] 21 | tempfile.workspace = true -------------------------------------------------------------------------------- /charts/rhio-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/minio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/minio 5 | patches: 6 | - patch: |- 7 | - op: replace 8 | path: /spec/source/helm/valuesObject/operator/env/1/value 9 | value: cluster1.local 10 | target: 11 | group: argoproj.io 12 | version: v1alpha1 13 | kind: Application 14 | name: minio-operator 15 | namespace: argocd -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/minio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/minio 5 | patches: 6 | - patch: |- 7 | - op: replace 8 | path: /spec/source/helm/valuesObject/operator/env/1/value 9 | value: cluster2.local 10 | target: 11 | group: argoproj.io 12 | version: v1alpha1 13 | kind: Application 14 | name: minio-operator 15 | namespace: argocd -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/minio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/minio 5 | patches: 6 | - patch: |- 7 | - op: replace 8 | path: /spec/source/helm/valuesObject/operator/env/1/value 9 | value: cluster2.local 10 | target: 11 | group: argoproj.io 12 | version: v1alpha1 13 | kind: Application 14 | name: minio-operator 15 | namespace: argocd -------------------------------------------------------------------------------- /rhio-http-api/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhio-http-api" 3 | description = "Rhio HTTP API" 4 | version = "0.1.1" 5 | edition = "2024" 6 | publish = false 7 | 8 | [lints] 9 | workspace = true 10 | 11 | [lib] 12 | name = "rhio_http_api" 13 | 14 | [dependencies] 15 | anyhow.workspace = true 16 | serde.workspace = true 17 | schemars.workspace = true 18 | async-trait.workspace = true 19 | reqwest.workspace = true 20 | tokio.workspace = true 21 | tracing.workspace = true 22 | axum.workspace = true -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1-alpine AS builder 2 | WORKDIR /usr/src/rhio 3 | COPY . . 4 | RUN apk add musl-dev libressl libressl-dev 5 | RUN cargo install --path ./rhio 6 | RUN cargo install --path ./rhio-operator 7 | 8 | FROM rust:1-alpine 9 | RUN apk add musl-dev libressl libressl-dev 10 | COPY --from=builder /usr/local/cargo/bin/rhio /usr/local/bin/rhio 11 | COPY --from=builder /usr/local/cargo/bin/rhio-operator /usr/local/bin/rhio-operator 12 | CMD ["/usr/local/bin/rhio", "-c", "/etc/rhio/config.yaml"] 13 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/rhio/rmss.cluster2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: cluster2-stream-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 11 | subscriptions: 12 | - stream: stream 13 | subject: cluster2 14 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/rhio/rmss.cluster3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: cluster3-stream-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 11 | subscriptions: 12 | - stream: stream 13 | subject: cluster3 14 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/rhio/rmss.cluster1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: cluster1-stream-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 11 | subscriptions: 12 | - stream: stream 13 | subject: cluster1 14 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/rhio/rmss.cluster3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: cluster3-stream-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 11 | subscriptions: 12 | - stream: stream 13 | subject: cluster3 14 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/rhio/rmss.cluster1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: cluster1-stream-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 11 | subscriptions: 12 | - stream: stream 13 | subject: cluster1 14 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/rhio/rmss.cluster2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedMessageStreamSubscription 3 | metadata: 4 | name: cluster2-stream-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 11 | subscriptions: 12 | - stream: stream 13 | subject: cluster2 14 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/rhio/ross.cluster2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: cluster2-bucket-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 11 | buckets: 12 | - remoteBucket: source 13 | localBucket: cluster2 14 | 15 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/rhio/ross.cluster3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: cluster3-bucket-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 11 | buckets: 12 | - remoteBucket: source 13 | localBucket: cluster3 14 | 15 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/rhio/ross.cluster1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: cluster1-bucket-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 11 | buckets: 12 | - remoteBucket: source 13 | localBucket: cluster1 14 | 15 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/rhio/ross.cluster3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: cluster3-bucket-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 11 | buckets: 12 | - remoteBucket: source 13 | localBucket: cluster3 14 | 15 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/rhio/ross.cluster1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: cluster1-bucket-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 11 | buckets: 12 | - remoteBucket: source 13 | localBucket: cluster1 14 | 15 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/rhio/ross.cluster2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: ReplicatedObjectStoreSubscription 3 | metadata: 4 | name: cluster2-bucket-subscription 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 11 | buckets: 12 | - remoteBucket: source 13 | localBucket: cluster2 14 | 15 | -------------------------------------------------------------------------------- /rhio/src/tests/utils.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Result, bail}; 2 | use std::time::{Duration, Instant}; 3 | 4 | pub fn wait_for_condition(timeout: Duration, condition: F) -> Result<()> 5 | where 6 | F: Fn() -> Result, 7 | { 8 | let start = Instant::now(); 9 | while Instant::now().duration_since(start) < timeout { 10 | if condition()? { 11 | return Ok(()); 12 | } 13 | std::thread::sleep(Duration::from_millis(100)); 14 | } 15 | bail!("timeout waiting condition") 16 | } 17 | -------------------------------------------------------------------------------- /s3-server/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "s3-server" 3 | description = "Simple S3 server for testing purposes" 4 | version = "0.1.1" 5 | edition = "2024" 6 | publish = false 7 | 8 | [lints] 9 | workspace = true 10 | 11 | [lib] 12 | name = "s3_server" 13 | 14 | [dependencies] 15 | anyhow.workspace = true 16 | s3s-fs.workspace = true 17 | s3s.workspace = true 18 | rust-s3.workspace = true 19 | hyper-util.workspace = true 20 | url.workspace = true 21 | tempfile.workspace = true 22 | tokio.workspace = true 23 | tracing.workspace = true 24 | tokio-util.workspace = true 25 | once_cell.workspace = true 26 | -------------------------------------------------------------------------------- /rhio-config/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhio-config" 3 | description = "Configuration for Rhio service" 4 | version = "0.1.1" 5 | edition = "2024" 6 | publish = false 7 | 8 | [lints] 9 | workspace = true 10 | 11 | [lib] 12 | name = "rhio_config" 13 | 14 | [dependencies] 15 | anyhow.workspace = true 16 | clap.workspace = true 17 | directories.workspace = true 18 | figment.workspace = true 19 | p2panda-core.workspace = true 20 | rust-s3.workspace = true 21 | serde.workspace = true 22 | rhio-core.workspace = true 23 | 24 | [dev-dependencies] 25 | figment = { workspace = true, features = ["test"] } 26 | -------------------------------------------------------------------------------- /rhio-operator/examples/rhio-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: RhioService 3 | metadata: 4 | name: test-service 5 | spec: 6 | clusterConfig: 7 | listenerClass: cluster-internal 8 | gracefulShutdownTimeout: 1s 9 | 10 | image: 11 | custom: ghcr.io/hiro-microdatacenters-bv/rhio:1.0.0 12 | productVersion: 1.0.0 13 | pullPolicy: IfNotPresent 14 | 15 | configuration: 16 | networkId: test 17 | privateKeySecret: secret-key 18 | nodes: [] 19 | s3: null 20 | nats: 21 | endpoint: nats://nats-jetstream.dkg-engine.svc.cluster.local:4222 22 | credentials: null 23 | -------------------------------------------------------------------------------- /rhio-core/src/private_key.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::Read; 3 | use std::path::PathBuf; 4 | 5 | use anyhow::Result; 6 | use p2panda_core::identity::PrivateKey; 7 | 8 | /// Loads a private key from a file at the given path and derives ed25519 private key from it. 9 | /// 10 | /// The private key in the file needs to be represented as a hex-encoded string. 11 | pub fn load_private_key_from_file(path: &PathBuf) -> Result { 12 | let mut file = File::open(path)?; 13 | let mut private_key_hex = String::new(); 14 | file.read_to_string(&mut private_key_hex)?; 15 | let private_key = PrivateKey::try_from(&hex::decode(&private_key_hex)?[..])?; 16 | Ok(private_key) 17 | } 18 | -------------------------------------------------------------------------------- /rhio/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod blobs; 2 | pub mod context; 3 | pub mod context_builder; 4 | mod http; 5 | pub mod metrics; 6 | mod nats; 7 | mod network; 8 | mod node; 9 | mod topic; 10 | pub mod tracing; 11 | mod utils; 12 | 13 | pub use nats::StreamName; 14 | pub use topic::{ 15 | FilesSubscription, FilteredMessageStream, MessagesSubscription, Publication, Subscription, 16 | }; 17 | 18 | pub use node::rhio::Node; 19 | 20 | pub mod built_info { 21 | // The file has been placed there by the build script. 22 | include!(concat!(env!("OUT_DIR"), "/built.rs")); 23 | } 24 | 25 | #[cfg(test)] 26 | mod tests; 27 | 28 | pub(crate) type JoinErrToStr = 29 | Box String + Send + Sync + 'static>; 30 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/minimal/rhio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: RhioService 3 | metadata: 4 | name: test-service 5 | uid: 1 6 | spec: 7 | clusterConfig: 8 | listenerClass: cluster-internal 9 | gracefulShutdownTimeout: 1s 10 | 11 | image: 12 | custom: ghcr.io/hiro-microdatacenters-bv/rhio-dev:1.0.1 13 | productVersion: 1.0.1 14 | pullPolicy: IfNotPresent 15 | 16 | configuration: 17 | networkId: test 18 | privateKeySecret: secret-key 19 | nodes: [] 20 | s3: null 21 | nats: 22 | endpoint: nats://nats-jetstream.dkg-engine.svc.cluster.local:4222 23 | credentialsSecret: null 24 | -------------------------------------------------------------------------------- /env/dev/apps/nats/nats.balancer.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: nats-service-external 6 | namespace: nats 7 | spec: 8 | type: LoadBalancer 9 | externalTrafficPolicy: Local 10 | selector: 11 | app.kubernetes.io/component: nats 12 | app.kubernetes.io/instance: nats-jetstream 13 | app.kubernetes.io/name: nats 14 | ports: 15 | - appProtocol: tcp 16 | name: nats 17 | port: 4222 18 | protocol: TCP 19 | targetPort: nats 20 | - appProtocol: http 21 | name: websocket 22 | port: 8080 23 | protocol: TCP 24 | targetPort: websocket 25 | - appProtocol: tcp 26 | name: mqtt 27 | port: 1883 28 | protocol: TCP 29 | targetPort: mqtt -------------------------------------------------------------------------------- /rhio-blobs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhio-blobs" 3 | version = "0.1.1" 4 | edition = "2024" 5 | publish = false 6 | 7 | [lints] 8 | workspace = true 9 | 10 | [dependencies] 11 | anyhow.workspace = true 12 | bytes.workspace = true 13 | futures-lite.workspace = true 14 | iroh-blobs.workspace = true 15 | iroh-io.workspace = true 16 | p2panda-core.workspace = true 17 | rust-s3.workspace = true 18 | serde.workspace = true 19 | serde_json.workspace = true 20 | thiserror.workspace = true 21 | tokio.workspace = true 22 | tracing.workspace = true 23 | url.workspace = true 24 | chrono.workspace = true 25 | dashmap.workspace = true 26 | 27 | [dev-dependencies] 28 | s3-server.workspace = true 29 | s3s.workspace = true 30 | url.workspace = true 31 | rand.workspace = true -------------------------------------------------------------------------------- /charts/rhio-operator/README.md: -------------------------------------------------------------------------------- 1 | # Helm Chart for Stackable Operator for Rhio Service 2 | 3 | This Helm Chart can be used to install Custom Resource Definitions and the Operator for Rhio Service provided by Hiro. 4 | 5 | ## Requirements 6 | 7 | - Create a [Kubernetes Cluster](../Readme.md) 8 | - Install [Helm](https://helm.sh/docs/intro/install/) 9 | 10 | ## Install the Stackable Operator for Rhio Service 11 | 12 | ```bash 13 | helm install rhio-operator charts/helm/rhio-operator 14 | ``` 15 | 16 | ## Usage of the CRDs 17 | 18 | The operator has example requests included in the [`/examples`](https://github.com/hiro-microdatacenters-bv/rhio/rhio-operator/tree/main/examples) directory. 19 | 20 | ## Links 21 | 22 | 23 | -------------------------------------------------------------------------------- /charts/rhio/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "app.fullname" . }} 5 | labels: 6 | {{- include "app.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.services.type }} 9 | ports: 10 | # the port for the p2p networking endpoint 11 | - port: {{ .Values.services.network.port }} 12 | {{- if .Values.services.network.nodePort }} 13 | nodePort: {{ .Values.services.network.nodePort }} 14 | {{- end }} 15 | protocol: UDP 16 | name: network 17 | # the port for the HTTP health endpoint 18 | - port: {{ .Values.services.health.port }} 19 | targetPort: http 20 | protocol: TCP 21 | name: http 22 | selector: 23 | {{- include "app.selectorLabels" . | nindent 4 }} 24 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/nats/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/nats 5 | patches: 6 | - patch: |- 7 | - op: replace 8 | path: /spec/source/helm/valuesObject/config/serverNamePrefix 9 | value: cluster1.local 10 | target: 11 | group: argoproj.io 12 | version: v1alpha1 13 | kind: Application 14 | name: nats-jetstream 15 | namespace: argocd 16 | - patch: |- 17 | - op: replace 18 | path: /spec/source/helm/valuesObject/config/websocket/ingress/hosts/0 19 | value: nats-websocket.cluster1.local 20 | target: 21 | group: argoproj.io 22 | version: v1alpha1 23 | kind: Application 24 | name: nats-jetstream 25 | namespace: argocd -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/nats/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/nats 5 | patches: 6 | - patch: |- 7 | - op: replace 8 | path: /spec/source/helm/valuesObject/config/serverNamePrefix 9 | value: cluster2.local 10 | target: 11 | group: argoproj.io 12 | version: v1alpha1 13 | kind: Application 14 | name: nats-jetstream 15 | namespace: argocd 16 | - patch: |- 17 | - op: replace 18 | path: /spec/source/helm/valuesObject/config/websocket/ingress/hosts/0 19 | value: nats-websocket.cluster2.local 20 | target: 21 | group: argoproj.io 22 | version: v1alpha1 23 | kind: Application 24 | name: nats-jetstream 25 | namespace: argocd -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/nats/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/nats 5 | patches: 6 | - patch: |- 7 | - op: replace 8 | path: /spec/source/helm/valuesObject/config/serverNamePrefix 9 | value: cluster3.local 10 | target: 11 | group: argoproj.io 12 | version: v1alpha1 13 | kind: Application 14 | name: nats-jetstream 15 | namespace: argocd 16 | - patch: |- 17 | - op: replace 18 | path: /spec/source/helm/valuesObject/config/websocket/ingress/hosts/0 19 | value: nats-websocket.cluster3.local 20 | target: 21 | group: argoproj.io 22 | version: v1alpha1 23 | kind: Application 24 | name: nats-jetstream 25 | namespace: argocd -------------------------------------------------------------------------------- /rhio-http-api/src/api.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use async_trait::async_trait; 3 | 4 | use crate::status::HealthStatus; 5 | 6 | pub const HTTP_HEALTH_ROUTE: &str = "/health"; 7 | pub const HTTP_METRICS_ROUTE: &str = "/metrics"; 8 | 9 | /// Trait representing the Rhio Http API, which provides methods for checking the health status 10 | /// and retrieving metrics of the service. 11 | /// 12 | /// # Methods 13 | /// 14 | /// * `health` - Asynchronously checks the health status of the service and returns a `HealthStatus` result. 15 | /// * `metrics` - Asynchronously retrieves metrics of the service and returns them as a `String` result. 16 | /// 17 | #[async_trait] 18 | pub trait RhioApi: Send + Sync { 19 | async fn health(&self) -> Result; 20 | 21 | async fn metrics(&self) -> Result; 22 | } 23 | -------------------------------------------------------------------------------- /rhio/src/metrics.rs: -------------------------------------------------------------------------------- 1 | pub const MESSAGE_RECEIVE_TOTAL: &str = "message_receive_total"; 2 | pub const LABEL_SUBJECT: &str = "subject"; 3 | pub const LABEL_REMOTE_BUCKET: &str = "remote_bucket"; 4 | pub const LABEL_LOCAL_BUCKET: &str = "local_bucket"; 5 | pub const LABEL_SOURCE: &str = "source"; 6 | pub const LABEL_SOURCE_NATS: &str = "nats"; 7 | pub const LABEL_SOURCE_BLOBS: &str = "blob"; 8 | pub const LABEL_SOURCE_NETWORK: &str = "network"; 9 | pub const LABEL_MSG_TYPE: &str = "msg_type"; 10 | pub const LABEL_NETWORK_MSG_TYPE_NATS_MESSAGE: &str = "nats_message"; 11 | pub const LABEL_NETWORK_MSG_TYPE_BLOB_ANNOUNCEMENT: &str = "blob_announcement"; 12 | pub const BLOBS_DOWNLOAD_TOTAL: &str = "blobs_download_total"; 13 | pub const LABEL_BLOB_MSG_TYPE_DONE: &str = "done"; 14 | pub const LABEL_BLOB_MSG_TYPE_ERROR: &str = "error"; 15 | -------------------------------------------------------------------------------- /rhio-operator/src/rhio/fixtures/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod statefulset { 2 | pub const RHIO: &str = include_str!(concat!( 3 | env!("CARGO_MANIFEST_DIR"), 4 | "/src/rhio/fixtures/rhio.yaml" 5 | )); 6 | 7 | pub const STS: &str = include_str!(concat!( 8 | env!("CARGO_MANIFEST_DIR"), 9 | "/src/rhio/fixtures/statefulset.yaml" 10 | )); 11 | 12 | pub const SVC: &str = include_str!(concat!( 13 | env!("CARGO_MANIFEST_DIR"), 14 | "/src/rhio/fixtures/service.yaml" 15 | )); 16 | } 17 | 18 | pub mod service { 19 | pub const RHIO: &str = include_str!(concat!( 20 | env!("CARGO_MANIFEST_DIR"), 21 | "/src/rhio/fixtures/rhio.yaml" 22 | )); 23 | 24 | pub const SVC: &str = include_str!(concat!( 25 | env!("CARGO_MANIFEST_DIR"), 26 | "/src/rhio/fixtures/service.yaml" 27 | )); 28 | } 29 | -------------------------------------------------------------------------------- /osv-scanner.toml: -------------------------------------------------------------------------------- 1 | [[IgnoredVulns]] 2 | id = "RUSTSEC-2024-0384" 3 | reason = "Low risk (unmaintained)" 4 | 5 | [[IgnoredVulns]] 6 | id = "RUSTSEC-2024-0436" 7 | reason = "Low risk (unmaintained)" 8 | 9 | [[IgnoredVulns]] 10 | id = "RUSTSEC-2024-0370" 11 | reason = "Low risk (unmaintained)" 12 | 13 | [[IgnoredVulns]] 14 | id = "RUSTSEC-2025-0056" 15 | reason = "Low risk (unmaintained)" 16 | 17 | [[IgnoredVulns]] 18 | id = "RUSTSEC-2023-0071" 19 | reason = "No solution yet" 20 | # https://osv.dev/vulnerability/RUSTSEC-2023-0071 21 | # Patches 22 | # No patch is yet available, however work is underway to migrate to a fully constant-time implementation. 23 | # Workarounds 24 | # The only currently available workaround is to avoid using the rsa crate in settings where attackers are 25 | # able to observe timing information, e.g. local use on a non-compromised computer is fine. 26 | 27 | -------------------------------------------------------------------------------- /env/dev/apps/nats/nats.nack.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Application 3 | metadata: 4 | name: nats-nack 5 | namespace: argocd 6 | annotations: 7 | argocd.argoproj.io/compare-options: ServerSideDiff=true 8 | argocd.argoproj.io/sync-wave: "-20" 9 | finalizers: 10 | - resources-finalizer.argocd.argoproj.io 11 | spec: 12 | destination: 13 | namespace: nats 14 | server: https://kubernetes.default.svc 15 | project: default 16 | source: 17 | repoURL: https://nats-io.github.io/k8s/helm/charts 18 | chart: nack 19 | targetRevision: 0.27.0 20 | helm: 21 | valuesObject: 22 | jetstream: 23 | nats: 24 | url: nats://nats-jetstream:4222 25 | syncPolicy: 26 | automated: 27 | prune: true 28 | selfHeal: true 29 | syncOptions: 30 | - CreateNamespace=true 31 | - ServerSideApply=true 32 | - RespectIgnoreDifferences=true 33 | -------------------------------------------------------------------------------- /.github/workflows/daily-security.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Security audit 3 | 4 | on: 5 | schedule: 6 | - cron: '15 4 * * *' 7 | workflow_dispatch: 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | RUST_TOOLCHAIN: "1.91" 12 | 13 | jobs: 14 | audit: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | contents: read 18 | pull-requests: write 19 | checks: write 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | name: Checkout repository 24 | with: 25 | fetch-depth: 0 26 | 27 | - name: Setup Rust toolchain 28 | uses: moonrepo/setup-rust@v1 29 | with: 30 | cache: false 31 | channel: ${{ env.RUST_TOOLCHAIN }} 32 | 33 | # build speedups 34 | - uses: Swatinem/rust-cache@v2 35 | 36 | - name: Install cargo-audit 37 | run: cargo install cargo-audit 38 | 39 | - name: Run audit 40 | run: cargo audit --json -------------------------------------------------------------------------------- /rhio-operator/src/rhio/fixtures/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: server 6 | app.kubernetes.io/instance: test-service 7 | app.kubernetes.io/managed-by: rhio.hiro.io_rhioservice 8 | app.kubernetes.io/name: rhio 9 | app.kubernetes.io/role-group: default 10 | app.kubernetes.io/version: 1.0.1-1.0.1 11 | stackable.tech/vendor: Stackable 12 | name: test-service 13 | ownerReferences: 14 | - apiVersion: rhio.hiro.io/v1 15 | controller: true 16 | kind: RhioService 17 | name: test-service 18 | uid: '1' 19 | spec: 20 | ports: 21 | - name: rhio 22 | port: 9102 23 | protocol: UDP 24 | - name: health 25 | port: 8080 26 | protocol: TCP 27 | selector: 28 | app.kubernetes.io/component: server 29 | app.kubernetes.io/instance: test-service 30 | app.kubernetes.io/name: rhio 31 | app.kubernetes.io/role-group: default 32 | type: ClusterIP -------------------------------------------------------------------------------- /rhio-blobs/src/paths.rs: -------------------------------------------------------------------------------- 1 | pub const RHIO_PREFIX: &str = ".rhio/"; 2 | 3 | pub const META_SUFFIX: &str = ".rhio.json"; 4 | 5 | pub const OUTBOARD_SUFFIX: &str = ".rhio.bao4"; 6 | 7 | pub const NO_PREFIX: String = String::new(); // Empty string. 8 | 9 | #[derive(Debug, Clone)] 10 | pub struct Paths { 11 | path: String, 12 | } 13 | 14 | impl Paths { 15 | pub fn new(path: &str) -> Self { 16 | let path = path.to_string().replace(RHIO_PREFIX, ""); 17 | Self { path } 18 | } 19 | 20 | pub fn from_meta(path: &str) -> Self { 21 | Self::new(&path[0..path.len().saturating_sub(META_SUFFIX.len())]) 22 | } 23 | 24 | pub fn data(&self) -> String { 25 | self.path.clone() 26 | } 27 | 28 | pub fn meta(&self) -> String { 29 | format!("{RHIO_PREFIX}{}{META_SUFFIX}", self.path) 30 | } 31 | 32 | pub fn outboard(&self) -> String { 33 | format!("{RHIO_PREFIX}{}{OUTBOARD_SUFFIX}", self.path) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /rhio-operator/src/operations/graceful_shutdown.rs: -------------------------------------------------------------------------------- 1 | use snafu::{ResultExt, Snafu}; 2 | use stackable_operator::builder::pod::PodBuilder; 3 | 4 | use crate::api::service::RhioServiceConfig; 5 | 6 | #[derive(Debug, Snafu)] 7 | pub enum Error { 8 | #[snafu(display("Failed to set terminationGracePeriod"))] 9 | SetTerminationGracePeriod { 10 | source: stackable_operator::builder::pod::Error, 11 | }, 12 | } 13 | 14 | pub fn add_graceful_shutdown_config( 15 | merged_config: &RhioServiceConfig, 16 | pod_builder: &mut PodBuilder, 17 | ) -> Result<(), Error> { 18 | // This must be always set by the merge mechanism, as we provide a default value, 19 | // users can not disable graceful shutdown. 20 | if let Some(graceful_shutdown_timeout) = merged_config.graceful_shutdown_timeout { 21 | pod_builder 22 | .termination_grace_period(&graceful_shutdown_timeout) 23 | .context(SetTerminationGracePeriodSnafu)?; 24 | } 25 | 26 | Ok(()) 27 | } 28 | -------------------------------------------------------------------------------- /rhio-operator/src/rhio/fixtures/rhio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: RhioService 3 | metadata: 4 | name: test-service 5 | uid: 1 6 | spec: 7 | image: 8 | custom: ghcr.io/hiro-microdatacenters-bv/rhio-dev:1.0.1 9 | productVersion: 1.0.1 10 | pullPolicy: IfNotPresent 11 | 12 | configuration: 13 | networkId: test 14 | privateKeySecret: rhio_private_key_secret 15 | nodes: 16 | - publicKey: b2030d8df6c0a8bc53513e1c1746446ff00424e39f0ba25441f76b3d68752b8c 17 | endpoints: 18 | - 10.0.1.2:9102 19 | - 10.0.1.3:9102 20 | 21 | s3: 22 | endpoint: http://localhost:32000 23 | region: eu-west-01 24 | credentialsSecret: rhio_s3_credentials_secret 25 | 26 | nats: 27 | endpoint: nats://nats-jetstream.dkg-engine.svc.cluster.local:4222 28 | credentialsSecret: rhio_nats_credentials_secret 29 | 30 | logLevel: =INFO 31 | 32 | clusterConfig: 33 | listenerClass: cluster-internal 34 | gracefulShutdownTimeout: 1s 35 | 36 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/full/rhio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: RhioService 3 | metadata: 4 | name: test-service 5 | uid: 1 6 | spec: 7 | image: 8 | custom: ghcr.io/hiro-microdatacenters-bv/rhio-dev:1.0.1 9 | productVersion: 1.0.1 10 | pullPolicy: IfNotPresent 11 | 12 | configuration: 13 | networkId: test 14 | privateKeySecret: rhio_private_key_secret 15 | nodes: 16 | - publicKey: b2030d8df6c0a8bc53513e1c1746446ff00424e39f0ba25441f76b3d68752b8c 17 | endpoints: 18 | - 10.0.1.2:9102 19 | - 10.0.1.3:9102 20 | 21 | s3: 22 | endpoint: http://localhost:32000 23 | region: eu-west-01 24 | credentialsSecret: rhio_s3_credentials_secret 25 | 26 | nats: 27 | endpoint: nats://nats-jetstream.dkg-engine.svc.cluster.local:4222 28 | credentialsSecret: rhio_nats_credentials_secret 29 | 30 | logLevel: =INFO 31 | 32 | clusterConfig: 33 | listenerClass: cluster-internal 34 | gracefulShutdownTimeout: 1s 35 | 36 | -------------------------------------------------------------------------------- /charts/rhio-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.serviceAccount.create -}} 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: {{ include "operator.fullname" . }}-serviceaccount 7 | labels: 8 | {{- include "operator.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | # This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. 16 | kind: ClusterRoleBinding 17 | metadata: 18 | name: {{ include "operator.fullname" . }}-clusterrolebinding 19 | labels: 20 | {{- include "operator.labels" . | nindent 4 }} 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ include "operator.fullname" . }}-serviceaccount 24 | namespace: {{ .Release.Namespace }} 25 | roleRef: 26 | kind: ClusterRole 27 | name: {{ include "operator.fullname" . }}-clusterrole 28 | apiGroup: rbac.authorization.k8s.io 29 | {{- end }} 30 | -------------------------------------------------------------------------------- /env/dev/apps/minio/minio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Application 3 | metadata: 4 | name: minio-operator 5 | namespace: argocd 6 | annotations: 7 | argocd.argoproj.io/compare-options: ServerSideDiff=true 8 | argocd.argoproj.io/sync-wave: "-30" 9 | finalizers: 10 | - resources-finalizer.argocd.argoproj.io 11 | spec: 12 | destination: 13 | namespace: minio 14 | server: https://kubernetes.default.svc 15 | project: default 16 | source: 17 | repoURL: https://operator.min.io 18 | chart: operator 19 | targetRevision: 5.0.14 20 | helm: 21 | valuesObject: 22 | operator: 23 | env: 24 | - name: OPERATOR_STS_ENABLED 25 | value: "on" 26 | - name: CLUSTER_DOMAIN 27 | value: cluster.local 28 | console: 29 | ingress: 30 | enabled: false 31 | tenants: [] 32 | syncPolicy: 33 | automated: 34 | prune: true 35 | selfHeal: true 36 | syncOptions: 37 | - CreateNamespace=true 38 | - ServerSideApply=true 39 | -------------------------------------------------------------------------------- /rhio-operator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhio-operator" 3 | version = "0.1.0" 4 | edition = "2024" 5 | publish = false 6 | default-run = "rhio-operator" 7 | 8 | [[bin]] 9 | doc = false 10 | name = "rhio-operator" 11 | path = "src/main.rs" 12 | 13 | [dependencies] 14 | # stackable operator framework 15 | stackable-operator.workspace = true 16 | product-config.workspace = true 17 | 18 | # rhio configuration generation 19 | rhio-config.workspace = true 20 | 21 | # due to rhio-config 22 | rhio-core.workspace = true 23 | rust-s3.workspace = true 24 | p2panda-core.workspace = true 25 | anyhow.workspace = true 26 | 27 | rhio-http-api.workspace = true 28 | 29 | clap.workspace = true 30 | futures.workspace = true 31 | tokio.workspace = true 32 | schemars.workspace = true 33 | serde.workspace = true 34 | serde_json.workspace = true 35 | serde_yaml.workspace = true 36 | tracing.workspace = true 37 | tracing-subscriber.workspace = true 38 | snafu.workspace = true 39 | strum.workspace = true 40 | rustls.workspace = true 41 | 42 | [dev-dependencies] 43 | 44 | [build-dependencies] 45 | built.workspace = true 46 | -------------------------------------------------------------------------------- /env/dev/apps/rhio/rhio-operator.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Application 3 | metadata: 4 | name: rhio-operator 5 | namespace: argocd 6 | annotations: 7 | argocd.argoproj.io/compare-options: ServerSideDiff=true 8 | argocd.argoproj.io/sync-wave: "-30" 9 | finalizers: 10 | - resources-finalizer.argocd.argoproj.io 11 | spec: 12 | destination: 13 | namespace: rhio 14 | server: https://kubernetes.default.svc 15 | project: default 16 | source: 17 | repoURL: https://HIRO-MicroDataCenters-BV.github.io/rhio/helm-charts/ 18 | chart: rhio-operator 19 | targetRevision: 0.*.* 20 | helm: 21 | version: v3 22 | parameters: 23 | - name: crds 24 | value: "false" 25 | valuesObject: 26 | kubernetesClusterDomain: cluster1.local 27 | image: 28 | repository: "ghcr.io/hiro-microdatacenters-bv/rhio" 29 | tag: "0.2.1" 30 | 31 | syncPolicy: 32 | automated: 33 | prune: true 34 | selfHeal: true 35 | syncOptions: 36 | - CreateNamespace=true 37 | - ServerSideApply=true 38 | -------------------------------------------------------------------------------- /rhio-operator/src/api/object_store.rs: -------------------------------------------------------------------------------- 1 | use rhio_http_api::status::ObjectStorePublishStatus; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | use stackable_operator::kube::CustomResource; 5 | 6 | #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] 7 | #[cfg_attr(test, derive(Default))] 8 | #[kube( 9 | kind = "ReplicatedObjectStore", 10 | group = "rhio.hiro.io", 11 | version = "v1", 12 | plural = "replicatedobjectstores", 13 | status = "ReplicatedObjectStoreStatus", 14 | shortname = "ros", 15 | namespaced, 16 | crates( 17 | kube_core = "stackable_operator::kube::core", 18 | k8s_openapi = "stackable_operator::k8s_openapi", 19 | schemars = "stackable_operator::schemars" 20 | ) 21 | )] 22 | #[serde(rename_all = "camelCase")] 23 | pub struct ReplicatedObjectStoreSpec { 24 | pub buckets: Vec, 25 | } 26 | 27 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 28 | #[serde(rename_all = "camelCase")] 29 | pub struct ReplicatedObjectStoreStatus { 30 | pub buckets: Vec, 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 HIRO-MicroDataCenters 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /rhio-operator/src/api/message_stream.rs: -------------------------------------------------------------------------------- 1 | use rhio_http_api::status::MessageStreamPublishStatus; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | use stackable_operator::kube::CustomResource; 5 | 6 | #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] 7 | #[cfg_attr(test, derive(Default))] 8 | #[kube( 9 | kind = "ReplicatedMessageStream", 10 | group = "rhio.hiro.io", 11 | version = "v1", 12 | plural = "replicatedmessagestreams", 13 | namespaced, 14 | status = "ReplicatedMessageStreamStatus", 15 | shortname = "rms", 16 | crates( 17 | kube_core = "stackable_operator::kube::core", 18 | k8s_openapi = "stackable_operator::k8s_openapi", 19 | schemars = "stackable_operator::schemars" 20 | ) 21 | )] 22 | #[serde(rename_all = "camelCase")] 23 | pub struct ReplicatedMessageStreamSpec { 24 | pub stream_name: String, 25 | pub subjects: Vec, 26 | } 27 | 28 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 29 | #[serde(rename_all = "camelCase")] 30 | pub struct ReplicatedMessageStreamStatus { 31 | pub subjects: Vec, 32 | } 33 | -------------------------------------------------------------------------------- /rhio/src/tests/fake_rhio_server.rs: -------------------------------------------------------------------------------- 1 | use crate::{context::Context, context_builder::ContextBuilder}; 2 | use anyhow::Result; 3 | use p2panda_core::PrivateKey; 4 | use rhio_config::configuration::Config; 5 | 6 | /// A fake server for testing purposes in the Rhio project. 7 | /// 8 | /// The `FakeRhioServer` struct provides methods to start and discard a fake server instance. 9 | /// 10 | /// # Fields 11 | /// 12 | /// * `context` - The context in which the server operates. 13 | /// 14 | /// # Methods 15 | /// 16 | /// * `try_start` - Attempts to start the fake server with the given configuration and private key. 17 | /// * `discard` - Shuts down the fake server and cleans up resources. 18 | pub struct FakeRhioServer { 19 | context: Context, 20 | } 21 | 22 | impl FakeRhioServer { 23 | pub fn try_start(config: Config, private_key: PrivateKey) -> Result { 24 | let builder = ContextBuilder::new(config, private_key); 25 | let context = builder.try_build_and_start()?; 26 | context.configure()?; 27 | context.log_configuration(); 28 | Ok(FakeRhioServer { context }) 29 | } 30 | 31 | pub fn discard(self) -> Result<()> { 32 | self.context.shutdown() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /env/dev/README.md: -------------------------------------------------------------------------------- 1 | # Development Environment Setup 2 | 3 | This repository contains shell scripts to set up and manage your development environment. 4 | 5 | ## Prerequisites 6 | - Ensure you have `kubectl`, `jq`, `kind`, `cloud-provider-kind` installed and configured. 7 | - Ensure you have the necessary permissions to create and manage Kubernetes clusters. 8 | 9 | ## Scripts 10 | 11 | ### clusters.sh 12 | This script is used to manage your Kubernetes clusters. 13 | 14 | **Usage:** 15 | ```sh 16 | ./clusters.sh [options] 17 | ``` 18 | 19 | **Options:** 20 | - `create` - Create new clusters 21 | - `delete` - Delete existing clusters 22 | - `cloudprovider` - Runs the cloud-provider-kind 23 | 24 | ### argocd.install.sh 25 | This script installs Argo CD, a declarative, GitOps continuous delivery tool for Kubernetes. 26 | 27 | **Usage:** 28 | ```sh 29 | ./argocd.install.sh 30 | ``` 31 | 32 | ### applications.sh 33 | This script manages the deployment of applications to your Kubernetes clusters. 34 | 35 | **Usage:** 36 | ```sh 37 | ./applications.sh [options] 38 | ``` 39 | 40 | **Options:** 41 | - `install` - Deploy applications 42 | - `uninstall` - Delete applications 43 | - `listsvc` - Lists all services of type LoadBalancer in all specified clusters 44 | 45 | -------------------------------------------------------------------------------- /env/dev/apps/rhio/rhio-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rhio.hiro.io/v1 2 | kind: RhioService 3 | metadata: 4 | name: rhio-service 5 | namespace: rhio 6 | annotations: 7 | argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true 8 | argocd.argoproj.io/sync-wave: "30" 9 | spec: 10 | clusterConfig: 11 | listenerClass: cluster-internal 12 | gracefulShutdownTimeout: 1s 13 | 14 | image: 15 | custom: ghcr.io/hiro-microdatacenters-bv/rhio:0.2.1 16 | productVersion: 0.2.1 17 | pullPolicy: IfNotPresent 18 | 19 | configuration: 20 | networkId: test 21 | privateKeySecret: private-key-secret 22 | nodes: 23 | - publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 24 | endpoints: 25 | - rhio-service.rhio.svc.cluster2.local:9102 26 | - publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 27 | endpoints: 28 | - rhio-service.rhio.svc.cluster3.local:9102 29 | s3: 30 | endpoint: http://minio-tenant-hl.minio.svc.cluster1.local:9000 31 | region: custom 32 | credentialsSecret: s3-credentials 33 | nats: 34 | endpoint: nats://nats-jetstream.nats.svc.cluster1.local:4222 35 | credentialsSecret: null 36 | -------------------------------------------------------------------------------- /rhio-operator/src/configuration/fixtures/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod minimal { 2 | pub const RHIO: &str = include_str!(concat!( 3 | env!("CARGO_MANIFEST_DIR"), 4 | "/src/configuration/fixtures/minimal/rhio.yaml" 5 | )); 6 | } 7 | 8 | pub mod full { 9 | pub const RHIO: &str = include_str!(concat!( 10 | env!("CARGO_MANIFEST_DIR"), 11 | "/src/configuration/fixtures/full/rhio.yaml" 12 | )); 13 | pub const RHIO_NATS: &str = include_str!(concat!( 14 | env!("CARGO_MANIFEST_DIR"), 15 | "/src/configuration/fixtures/full/rhio_nats_credentials_secret.yaml" 16 | )); 17 | pub const RHIO_S3: &str = include_str!(concat!( 18 | env!("CARGO_MANIFEST_DIR"), 19 | "/src/configuration/fixtures/full/rhio_s3_credentials_secret.yaml" 20 | )); 21 | 22 | pub const RMS: &str = include_str!(concat!( 23 | env!("CARGO_MANIFEST_DIR"), 24 | "/src/configuration/fixtures/full/rms.yaml" 25 | )); 26 | pub const RMSS: &str = include_str!(concat!( 27 | env!("CARGO_MANIFEST_DIR"), 28 | "/src/configuration/fixtures/full/rmss.yaml" 29 | )); 30 | pub const ROS: &str = include_str!(concat!( 31 | env!("CARGO_MANIFEST_DIR"), 32 | "/src/configuration/fixtures/full/ros.yaml" 33 | )); 34 | pub const ROSS: &str = include_str!(concat!( 35 | env!("CARGO_MANIFEST_DIR"), 36 | "/src/configuration/fixtures/full/ross.yaml" 37 | )); 38 | } 39 | -------------------------------------------------------------------------------- /rhio-http-api/src/blocking.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use crate::{api::RhioApi, status::HealthStatus}; 4 | use anyhow::Result; 5 | use tokio::runtime::Runtime; 6 | 7 | /// A blocking client for interacting with the Rhio HTTP API. 8 | /// 9 | /// The `BlockingClient` struct provides a synchronous interface to the asynchronous 10 | /// `RhioApi` by using a Tokio runtime to block on asynchronous calls. This allows 11 | /// users to interact with the API in a blocking manner, which can be useful in 12 | /// contexts where asynchronous code is not suitable. 13 | /// 14 | /// # Type Parameters 15 | /// 16 | /// * `A` - A type that implements the `RhioApi` trait. 17 | /// 18 | /// # Fields 19 | /// 20 | /// * `inner` - The inner API client that implements the `RhioApi` trait. 21 | /// * `runtime` - An `Arc` wrapped Tokio runtime used to block on asynchronous calls. 22 | /// 23 | pub struct BlockingClient 24 | where 25 | A: RhioApi, 26 | { 27 | inner: A, 28 | runtime: Arc, 29 | } 30 | 31 | impl BlockingClient 32 | where 33 | A: RhioApi, 34 | { 35 | pub fn new(inner: A, runtime: Arc) -> Self { 36 | Self { inner, runtime } 37 | } 38 | 39 | pub fn health(&self) -> Result { 40 | self.runtime.block_on(async { self.inner.health().await }) 41 | } 42 | 43 | pub fn metrics(&self) -> Result { 44 | self.runtime.block_on(async { self.inner.metrics().await }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /rhio-operator/src/api/message_stream_subscription.rs: -------------------------------------------------------------------------------- 1 | use rhio_http_api::status::MessageStreamSubscribeStatus; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | use stackable_operator::kube::CustomResource; 5 | 6 | #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] 7 | #[cfg_attr(test, derive(Default))] 8 | #[kube( 9 | kind = "ReplicatedMessageStreamSubscription", 10 | group = "rhio.hiro.io", 11 | version = "v1", 12 | plural = "replicatedmessagestreamsubscriptions", 13 | status = "ReplicatedMessageStreamSubscriptionStatus", 14 | shortname = "rmss", 15 | namespaced, 16 | crates( 17 | kube_core = "stackable_operator::kube::core", 18 | k8s_openapi = "stackable_operator::k8s_openapi", 19 | schemars = "stackable_operator::schemars" 20 | ) 21 | )] 22 | #[serde(rename_all = "camelCase")] 23 | pub struct ReplicatedMessageStreamSubscriptionSpec { 24 | pub public_key: String, 25 | pub subscriptions: Vec, 26 | } 27 | 28 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 29 | #[serde(rename_all = "camelCase")] 30 | pub struct SubjectSpec { 31 | pub subject: String, 32 | pub stream: String, 33 | } 34 | 35 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 36 | #[serde(rename_all = "camelCase")] 37 | pub struct ReplicatedMessageStreamSubscriptionStatus { 38 | pub subjects: Vec, 39 | } 40 | -------------------------------------------------------------------------------- /rhio-operator/src/api/role.rs: -------------------------------------------------------------------------------- 1 | use super::service::RhioService; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | use stackable_operator::{kube::runtime::reflector::ObjectRef, role_utils::RoleGroupRef}; 5 | use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; 6 | 7 | #[derive( 8 | Clone, 9 | Debug, 10 | Deserialize, 11 | Display, 12 | EnumIter, 13 | Eq, 14 | Hash, 15 | JsonSchema, 16 | PartialEq, 17 | Serialize, 18 | EnumString, 19 | )] 20 | pub enum RhioRole { 21 | #[strum(serialize = "server")] 22 | Server, 23 | } 24 | 25 | /// Group roles are needed for the cases when the instance may have several functional components, e.g. master and workers 26 | /// In Rhio we don't have multiple roles. Role is used for compatibility with stackable-operator framework. 27 | impl RhioRole { 28 | /// Metadata about a rolegroup 29 | pub fn rolegroup_ref( 30 | &self, 31 | service: &RhioService, 32 | group_name: impl Into, 33 | ) -> RoleGroupRef { 34 | RoleGroupRef { 35 | cluster: ObjectRef::from_obj(service), 36 | role: self.to_string(), 37 | role_group: group_name.into(), 38 | } 39 | } 40 | 41 | pub fn roles() -> Vec { 42 | let mut roles = vec![]; 43 | for role in Self::iter() { 44 | roles.push(role.to_string()) 45 | } 46 | roles 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /rhio-operator/src/api/object_store_subscription.rs: -------------------------------------------------------------------------------- 1 | use rhio_http_api::status::ObjectStoreSubscribeStatus; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | use stackable_operator::kube::CustomResource; 5 | 6 | #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] 7 | #[cfg_attr(test, derive(Default))] 8 | #[kube( 9 | kind = "ReplicatedObjectStoreSubscription", 10 | group = "rhio.hiro.io", 11 | version = "v1", 12 | plural = "replicatedobjectstoresubscriptions", 13 | status = "ReplicatedObjectStoreSubscriptionStatus", 14 | shortname = "ross", 15 | namespaced, 16 | crates( 17 | kube_core = "stackable_operator::kube::core", 18 | k8s_openapi = "stackable_operator::k8s_openapi", 19 | schemars = "stackable_operator::schemars" 20 | ) 21 | )] 22 | #[serde(rename_all = "camelCase")] 23 | pub struct ReplicatedObjectStoreSubscriptionSpec { 24 | #[serde(default)] 25 | pub public_key: String, 26 | pub buckets: Vec, 27 | } 28 | 29 | #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, Default)] 30 | #[serde(rename_all = "camelCase")] 31 | pub struct BucketSpec { 32 | pub remote_bucket: String, 33 | pub local_bucket: String, 34 | } 35 | 36 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 37 | #[serde(rename_all = "camelCase")] 38 | pub struct ReplicatedObjectStoreSubscriptionStatus { 39 | pub buckets: Vec, 40 | } 41 | -------------------------------------------------------------------------------- /charts/rhio-operator/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for rhio-operator. 2 | --- 3 | image: 4 | repository: "" 5 | tag: "" 6 | pullPolicy: IfNotPresent 7 | pullSecrets: [] 8 | 9 | nameOverride: "" 10 | fullnameOverride: "" 11 | 12 | serviceAccount: 13 | # Specifies whether a service account should be created 14 | create: true 15 | # Annotations to add to the service account 16 | annotations: {} 17 | # The name of the service account to use. 18 | # If not set and create is true, a name is generated using the fullname template 19 | name: "" 20 | 21 | # Provide additional labels which get attached to all deployed resources 22 | labels: 23 | stackable.tech/vendor: Hiro 24 | 25 | # stackable-operator framework depends on listeners crds. 26 | # We are not using Listeners (at least at the moment), but we have to deploy crds so that the framework functioning as expected. 27 | # This is the switch to enable/disable Listeners crds. 28 | enableListeners: true 29 | 30 | podAnnotations: {} 31 | 32 | podSecurityContext: {} 33 | # fsGroup: 2000 34 | 35 | securityContext: {} 36 | # capabilities: 37 | # drop: 38 | # - ALL 39 | # readOnlyRootFilesystem: true 40 | # runAsNonRoot: true 41 | # runAsUser: 1000 42 | 43 | resources: 44 | limits: 45 | cpu: 100m 46 | memory: 128Mi 47 | requests: 48 | cpu: 100m 49 | memory: 128Mi 50 | 51 | nodeSelector: {} 52 | 53 | tolerations: [] 54 | 55 | affinity: {} 56 | 57 | # When running on a non-default Kubernetes cluster domain, the cluster domain can be configured here. 58 | # See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details. 59 | # kubernetesClusterDomain: my-cluster.local 60 | -------------------------------------------------------------------------------- /rhio/src/tracing.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use tracing::Level; 4 | use tracing_subscriber::EnvFilter; 5 | use tracing_subscriber::fmt::Layer; 6 | use tracing_subscriber::layer::SubscriberExt; 7 | use tracing_subscriber::util::SubscriberInitExt; 8 | 9 | /// Setup logging with the help of the `tracing` crate. 10 | /// 11 | /// The verbosity and targets can be configured with a filter string: 12 | /// 13 | /// 1. When no filter is set the default "rhio=INFO" filter will be applied 14 | /// 2. When only a level was given ("INFO", "DEBUG", etc.) it will be used for the rhio target 15 | /// "rhio={level}" 16 | /// 3. When the string begins with an "=", then the log level will be applied for _all_ targets. 17 | /// This is equivalent to only setting the level "{level}" 18 | /// 4. When the string specifies a target and a level it will be used as-is: "{filter}", for 19 | /// example "tokio=TRACE" 20 | pub fn setup_tracing(filter: Option) { 21 | let default = "rhio=INFO" 22 | .parse() 23 | .expect("hard-coded default directive should be valid"); 24 | 25 | let builder = EnvFilter::builder().with_default_directive(default); 26 | 27 | let filter = if let Some(filter) = filter { 28 | if let Some(all_targets_level) = filter.strip_prefix("=") { 29 | all_targets_level.to_string() 30 | } else { 31 | match Level::from_str(&filter) { 32 | Ok(level) => format!("rhio={level}"), 33 | Err(_) => filter, 34 | } 35 | } 36 | } else { 37 | String::default() 38 | }; 39 | let filter = builder.parse_lossy(filter); 40 | 41 | tracing_subscriber::registry() 42 | .with(Layer::default()) 43 | .with(filter) 44 | .try_init() 45 | .ok(); 46 | } 47 | -------------------------------------------------------------------------------- /rhio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhio" 3 | description = "Peer-to-peer NATS message routing and S3 object sync solution" 4 | version = "0.1.1" 5 | edition = "2024" 6 | publish = false 7 | default-run = "rhio" 8 | 9 | [lints] 10 | workspace = true 11 | 12 | [lib] 13 | name = "rhio" 14 | 15 | [[bin]] 16 | name = "rhio" 17 | path = "src/main.rs" 18 | 19 | [dependencies] 20 | anyhow.workspace = true 21 | async-nats.workspace = true 22 | async-trait.workspace = true 23 | axum.workspace = true 24 | axum-prometheus.workspace = true 25 | ciborium.workspace = true 26 | clap = { workspace = true, features = ["derive"] } 27 | directories.workspace = true 28 | figment.workspace = true 29 | futures-util.workspace = true 30 | hex.workspace = true 31 | loole.workspace = true 32 | p2panda-blobs.workspace = true 33 | p2panda-core.workspace = true 34 | p2panda-net.workspace = true 35 | p2panda-sync.workspace = true 36 | p2panda-discovery.workspace = true 37 | iroh-base.workspace = true 38 | rand.workspace = true 39 | rhio-blobs.workspace = true 40 | rhio-core.workspace = true 41 | rhio-config.workspace = true 42 | rhio-http-api.workspace = true 43 | rust-s3.workspace = true 44 | serde.workspace = true 45 | tokio.workspace = true 46 | tokio-stream.workspace = true 47 | tokio-util.workspace = true 48 | tracing.workspace = true 49 | tracing-subscriber.workspace = true 50 | pin-project.workspace = true 51 | bytes.workspace = true 52 | futures.workspace = true 53 | pin-project-lite.workspace = true 54 | thiserror.workspace = true 55 | 56 | [dev-dependencies] 57 | figment = { workspace = true, features = ["test"] } 58 | once_cell.workspace = true 59 | dashmap.workspace = true 60 | serde_json.workspace = true 61 | s3-server.workspace = true 62 | s3s.workspace = true 63 | url.workspace = true 64 | 65 | [build-dependencies] 66 | built.workspace = true 67 | -------------------------------------------------------------------------------- /rhio-operator/src/operations/pdb.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | api::{role::RhioRole, service::RhioService}, 3 | rhio::controller::{APP_NAME, OPERATOR_NAME, RHIO_CONTROLLER_NAME}, 4 | }; 5 | use snafu::{ResultExt, Snafu}; 6 | use stackable_operator::{ 7 | builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, 8 | commons::pdb::PdbConfig, kube::ResourceExt, 9 | }; 10 | 11 | #[derive(Snafu, Debug)] 12 | pub enum Error { 13 | #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] 14 | CreatePdb { 15 | source: stackable_operator::builder::pdb::Error, 16 | role: String, 17 | }, 18 | #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] 19 | ApplyPdb { 20 | source: stackable_operator::cluster_resources::Error, 21 | name: String, 22 | }, 23 | } 24 | 25 | pub async fn add_pdbs( 26 | pdb: &PdbConfig, 27 | rhio: &RhioService, 28 | role: &RhioRole, 29 | client: &Client, 30 | cluster_resources: &mut ClusterResources, 31 | ) -> Result<(), Error> { 32 | if !pdb.enabled { 33 | return Ok(()); 34 | } 35 | let max_unavailable = pdb.max_unavailable.unwrap_or(match role { 36 | RhioRole::Server => max_unavailable_servers(), 37 | }); 38 | let pdb = PodDisruptionBudgetBuilder::new_with_role( 39 | rhio, 40 | APP_NAME, 41 | &role.to_string(), 42 | OPERATOR_NAME, 43 | RHIO_CONTROLLER_NAME, 44 | ) 45 | .with_context(|_| CreatePdbSnafu { 46 | role: role.to_string(), 47 | })? 48 | .with_max_unavailable(max_unavailable) 49 | .build(); 50 | let pdb_name = pdb.name_any(); 51 | cluster_resources 52 | .add(client, pdb) 53 | .await 54 | .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; 55 | 56 | Ok(()) 57 | } 58 | 59 | fn max_unavailable_servers() -> u16 { 60 | 1 61 | } 62 | -------------------------------------------------------------------------------- /rhio-http-api/src/client.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | api::RhioApi, 3 | api::{HTTP_HEALTH_ROUTE, HTTP_METRICS_ROUTE}, 4 | status::HealthStatus, 5 | }; 6 | use anyhow::{Context, Result}; 7 | use async_trait::async_trait; 8 | 9 | /// `RhioApiClient` is a client for interacting with the Rhio HTTP API. 10 | /// 11 | /// # Fields 12 | /// 13 | /// * `endpoint` - The base URL of the Rhio API. 14 | /// 15 | /// # Examples 16 | /// 17 | /// ```rust 18 | /// use rhio_http_api::client::RhioApiClient; 19 | /// 20 | /// let client = RhioApiClient::new("http://localhost:8080".to_string()); 21 | /// ``` 22 | /// 23 | /// # Methods 24 | /// 25 | /// * `new` - Creates a new instance of `RhioApiClient`. 26 | /// 27 | /// # Implementations 28 | /// 29 | /// This struct implements the `RhioApi` trait, providing methods to check the health status 30 | /// and retrieve metrics from the Rhio API. 31 | pub struct RhioApiClient { 32 | endpoint: String, 33 | } 34 | 35 | impl RhioApiClient { 36 | pub fn new(endpoint: String) -> RhioApiClient { 37 | RhioApiClient { endpoint } 38 | } 39 | } 40 | 41 | #[async_trait] 42 | impl RhioApi for RhioApiClient { 43 | async fn health(&self) -> Result { 44 | let url = format!("{}{}", self.endpoint, HTTP_HEALTH_ROUTE); 45 | let response = reqwest::get(url) 46 | .await 47 | .context("health request")? 48 | .json::() 49 | .await 50 | .context("health response deserialization")?; 51 | Ok(response) 52 | } 53 | 54 | async fn metrics(&self) -> Result { 55 | let url = format!("{}{}", self.endpoint, HTTP_METRICS_ROUTE); 56 | let response = reqwest::get(url) 57 | .await 58 | .context("metrics request")? 59 | .text() 60 | .await 61 | .context("metrics response deserialization")?; 62 | Ok(response) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /charts/rhio/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "app.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "app.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "app.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "app.labels" -}} 37 | helm.sh/chart: {{ include "app.chart" . }} 38 | {{ include "app.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "app.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "app.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "app.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "app.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /rhio/src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use rhio::{built_info, context_builder::ContextBuilder}; 3 | use tracing::info; 4 | 5 | fn main() -> Result<()> { 6 | let context_builder = ContextBuilder::from_cli()?; 7 | let context = context_builder.try_build_and_start()?; 8 | 9 | context.configure()?; 10 | 11 | log_hello_rhio(); 12 | context.log_configuration(); 13 | 14 | context.wait_for_termination()?; 15 | 16 | info!("shutting down"); 17 | 18 | context.shutdown() 19 | } 20 | 21 | fn log_hello_rhio() { 22 | r#" ___ ___ ___ 23 | /\ \ /\__\ ___ /\ \ 24 | /::\ \ /:/ / /\ \ /::\ \ 25 | /:/\:\ \ /:/__/ \:\ \ /:/\:\ \ 26 | /::\~\:\ \ /::\ \ ___ /::\__\ /:/ \:\ \ 27 | /:/\:\ \:\__\ /:/\:\ /\__\ __/:/\/__/ /:/__/ \:\__\ 28 | \/_|::\/:/ / \/__\:\/:/ / /\/:/ / \:\ \ /:/ / 29 | |:|::/ / \::/ / \::/__/ \:\ /:/ / 30 | |:|\/__/ /:/ / \:\__\ \:\/:/ / 31 | |:| | /:/ / \/__/ \::/ / 32 | \|__| \/__/ \/__/ 33 | "# 34 | .split("\n") 35 | .for_each(|line| info!("{}", line)); 36 | print_startup_string( 37 | env!("CARGO_PKG_DESCRIPTION"), 38 | env!("CARGO_PKG_VERSION"), 39 | built_info::GIT_VERSION, 40 | built_info::TARGET, 41 | built_info::BUILT_TIME_UTC, 42 | built_info::RUSTC_VERSION, 43 | ); 44 | info!(""); 45 | } 46 | 47 | pub fn print_startup_string( 48 | pkg_description: &str, 49 | pkg_version: &str, 50 | git_version: Option<&str>, 51 | target: &str, 52 | built_time: &str, 53 | rustc_version: &str, 54 | ) { 55 | let git = match git_version { 56 | None => "".to_string(), 57 | Some(git) => format!(" (Git information: {git})"), 58 | }; 59 | info!("Starting {pkg_description}"); 60 | info!( 61 | "This is version {pkg_version}{git}, built for {target} by {rustc_version} at {built_time}", 62 | ) 63 | } 64 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster1/rhio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/rhio 5 | - private-key-secret.yaml 6 | - rmss.cluster2.yaml 7 | - rmss.cluster3.yaml 8 | - ross.cluster2.yaml 9 | - ross.cluster3.yaml 10 | patches: 11 | - patch: |- 12 | - op: replace 13 | path: /spec/configuration/s3/endpoint 14 | value: http://minio-tenant-hl.minio.svc.cluster1.local:9000 15 | target: 16 | group: rhio.hiro.io 17 | version: v1 18 | kind: RhioService 19 | name: rhio-service 20 | namespace: rhio 21 | - patch: |- 22 | - op: replace 23 | path: /spec/configuration/nats/endpoint 24 | value: nats://nats-jetstream.nats.svc.cluster1.local:4222 25 | target: 26 | group: rhio.hiro.io 27 | version: v1 28 | kind: RhioService 29 | name: rhio-service 30 | namespace: rhio 31 | - patch: |- 32 | - op: replace 33 | path: /spec/source/helm/valuesObject/kubernetesClusterDomain 34 | value: cluster1.local 35 | target: 36 | group: argoproj.io 37 | version: v1alpha1 38 | kind: Application 39 | name: rhio-operator 40 | namespace: argocd 41 | - patch: |- 42 | - op: replace 43 | path: /spec/configuration/nodes/0 44 | value: 45 | publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 46 | endpoints: 47 | - rhio-service.rhio.svc.cluster2.local:9102 48 | target: 49 | group: rhio.hiro.io 50 | version: v1 51 | kind: RhioService 52 | name: rhio-service 53 | namespace: rhio 54 | - patch: |- 55 | - op: replace 56 | path: /spec/configuration/nodes/1 57 | value: 58 | publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 59 | endpoints: 60 | - rhio-service.rhio.svc.cluster3.local:9102 61 | target: 62 | group: rhio.hiro.io 63 | version: v1 64 | kind: RhioService 65 | name: rhio-service 66 | namespace: rhio 67 | - patch: |- 68 | - op: replace 69 | path: /spec/subjects/0 70 | value: cluster1 71 | target: 72 | group: rhio.hiro.io 73 | version: v1 74 | kind: ReplicatedMessageStream 75 | name: test-stream 76 | namespace: rhio 77 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster2/rhio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/rhio 5 | - private-key-secret.yaml 6 | - rmss.cluster1.yaml 7 | - rmss.cluster3.yaml 8 | - ross.cluster1.yaml 9 | - ross.cluster3.yaml 10 | patches: 11 | - patch: |- 12 | - op: replace 13 | path: /spec/configuration/s3/endpoint 14 | value: http://minio-tenant-hl.minio.svc.cluster2.local:9000 15 | target: 16 | group: rhio.hiro.io 17 | version: v1 18 | kind: RhioService 19 | name: rhio-service 20 | namespace: rhio 21 | - patch: |- 22 | - op: replace 23 | path: /spec/configuration/nats/endpoint 24 | value: nats://nats-jetstream.nats.svc.cluster2.local:4222 25 | target: 26 | group: rhio.hiro.io 27 | version: v1 28 | kind: RhioService 29 | name: rhio-service 30 | namespace: rhio 31 | - patch: |- 32 | - op: replace 33 | path: /spec/configuration/nodes/0 34 | value: 35 | publicKey: 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 36 | endpoints: 37 | - rhio-service.rhio.svc.cluster1.local:9102 38 | target: 39 | group: rhio.hiro.io 40 | version: v1 41 | kind: RhioService 42 | name: rhio-service 43 | namespace: rhio 44 | - patch: |- 45 | - op: replace 46 | path: /spec/configuration/nodes/1 47 | value: 48 | publicKey: 43b2bb39061bc3267e869303268a81734fb8767d3a17ee490813955bd734fd3a 49 | endpoints: 50 | - rhio-service.rhio.svc.cluster3.local:9102 51 | target: 52 | group: rhio.hiro.io 53 | version: v1 54 | kind: RhioService 55 | name: rhio-service 56 | namespace: rhio 57 | - patch: |- 58 | - op: replace 59 | path: /spec/source/helm/valuesObject/kubernetesClusterDomain 60 | value: cluster2.local 61 | target: 62 | group: argoproj.io 63 | version: v1alpha1 64 | kind: Application 65 | name: rhio-operator 66 | namespace: argocd 67 | - patch: |- 68 | - op: replace 69 | path: /spec/subjects/0 70 | value: cluster2 71 | target: 72 | group: rhio.hiro.io 73 | version: v1 74 | kind: ReplicatedMessageStream 75 | name: test-stream 76 | namespace: rhio 77 | -------------------------------------------------------------------------------- /env/dev/overlays/kind-cluster3/rhio/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ../../../apps/rhio 5 | - private-key-secret.yaml 6 | - rmss.cluster1.yaml 7 | - rmss.cluster2.yaml 8 | - ross.cluster1.yaml 9 | - ross.cluster2.yaml 10 | patches: 11 | - patch: |- 12 | - op: replace 13 | path: /spec/configuration/s3/endpoint 14 | value: http://minio-tenant-hl.minio.svc.cluster3.local:9000 15 | target: 16 | group: rhio.hiro.io 17 | version: v1 18 | kind: RhioService 19 | name: rhio-service 20 | namespace: rhio 21 | - patch: |- 22 | - op: replace 23 | path: /spec/configuration/nats/endpoint 24 | value: nats://nats-jetstream.nats.svc.cluster3.local:4222 25 | target: 26 | group: rhio.hiro.io 27 | version: v1 28 | kind: RhioService 29 | name: rhio-service 30 | namespace: rhio 31 | - patch: |- 32 | - op: replace 33 | path: /spec/configuration/nodes/0 34 | value: 35 | publicKey: 3f0ae398f8db1ee6b85607f7e54f4dbcf023b90e052dc45e43a4192e16e02386 36 | endpoints: 37 | - rhio-service.rhio.svc.cluster1.local:9102 38 | target: 39 | group: rhio.hiro.io 40 | version: v1 41 | kind: RhioService 42 | name: rhio-service 43 | namespace: rhio 44 | - patch: |- 45 | - op: replace 46 | path: /spec/configuration/nodes/1 47 | value: 48 | publicKey: b01854865341ac6db10b6aa9646045d65ddc2ac8e5e198ffd2d04ceca045ddf9 49 | endpoints: 50 | - rhio-service.rhio.svc.cluster2.local:9102 51 | target: 52 | group: rhio.hiro.io 53 | version: v1 54 | kind: RhioService 55 | name: rhio-service 56 | namespace: rhio 57 | - patch: |- 58 | - op: replace 59 | path: /spec/source/helm/valuesObject/kubernetesClusterDomain 60 | value: cluster3.local 61 | target: 62 | group: argoproj.io 63 | version: v1alpha1 64 | kind: Application 65 | name: rhio-operator 66 | namespace: argocd 67 | - patch: |- 68 | - op: replace 69 | path: /spec/subjects/0 70 | value: cluster3 71 | target: 72 | group: rhio.hiro.io 73 | version: v1 74 | kind: ReplicatedMessageStream 75 | name: test-stream 76 | namespace: rhio 77 | -------------------------------------------------------------------------------- /env/dev/apps/minio/minio-tenant.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Application 3 | metadata: 4 | name: minio-tenant 5 | namespace: argocd 6 | annotations: 7 | argocd.argoproj.io/compare-options: ServerSideDiff=true 8 | argocd.argoproj.io/sync-wave: "10" 9 | finalizers: 10 | - resources-finalizer.argocd.argoproj.io 11 | spec: 12 | project: default 13 | sources: 14 | - repoURL: https://operator.min.io 15 | chart: tenant 16 | targetRevision: 5.0.14 17 | helm: 18 | valuesObject: 19 | # Secret for configuring the root MinIO user 20 | secrets: 21 | name: minio-env-configuration 22 | accessKey: minio 23 | secretKey: minio123 24 | 25 | tenant: 26 | name: minio-tenant 27 | # Kubernetes secret name that contains MinIO environment variable 28 | # configurations 29 | configuration: 30 | name: minio-env-configuration 31 | pools: 32 | - name: minio-pool 33 | # Number of MinIO Tenant pods 34 | servers: 1 35 | # Number of volumes per MinIO Tenant pod 36 | volumesPerServer: 4 37 | # Size of each volume 38 | size: 500Mi 39 | # Storage class of the volumes 40 | storageClassName: standard 41 | # Minimum and maximum resources requested for each pod 42 | resources: 43 | requests: 44 | cpu: 500m 45 | memory: 2Gi 46 | limits: 47 | cpu: 1 48 | memory: 2Gi 49 | # Enable automatic certificate generation and signing 50 | certificate: 51 | requestAutoCert: false 52 | # Buckets to create during Tenant provisioning 53 | buckets: 54 | - name: source 55 | - name: cluster1 56 | - name: cluster2 57 | - name: cluster3 58 | ingress: 59 | api: 60 | enabled: false 61 | console: 62 | enabled: false 63 | exposeServices: 64 | console: true 65 | destination: 66 | namespace: minio 67 | server: https://kubernetes.default.svc 68 | syncPolicy: 69 | automated: 70 | prune: true 71 | selfHeal: true 72 | syncOptions: 73 | - ServerSideApply=true 74 | -------------------------------------------------------------------------------- /.github/workflows/rust.yaml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: push 4 | 5 | env: 6 | CARGO_TERM_COLOR: always 7 | RUST_TOOLCHAIN: "1.91" 8 | 9 | jobs: 10 | 11 | check: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v4 17 | 18 | - name: Setup Rust toolchain 19 | uses: moonrepo/setup-rust@v1 20 | with: 21 | cache: false 22 | channel: ${{ env.RUST_TOOLCHAIN }} 23 | 24 | # build speedups 25 | - uses: Swatinem/rust-cache@v2 26 | 27 | - name: Check project and dependencies 28 | run: cargo check 29 | 30 | fmt: 31 | runs-on: ubuntu-latest 32 | 33 | steps: 34 | - name: Checkout repository 35 | uses: actions/checkout@v4 36 | 37 | - name: Setup Rust toolchain 38 | uses: moonrepo/setup-rust@v1 39 | with: 40 | cache: false 41 | components: rustfmt 42 | channel: ${{ env.RUST_TOOLCHAIN }} 43 | 44 | # build speedups 45 | - uses: Swatinem/rust-cache@v2 46 | 47 | - name: Check formatting 48 | run: cargo fmt -- --check 49 | 50 | clippy: 51 | runs-on: ubuntu-latest 52 | 53 | steps: 54 | - name: Checkout repository 55 | uses: actions/checkout@v4 56 | 57 | - name: Setup Rust toolchain 58 | uses: moonrepo/setup-rust@v1 59 | with: 60 | cache: false 61 | components: clippy 62 | channel: ${{ env.RUST_TOOLCHAIN }} 63 | 64 | # build speedups 65 | - uses: Swatinem/rust-cache@v2 66 | 67 | - name: Check code with clippy 68 | run: cargo clippy -- -D warnings --no-deps 69 | 70 | sbom: 71 | runs-on: ubuntu-latest 72 | 73 | steps: 74 | - name: Checkout repository 75 | uses: actions/checkout@v4 76 | 77 | - name: Setup Rust toolchain 78 | uses: moonrepo/setup-rust@v1 79 | with: 80 | cache: false 81 | channel: ${{ env.RUST_TOOLCHAIN }} 82 | 83 | - name: Install SBOM 84 | uses: psastras/sbom-rs/actions/install-cargo-sbom@cargo-sbom-v0.10.0 85 | 86 | # build speedups 87 | - uses: Swatinem/rust-cache@v2 88 | 89 | - name: Run cargo-sbom 90 | run: cargo-sbom --output-format=cyclone_dx_json_1_6 > sbom.cdx.json 91 | 92 | osv-scanner: 93 | uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v2.2.1 94 | needs: [sbom] 95 | -------------------------------------------------------------------------------- /charts/rhio-operator/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "operator.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-operator" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Expand the name of the chart. 10 | */}} 11 | {{- define "operator.appname" -}} 12 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 13 | {{- end }} 14 | 15 | {{/* 16 | Create a default fully qualified app name. 17 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 18 | If release name contains chart name it will be used as a full name. 19 | */}} 20 | {{- define "operator.fullname" -}} 21 | {{- if .Values.fullnameOverride }} 22 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 23 | {{- else }} 24 | {{- $name := default .Chart.Name .Values.nameOverride }} 25 | {{- if contains $name .Release.Name }} 26 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 27 | {{- else }} 28 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 29 | {{- end }} 30 | {{- end }} 31 | {{- end }} 32 | 33 | {{/* 34 | Create chart name and version as used by the chart label. 35 | */}} 36 | {{- define "operator.chart" -}} 37 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 38 | {{- end }} 39 | 40 | {{/* 41 | Common labels 42 | */}} 43 | {{- define "operator.labels" -}} 44 | helm.sh/chart: {{ include "operator.chart" . }} 45 | {{ include "operator.selectorLabels" . }} 46 | {{- if .Chart.AppVersion }} 47 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 48 | {{- end }} 49 | app.kubernetes.io/managed-by: {{ .Release.Service }} 50 | {{- end }} 51 | 52 | {{/* 53 | Selector labels 54 | */}} 55 | {{- define "operator.selectorLabels" -}} 56 | app.kubernetes.io/name: {{ include "operator.appname" . }} 57 | app.kubernetes.io/instance: {{ .Release.Name }} 58 | {{- with .Values.labels }} 59 | {{ toYaml . }} 60 | {{- end }} 61 | {{- end }} 62 | 63 | {{/* 64 | Create the name of the service account to use 65 | */}} 66 | {{- define "operator.serviceAccountName" -}} 67 | {{- if .Values.serviceAccount.create }} 68 | {{- default (include "operator.fullname" .) .Values.serviceAccount.name }} 69 | {{- else }} 70 | {{- default "default" .Values.serviceAccount.name }} 71 | {{- end }} 72 | {{- end }} 73 | 74 | {{/* 75 | Labels for Kubernetes objects created by helm test 76 | */}} 77 | {{- define "operator.testLabels" -}} 78 | helm.sh/test: {{ include "operator.chart" . }} 79 | {{- end }} 80 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ### Added 11 | 12 | - Improved NATS message replication logic and configuration ([#86](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/86) [#82](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/82), [#75](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/75)) 13 | - Improved S3 objects replication logic and configuration ([#92](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/92), [#79](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/79), [#74](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/74)) 14 | - Direct p2p sync between S3 buckets with bao-encoding ([#73](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/73)) 15 | - Resolve FQDN endpoints in config ([#96](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/96)) 16 | - HTTP `/health` endpoint ([#90](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/90)) 17 | - Kubernetes Support ([#91](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/91)) 18 | 19 | ### Changed 20 | 21 | - De-duplicate topic id's for gossip overlays ([#86](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/86)) 22 | - Removed FFI bindings for Python for now 23 | - Removed `rhio-client` as it is not required anymore 24 | 25 | ### Fixed 26 | 27 | - Fix NATS consumers after message sync changes ([#89](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/89)) 28 | - Fix remote bucket logic after S3 sync changes ([#94](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/94)) 29 | 30 | ## [0.1.0] - 2024-08-30 31 | 32 | ### Added 33 | 34 | - GitHub CI for Rust tests and linters ([#65](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/65)) 35 | - NATS JetStream integration ([#58](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/58)) 36 | - Introduce `rhio-core` and `rhio-client` crates for client development ([#57](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/57)) 37 | - Configuration interface ([#53](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/53)) 38 | - Support for MinIO storage ([#47](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/47)) 39 | - Experimental FFI bindings for Python ([#37](https://github.com/HIRO-MicroDataCenters-BV/rhio/pull/37)) 40 | 41 | [unreleased]: https://github.com/HIRO-MicroDataCenters-BV/rhio/compare/v0.1.0...HEAD 42 | [0.1.0]: https://github.com/HIRO-MicroDataCenters-BV/rhio/releases/tag/v0.1.0 43 | -------------------------------------------------------------------------------- /rhio/src/network/mod.rs: -------------------------------------------------------------------------------- 1 | mod actor; 2 | pub mod membership; 3 | pub mod sync; 4 | 5 | use anyhow::Result; 6 | use futures_util::future::{MapErr, Shared}; 7 | use futures_util::{FutureExt, TryFutureExt}; 8 | use p2panda_net::Network; 9 | use p2panda_net::network::FromNetwork; 10 | use tokio::sync::{mpsc, oneshot}; 11 | use tokio::task::JoinError; 12 | use tokio_util::task::AbortOnDropHandle; 13 | use tracing::error; 14 | 15 | use crate::JoinErrToStr; 16 | use crate::network::actor::{PandaActor, ToPandaActor}; 17 | use crate::topic::Query; 18 | 19 | #[derive(Debug)] 20 | pub struct Panda { 21 | panda_actor_tx: mpsc::Sender, 22 | #[allow(dead_code)] 23 | actor_handle: Shared, JoinErrToStr>>, 24 | } 25 | 26 | impl Panda { 27 | pub fn new(network: Network) -> Self { 28 | let (panda_actor_tx, panda_actor_rx) = mpsc::channel(512); 29 | let panda_actor = PandaActor::new(network, panda_actor_rx); 30 | 31 | let actor_handle = tokio::task::spawn(async move { 32 | if let Err(err) = panda_actor.run().await { 33 | error!("p2panda actor failed: {err:?}"); 34 | } 35 | }); 36 | 37 | let actor_drop_handle = AbortOnDropHandle::new(actor_handle) 38 | .map_err(Box::new(|e: JoinError| e.to_string()) as JoinErrToStr) 39 | .shared(); 40 | 41 | Self { 42 | panda_actor_tx, 43 | actor_handle: actor_drop_handle, 44 | } 45 | } 46 | 47 | /// Subscribe to a data stream in the network. 48 | pub async fn subscribe(&self, query: Query) -> Result>> { 49 | let (reply, reply_rx) = oneshot::channel(); 50 | self.panda_actor_tx 51 | .send(ToPandaActor::Subscribe { query, reply }) 52 | .await?; 53 | let rx = reply_rx.await?; 54 | Ok(rx) 55 | } 56 | 57 | /// Broadcasts a message in the gossip overlay-network with this topic id. 58 | pub async fn broadcast(&self, payload: Vec, topic_id: [u8; 32]) -> Result<()> { 59 | let (reply, reply_rx) = oneshot::channel(); 60 | self.panda_actor_tx 61 | .send(ToPandaActor::Broadcast { 62 | payload, 63 | topic_id, 64 | reply, 65 | }) 66 | .await?; 67 | reply_rx.await? 68 | } 69 | 70 | pub async fn shutdown(&self) -> Result<()> { 71 | let (reply, reply_rx) = oneshot::channel(); 72 | self.panda_actor_tx 73 | .send(ToPandaActor::Shutdown { reply }) 74 | .await?; 75 | reply_rx.await?; 76 | Ok(()) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /env/dev/apps/nats/nats.nats.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: argoproj.io/v1alpha1 2 | kind: Application 3 | metadata: 4 | name: nats-jetstream 5 | namespace: argocd 6 | annotations: 7 | argocd.argoproj.io/compare-options: ServerSideDiff=true 8 | argocd.argoproj.io/sync-wave: "-30" 9 | finalizers: 10 | - resources-finalizer.argocd.argoproj.io 11 | spec: 12 | destination: 13 | namespace: nats 14 | server: https://kubernetes.default.svc 15 | project: default 16 | source: 17 | repoURL: https://nats-io.github.io/k8s/helm/charts 18 | chart: nats 19 | targetRevision: 1.2.6 20 | helm: 21 | valuesObject: 22 | config: 23 | serverNamePrefix: cluster.local 24 | 25 | cluster: 26 | enabled: true 27 | replicas: 2 28 | 29 | jetstream: 30 | enabled: true 31 | 32 | fileStore: 33 | enabled: true 34 | dir: /data 35 | 36 | ############################################################ 37 | # stateful set -> volume claim templates -> jetstream pvc 38 | ############################################################ 39 | pvc: 40 | enabled: true 41 | size: 30Mi 42 | storageClassName: standard 43 | 44 | memoryStore: 45 | enabled: false 46 | # ensure that container has a sufficient memory limit greater than maxSize 47 | maxSize: 30Mi 48 | 49 | # merge: 50 | # server_tags: az:integration 51 | 52 | mqtt: 53 | enabled: true 54 | 55 | websocket: 56 | enabled: true 57 | port: 8080 58 | noTLS: true 59 | 60 | sameOrigin: false 61 | allowedOrigins: [] 62 | ingress: 63 | className: nginx 64 | path: / 65 | pathType: Prefix 66 | hosts: 67 | - nats-websocket.cluster1.local 68 | 69 | # This will optionally specify what host:port for websocket 70 | # connections to be advertised in the cluster. 71 | # advertise: "host:port" 72 | 73 | # Set the handshake timeout for websocket connections 74 | # handshakeTimeout: 5s 75 | 76 | ignoreDifferences: 77 | - group: apps 78 | kind: StatefulSet 79 | jqPathExpressions: 80 | - '.spec.volumeClaimTemplates[]?' 81 | syncPolicy: 82 | automated: 83 | prune: true 84 | selfHeal: true 85 | syncOptions: 86 | - CreateNamespace=true 87 | - ServerSideApply=true 88 | - RespectIgnoreDifferences=true 89 | -------------------------------------------------------------------------------- /rhio/src/blobs/proxy.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use futures_util::future::{MapErr, Shared}; 3 | use futures_util::{FutureExt, TryFutureExt}; 4 | use p2panda_blobs::Blobs as BlobsHandler; 5 | use rhio_blobs::{NotImportedObject, S3Store, SignedBlobInfo}; 6 | use tokio::sync::{mpsc, oneshot}; 7 | use tokio::task::JoinError; 8 | use tokio_util::task::{AbortOnDropHandle, LocalPoolHandle}; 9 | use tracing::error; 10 | 11 | use crate::JoinErrToStr; 12 | use crate::blobs::actor::{BlobsActor, ToBlobsActor}; 13 | use crate::topic::Query; 14 | 15 | #[derive(Debug)] 16 | pub struct BlobsActorProxy { 17 | blobs_actor_tx: mpsc::Sender, 18 | #[allow(dead_code)] 19 | actor_handle: Shared, JoinErrToStr>>, 20 | } 21 | 22 | impl BlobsActorProxy { 23 | pub fn new(blob_store: S3Store, blobs_handler: BlobsHandler) -> Self { 24 | let (blobs_actor_tx, blobs_actor_rx) = mpsc::channel(512); 25 | let blobs_actor = BlobsActor::new(blob_store.clone(), blobs_handler, blobs_actor_rx); 26 | 27 | let pool = LocalPoolHandle::new(1); 28 | let actor_handle = pool.spawn_pinned(|| async move { 29 | if let Err(err) = blobs_actor.run().await { 30 | error!("blobs actor failed: {err:?}"); 31 | } 32 | }); 33 | 34 | let actor_drop_handle = AbortOnDropHandle::new(actor_handle) 35 | .map_err(Box::new(|e: JoinError| e.to_string()) as JoinErrToStr) 36 | .shared(); 37 | 38 | Self { 39 | blobs_actor_tx, 40 | actor_handle: actor_drop_handle, 41 | } 42 | } 43 | 44 | /// Download a blob from the network. 45 | /// 46 | /// Attempt to download a blob from peers on the network and place it into the nodes MinIO 47 | /// bucket. 48 | pub async fn download(&self, blob: SignedBlobInfo) -> Result<()> { 49 | let (reply, reply_rx) = oneshot::channel(); 50 | self.blobs_actor_tx 51 | .send(ToBlobsActor::DownloadBlob { blob, reply }) 52 | .await?; 53 | let result = reply_rx.await?; 54 | result?; 55 | Ok(()) 56 | } 57 | 58 | /// Import an existing, local S3 object into the blob store, preparing it for p2p sync. 59 | pub async fn import_s3_object(&self, object: NotImportedObject) -> Result<()> { 60 | let (reply, reply_rx) = oneshot::channel(); 61 | self.blobs_actor_tx 62 | .send(ToBlobsActor::ImportS3Object { object, reply }) 63 | .await?; 64 | reply_rx.await??; 65 | Ok(()) 66 | } 67 | 68 | pub async fn shutdown(&self) -> Result<()> { 69 | let (reply, reply_rx) = oneshot::channel(); 70 | self.blobs_actor_tx 71 | .send(ToBlobsActor::Shutdown { reply }) 72 | .await?; 73 | reply_rx.await?; 74 | Ok(()) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /env/dev/argocd.install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This script automates the installation and setup of ArgoCD on multiple Kubernetes clusters. 4 | # It performs the following tasks: 5 | # 1. Installs ArgoCD on all specified clusters. 6 | # 2. Sets up a LoadBalancer service for ArgoCD on all specified clusters. 7 | # 3. Waits for all pods to be ready on all specified clusters. 8 | # 4. Retrieves and displays the initial admin password for ArgoCD on all specified clusters. 9 | # 10 | # 11 | CLUSTERS=("kind-cluster1" "kind-cluster2" "kind-cluster3") 12 | 13 | export KUBECONFIG=$(pwd)/target/merged-kubeconfig.yaml 14 | 15 | main() { 16 | install_argo_all 17 | install_balancer_all 18 | wait_pods_ready_all 19 | list_argo_credentials_all 20 | } 21 | 22 | install_argo_all() { 23 | echo "### Installing ArgoCD... ###" 24 | for i in "${!CLUSTERS[@]}"; do 25 | cluster="${CLUSTERS[$i]}" 26 | install_argo $cluster 27 | done 28 | echo "" 29 | } 30 | 31 | install_argo() { 32 | local cluster=$1 33 | 34 | echo "Installing ArgoCD on $cluster" 35 | kubectl --context kind-${cluster} create namespace argocd 36 | kubectl --context kind-${cluster} apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml 37 | } 38 | 39 | install_balancer_all() { 40 | echo "### Installing ArgoCD Load Balancer... ###" 41 | for i in "${!CLUSTERS[@]}"; do 42 | cluster="${CLUSTERS[$i]}" 43 | install_balancer $cluster 44 | done 45 | echo "" 46 | } 47 | 48 | install_balancer() { 49 | local cluster=$1 50 | 51 | echo "Installing ArgoCD Load Balancer on $cluster" 52 | kubectl --context kind-${cluster} create -f -< Result<()> { 12 | let response = bucket 13 | .put_object_with_content_type(paths.meta(), &meta.to_bytes(), META_CONTENT_TYPE) 14 | .await?; 15 | if response.status_code() != 200 { 16 | return Err(anyhow!("failed to create blob meta file in s3 bucket")); 17 | } 18 | trace!( 19 | key = %paths.meta(), 20 | complete = %meta.complete, 21 | bucket_name = bucket.name(), 22 | "created meta file in S3 bucket", 23 | ); 24 | Ok(()) 25 | } 26 | 27 | /// Creates an outboard file in S3 bucket. 28 | pub async fn put_outboard(bucket: &Bucket, paths: &Paths, outboard: &[u8]) -> Result<()> { 29 | let response = bucket.put_object(paths.outboard(), outboard).await?; 30 | if response.status_code() != 200 { 31 | return Err(anyhow!("failed to create blob outboard file in s3 bucket")); 32 | } 33 | trace!( 34 | key = %paths.outboard(), 35 | bytes = %outboard.len(), 36 | bucket_name = bucket.name(), 37 | "created outboard file in S3 bucket", 38 | ); 39 | Ok(()) 40 | } 41 | 42 | /// Loads a meta file from S3 bucket. 43 | pub async fn get_meta(bucket: &Bucket, paths: &Paths) -> Result { 44 | let response = bucket.get_object(paths.meta()).await?; 45 | if response.status_code() != 200 { 46 | return Err(anyhow!("Failed to get blob meta file to s3 bucket")); 47 | } 48 | let meta = BaoMeta::from_bytes(response.as_slice())?; 49 | Ok(meta) 50 | } 51 | 52 | /// Loads an outboard file from S3 bucket. 53 | pub async fn get_outboard(bucket: &Bucket, paths: &Paths) -> Result { 54 | let response = bucket.get_object(paths.outboard()).await?; 55 | if response.status_code() != 200 { 56 | return Err(anyhow!("Failed to get blob outboard file to s3 bucket")); 57 | } 58 | let mut outboard = SparseMemFile::new(); 59 | outboard.write_all_at(0, response.as_slice())?; 60 | Ok(outboard) 61 | } 62 | 63 | /// Remove meta file from S3 bucket. 64 | pub async fn remove_meta(bucket: &Bucket, paths: &Paths) -> Result<()> { 65 | let response = bucket.delete_object(paths.meta()).await?; 66 | if response.status_code() != 200 { 67 | return Err(anyhow!("failed to remove blob meta file from s3 bucket")); 68 | } 69 | Ok(()) 70 | } 71 | 72 | /// Remove outboard file from S3 bucket. 73 | pub async fn remove_outboard(bucket: &Bucket, paths: &Paths) -> Result<()> { 74 | let response = bucket.delete_object(paths.outboard()).await?; 75 | if response.status_code() != 200 { 76 | return Err(anyhow!( 77 | "failed to remove blob outboard file from s3 bucket" 78 | )); 79 | } 80 | Ok(()) 81 | } 82 | -------------------------------------------------------------------------------- /rhio-http-api/src/server.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | net::{IpAddr, Ipv4Addr, SocketAddr}, 3 | sync::Arc, 4 | }; 5 | 6 | use anyhow::{Context, Result}; 7 | use axum::Router; 8 | use axum::http::StatusCode; 9 | use axum::response::IntoResponse; 10 | use axum::routing::get; 11 | use axum::{Json, extract::State}; 12 | use tokio::net::TcpListener; 13 | use tracing::debug; 14 | 15 | use crate::{ 16 | api::{HTTP_HEALTH_ROUTE, HTTP_METRICS_ROUTE, RhioApi}, 17 | status::HealthStatus, 18 | }; 19 | 20 | /// `RhioHTTPServer` is a struct that represents an HTTP server for the Rhio application. 21 | /// It is responsible for serving health and metrics endpoints over HTTP. 22 | /// 23 | /// # Fields 24 | /// - `port`: The port on which the server will listen for incoming HTTP requests. 25 | /// - `api`: An `Arc` to a trait object implementing `RhioApi`, which provides the health and metrics functionality. 26 | /// 27 | /// # Methods 28 | /// - `new(port: u16, api: Arc) -> RhioHTTPServer` 29 | /// - Creates a new instance of `RhioHTTPServer` with the specified port and API implementation. 30 | /// - `run(&self) -> Result<()>` 31 | /// - Asynchronously runs the HTTP server, binding to the specified port and serving the health and metrics endpoints. 32 | /// - Returns a `Result` indicating success or failure of the server operation. 33 | /// 34 | pub struct RhioHTTPServer { 35 | port: u16, 36 | api: Arc, 37 | } 38 | 39 | impl RhioHTTPServer { 40 | pub fn new(port: u16, api: Arc) -> RhioHTTPServer { 41 | RhioHTTPServer { port, api } 42 | } 43 | 44 | pub async fn run(&self) -> Result<()> { 45 | let listener = TcpListener::bind(SocketAddr::new( 46 | IpAddr::V4(Ipv4Addr::UNSPECIFIED), 47 | self.port, 48 | )) 49 | .await 50 | .context("TCP Listener binding")?; 51 | debug!( 52 | "HTTP health and metrics endpoint listening on {}", 53 | listener.local_addr()? 54 | ); 55 | let state = ServerState { 56 | api: self.api.clone(), 57 | }; 58 | 59 | let app = Router::new() 60 | .route(HTTP_HEALTH_ROUTE, get(health)) 61 | .route(HTTP_METRICS_ROUTE, get(metrics)) 62 | .with_state(state); 63 | axum::serve(listener, app) 64 | .await 65 | .context("HTTP metrics and health serving")?; 66 | 67 | Ok(()) 68 | } 69 | } 70 | 71 | async fn health(State(state): State) -> impl IntoResponse { 72 | state 73 | .api 74 | .health() 75 | .await 76 | .map(|result| (StatusCode::OK, Json(result))) 77 | .map_err(|e| { 78 | ( 79 | StatusCode::INTERNAL_SERVER_ERROR, 80 | Json(Into::::into(e)), 81 | ) 82 | }) 83 | } 84 | 85 | async fn metrics(State(state): State) -> impl IntoResponse { 86 | state 87 | .api 88 | .metrics() 89 | .await 90 | .map(|result| (StatusCode::OK, result)) 91 | .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("{:?}", e))) 92 | } 93 | 94 | #[derive(Clone)] 95 | pub struct ServerState { 96 | pub api: Arc, 97 | } 98 | -------------------------------------------------------------------------------- /rhio-operator/src/rhio/fixtures/statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: server 6 | app.kubernetes.io/instance: test-service 7 | app.kubernetes.io/managed-by: rhio.hiro.io_rhioservice 8 | app.kubernetes.io/name: rhio 9 | app.kubernetes.io/role-group: default 10 | app.kubernetes.io/version: 1.0.1-1.0.1 11 | stackable.tech/vendor: HIRO 12 | name: test-service-server-default 13 | ownerReferences: 14 | - apiVersion: rhio.hiro.io/v1 15 | controller: true 16 | kind: RhioService 17 | name: test-service 18 | uid: '1' 19 | spec: 20 | podManagementPolicy: Parallel 21 | replicas: 1 22 | selector: 23 | matchLabels: 24 | app.kubernetes.io/component: server 25 | app.kubernetes.io/instance: test-service 26 | app.kubernetes.io/name: rhio 27 | app.kubernetes.io/role-group: default 28 | serviceName: test-service-server-default 29 | template: 30 | metadata: 31 | annotations: 32 | rhio.hiro.io/config-hash: test_hash 33 | labels: 34 | app.kubernetes.io/component: server 35 | app.kubernetes.io/instance: test-service 36 | app.kubernetes.io/managed-by: rhio.hiro.io_rhioservice 37 | app.kubernetes.io/name: rhio 38 | app.kubernetes.io/role-group: default 39 | app.kubernetes.io/version: 1.0.1-1.0.1 40 | stackable.tech/vendor: HIRO 41 | spec: 42 | affinity: {} 43 | containers: 44 | - args: 45 | - /usr/local/bin/rhio 46 | - -c 47 | - /etc/rhio/config.yaml 48 | env: 49 | - name: PRIVATE_KEY 50 | valueFrom: 51 | secretKeyRef: 52 | key: secretKey 53 | name: rhio_private_key_secret 54 | image: ghcr.io/hiro-microdatacenters-bv/rhio-dev:1.0.1 55 | imagePullPolicy: IfNotPresent 56 | livenessProbe: 57 | failureThreshold: 3 58 | httpGet: 59 | path: /health 60 | port: 8080 61 | scheme: HTTP 62 | initialDelaySeconds: 5 63 | periodSeconds: 10 64 | timeoutSeconds: 5 65 | name: rhio 66 | ports: 67 | - containerPort: 8080 68 | name: health 69 | protocol: TCP 70 | - containerPort: 9102 71 | name: rhio 72 | protocol: UDP 73 | readinessProbe: 74 | failureThreshold: 3 75 | httpGet: 76 | path: /health 77 | port: 8080 78 | scheme: HTTP 79 | initialDelaySeconds: 5 80 | periodSeconds: 10 81 | timeoutSeconds: 5 82 | resources: 83 | limits: 84 | cpu: '1' 85 | memory: 1Gi 86 | requests: 87 | cpu: 250m 88 | memory: 128Mi 89 | volumeMounts: 90 | - mountPath: /etc/rhio/config.yaml 91 | name: config 92 | subPath: config.yaml 93 | enableServiceLinks: false 94 | serviceAccountName: '' 95 | terminationGracePeriodSeconds: 1 96 | volumes: 97 | - configMap: 98 | name: test-service 99 | name: config 100 | -------------------------------------------------------------------------------- /rhio-operator/src/cli.rs: -------------------------------------------------------------------------------- 1 | use rhio_config::configuration::NatsCredentials; 2 | use s3::creds::Credentials; 3 | 4 | use crate::{configuration::secret::Secret, rhio::private_key::PrivateKey}; 5 | 6 | /// Enum representing the various commands that can be executed by the Rhio CLI. 7 | /// 8 | /// The available commands are: 9 | /// 10 | /// - `CreatePrivateKeySecret`: Generates a secret containing a private key. 11 | /// - `CreateNatsSecret`: Generates a secret containing NATS credentials. 12 | /// - `CreateS3Secret`: Generates a secret containing S3 credentials. 13 | /// - `Framework`: Executes a command from the stackable operator framework, which includes: 14 | /// - `Crd`: prints CRD yaml. 15 | /// - `Run`: runs the operator. 16 | /// 17 | #[derive(clap::Parser)] 18 | pub enum RhioCommand { 19 | CreatePrivateKeySecret(PrivateKeySecretArgs), 20 | CreateNatsSecret(NatsSecretArgs), 21 | CreateS3Secret(S3SecretArgs), 22 | #[clap(flatten)] 23 | Framework(stackable_operator::cli::Command), 24 | } 25 | 26 | #[derive(clap::Parser, Debug, PartialEq, Eq)] 27 | #[command(long_about = "")] 28 | pub struct PrivateKeySecretArgs {} 29 | 30 | impl PrivateKeySecretArgs { 31 | pub fn generate_secret(&self) -> anyhow::Result<()> { 32 | let private_key = p2panda_core::PrivateKey::new(); 33 | let private_key = PrivateKey { 34 | secret_key: private_key.to_hex(), 35 | public_key: private_key.public_key().to_hex(), 36 | }; 37 | Secret::::new("private_key_secret".into(), "default".into(), private_key) 38 | .print_yaml()?; 39 | Ok(()) 40 | } 41 | } 42 | 43 | #[derive(clap::Parser, Debug, PartialEq, Eq)] 44 | #[command(long_about = "")] 45 | pub struct NatsSecretArgs { 46 | #[arg(long, short = 'u')] 47 | pub username: String, 48 | 49 | #[arg(long, short = 'p')] 50 | pub password: String, 51 | } 52 | 53 | impl NatsSecretArgs { 54 | pub fn generate_secret(&self) -> anyhow::Result<()> { 55 | let credentials = NatsCredentials { 56 | nkey: None, 57 | username: Some(self.username.to_owned()), 58 | password: Some(self.password.to_owned()), 59 | token: None, 60 | }; 61 | Secret::::new("nats_credentials".into(), "default".into(), credentials) 62 | .print_yaml()?; 63 | Ok(()) 64 | } 65 | } 66 | 67 | #[derive(clap::Parser, Debug, PartialEq, Eq)] 68 | #[command(long_about = "")] 69 | pub struct S3SecretArgs { 70 | #[arg(long, short = 'a')] 71 | pub access_key: String, 72 | 73 | #[arg(long, short = 's')] 74 | pub secret_key: String, 75 | } 76 | 77 | impl S3SecretArgs { 78 | pub fn generate_secret(&self) -> anyhow::Result<()> { 79 | let credentials = Credentials { 80 | access_key: Some(self.access_key.to_owned()), 81 | secret_key: Some(self.secret_key.to_owned()), 82 | security_token: None, 83 | session_token: None, 84 | expiration: None, 85 | }; 86 | Secret::::new("s3_credentials".into(), "default".into(), credentials) 87 | .print_yaml()?; 88 | Ok(()) 89 | } 90 | } 91 | 92 | #[derive(clap::Parser)] 93 | #[clap(about, author)] 94 | pub struct Opts { 95 | #[clap(subcommand)] 96 | pub cmd: RhioCommand, 97 | } 98 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "rhio", 5 | "rhio-blobs", 6 | "rhio-core", 7 | "s3-server", 8 | "rhio-operator", 9 | "rhio-config", 10 | "rhio-http-api", 11 | ] 12 | 13 | [workspace.lints.rust] 14 | 15 | [workspace.dependencies] 16 | anyhow = "1.0.100" 17 | axum = "0.8.6" 18 | axum-prometheus = "0.9.0" 19 | async-nats = "0.45.0" 20 | async-trait = "0.1.89" 21 | bytes = "1.10.1" 22 | ciborium = "0.2.2" 23 | hex = "0.4.3" 24 | clap = { version = "4.5.51", features = ["derive"] } 25 | p2panda-core = { git = "https://github.com/p2panda/p2panda.git", tag="v0.4.0" } 26 | p2panda-net = { git = "https://github.com/p2panda/p2panda.git", default-features = false, tag="v0.4.0" } 27 | p2panda-sync = { git = "https://github.com/p2panda/p2panda.git", features = ["cbor", "log-sync"], default-features = false, tag="v0.4.0" } 28 | p2panda-discovery = { git = "https://github.com/p2panda/p2panda.git", tag="v0.4.0" } 29 | p2panda-store = { git = "https://github.com/p2panda/p2panda.git", features=["memory"], tag="v0.4.0" } 30 | p2panda-blobs = { git = "https://github.com/p2panda/p2panda.git", tag="v0.4.0" } 31 | iroh-io = { version = "0.6.2", features = ["x-http"] } 32 | iroh-base = "0.34.1" 33 | iroh-blobs = "0.34.1" 34 | rhio-blobs = { path = "rhio-blobs" } 35 | rhio-core = { path = "rhio-core" } 36 | rhio-config = { path = "rhio-config" } 37 | rhio-http-api = { path = "rhio-http-api" } 38 | s3-server = { path = "s3-server" } 39 | serde = { version = "1.0.215", features = ["derive"] } 40 | serde_json = "1.0.145" 41 | serde_yaml = "0.9.34-deprecated" 42 | futures-lite = "2.6.1" 43 | futures-util = "0.3.31" 44 | rust-s3 = { version = "0.37.0", features = ["tokio", "blocking"] } 45 | thiserror = "2.0.17" 46 | tokio = { version = "1.48.0", features = ["full"] } 47 | tokio-stream = "0.1.17" 48 | tokio-util = "0.7.17" 49 | url = "2.5.7" 50 | directories = "6.0.0" 51 | figment = { version = "0.10.19", features = ["env", "yaml"] } 52 | rand = "0.9.2" 53 | loole = "0.4.1" 54 | tracing = "0.1.41" 55 | tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } 56 | tempfile = "3.23.0" 57 | rusty-hook = "^0.11.2" 58 | once_cell = "1.21.3" 59 | dashmap = "6.1.0" 60 | futures = { version = "0.3.31", default-features = false, features = ["std"] } 61 | pin-project = "1.1.10" 62 | s3s-fs = { version = "0.11.1" } 63 | s3s = { version = "0.11.1" } 64 | hyper-util = { version = "0.1.17", features = [ 65 | "server", 66 | "http1", 67 | "http2", 68 | "tokio", 69 | ] } 70 | built = { version = "0.7", features = ["chrono", "git2"] } 71 | stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.100.3" } 72 | product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } 73 | 74 | snafu = { version = "0.8.9" } 75 | strum = { version = "0.27.2", features = ["derive"] } 76 | reqwest = { version = "0.12.24", features = ["json"] } 77 | pin-project-lite = { version = "0.2.16" } 78 | 79 | # pinned version of rusttls due to iroh-net and stackable-operator/kube-rs 80 | rustls = { version = "0.23.35" } 81 | chrono = { version = "0.4.42" } 82 | 83 | # pinned version due to product-config 84 | schemars = { version = "1.0.0" } 85 | 86 | # pinned version due to vulnerabilities 87 | openssl = "=0.10.74" 88 | slab = "=0.4.11" 89 | crossbeam-channel = "=0.5.15" 90 | ring = "=0.17.8" 91 | quinn-proto = "=0.11.7" -------------------------------------------------------------------------------- /rhio/src/tests/http_api.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | 3 | use super::fake_rhio_server::FakeRhioServer; 4 | use crate::{ 5 | tests::configuration::{generate_nats_config, generate_rhio_config}, 6 | tracing::setup_tracing, 7 | }; 8 | use anyhow::{Context, Result}; 9 | use p2panda_core::PrivateKey; 10 | use rhio_http_api::{ 11 | blocking::BlockingClient, 12 | client::RhioApiClient, 13 | status::{HealthStatus, ServiceStatus}, 14 | }; 15 | use tokio::runtime::Builder; 16 | use tracing::info; 17 | 18 | #[test] 19 | pub fn test_health_status() -> Result<()> { 20 | setup_tracing(Some("=INFO".into())); 21 | 22 | let SingleServerSetup { rhio, http_api } = create_setup()?; 23 | 24 | let status = http_api.health()?; 25 | assert_eq!( 26 | HealthStatus { 27 | status: ServiceStatus::Running, 28 | msg: None, 29 | ..HealthStatus::default() 30 | }, 31 | status 32 | ); 33 | 34 | rhio.discard()?; 35 | Ok(()) 36 | } 37 | 38 | #[test] 39 | pub fn test_metrics() -> Result<()> { 40 | setup_tracing(Some("=INFO".into())); 41 | 42 | let SingleServerSetup { rhio, http_api } = create_setup()?; 43 | 44 | let metrics = http_api.metrics()?; 45 | assert_eq!("", metrics); 46 | 47 | rhio.discard()?; 48 | Ok(()) 49 | } 50 | 51 | struct SingleServerSetup { 52 | rhio: FakeRhioServer, 53 | http_api: BlockingClient, 54 | } 55 | 56 | /// Creates a setup for testing with a single server instance. 57 | /// 58 | /// This function generates the necessary configurations for NATS and Rhio, 59 | /// initializes a Tokio runtime, and starts a fake Rhio server. It also 60 | /// creates an HTTP API client for interacting with the Rhio server. 61 | /// 62 | /// # Returns 63 | /// 64 | /// A `Result` containing a `SingleServerSetup` struct with the initialized 65 | /// Rhio server and HTTP API client, or an error if the setup fails. 66 | /// 67 | /// # Errors 68 | /// 69 | /// This function will return an error if the Rhio server fails to start or 70 | /// if there is an issue with the configurations. 71 | /// 72 | /// # Example 73 | /// 74 | /// ```rust 75 | /// let setup = create_setup()?; 76 | /// ``` 77 | fn create_setup() -> Result { 78 | let nats_config = generate_nats_config(); 79 | info!("nats config {:?}", nats_config); 80 | 81 | let rhio_config = generate_rhio_config(&nats_config, &None); 82 | let rhio_private_key = PrivateKey::new(); 83 | 84 | info!("rhio config {:?} ", rhio_config.node); 85 | 86 | let test_runtime = Arc::new( 87 | Builder::new_multi_thread() 88 | .enable_io() 89 | .enable_time() 90 | .thread_name("test-runtime") 91 | .worker_threads(2) 92 | .build() 93 | .expect("test tokio runtime"), 94 | ); 95 | 96 | let http_api = BlockingClient::new( 97 | RhioApiClient::new(format!( 98 | "http://127.0.0.1:{}", 99 | rhio_config.node.http_bind_port 100 | )), 101 | test_runtime, 102 | ); 103 | let rhio = FakeRhioServer::try_start(rhio_config.clone(), rhio_private_key.clone()) 104 | .context("RhioServer")?; 105 | 106 | let setup = SingleServerSetup { rhio, http_api }; 107 | 108 | Ok(setup) 109 | } 110 | -------------------------------------------------------------------------------- /rhio/src/nats/client/types.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Result, anyhow}; 2 | use async_nats::HeaderMap; 3 | use async_nats::Message; 4 | use async_nats::jetstream::consumer::{DeliverPolicy, Info}; 5 | use async_trait::async_trait; 6 | use bytes::Bytes; 7 | use futures::Stream; 8 | use rhio_core::Subject; 9 | 10 | use crate::StreamName; 11 | 12 | /// Represents a stream of NATS JetStream messages. 13 | /// 14 | /// This trait is used to define a stream of messages that can be consumed from a NATS JetStream consumer. 15 | /// The stream yields `Result` items. 16 | pub trait NatsMessageStream: Stream + Sized {} 17 | 18 | /// A trait for a NATS client that interacts with NATS JetStream. 19 | /// 20 | /// This trait defines the necessary methods for creating a consumer stream and publishing messages to NATS. 21 | /// 22 | /// # Type Parameters 23 | /// 24 | /// * `M` - A type that implements the `NatsMessageStream` trait. 25 | /// 26 | /// # Methods 27 | /// 28 | /// * `create_consumer_stream` - Creates a consumer stream for a given stream name, filter subjects, and delivery policy. 29 | /// * `publish` - Publishes a message to a given subject with an optional payload and headers. 30 | #[async_trait] 31 | pub trait NatsClient: Sized { 32 | /// Creates a consumer stream for a given stream name, filter subjects, and delivery policy. 33 | /// 34 | /// # Arguments 35 | /// 36 | /// * `stream_name` - The name of the stream to consume from. 37 | /// * `filter_subjects` - A vector of subjects to filter the messages. 38 | /// * `deliver_policy` - The delivery policy for the consumer. 39 | /// 40 | /// # Returns 41 | /// 42 | /// A tuple containing the consumer stream and its information. 43 | async fn create_consumer_stream( 44 | &self, 45 | consumer_name: String, 46 | stream_name: StreamName, 47 | filter_subjects: Vec, 48 | deliver_policy: DeliverPolicy, 49 | ) -> Result<(M, Info)>; 50 | 51 | /// Publishes a message to a given subject with an optional payload and headers. 52 | /// 53 | /// # Arguments 54 | /// 55 | /// * `wait_for_ack` - Whether to wait for an acknowledgment from the server. 56 | /// * `subject` - The subject to publish the message to. 57 | /// * `payload` - The payload of the message. 58 | /// * `headers` - Optional headers to include with the message. 59 | /// 60 | /// # Returns 61 | /// 62 | /// A result indicating success or failure. 63 | async fn publish( 64 | &self, 65 | wait_for_ack: bool, 66 | subject: String, 67 | payload: Bytes, 68 | headers: Option, 69 | ) -> Result<()>; 70 | } 71 | 72 | #[derive(Clone, Debug)] 73 | pub enum NatsStreamProtocol { 74 | Msg { msg: Message, seq: Option }, 75 | Error { msg: String }, 76 | ServerDisconnect, 77 | } 78 | 79 | impl From for Result { 80 | fn from(value: NatsStreamProtocol) -> Self { 81 | match value { 82 | NatsStreamProtocol::Msg { msg, .. } => Ok(msg), 83 | NatsStreamProtocol::ServerDisconnect => { 84 | Err(anyhow!("NatsStreamProtocol::ServerDisconnect")) 85 | } 86 | NatsStreamProtocol::Error { msg } => Err(anyhow!("NatsStreamProtocol::Error {msg}")), 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /rhio/src/tests/service_configuration.rs: -------------------------------------------------------------------------------- 1 | use std::{sync::Arc, time::Duration}; 2 | 3 | use crate::{ 4 | nats::client::fake::{ 5 | blocking::BlockingClient, 6 | client::{FakeNatsClient, FakeNatsMessages}, 7 | server::FakeNatsServer, 8 | }, 9 | tests::{ 10 | configuration::{configure_message_publisher, generate_nats_config, generate_rhio_config}, 11 | fake_rhio_server::FakeRhioServer, 12 | }, 13 | tracing::setup_tracing, 14 | }; 15 | use anyhow::{Context, Result}; 16 | use p2panda_core::PrivateKey; 17 | use rhio_config::configuration::Config; 18 | use tokio::runtime::Builder; 19 | use tracing::info; 20 | 21 | #[test] 22 | pub fn test_start_rhio_if_nats_not_available() -> Result<()> { 23 | setup_tracing(Some("=INFO".into())); 24 | 25 | let SingleNodeMessagingSetup { 26 | rhio_config, 27 | rhio_private_key, 28 | nats_server, 29 | test_runtime, 30 | .. 31 | } = create_single_node_messaging_setup()?; 32 | 33 | nats_server.enable_connection_error(); 34 | 35 | let rhio = FakeRhioServer::try_start(rhio_config.clone(), rhio_private_key.clone()) 36 | .context("Source RhioServer")?; 37 | 38 | test_runtime 39 | .block_on(async { 40 | nats_server 41 | .wait_for_connection_error(Duration::from_secs(10)) 42 | .await 43 | }) 44 | .context("waiting for connection errors")?; 45 | nats_server.disable_connection_error(); 46 | 47 | test_runtime 48 | .block_on(async { 49 | nats_server 50 | .wait_for_connections(Duration::from_secs(5)) 51 | .await 52 | }) 53 | .context("waiting rhio reconnecting")?; 54 | 55 | rhio.discard()?; 56 | 57 | Ok(()) 58 | } 59 | 60 | struct SingleNodeMessagingSetup { 61 | pub(crate) rhio_config: Config, 62 | pub(crate) rhio_private_key: PrivateKey, 63 | 64 | #[allow(dead_code)] 65 | pub(crate) nats_client: BlockingClient, 66 | pub(crate) nats_server: Arc, 67 | 68 | pub(crate) test_runtime: Arc, 69 | } 70 | 71 | fn create_single_node_messaging_setup() -> Result { 72 | let nats_config = generate_nats_config(); 73 | info!("nats config {:?}", nats_config); 74 | 75 | let mut rhio_config = generate_rhio_config(&nats_config, &None); 76 | let rhio_private_key = PrivateKey::new(); 77 | 78 | info!("rhio source config {:?} ", rhio_config.node); 79 | 80 | configure_message_publisher(&mut rhio_config, "stream", "subject"); 81 | 82 | let test_runtime = Arc::new( 83 | Builder::new_multi_thread() 84 | .enable_io() 85 | .enable_time() 86 | .thread_name("test-runtime") 87 | .worker_threads(5) 88 | .build() 89 | .expect("test tokio runtime"), 90 | ); 91 | 92 | let nats_client = BlockingClient::new( 93 | FakeNatsClient::new(rhio_config.nats.clone()).context("Source FakeNatsClient")?, 94 | test_runtime.clone(), 95 | ); 96 | 97 | let nats_server = 98 | FakeNatsServer::get_by_config(&nats_config).context("no fake NATS server exists")?; 99 | 100 | let setup = SingleNodeMessagingSetup { 101 | nats_server, 102 | nats_client, 103 | rhio_config, 104 | rhio_private_key, 105 | test_runtime, 106 | }; 107 | 108 | Ok(setup) 109 | } 110 | -------------------------------------------------------------------------------- /charts/rhio-operator/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "operator.fullname" . }}-deployment 6 | labels: 7 | {{- include "operator.labels" . | nindent 4 }} 8 | spec: 9 | replicas: 1 10 | strategy: 11 | type: Recreate 12 | selector: 13 | matchLabels: 14 | {{- include "operator.selectorLabels" . | nindent 6 }} 15 | template: 16 | metadata: 17 | annotations: 18 | internal.stackable.tech/image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 19 | {{- with .Values.podAnnotations }} 20 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} 21 | {{- toYaml . | nindent 8 }} 22 | {{- end }} 23 | labels: 24 | {{- include "operator.selectorLabels" . | nindent 8 }} 25 | spec: 26 | {{- with .Values.image.pullSecrets }} 27 | imagePullSecrets: 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | serviceAccountName: {{ include "operator.fullname" . }}-serviceaccount 31 | securityContext: 32 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 33 | containers: 34 | - name: {{ include "operator.appname" . }} 35 | securityContext: 36 | {{- toYaml .Values.securityContext | nindent 12 }} 37 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 38 | imagePullPolicy: {{ .Values.image.pullPolicy }} 39 | args: 40 | - /usr/local/bin/rhio-operator 41 | - run 42 | resources: 43 | {{- toYaml .Values.resources | nindent 12 }} 44 | volumeMounts: 45 | - mountPath: /etc/hiro/{{ include "operator.appname" . }}/config-spec 46 | name: config-spec 47 | env: 48 | - name: OPERATOR_IMAGE 49 | # Tilt can use annotations as image paths, but not env variables 50 | valueFrom: 51 | fieldRef: 52 | fieldPath: metadata.annotations['internal.stackable.tech/image'] 53 | # Namespace the operator Pod is running in, e.g. used to construct the conversion 54 | # webhook endpoint. 55 | - name: OPERATOR_NAMESPACE 56 | valueFrom: 57 | fieldRef: 58 | fieldPath: metadata.namespace 59 | 60 | # The name of the Kubernetes Service that point to the operator Pod, e.g. used to 61 | # construct the conversion webhook endpoint. 62 | - name: OPERATOR_SERVICE_NAME 63 | value: {{ include "operator.fullname" . }} 64 | 65 | # Operators need to know the node name they are running on, to e.g. discover the 66 | # Kubernetes domain name from the kubelet API. 67 | - name: KUBERNETES_NODE_NAME 68 | valueFrom: 69 | fieldRef: 70 | fieldPath: spec.nodeName 71 | {{- if .Values.kubernetesClusterDomain }} 72 | - name: KUBERNETES_CLUSTER_DOMAIN 73 | value: {{ .Values.kubernetesClusterDomain | quote }} 74 | {{- end }} 75 | volumes: 76 | - name: config-spec 77 | configMap: 78 | name: {{ include "operator.fullname" . }}-configmap 79 | {{- with .Values.nodeSelector }} 80 | nodeSelector: 81 | {{- toYaml . | nindent 8 }} 82 | {{- end }} 83 | {{- with .Values.affinity }} 84 | affinity: 85 | {{- toYaml . | nindent 8 }} 86 | {{- end }} 87 | {{- with .Values.tolerations }} 88 | tolerations: 89 | {{- toYaml . | nindent 8 }} 90 | {{- end }} 91 | -------------------------------------------------------------------------------- /rhio-core/src/nats.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use async_nats::{HeaderMap, Message as NatsMessage}; 3 | use p2panda_core::{PrivateKey, PublicKey, Signature}; 4 | 5 | use crate::NetworkMessage; 6 | 7 | /// Custom NATS header used by rhio to indicate that message was authored by this public key. 8 | pub const NATS_RHIO_PUBLIC_KEY: &str = "X-Rhio-PublicKey"; 9 | 10 | /// Custom NATS header used by rhio to cryptographically authenticate this message. 11 | pub const NATS_RHIO_SIGNATURE: &str = "X-Rhio-Signature"; 12 | 13 | pub fn has_nats_signature(headers: &Option) -> bool { 14 | if let Some(headers) = &headers 15 | && (headers.get(NATS_RHIO_SIGNATURE).is_some() 16 | || headers.get(NATS_RHIO_PUBLIC_KEY).is_some()) 17 | { 18 | return true; 19 | } 20 | false 21 | } 22 | 23 | /// This method checks if the public key given in the NATS header matches. 24 | /// 25 | /// Important: This method does _not_ do any integrity or cryptography checks. We assume that this 26 | /// method is used in contexts _after_ all validation took already place. 27 | pub fn matching_public_key(message: &NatsMessage, expected_public_key: &PublicKey) -> bool { 28 | if let Some(headers) = &message.headers { 29 | let Some(_) = headers.get(NATS_RHIO_SIGNATURE) else { 30 | return false; 31 | }; 32 | 33 | let Some(given_public_key) = headers.get(NATS_RHIO_PUBLIC_KEY) else { 34 | return false; 35 | }; 36 | 37 | return given_public_key.to_string() == expected_public_key.to_string(); 38 | } 39 | 40 | false 41 | } 42 | 43 | /// Add signature and public key as custom rhio NATS headers to message if they do not exist yet. 44 | pub fn add_custom_nats_headers( 45 | previous_header: &Option, 46 | signature: Signature, 47 | public_key: PublicKey, 48 | ) -> HeaderMap { 49 | if has_nats_signature(previous_header) { 50 | return previous_header 51 | .to_owned() 52 | .expect("at this point we know there's a header"); 53 | } 54 | 55 | let mut headers = match &previous_header { 56 | Some(headers) => headers.clone(), 57 | None => HeaderMap::new(), 58 | }; 59 | headers.insert(NATS_RHIO_SIGNATURE, signature.to_string()); 60 | headers.insert(NATS_RHIO_PUBLIC_KEY, public_key.to_string()); 61 | headers 62 | } 63 | 64 | /// Remove potentially existing custom rhio NATS headers from a message. 65 | pub fn remove_custom_nats_headers(previous_header: &HeaderMap) -> Option { 66 | let mut header = HeaderMap::new(); 67 | for (key, values) in previous_header.iter() { 68 | if key.to_string() != NATS_RHIO_PUBLIC_KEY && key.to_string() != NATS_RHIO_SIGNATURE { 69 | for value in values { 70 | header.append(key.to_owned(), value.to_owned()); 71 | } 72 | } 73 | } 74 | 75 | if header.is_empty() { 76 | None 77 | } else { 78 | Some(header) 79 | } 80 | } 81 | 82 | /// Helper method automatically signing and wrapping NATS messages in a network message in case 83 | /// they are not yet signed. Already signed messages (probably from other authors) just get 84 | /// wrapped. 85 | pub fn wrap_and_sign_nats_message( 86 | message: NatsMessage, 87 | private_key: &PrivateKey, 88 | ) -> Result { 89 | // Wrap already signed NATS messages in a network message. 90 | if has_nats_signature(&message.headers) { 91 | return NetworkMessage::new_signed_nats(message); 92 | } 93 | 94 | // Sign them with our private key in case they are not. 95 | let mut network_message = NetworkMessage::new_nats(message, &private_key.public_key()); 96 | network_message.sign(private_key); 97 | Ok(network_message) 98 | } 99 | -------------------------------------------------------------------------------- /charts/rhio-operator/templates/roles.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: {{ include "operator.fullname" . }}-clusterrole 6 | labels: 7 | {{- include "operator.labels" . | nindent 4 }} 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - nodes 13 | verbs: 14 | - list 15 | - watch 16 | # For automatic cluster domain detection 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - nodes/proxy 21 | verbs: 22 | - get 23 | - apiGroups: 24 | - "" 25 | resources: 26 | - pods 27 | - configmaps 28 | - secrets 29 | - services 30 | - endpoints 31 | - serviceaccounts 32 | verbs: 33 | - create 34 | - delete 35 | - get 36 | - list 37 | - patch 38 | - update 39 | - watch 40 | - apiGroups: 41 | - rbac.authorization.k8s.io 42 | resources: 43 | - rolebindings 44 | verbs: 45 | - create 46 | - delete 47 | - get 48 | - list 49 | - patch 50 | - update 51 | - watch 52 | - apiGroups: 53 | - rbac.authorization.k8s.io 54 | resources: 55 | - clusterroles 56 | verbs: 57 | - bind 58 | resourceNames: 59 | - {{ include "operator.name" . }}-clusterrole 60 | - apiGroups: 61 | - apps 62 | resources: 63 | - statefulsets 64 | verbs: 65 | - get 66 | - create 67 | - delete 68 | - list 69 | - patch 70 | - update 71 | - watch 72 | - apiGroups: 73 | - batch 74 | resources: 75 | - jobs 76 | verbs: 77 | - create 78 | - delete 79 | - get 80 | - list 81 | - patch 82 | - update 83 | - watch 84 | - apiGroups: 85 | - policy 86 | resources: 87 | - poddisruptionbudgets 88 | verbs: 89 | - create 90 | - delete 91 | - get 92 | - list 93 | - patch 94 | - update 95 | - watch 96 | - apiGroups: 97 | - apiextensions.k8s.io 98 | resources: 99 | - customresourcedefinitions 100 | verbs: 101 | - get 102 | - apiGroups: 103 | - events.k8s.io 104 | resources: 105 | - events 106 | verbs: 107 | - create 108 | - patch 109 | - apiGroups: 110 | - rhio.hiro.io 111 | resources: 112 | - rhioservices 113 | - replicatedmessagestreams 114 | - replicatedmessagestreamsubscriptions 115 | - replicatedobjectstores 116 | - replicatedobjectstoresubscriptions 117 | verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] 118 | - apiGroups: 119 | - rhio.hiro.io 120 | resources: 121 | - rhioservices/status 122 | - replicatedmessagestreams/status 123 | - replicatedmessagestreamsubscriptions/status 124 | - replicatedobjectstores/status 125 | - replicatedobjectstoresubscriptions/status 126 | verbs: 127 | - patch 128 | --- 129 | apiVersion: rbac.authorization.k8s.io/v1 130 | kind: ClusterRole 131 | metadata: 132 | name: {{ include "operator.name" . }}-clusterrole 133 | labels: 134 | {{- include "operator.labels" . | nindent 4 }} 135 | rules: 136 | - apiGroups: 137 | - "" 138 | resources: 139 | - configmaps 140 | - secrets 141 | - serviceaccounts 142 | verbs: 143 | - get 144 | - apiGroups: 145 | - events.k8s.io 146 | resources: 147 | - events 148 | verbs: 149 | - create 150 | {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} 151 | - apiGroups: 152 | - security.openshift.io 153 | resources: 154 | - securitycontextconstraints 155 | resourceNames: 156 | - nonroot-v2 157 | verbs: 158 | - use 159 | {{ end }} 160 | -------------------------------------------------------------------------------- /rhio/src/utils/retry/types.rs: -------------------------------------------------------------------------------- 1 | use futures::Stream; 2 | use futures::TryFuture; 3 | use std::time::Duration; 4 | 5 | /// Stream Item sequence number 6 | pub type SeqNo = u64; 7 | 8 | /// A trait for creating streams with optional sequence number starting points. 9 | /// 10 | /// The `StreamFactory` trait provides an abstraction for creating streams that can start 11 | /// from a specific sequence number. It is designed to be generic and flexible, allowing 12 | /// implementers to define the types of streams, items, and errors they work with. 13 | /// 14 | /// # Associated Types 15 | /// 16 | /// * `T` - The type of the items produced by the stream. 17 | /// * `ErrorT` - The type of the error that can occur during stream creation. 18 | /// * `StreamT` - The type of the stream to be created, which must implement the `Stream` trait 19 | /// and produce items of type `T`. 20 | /// * `Fut` - The type of the future returned by the `create` method, which must implement the 21 | /// `TryFuture` trait and resolve to a `Result` containing the created stream or an error. 22 | /// 23 | /// # Required Methods 24 | /// 25 | /// * `create` - Creates a new stream, optionally starting from the given sequence number. 26 | /// This method returns a future that resolves to the created stream or an error. 27 | /// 28 | pub trait StreamFactory { 29 | type T; 30 | type ErrorT; 31 | type StreamT: Stream; 32 | type Fut: TryFuture< 33 | Ok = Self::StreamT, 34 | Error = Self::ErrorT, 35 | Output = Result, 36 | >; 37 | 38 | fn create(&self, seq_no: Option) -> Self::Fut; 39 | } 40 | 41 | /// 42 | /// This enum represents the different strategies that can be used when a stream error occurs. 43 | /// It provides two variants: 44 | /// 45 | /// * `Forward(T)` - This variant indicates that the message should be forwarded as-is. 46 | /// * `WaitRetry(Duration, Option)` - This variant indicates that the system should wait for a specified duration 47 | /// before making another attempt to recreate the stream, optionally starting from a given sequence number. 48 | /// 49 | /// # Variants 50 | /// 51 | /// * `Forward(T)` - Forward the message. 52 | /// * `WaitRetry(Duration, Option)` - Wait for the specified duration and retry, optionally starting from the given sequence number. 53 | /// 54 | /// # Type Parameters 55 | /// 56 | /// * `T` - The type of the message to be forwarded. 57 | /// 58 | #[derive(Debug, Eq, PartialEq)] 59 | pub enum RetryPolicy { 60 | /// Message Forwarding 61 | Forward(T), 62 | /// Wait for a given duration and make another attempt then starting with a sequence number. 63 | WaitRetry(Duration, Option), 64 | } 65 | 66 | /// This trait defines methods for handling stream messages and factory errors. 67 | /// 68 | /// # Type Parameters 69 | /// 70 | /// * `T` - The type of the protocol message to be handled. 71 | /// * `FactoryError` - The type of the error that can occur during stream creation. 72 | /// 73 | /// # Associated Types 74 | /// 75 | /// * `Out` - The type of the output produced by the error handler. 76 | /// 77 | /// # Methods 78 | /// 79 | /// * `on_stream_msg` - Handles a stream message and returns a `RetryPolicy` indicating the action to be taken. 80 | /// * `on_factory_error` - Handles a factory error and returns a `RetryPolicy` indicating the action to be taken. 81 | /// 82 | /// # Methods 83 | /// 84 | /// * `on_stream_msg` - Handles a stream message and returns a `RetryPolicy` indicating the action to be taken. 85 | /// * `on_factory_error` - Handles a factory error and returns a `RetryPolicy` indicating the action to be taken. 86 | /// 87 | pub trait ErrorHandler { 88 | type Out; 89 | 90 | fn on_stream_msg(&mut self, protocol: T) -> RetryPolicy; 91 | 92 | fn on_factory_error(&mut self, attempt: usize, error: FactoryError) -> RetryPolicy; 93 | } 94 | -------------------------------------------------------------------------------- /rhio/src/nats/mod.rs: -------------------------------------------------------------------------------- 1 | mod actor; 2 | pub mod client; 3 | mod consumer; 4 | use anyhow::Result; 5 | use async_nats::HeaderMap; 6 | use async_nats::jetstream::consumer::DeliverPolicy; 7 | use client::types::{NatsClient, NatsMessageStream}; 8 | use futures_util::future::{MapErr, Shared}; 9 | use futures_util::{FutureExt, TryFutureExt}; 10 | use rhio_core::Subject; 11 | use tokio::sync::{mpsc, oneshot}; 12 | use tokio::task::JoinError; 13 | use tokio_util::task::AbortOnDropHandle; 14 | use tracing::error; 15 | 16 | use crate::JoinErrToStr; 17 | use crate::nats::actor::{NatsActor, ToNatsActor}; 18 | pub use crate::nats::consumer::{ConsumerId, JetStreamEvent, StreamName}; 19 | 20 | #[derive(Clone, Debug)] 21 | pub struct Nats { 22 | nats_actor_tx: mpsc::Sender, 23 | #[allow(dead_code)] 24 | actor_handle: Shared, JoinErrToStr>>, 25 | } 26 | 27 | impl Nats { 28 | pub async fn new(client: N) -> Result 29 | where 30 | M: NatsMessageStream + Send + Sync + Unpin + 'static, 31 | N: NatsClient + 'static + Send + Sync, 32 | { 33 | // Start the main NATS JetStream actor to dynamically maintain "stream consumers". 34 | let (nats_actor_tx, nats_actor_rx) = mpsc::channel(512); 35 | let nats_actor = NatsActor::new(client, nats_actor_rx); 36 | 37 | let actor_handle = tokio::task::spawn(async move { 38 | if let Err(err) = nats_actor.run().await { 39 | error!("engine actor failed: {err:?}"); 40 | } 41 | }); 42 | 43 | let actor_drop_handle = AbortOnDropHandle::new(actor_handle) 44 | .map_err(Box::new(|e: JoinError| e.to_string()) as JoinErrToStr) 45 | .shared(); 46 | 47 | Ok(Self { 48 | nats_actor_tx, 49 | actor_handle: actor_drop_handle, 50 | }) 51 | } 52 | 53 | /// Subscribes to a NATS Jetstream "subject" by creating a consumer hooking into a stream 54 | /// provided by the NATS server. 55 | /// 56 | /// All consumers in rhio are push-based, ephemeral and do not ack when a message was received. 57 | /// With this design we can download any past messages from the stream at any point. 58 | /// 59 | /// This method creates a consumer and fails if something goes wrong. It proceeds with 60 | /// downloading all past data from the server when configured like that via a "delivery 61 | /// policy"; the returned channel can be used to await when that download has been finished. 62 | /// Finally it keeps the consumer alive in the background for handling future messages. All 63 | /// past and future messages are sent to the returned stream. 64 | pub async fn subscribe( 65 | &self, 66 | stream_name: StreamName, 67 | subjects: Vec, 68 | deliver_policy: DeliverPolicy, 69 | topic_id: [u8; 32], 70 | ) -> Result<(ConsumerId, loole::Receiver)> { 71 | let (reply, reply_rx) = oneshot::channel(); 72 | self.nats_actor_tx 73 | .send(ToNatsActor::Subscribe { 74 | stream_name, 75 | subjects, 76 | deliver_policy, 77 | topic_id, 78 | reply, 79 | }) 80 | .await?; 81 | reply_rx.await? 82 | } 83 | 84 | pub async fn unsubscribe(&self, consumer_id: ConsumerId) -> Result<()> { 85 | self.nats_actor_tx 86 | .send(ToNatsActor::Unsubscribe { consumer_id }) 87 | .await?; 88 | Ok(()) 89 | } 90 | 91 | pub async fn publish( 92 | &self, 93 | wait_for_ack: bool, 94 | subject: String, 95 | headers: Option, 96 | payload: Vec, 97 | ) -> Result<()> { 98 | let (reply, reply_rx) = oneshot::channel(); 99 | self.nats_actor_tx 100 | .send(ToNatsActor::Publish { 101 | wait_for_ack, 102 | subject, 103 | headers, 104 | payload, 105 | reply, 106 | }) 107 | .await?; 108 | reply_rx.await? 109 | } 110 | 111 | pub async fn shutdown(&self) -> Result<()> { 112 | let (reply, reply_rx) = oneshot::channel(); 113 | self.nats_actor_tx 114 | .send(ToNatsActor::Shutdown { reply }) 115 | .await?; 116 | reply_rx.await?; 117 | Ok(()) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /rhio/src/blobs/mod.rs: -------------------------------------------------------------------------------- 1 | mod actor; 2 | mod proxy; 3 | pub mod watcher; 4 | 5 | use std::collections::HashMap; 6 | use std::time::Duration; 7 | 8 | use anyhow::{Context, Result, anyhow}; 9 | use p2panda_blobs::{Blobs as BlobsHandler, Config as BlobsConfig}; 10 | use proxy::BlobsActorProxy; 11 | use rhio_blobs::{NotImportedObject, S3Store, SignedBlobInfo}; 12 | use s3::{Bucket, Region}; 13 | use tokio::sync::mpsc; 14 | use watcher::{S3Event, S3WatcherOptions}; 15 | 16 | use crate::topic::Query; 17 | use rhio_config::configuration::Config; 18 | 19 | use crate::blobs::watcher::S3Watcher; 20 | use s3::error::S3Error; 21 | 22 | #[derive(Debug)] 23 | pub struct Blobs { 24 | blobs: BlobsActorProxy, 25 | #[allow(dead_code)] 26 | watcher: S3Watcher, 27 | } 28 | 29 | impl Blobs { 30 | pub fn new( 31 | blob_store: S3Store, 32 | blobs_handler: BlobsHandler, 33 | watcher_tx: mpsc::Sender>, 34 | options: S3WatcherOptions, 35 | ) -> Blobs { 36 | let blobs = BlobsActorProxy::new(blob_store.clone(), blobs_handler); 37 | let watcher = S3Watcher::new(blob_store, watcher_tx, options); 38 | Blobs { blobs, watcher } 39 | } 40 | 41 | /// Download a blob from the network. 42 | /// 43 | /// Attempt to download a blob from peers on the network and place it into the nodes MinIO 44 | /// bucket. 45 | pub async fn download(&self, blob: SignedBlobInfo) -> Result<()> { 46 | self.blobs.download(blob).await 47 | } 48 | 49 | /// Import an existing, local S3 object into the blob store, preparing it for p2p sync. 50 | pub async fn import_s3_object(&self, object: NotImportedObject) -> Result<()> { 51 | self.blobs.import_s3_object(object).await 52 | } 53 | 54 | pub async fn shutdown(&self) -> Result<()> { 55 | self.blobs.shutdown().await 56 | } 57 | } 58 | 59 | pub fn blobs_config() -> BlobsConfig { 60 | BlobsConfig { 61 | // Max. number of nodes we connect to for blob download. 62 | // 63 | // @TODO: This needs to be set to 1 as we're currently not allowed to write bytes 64 | // out-of-order. See comment in `s3_file.rs` in `rhio-blobs` for more details. 65 | max_concurrent_dials_per_hash: 1, 66 | initial_retry_delay: Duration::from_secs(10), 67 | ..Default::default() 68 | } 69 | } 70 | 71 | /// Initiates and returns a blob store for S3 buckets based on the rhio config. 72 | /// 73 | /// This method fails when we couldn't connect to the S3 buckets due to invalid configuration 74 | /// values, authentication or connection errors. 75 | pub fn store_from_config(config: &Config) -> Result { 76 | if let Some(s3_config) = config.s3.as_ref() { 77 | let mut buckets: HashMap = HashMap::new(); 78 | 79 | let credentials = s3_config 80 | .credentials 81 | .as_ref() 82 | .ok_or(anyhow!("s3 credentials are not set")) 83 | .context("reading s3 credentials from config")?; 84 | let region: Region = Region::Custom { 85 | region: s3_config.region.clone(), 86 | endpoint: s3_config.endpoint.clone(), 87 | }; 88 | 89 | // Merge all buckets mentioned in the regarding publish and subscribe config sections and 90 | // de-duplicate them. On this level we want them to be all handled by the same interface. 91 | if let Some(publish) = &config.publish { 92 | for bucket_name in &publish.s3_buckets { 93 | let bucket = Bucket::new(bucket_name, region.clone(), credentials.clone())? 94 | .with_path_style(); 95 | buckets.insert(bucket_name.clone(), *bucket); 96 | } 97 | } 98 | 99 | if let Some(subscribe) = &config.subscribe { 100 | for remote_bucket in &subscribe.s3_buckets { 101 | let bucket = Bucket::new( 102 | &remote_bucket.local_bucket_name, 103 | region.clone(), 104 | credentials.clone(), 105 | )? 106 | .with_path_style(); 107 | buckets.insert(remote_bucket.local_bucket_name.clone(), *bucket); 108 | } 109 | } 110 | 111 | let buckets: Vec = buckets.values().cloned().collect(); 112 | let store = S3Store::new(buckets); 113 | Ok(store) 114 | } else { 115 | Ok(S3Store::empty()) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /rhio/src/network/membership.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, time::Duration}; 2 | 3 | use anyhow::Result; 4 | use iroh_base::{NodeAddr, NodeId}; 5 | use loole::Receiver; 6 | use p2panda_core::PublicKey; 7 | use p2panda_discovery::DiscoveryEvent; 8 | use p2panda_discovery::{BoxedStream, Discovery}; 9 | use rhio_config::configuration::{DiscoveryOptions, KnownNode}; 10 | use tokio_util::task::AbortOnDropHandle; 11 | use tracing::trace; 12 | 13 | type SubscribeReceiver = Receiver>; 14 | 15 | struct NodeInfo { 16 | pub addresses: Vec, 17 | pub addr: Option, 18 | } 19 | 20 | /// This is a simple implementation of peer discovery built specifically to workaround cases 21 | /// where peers may not be running or being redeployed in kubernetes. 22 | /// In those cases the DNS will not be able to resolve IP addresses of peers, 23 | /// because kubernetes services will not be available and therefore FQDN records will not exist. 24 | /// 25 | /// # Responsibilities 26 | /// - Maintains a list of known peers and their associated information, such as 27 | /// addresses and resolved `NodeAddr`. 28 | /// - Periodically performs peer discovery by resolving addresses of known peers 29 | /// and updating their `NodeAddr` if changes are detected. 30 | /// - Sends discovery events to subscribers when new peers are discovered. 31 | /// 32 | #[derive(Debug)] 33 | pub struct Membership { 34 | #[allow(dead_code)] 35 | handle: AbortOnDropHandle<()>, 36 | rx: SubscribeReceiver, 37 | } 38 | 39 | impl Membership { 40 | pub fn new(known_nodes: &Vec, options: DiscoveryOptions) -> Self { 41 | let (tx, rx) = loole::bounded(64); 42 | let mut known_peers: HashMap = HashMap::new(); 43 | for node in known_nodes { 44 | let node_info = NodeInfo { 45 | addr: None, 46 | addresses: node.direct_addresses.clone(), 47 | }; 48 | known_peers.insert(node.public_key, node_info); 49 | } 50 | 51 | let sender = tx.clone(); 52 | 53 | let handle = tokio::task::spawn(async move { 54 | let query_interval = Duration::from_secs(options.query_interval_seconds); 55 | let mut interval = tokio::time::interval(query_interval); 56 | 57 | loop { 58 | tokio::select! { 59 | _ = interval.tick() => { 60 | let discovery_result = Membership::discover_peers(&mut known_peers).await; 61 | trace!("Peer discovery result: {:?}", discovery_result); 62 | for node_addr in discovery_result { 63 | sender.send(Ok(DiscoveryEvent{ provenance: "peer_discovery", node_addr })).ok(); 64 | } 65 | }, 66 | } 67 | } 68 | }); 69 | 70 | Self { 71 | handle: AbortOnDropHandle::new(handle), 72 | rx, 73 | } 74 | } 75 | 76 | async fn discover_peers(known_peers: &mut HashMap) -> Vec { 77 | let mut discovered = vec![]; 78 | for (public_key, info) in known_peers.iter_mut() { 79 | let mut direct_addresses = vec![]; 80 | for fqdn in &info.addresses { 81 | let maybe_peers = tokio::net::lookup_host(fqdn).await; 82 | if let Ok(peers) = maybe_peers { 83 | for resolved in peers { 84 | direct_addresses.push(resolved); 85 | } 86 | } 87 | } 88 | if direct_addresses.is_empty() { 89 | continue; 90 | } 91 | let key = NodeId::from_bytes(public_key.as_bytes()).expect("invalid public key"); 92 | let node_addr = Some(NodeAddr::from_parts(key, None, direct_addresses)); 93 | if node_addr != info.addr { 94 | info.addr = node_addr.clone(); 95 | if let Some(addr) = node_addr { 96 | discovered.push(addr); 97 | } 98 | } 99 | } 100 | discovered 101 | } 102 | } 103 | 104 | impl Discovery for Membership { 105 | fn update_local_address(&self, _node_addr: &NodeAddr) -> Result<()> { 106 | Ok(()) 107 | } 108 | 109 | fn subscribe(&self, _network_id: [u8; 32]) -> Option>> { 110 | let rx_stream = Box::pin(self.rx.clone().into_stream()); 111 | Some(rx_stream) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /rhio/src/nats/client/fake/blocking.rs: -------------------------------------------------------------------------------- 1 | use crate::nats::HeaderMap; 2 | use anyhow::Result; 3 | use async_nats::Message; 4 | use async_nats::jetstream::consumer::DeliverPolicy; 5 | use bytes::Bytes; 6 | use rhio_core::Subject; 7 | use std::marker::PhantomData; 8 | use std::sync::Arc; 9 | use tokio::runtime::Runtime; 10 | 11 | use crate::StreamName; 12 | 13 | use super::{ 14 | super::types::{NatsClient, NatsMessageStream}, 15 | client::Consumer, 16 | }; 17 | 18 | /// A blocking client for interacting with NATS. 19 | /// 20 | /// This client wraps an asynchronous NATS client and provides blocking 21 | /// methods for publishing messages and creating consumers. 22 | /// 23 | /// # Type Parameters 24 | /// 25 | /// * `T` - The type of the NATS client. 26 | /// * `M` - The type of the NATS message stream. 27 | /// 28 | /// # Methods 29 | /// 30 | /// * `new(inner: T, runtime: Arc) -> Self` 31 | /// - Creates a new `BlockingClient`. 32 | /// 33 | /// * `publish(&self, subject: String, payload: Bytes, headers: Option) -> Result<()>` 34 | /// - Publishes a message to the given subject with the specified payload and headers. 35 | /// 36 | /// * `create_consumer(&self, stream_name: StreamName, filter_subjects: Vec, _deliver_policy: DeliverPolicy) -> Result>` 37 | /// - Creates a new consumer for the specified stream and subjects with the given delivery policy. 38 | /// 39 | /// A blocking consumer for receiving messages from NATS. 40 | /// 41 | /// This consumer wraps an asynchronous consumer and provides a blocking 42 | /// method for receiving messages. 43 | /// 44 | /// # Type Parameters 45 | /// 46 | /// * `M` - The type of the NATS message stream. 47 | /// 48 | /// # Fields 49 | /// 50 | /// * `consumer` - The inner asynchronous consumer. 51 | /// * `runtime` - The Tokio runtime used for blocking operations. 52 | /// 53 | /// # Methods 54 | /// 55 | /// * `new(consumer: Consumer, runtime: Arc) -> BlockingConsumer` 56 | /// - Creates a new `BlockingConsumer`. 57 | /// 58 | /// * `recv_count(&mut self, timeout: std::time::Duration, count: usize) -> Result>` 59 | /// - Receives a specified number of messages with a timeout. 60 | pub struct BlockingClient 61 | where 62 | T: NatsClient, 63 | M: NatsMessageStream, 64 | { 65 | /// The inner asynchronous NATS client. 66 | inner: T, 67 | /// The Tokio runtime used for blocking operations. 68 | runtime: Arc, 69 | phantom: PhantomData, 70 | } 71 | 72 | impl BlockingClient 73 | where 74 | T: NatsClient, 75 | M: NatsMessageStream, 76 | { 77 | pub fn new(inner: T, runtime: Arc) -> Self { 78 | Self { 79 | inner, 80 | runtime, 81 | phantom: PhantomData, 82 | } 83 | } 84 | 85 | pub fn publish( 86 | &self, 87 | subject: String, 88 | payload: Bytes, 89 | headers: Option, 90 | ) -> Result<()> { 91 | self.runtime 92 | .block_on(async { self.inner.publish(false, subject, payload, headers).await }) 93 | } 94 | 95 | pub fn create_consumer( 96 | &self, 97 | consumer_name: String, 98 | stream_name: StreamName, 99 | filter_subjects: Vec, 100 | _deliver_policy: DeliverPolicy, 101 | ) -> Result> { 102 | self.runtime.block_on(async { 103 | let (messages, _) = self 104 | .inner 105 | .create_consumer_stream( 106 | consumer_name, 107 | stream_name, 108 | filter_subjects, 109 | _deliver_policy, 110 | ) 111 | .await?; 112 | Ok(BlockingConsumer::new( 113 | Consumer::new(messages), 114 | self.runtime.clone(), 115 | )) 116 | }) 117 | } 118 | } 119 | 120 | pub struct BlockingConsumer { 121 | consumer: Consumer, 122 | runtime: Arc, 123 | } 124 | 125 | impl BlockingConsumer { 126 | pub fn new(consumer: Consumer, runtime: Arc) -> BlockingConsumer { 127 | BlockingConsumer { consumer, runtime } 128 | } 129 | 130 | pub fn recv_count( 131 | &mut self, 132 | timeout: std::time::Duration, 133 | count: usize, 134 | ) -> Result> { 135 | self.runtime 136 | .block_on(async { self.consumer.recv_timeout(timeout, count).await }) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /rhio-core/src/subject.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::str::FromStr; 3 | 4 | use anyhow::bail; 5 | use async_nats::subject::Subject as NatsSubject; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | const DELIMITER_TOKEN: &str = "."; 9 | const WILDCARD_TOKEN: &str = "*"; 10 | 11 | type Token = String; 12 | 13 | /// Filterable NATS subject. 14 | /// 15 | /// For example: `hello.*.world` 16 | #[derive(Clone, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)] 17 | pub struct Subject(Vec); 18 | 19 | impl Subject { 20 | fn new(value: &str) -> Self { 21 | Self( 22 | value 23 | .split(DELIMITER_TOKEN) 24 | .map(|value| value.to_string()) 25 | .collect(), 26 | ) 27 | } 28 | 29 | /// Returns true if a given, second subject is a sub-set of this one. 30 | /// 31 | /// The matching rules are quite simple in NATS and do not resemble typical "globbing" rules, 32 | /// on top they are not hierarchical. 33 | /// 34 | /// For example `hello.*` does not match `hello.*.world` as both subjects need to have the same 35 | /// lenght. `hello.*.*` would match though. 36 | pub fn is_matching(&self, other_subject: &Subject) -> bool { 37 | if self.0.len() != other_subject.0.len() { 38 | return false; 39 | } 40 | 41 | for (index, token) in self.0.iter().enumerate() { 42 | let Some(other_token) = other_subject.0.get(index) else { 43 | unreachable!("compared subjects should be of same length"); 44 | }; 45 | 46 | if token == other_token || token == WILDCARD_TOKEN || other_token == WILDCARD_TOKEN { 47 | continue; 48 | } 49 | 50 | return false; 51 | } 52 | 53 | true 54 | } 55 | } 56 | 57 | impl fmt::Display for Subject { 58 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 59 | write!(f, "{}", self.0.join(DELIMITER_TOKEN)) 60 | } 61 | } 62 | 63 | impl FromStr for Subject { 64 | type Err = anyhow::Error; 65 | 66 | fn from_str(value: &str) -> Result { 67 | if value.is_empty() { 68 | bail!("can't have empty nats subject string"); 69 | } 70 | Ok(Self::new(value)) 71 | } 72 | } 73 | 74 | impl TryFrom for Subject { 75 | type Error = anyhow::Error; 76 | 77 | fn try_from(value: NatsSubject) -> Result { 78 | Self::from_str(&value) 79 | } 80 | } 81 | 82 | impl Serialize for Subject { 83 | fn serialize(&self, serializer: S) -> Result 84 | where 85 | S: serde::Serializer, 86 | { 87 | serializer.serialize_str(&self.to_string()) 88 | } 89 | } 90 | 91 | impl<'de> Deserialize<'de> for Subject { 92 | fn deserialize(deserializer: D) -> Result 93 | where 94 | D: serde::Deserializer<'de>, 95 | { 96 | let value: String = String::deserialize(deserializer)?; 97 | Self::from_str(&value).map_err(|err| serde::de::Error::custom(err.to_string())) 98 | } 99 | } 100 | 101 | pub fn subjects_to_str(mut subjects: Vec) -> String { 102 | subjects.sort(); 103 | subjects 104 | .iter() 105 | .map(|subject| subject.to_string()) 106 | .collect::>() 107 | .join(", ") 108 | } 109 | 110 | #[cfg(test)] 111 | mod tests { 112 | use std::str::FromStr; 113 | 114 | use super::Subject; 115 | 116 | #[test] 117 | fn subject_filter_matching() { 118 | let assert_filter = |a: String, b: String, expected: bool| { 119 | assert!( 120 | Subject::from_str(&a) 121 | .unwrap() 122 | .is_matching(&Subject::from_str(&b).unwrap()) 123 | == expected, 124 | "{a} does not match {b}" 125 | ); 126 | }; 127 | 128 | assert_filter("a.*".to_string(), "a.b".to_string(), true); 129 | assert_filter("a.*".to_string(), "a.b.c".to_string(), false); 130 | assert_filter("a.*.c".to_string(), "a.b.c".to_string(), true); 131 | assert_filter("*.*".to_string(), "a.b".to_string(), true); 132 | assert_filter("*.c".to_string(), "a.b.c".to_string(), false); 133 | assert_filter("*.*".to_string(), "a.b.c".to_string(), false); 134 | assert_filter("*.*.*".to_string(), "a.b.c".to_string(), true); 135 | assert_filter("a.b".to_string(), "a.b.c".to_string(), false); 136 | assert_filter("a.b.*".to_string(), "a.*.c".to_string(), true); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /rhio-blobs/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod bao_file; 2 | mod paths; 3 | mod s3_file; 4 | mod store; 5 | mod utils; 6 | 7 | // For blobs we still use BLAKE3 hashes, same as p2panda, but use the implementation of 8 | // `iroh-blobs` instead to ease using their APIs. On top we're getting a different encoding 9 | // (p2panda uses hexadecimal encoding, iroh-blobs uses base32) so we also get a nice "visual 10 | // difference" when logging blob hashes and can distinct them better from public keys or other 11 | // hexadecimal representations. 12 | pub use iroh_blobs::Hash as BlobHash; 13 | use p2panda_core::{PublicKey, Signature}; 14 | 15 | pub use paths::{META_SUFFIX, NO_PREFIX, OUTBOARD_SUFFIX, Paths}; 16 | use serde::{Deserialize, Serialize}; 17 | pub use store::BucketState; 18 | pub use store::BucketStatus; 19 | pub use store::S3Store; 20 | 21 | /// Name of the S3 bucket which can contain S3 objects stored under keys. 22 | pub type BucketName = String; 23 | 24 | /// Size in bytes of the S3 object. 25 | pub type ObjectSize = u64; 26 | 27 | /// Key of the S3 object, this is essentially it's "path". 28 | pub type ObjectKey = String; 29 | 30 | /// Object in S3 bucket which has not yet been imported to blob store. 31 | /// 32 | /// These are objects the user has locally uploaded into the S3 bucket, outside of rhio. We detect 33 | /// these new objects with a special "watcher" service, monitoring the S3 buckets. 34 | #[derive(Clone, Debug)] 35 | pub struct NotImportedObject { 36 | pub local_bucket_name: BucketName, 37 | pub key: ObjectKey, 38 | pub size: ObjectSize, 39 | } 40 | 41 | /// Blobs which have been imported but are still not complete are downloads from remote peers. 42 | /// 43 | /// Since we received them from an external source we already know their authentication info, thus 44 | /// they are always signed. 45 | pub type IncompleteBlob = SignedBlobInfo; 46 | 47 | /// Completed blobs have either been imported from local S3 objects or downloaded from remote 48 | /// peers. 49 | /// 50 | /// For downloaded items we always know the signature, as it has been delivered to us with the blob 51 | /// announcement. For locally uploaded items we don't have the signature as we're signing the blobs 52 | /// when announcing them on the network. 53 | #[allow(clippy::large_enum_variant)] 54 | #[derive(Clone, Debug)] 55 | pub enum CompletedBlob { 56 | Unsigned(UnsignedBlobInfo), 57 | Signed(SignedBlobInfo), 58 | } 59 | 60 | impl CompletedBlob { 61 | pub fn local_bucket_name(&self) -> BucketName { 62 | match self { 63 | CompletedBlob::Unsigned(UnsignedBlobInfo { 64 | local_bucket_name, .. 65 | }) => local_bucket_name.clone(), 66 | CompletedBlob::Signed(SignedBlobInfo { 67 | local_bucket_name, .. 68 | }) => local_bucket_name.clone(), 69 | } 70 | } 71 | 72 | pub fn size(&self) -> ObjectSize { 73 | match self { 74 | CompletedBlob::Unsigned(UnsignedBlobInfo { size, .. }) => *size, 75 | CompletedBlob::Signed(SignedBlobInfo { size, .. }) => *size, 76 | } 77 | } 78 | 79 | pub fn key(&self) -> ObjectKey { 80 | match self { 81 | CompletedBlob::Unsigned(UnsignedBlobInfo { key, .. }) => key.clone(), 82 | CompletedBlob::Signed(SignedBlobInfo { key, .. }) => key.clone(), 83 | } 84 | } 85 | 86 | pub fn hash(&self) -> BlobHash { 87 | match self { 88 | CompletedBlob::Unsigned(UnsignedBlobInfo { hash, .. }) => *hash, 89 | CompletedBlob::Signed(SignedBlobInfo { hash, .. }) => *hash, 90 | } 91 | } 92 | } 93 | 94 | /// An imported but unsigned blob which indicates that it must be from our local peer. 95 | /// 96 | /// As soon as we announce this local blob in the network we sign it but that state does not get 97 | /// persisted in the meta file. 98 | #[derive(Clone, Debug, Serialize, Deserialize)] 99 | pub struct UnsignedBlobInfo { 100 | pub hash: BlobHash, 101 | pub local_bucket_name: BucketName, 102 | pub key: ObjectKey, 103 | pub size: ObjectSize, 104 | } 105 | 106 | /// An imported and signed blob which indicates that it must have been downloaded from an remote 107 | /// peer. 108 | #[derive(Clone, Debug)] 109 | pub struct SignedBlobInfo { 110 | pub hash: BlobHash, 111 | // Name of the bucket where we store this S3 object locally. 112 | pub local_bucket_name: BucketName, 113 | // Name of the bucket where this S3 object originated from. 114 | pub remote_bucket_name: BucketName, 115 | pub key: ObjectKey, 116 | pub size: ObjectSize, 117 | pub public_key: PublicKey, 118 | pub signature: Signature, 119 | } 120 | -------------------------------------------------------------------------------- /rhio-http-api/src/status.rs: -------------------------------------------------------------------------------- 1 | use schemars::JsonSchema; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 5 | pub struct HealthStatus { 6 | pub status: ServiceStatus, 7 | pub msg: Option, 8 | pub streams: MessageStreams, 9 | pub stores: ObjectStores, 10 | } 11 | 12 | impl From for HealthStatus { 13 | fn from(error: anyhow::Error) -> Self { 14 | HealthStatus { 15 | status: ServiceStatus::Error, 16 | msg: Some(format!("{:?}", error)), 17 | ..HealthStatus::default() 18 | } 19 | } 20 | } 21 | 22 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 23 | pub enum ServiceStatus { 24 | Running, 25 | #[default] 26 | Unknown, 27 | Error, 28 | } 29 | 30 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 31 | pub struct MessageStreams { 32 | pub published: Vec, 33 | pub subscribed: Vec, 34 | } 35 | 36 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 37 | pub struct ObjectStores { 38 | pub published: Vec, 39 | pub subscribed: Vec, 40 | } 41 | 42 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 43 | pub struct MessageStreamPublishStatus { 44 | pub stream: String, 45 | pub subject: String, 46 | pub status: ObjectStatus, 47 | pub last_error: Option, 48 | pub last_check_time: Option, 49 | } 50 | 51 | impl MessageStreamPublishStatus { 52 | pub fn to_unknown(stream: &String, subject: &String) -> MessageStreamPublishStatus { 53 | MessageStreamPublishStatus { 54 | status: ObjectStatus::Unknown, 55 | stream: stream.to_owned(), 56 | subject: subject.to_owned(), 57 | last_error: None, 58 | last_check_time: None, 59 | } 60 | } 61 | } 62 | 63 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 64 | pub struct MessageStreamSubscribeStatus { 65 | pub source: String, 66 | pub stream: String, 67 | pub subject: String, 68 | pub status: ObjectStatus, 69 | pub last_error: Option, 70 | pub last_check_time: Option, 71 | } 72 | 73 | impl MessageStreamSubscribeStatus { 74 | pub fn to_unknown( 75 | source: &String, 76 | stream: &String, 77 | subject: &String, 78 | ) -> MessageStreamSubscribeStatus { 79 | MessageStreamSubscribeStatus { 80 | status: ObjectStatus::Unknown, 81 | source: source.to_owned(), 82 | stream: stream.to_owned(), 83 | subject: subject.to_owned(), 84 | last_error: None, 85 | last_check_time: None, 86 | } 87 | } 88 | } 89 | 90 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 91 | pub struct ObjectStorePublishStatus { 92 | pub bucket: String, 93 | pub status: ObjectStatus, 94 | pub last_error: Option, 95 | pub last_check_time: Option, 96 | } 97 | 98 | impl ObjectStorePublishStatus { 99 | pub fn to_unknown(bucket: &String) -> ObjectStorePublishStatus { 100 | ObjectStorePublishStatus { 101 | status: ObjectStatus::Unknown, 102 | bucket: bucket.to_owned(), 103 | last_error: None, 104 | last_check_time: None, 105 | } 106 | } 107 | } 108 | 109 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 110 | pub struct ObjectStoreSubscribeStatus { 111 | pub source: String, 112 | pub remote_bucket: String, 113 | pub local_bucket: String, 114 | pub status: ObjectStatus, 115 | pub last_error: Option, 116 | pub last_check_time: Option, 117 | } 118 | 119 | impl ObjectStoreSubscribeStatus { 120 | pub fn to_unknown( 121 | source: &String, 122 | remote_bucket: &String, 123 | local_bucket: &String, 124 | ) -> ObjectStoreSubscribeStatus { 125 | ObjectStoreSubscribeStatus { 126 | status: ObjectStatus::Unknown, 127 | source: source.to_owned(), 128 | remote_bucket: remote_bucket.to_owned(), 129 | local_bucket: local_bucket.to_owned(), 130 | last_error: None, 131 | last_check_time: None, 132 | } 133 | } 134 | } 135 | 136 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq)] 137 | pub enum ObjectStatus { 138 | New, 139 | #[default] 140 | Active, 141 | Inactive, 142 | Unknown, 143 | } 144 | -------------------------------------------------------------------------------- /rhio/src/network/actor.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{HashMap, hash_map}; 2 | 3 | use anyhow::{Result, bail}; 4 | use p2panda_net::network::{FromNetwork, ToNetwork}; 5 | use p2panda_net::{Network, TopicId}; 6 | use tokio::sync::{mpsc, oneshot}; 7 | use tracing::{error, trace, warn}; 8 | 9 | use crate::topic::Query; 10 | 11 | pub enum ToPandaActor { 12 | Broadcast { 13 | payload: Vec, 14 | topic_id: [u8; 32], 15 | reply: oneshot::Sender>, 16 | }, 17 | Subscribe { 18 | query: Query, 19 | reply: oneshot::Sender>>, 20 | }, 21 | Shutdown { 22 | reply: oneshot::Sender<()>, 23 | }, 24 | } 25 | 26 | pub struct PandaActor { 27 | /// p2panda-net network. 28 | network: Network, 29 | 30 | /// Map containing senders for all subscribed topics. Messages sent on this channels will be 31 | /// broadcast to peers interested in the same topic. 32 | topic_gossip_tx_map: HashMap<[u8; 32], mpsc::Sender>, 33 | 34 | /// Actor inbox. 35 | inbox: mpsc::Receiver, 36 | } 37 | 38 | impl PandaActor { 39 | pub fn new(network: Network, inbox: mpsc::Receiver) -> Self { 40 | Self { 41 | network, 42 | topic_gossip_tx_map: HashMap::default(), 43 | inbox, 44 | } 45 | } 46 | 47 | pub async fn run(mut self) -> Result<()> { 48 | // Take oneshot sender from outside API awaited by `shutdown` call and fire it as soon as 49 | // shutdown completed. 50 | let shutdown_completed_signal = self.run_inner().await; 51 | if let Err(err) = self.shutdown().await { 52 | error!(?err, "error during shutdown"); 53 | } 54 | 55 | match shutdown_completed_signal { 56 | Ok(reply_tx) => { 57 | reply_tx.send(()).ok(); 58 | Ok(()) 59 | } 60 | Err(err) => Err(err), 61 | } 62 | } 63 | 64 | async fn run_inner(&mut self) -> Result> { 65 | loop { 66 | tokio::select! { 67 | Some(msg) = self.inbox.recv() => { 68 | match msg { 69 | ToPandaActor::Shutdown { reply } => { 70 | break Ok(reply); 71 | } 72 | msg => { 73 | if let Err(err) = self.on_actor_message(msg).await { 74 | warn!(err = ?err, "error processing actor message"); 75 | } 76 | } 77 | } 78 | }, 79 | } 80 | } 81 | } 82 | 83 | async fn on_actor_message(&mut self, msg: ToPandaActor) -> Result { 84 | match msg { 85 | ToPandaActor::Subscribe { query, reply } => { 86 | let result = self.on_subscribe(query).await?; 87 | reply.send(result).ok(); 88 | } 89 | ToPandaActor::Broadcast { 90 | payload, 91 | topic_id, 92 | reply, 93 | } => { 94 | let result = self.on_broadcast(payload, topic_id).await; 95 | reply.send(result).ok(); 96 | } 97 | ToPandaActor::Shutdown { .. } => { 98 | unreachable!("handled in run_inner"); 99 | } 100 | } 101 | 102 | Ok(true) 103 | } 104 | 105 | async fn on_broadcast(&mut self, bytes: Vec, topic_id: [u8; 32]) -> Result<()> { 106 | match self.topic_gossip_tx_map.get_mut(&topic_id) { 107 | Some(tx) => Ok(tx.send(ToNetwork::Message { bytes }).await?), 108 | None => { 109 | bail!("attempted to send operation on unknown topic id {topic_id:?}"); 110 | } 111 | } 112 | } 113 | 114 | async fn on_subscribe(&mut self, query: Query) -> Result>> { 115 | let topic_id = query.id(); 116 | 117 | if query.is_no_sync() && self.topic_gossip_tx_map.contains_key(&topic_id) { 118 | return Ok(None); 119 | } 120 | 121 | trace!( 122 | topic_id = hex::encode(topic_id), 123 | %query, 124 | "subscribe to topic" 125 | ); 126 | let (tx, rx, _) = self.network.subscribe(query).await?; 127 | if let hash_map::Entry::Vacant(entry) = self.topic_gossip_tx_map.entry(topic_id) { 128 | entry.insert(tx); 129 | } 130 | Ok(Some(rx)) 131 | } 132 | 133 | async fn shutdown(self) -> Result<()> { 134 | self.network.shutdown().await?; 135 | Ok(()) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /rhio-operator/src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::result_large_err)] 2 | use clap::{Parser, crate_description, crate_version}; 3 | use futures::FutureExt; 4 | use rhio_operator::api::message_stream::ReplicatedMessageStream; 5 | use rhio_operator::api::message_stream_subscription::ReplicatedMessageStreamSubscription; 6 | use rhio_operator::api::object_store::ReplicatedObjectStore; 7 | use rhio_operator::api::object_store_subscription::ReplicatedObjectStoreSubscription; 8 | use rhio_operator::api::service::RhioService; 9 | use rhio_operator::built_info; 10 | use rhio_operator::cli::{Opts, RhioCommand}; 11 | use rhio_operator::configuration::controllers::{ 12 | create_rms_controller, create_rmss_controller, create_ros_controller, create_ross_controller, 13 | }; 14 | use rhio_operator::rhio::controller::{OPERATOR_NAME, create_rhio_controller}; 15 | use stackable_operator::cli::Command; 16 | use stackable_operator::cli::{CommonOptions, RunArguments}; 17 | use stackable_operator::telemetry::Tracing; 18 | use stackable_operator::{CustomResourceExt, client}; 19 | 20 | const RHIO_OPERATOR_PRODUCT_PROPERTIES: &str = 21 | "/etc/hiro/rhio-operator/config-spec/properties.yaml"; 22 | const RHIO_OPERATOR_LOCAL_PRODUCT_PROPERTIES: &str = "./config-spec/properties.yaml"; 23 | 24 | #[tokio::main] 25 | async fn main() -> anyhow::Result<()> { 26 | // We have to initialize default rustls crypto provider 27 | // because it is used from inside of stackable-operator -> kube-rs -> tokio-tls 28 | rustls::crypto::aws_lc_rs::default_provider() 29 | .install_default() 30 | .expect("Failed to setup rustls default crypto provider [aws_lc_rs]"); 31 | 32 | let opts = Opts::parse(); 33 | match opts.cmd { 34 | RhioCommand::CreatePrivateKeySecret(cmd) => cmd.generate_secret()?, 35 | RhioCommand::CreateS3Secret(cmd) => cmd.generate_secret()?, 36 | RhioCommand::CreateNatsSecret(cmd) => cmd.generate_secret()?, 37 | RhioCommand::Framework(Command::Crd) => { 38 | RhioService::print_yaml_schema(built_info::PKG_VERSION)?; 39 | ReplicatedMessageStream::print_yaml_schema(built_info::PKG_VERSION)?; 40 | ReplicatedMessageStreamSubscription::print_yaml_schema(built_info::PKG_VERSION)?; 41 | ReplicatedObjectStore::print_yaml_schema(built_info::PKG_VERSION)?; 42 | ReplicatedObjectStoreSubscription::print_yaml_schema(built_info::PKG_VERSION)?; 43 | } 44 | RhioCommand::Framework(Command::Run(RunArguments { 45 | common: 46 | CommonOptions { 47 | cluster_info, 48 | telemetry, 49 | }, 50 | product_config, 51 | watch_namespace, 52 | .. 53 | })) => { 54 | let _tracing_guard = Tracing::pre_configured(built_info::PKG_NAME, telemetry).init()?; 55 | 56 | tracing::info!( 57 | built_info.pkg_version = built_info::PKG_VERSION, 58 | built_info.git_version = built_info::GIT_VERSION, 59 | built_info.target = built_info::TARGET, 60 | built_info.built_time_utc = built_info::BUILT_TIME_UTC, 61 | built_info.rustc_version = built_info::RUSTC_VERSION, 62 | "Starting {description}", 63 | description = built_info::PKG_DESCRIPTION 64 | ); 65 | stackable_operator::utils::print_startup_string( 66 | crate_description!(), 67 | crate_version!(), 68 | built_info::GIT_VERSION, 69 | built_info::TARGET, 70 | built_info::BUILT_TIME_UTC, 71 | built_info::RUSTC_VERSION, 72 | ); 73 | let product_config = product_config.load(&[ 74 | RHIO_OPERATOR_LOCAL_PRODUCT_PROPERTIES, 75 | RHIO_OPERATOR_PRODUCT_PROPERTIES, 76 | ])?; 77 | let client = 78 | client::initialize_operator(Some(OPERATOR_NAME.to_string()), &cluster_info).await?; 79 | let rhio_controller = 80 | create_rhio_controller(&client, product_config, watch_namespace.clone()).boxed(); 81 | let rms_controller = create_rms_controller(&client, watch_namespace.clone()).boxed(); 82 | let rmss_controller = create_rmss_controller(&client, watch_namespace.clone()).boxed(); 83 | let ros_controller = create_ros_controller(&client, watch_namespace.clone()).boxed(); 84 | let ross_controller = create_ross_controller(&client, watch_namespace.clone()).boxed(); 85 | 86 | let futures = vec![ 87 | rms_controller, 88 | rmss_controller, 89 | rhio_controller, 90 | ros_controller, 91 | ross_controller, 92 | ]; 93 | futures::future::select_all(futures).await; 94 | } 95 | }; 96 | 97 | Ok(()) 98 | } 99 | -------------------------------------------------------------------------------- /env/dev/applications.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script manages the installation, uninstallation, and listing of services for Kubernetes clusters. 4 | # 5 | # It supports the following commands: 6 | # - install: Installs applications on all specified clusters. 7 | # - uninstall: Uninstalls applications from all specified clusters. 8 | # - listsvc: Lists all services of type LoadBalancer in all specified clusters. 9 | # 10 | # Usage: 11 | # ./applications.sh {install|uninstall|listsvc} 12 | # 13 | # The script performs the following main functions: 14 | # - Validates the presence of required tools (e.g., kubectl). 15 | # - Generates Kubernetes secrets for each cluster. 16 | # - Bundles Kubernetes manifests using kustomize. 17 | # - Installs or uninstalls applications on the specified clusters. 18 | # - Lists services of type LoadBalancer in the specified clusters. 19 | # 20 | # Environment Variables: 21 | # - KUBECONFIG: Path to the merged kubeconfig file. 22 | # 23 | 24 | CLUSTERS=("kind-cluster1" "kind-cluster2" "kind-cluster3") 25 | 26 | export KUBECONFIG=$(pwd)/target/merged-kubeconfig.yaml 27 | 28 | usage() { 29 | echo "Usage: $0 {install|uninstall|listsvc}" 30 | exit 1 31 | } 32 | 33 | main() { 34 | if [ $# -ne 1 ]; then 35 | usage 36 | fi 37 | 38 | case "$1" in 39 | install) 40 | applications_install 41 | ;; 42 | uninstall) 43 | applications_uninstall 44 | ;; 45 | listsvc) 46 | services_list_all 47 | ;; 48 | *) 49 | echo "Error: Invalid command '$1'" 50 | usage 51 | ;; 52 | esac 53 | } 54 | 55 | applications_install() { 56 | validate 57 | generate_secrets 58 | kustomize_bundle_all 59 | install_applications_all 60 | } 61 | 62 | validate() { 63 | 64 | if ! command -v kubectl &> /dev/null; then 65 | echo "kubectl is not installed. Please install it before running this script." 66 | exit 1 67 | fi 68 | 69 | mkdir -p target 70 | } 71 | 72 | generate_secrets() { 73 | echo "### Generating secrets ... ###" 74 | for i in "${!CLUSTERS[@]}"; do 75 | cluster="${CLUSTERS[$i]}" 76 | generate_secret $cluster 77 | done 78 | } 79 | 80 | generate_secret() { 81 | local cluster=$1 82 | echo "Generatin Rhio secret for $cluster" 83 | 84 | private_key=$(cat ./overlays/${cluster}/pk.txt | base64) 85 | public_key=$(cat ./overlays/${cluster}/pb.txt | base64) 86 | 87 | cat > ./overlays/${cluster}/rhio/private-key-secret.yaml < ./target/${cluster}.bundle.yaml 112 | } 113 | 114 | install_applications_all() { 115 | echo "### Installing Applications ... ###" 116 | for i in "${!CLUSTERS[@]}"; do 117 | cluster="${CLUSTERS[$i]}" 118 | install_applications $cluster 119 | done 120 | } 121 | 122 | install_applications() { 123 | local cluster=$1 124 | echo "Installing Applications on $cluster" 125 | kubectl --context kind-${cluster} create ns rhio 126 | kubectl --context kind-${cluster} create ns nats 127 | kubectl --context kind-${cluster} create ns minio 128 | kubectl --context kind-${cluster} create -f ./target/${cluster}.bundle.yaml 129 | } 130 | 131 | applications_uninstall() { 132 | validate 133 | delete_applications_all 134 | } 135 | 136 | delete_applications_all() { 137 | echo "### Uninstalling Applications ... ###" 138 | 139 | for i in "${!CLUSTERS[@]}"; do 140 | cluster="${CLUSTERS[$i]}" 141 | delete_applications $cluster 142 | done 143 | echo "" 144 | } 145 | 146 | delete_applications() { 147 | local cluster=$1 148 | 149 | echo "Uninstalling Applications from $cluster" 150 | kubectl --context kind-${cluster} delete -f ./target/${cluster}.bundle.yaml 151 | } 152 | 153 | services_list_all() { 154 | echo "### List Services ... ###" 155 | validate 156 | for i in "${!CLUSTERS[@]}"; do 157 | cluster="${CLUSTERS[$i]}" 158 | services_list $cluster 159 | done 160 | echo "" 161 | } 162 | 163 | services_list() { 164 | local cluster=$1 165 | 166 | echo "### List Services in cluster $cluster ..." 167 | echo "-----------------------------------------------" 168 | echo "SERVICE | EXTERNAL-IP:PORT" 169 | echo "-----------------------------------------------" 170 | 171 | kubectl --context kind-$cluster get svc -A --field-selector spec.type=LoadBalancer -o json \ 172 | | jq -r '.items[] | select(.status.loadBalancer.ingress) | "\(.metadata.name) \(.status.loadBalancer.ingress[0].ip):\(.spec.ports[].port)"' \ 173 | | column -t -s' ' 174 | } 175 | 176 | main "$@" -------------------------------------------------------------------------------- /rhio-operator/src/configuration/error.rs: -------------------------------------------------------------------------------- 1 | use p2panda_core::IdentityError; 2 | use snafu::Snafu; 3 | use stackable_operator::{ 4 | k8s_openapi::api::core::v1::Secret, 5 | kube::{api::DynamicObject, core::error_boundary, runtime::reflector::ObjectRef}, 6 | logging::controller::ReconcilerError, 7 | }; 8 | use std::string::FromUtf8Error; 9 | use strum::{EnumDiscriminants, IntoStaticStr}; 10 | 11 | use crate::api::service::RhioService; 12 | 13 | #[derive(Snafu, Debug, EnumDiscriminants)] 14 | #[strum_discriminants(derive(IntoStaticStr))] 15 | #[allow(clippy::enum_variant_names)] 16 | #[snafu(visibility(pub))] 17 | pub enum Error { 18 | #[snafu(display("Configuration resource is invalid"))] 19 | InvalidReplicatedResource { 20 | source: error_boundary::InvalidObject, 21 | }, 22 | 23 | #[snafu(display("object defines no namespace"))] 24 | ObjectHasNoNamespace, 25 | 26 | #[snafu(display("object has no name"))] 27 | ObjectHasNoName, 28 | 29 | #[snafu(display("failed to get rhio services"))] 30 | GetRhioService { 31 | source: stackable_operator::client::Error, 32 | }, 33 | 34 | #[snafu(display("failed to get rhio services"))] 35 | RhioIsAbsent, 36 | 37 | #[snafu(display("failed to update status"))] 38 | ApplyStatus { 39 | source: stackable_operator::client::Error, 40 | }, 41 | 42 | #[snafu(display("failed to serialize Rhio config"))] 43 | RhioConfigurationSerialization { source: serde_json::Error }, 44 | 45 | #[snafu(display("failed to deserialize secret"))] 46 | SecretDeserialization { source: serde_json::Error }, 47 | 48 | #[snafu(display("failed to serialize secret"))] 49 | SecretSerialization { source: serde_json::Error }, 50 | 51 | #[snafu(display("failed to serialize secret"))] 52 | YamlSerialization { source: serde_yaml::Error }, 53 | 54 | #[snafu(display("Unable to make string from bytes"))] 55 | StringConversion { source: FromUtf8Error }, 56 | 57 | #[snafu(display("secret has no string data"))] 58 | SecretHasNoData { secret: ObjectRef }, 59 | 60 | #[snafu(display("object {} is missing metadata to build owner reference", rhio))] 61 | ObjectMissingMetadataForOwnerRef { 62 | source: stackable_operator::builder::meta::Error, 63 | rhio: ObjectRef, 64 | }, 65 | 66 | #[snafu(display("failed to build Metadata"))] 67 | MetadataBuild { 68 | source: stackable_operator::builder::meta::Error, 69 | }, 70 | 71 | #[snafu(display("invalid NATS subject {subject}"))] 72 | InvalidNatsSubject { 73 | source: anyhow::Error, 74 | subject: String, 75 | }, 76 | 77 | #[snafu(display("failed to build ConfigMap"))] 78 | BuildConfigMap { 79 | source: stackable_operator::builder::configmap::Error, 80 | }, 81 | 82 | #[snafu(display("failed to parse public key {key}"))] 83 | PublicKeyParsing { source: IdentityError, key: String }, 84 | 85 | #[snafu(display("failed to fetch secret {name}"))] 86 | GetSecret { 87 | source: stackable_operator::client::Error, 88 | name: String, 89 | }, 90 | 91 | #[snafu(display("fail to write YAML to stdout"))] 92 | WriteToStdout { source: std::io::Error }, 93 | 94 | #[snafu(display("multiple rhio services in the same namespace"))] 95 | MultipleServicesInTheSameNamespace { rhio: ObjectRef }, 96 | 97 | #[snafu(display("Unable to resolve rhio service endpoint"))] 98 | GetRhioServiceEndpoint, 99 | } 100 | 101 | impl ReconcilerError for Error { 102 | fn category(&self) -> &'static str { 103 | ErrorDiscriminants::from(self).into() 104 | } 105 | 106 | fn secondary_object(&self) -> Option> { 107 | match self { 108 | Error::ObjectHasNoName => None, 109 | Error::InvalidReplicatedResource { .. } => None, 110 | Error::ObjectHasNoNamespace => None, 111 | Error::GetRhioService { .. } => None, 112 | Error::RhioIsAbsent => None, 113 | Error::MultipleServicesInTheSameNamespace { rhio } => Some(rhio.clone().erase()), 114 | Error::ApplyStatus { .. } => None, 115 | Error::RhioConfigurationSerialization { .. } => None, 116 | Error::ObjectMissingMetadataForOwnerRef { rhio, .. } => Some(rhio.clone().erase()), 117 | Error::MetadataBuild { .. } => None, 118 | Error::InvalidNatsSubject { .. } => None, 119 | Error::BuildConfigMap { .. } => None, 120 | Error::PublicKeyParsing { .. } => None, 121 | Error::GetSecret { .. } => None, 122 | Error::SecretDeserialization { .. } => None, 123 | Error::SecretSerialization { .. } => None, 124 | Error::SecretHasNoData { secret } => Some(secret.clone().erase()), 125 | Error::WriteToStdout { .. } => None, 126 | Error::YamlSerialization { .. } => None, 127 | Error::GetRhioServiceEndpoint => None, 128 | Error::StringConversion { .. } => None, 129 | } 130 | } 131 | } 132 | 133 | pub type Result = std::result::Result; 134 | -------------------------------------------------------------------------------- /rhio/src/utils/nats/factory.rs: -------------------------------------------------------------------------------- 1 | use crate::nats::ConsumerId; 2 | use crate::nats::client::types::NatsClient; 3 | use crate::nats::client::types::NatsMessageStream; 4 | use crate::nats::client::types::NatsStreamProtocol; 5 | use crate::utils::retry::types::SeqNo; 6 | use crate::utils::retry::types::StreamFactory; 7 | use anyhow::Context as AnyhowContext; 8 | use async_nats::jetstream::consumer::DeliverPolicy; 9 | use futures::Stream; 10 | use std::future::Future; 11 | use std::marker::PhantomData; 12 | use std::pin::Pin; 13 | use std::sync::Arc; 14 | use tracing::Span; 15 | use tracing::info; 16 | 17 | #[derive(Clone)] 18 | pub struct NatsStreamFactory { 19 | inner: Arc>, 20 | } 21 | 22 | impl NatsStreamFactory 23 | where 24 | N: NatsClient + Send + Sync + 'static, 25 | M: NatsMessageStream + Send + Sync + 'static, 26 | { 27 | pub fn new( 28 | client: Arc, 29 | consumer_id: ConsumerId, 30 | stream_name: String, 31 | filter_subjects: Vec, 32 | deliver_policy: DeliverPolicy, 33 | span: Span, 34 | ) -> NatsStreamFactory { 35 | NatsStreamFactory { 36 | inner: Arc::new(NatsStreamFactoryInner { 37 | client, 38 | consumer_id, 39 | stream_name, 40 | filter_subjects, 41 | deliver_policy, 42 | span, 43 | phantom: PhantomData, 44 | }), 45 | } 46 | } 47 | } 48 | 49 | pub struct NatsStreamFactoryInner { 50 | client: Arc, 51 | consumer_id: ConsumerId, 52 | stream_name: String, 53 | filter_subjects: Vec, 54 | deliver_policy: DeliverPolicy, 55 | span: Span, 56 | phantom: PhantomData, 57 | } 58 | 59 | /// 60 | /// Implementation of the `StreamFactory` trait for `NatsStreamFactory`. 61 | /// 62 | /// This implementation provides a mechanism to create a stream of NATS messages 63 | /// with optional support for starting from a specific sequence number. 64 | /// 65 | /// # Type Parameters 66 | /// - `N`: A type that implements the `NatsClient` trait, responsible for handling 67 | /// NATS client operations. 68 | /// - `M`: A type that implements the `NatsMessageStream` trait, representing the 69 | /// stream of NATS messages. 70 | /// 71 | /// # Associated Types 72 | /// - `T`: The protocol type, which is `NatsStreamProtocol`. 73 | /// - `ErrorT`: The error type, which is `anyhow::Error`. 74 | /// - `StreamT`: The stream type, which is `M`. 75 | /// - `Fut`: The future type, which is a pinned boxed future resolving to a result 76 | /// containing the stream (`M`) or an error (`anyhow::Error`). 77 | /// 78 | /// # Methods 79 | /// - `create(&self, seq_no: Option) -> Self::Fut` 80 | /// 81 | /// This method creates a new NATS message stream. It accepts an optional sequence 82 | /// number (`seq_no`) to specify the starting point of the stream. If a sequence 83 | /// number is provided, the delivery policy is set to start from that sequence. 84 | /// Otherwise, the default delivery policy is used. 85 | /// 86 | /// The method returns a pinned boxed future that resolves to either the created 87 | /// message stream (`M`) or an error (`anyhow::Error`). 88 | /// 89 | /// # Parameters 90 | /// - `seq_no`: An optional sequence number (`SeqNo`) indicating the starting point 91 | /// of the message stream. 92 | /// 93 | /// # Returns 94 | /// - `Self::Fut`: A pinned boxed future resolving to a result containing the message 95 | /// stream (`M`) or an error (`anyhow::Error`). 96 | /// 97 | /// # Errors 98 | /// - Returns an error if the consumer stream cannot be created. 99 | /// - Errors may occur due to issues with the NATS client, invalid parameters, or 100 | /// other runtime conditions. 101 | /// 102 | impl StreamFactory for NatsStreamFactory 103 | where 104 | N: NatsClient + Send + Sync + 'static, 105 | M: NatsMessageStream + Send + Sync + 'static, 106 | { 107 | type T = NatsStreamProtocol; 108 | type ErrorT = anyhow::Error; 109 | type StreamT = M; 110 | type Fut = Pin> + Send>>; 111 | 112 | fn create(&self, seq_no: Option) -> Self::Fut { 113 | let this = Arc::new(self.inner.clone()); 114 | Box::pin(async move { 115 | let delivery_policy = seq_no 116 | .map(|s| DeliverPolicy::ByStartSequence { start_sequence: s }) 117 | .unwrap_or(this.deliver_policy); 118 | 119 | let (messages, consumer_info) = this 120 | .client 121 | .create_consumer_stream( 122 | this.consumer_id.to_string(), 123 | this.stream_name.clone(), 124 | this.filter_subjects.clone(), 125 | delivery_policy, 126 | ) 127 | .await 128 | .context("Cannot create consumer stream")?; 129 | 130 | if consumer_info.num_pending > 0 { 131 | info!(parent: &this.span, "pending message count {}", consumer_info.num_pending); 132 | } 133 | Ok::(messages) 134 | }) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /config.example.yaml: -------------------------------------------------------------------------------- 1 | # rhio configuration file 2 | # 3 | # 1. Copy this file to the location where a) rhio will run b) in an XDG 4 | # compliant config directory (for example "$HOME/.config/rhio/config.yaml" 5 | # on Linux) or c) refer to it using the -c command line argument when running 6 | # rhio. 7 | # 2. Replace the example values with your own desired configuration. All values 8 | # in this template resemble the defaults 9 | 10 | # ﾟ･｡+☆+｡ 11 | # NETWORK 12 | # ﾟ･｡+☆+｡ 13 | 14 | # Port for node-node communication and data replication. Defaults to 9102. 15 | # 16 | # When port is taken the node will automatically pick a random, free port. 17 | # 18 | # bind_port: 9102 19 | 20 | # 21 | # http_bind_port: 3000 22 | # 23 | # Port for HTTP server exposing the /health endpoint. Defaults to 3000. 24 | 25 | # Nodes will only communicate with each other if they know the same network id. 26 | # 27 | # network_id: "rhio-default-network-1" 28 | 29 | # ﾟ･｡+☆+｡･ 30 | # IDENTITY 31 | # ﾟ･｡+☆+｡･ 32 | 33 | # Path to file containing hexadecimal-encoded Ed25519 private key. The key is 34 | # used to identify the node towards other nodes during network discovery and 35 | # replication. 36 | # 37 | # private_key_path: "private-key.txt" 38 | 39 | # ﾟ･ 40 | # S3 41 | # ﾟ･ 42 | 43 | # Endpoint of the local S3-compatible Database (for example MinIO). 44 | # 45 | # s3: 46 | # endpoint: "http://localhost:9000" 47 | # region: "eu-central-1" 48 | # credentials: 49 | # access_key: "rhio" 50 | # secret_key: "rhio_password" 51 | 52 | # ﾟ･｡+ 53 | # NATS 54 | # ﾟ･｡+ 55 | 56 | # Endpoint of the local NATS Server with JetStreams enabled. 57 | # 58 | # nats: 59 | # endpoint: "localhost:4222" 60 | # credentials: 61 | # username: "rhio" 62 | # password: "rhio_password" 63 | # # Alternative authentication strategies: 64 | # nkey: "..." 65 | # token: "..." 66 | 67 | # ﾟ･｡+☆ 68 | # NODES 69 | # ﾟ･｡+☆ 70 | 71 | # List of known node addresses we want to connect to directly. Addresses have 72 | # to be FQDN's (absolute domain names) or IP addresses with a port number. 73 | # 74 | # NOTE: Make sure that nodes mentioned in this list are directly reachable 75 | # (they need to be hosted with a static IP address). 76 | # 77 | # nodes: 78 | # - public_key: "" 79 | # endpoints: 80 | # - "staging.rhio.org" 81 | # - "192.168.178.100:9102" 82 | # - "[2a02:8109:9c9a:4200:eb13:7c0a:4201:8128]:9103" 83 | 84 | # ﾟ･｡+☆+｡ 85 | # STREAMS 86 | # ﾟ･｡+☆+｡ 87 | 88 | # Share past and future NATS messages or files stored in S3 buckets from us 89 | # with other nodes in the network. 90 | # 91 | # publish: 92 | # # We're publishing all files given in our local bucket. 93 | # # 94 | # s3_buckets: 95 | # - "bucket-out-1" 96 | # - "bucket-out-2" 97 | # 98 | # # We're publishing any NATS message matching this NATS subject (wildcards 99 | # # supported) coming from our local JetStream. 100 | # # 101 | # nats_subjects: 102 | # - subject: "workload.berlin.energy" 103 | # stream: "workload" 104 | # - subject: "workload.rotterdam.energy" 105 | # stream: "workload" 106 | 107 | # Download data of interest from other nodes, this includes past and future 108 | # NATS messages or files stored in S3 buckets. 109 | # 110 | # Subscriptions join the network and actively try to initiate sync sessions 111 | # with nodes who published data of our interest - or with nodes who are 112 | # subscribed to this data as well and can "forward" it to us. 113 | # 114 | # subscribe: 115 | # # We're interested in files from this particular node and their (remote) S3 116 | # # bucket and would like to store them in the our local S3 bucket. 117 | # # 118 | # # NOTE: It is not possible to publish from and download into the same 119 | # # local bucket, also re-using the same bucket for multiple subscriptions is not 120 | # # allowed. This might lead to undefined behaviour! 121 | # # 122 | # s3_buckets: 123 | # - remote_bucket: "bucket-out-1" 124 | # local_bucket: "bucket-in-1" 125 | # public_key: "" 126 | # - remote_bucket: "bucket-out-2" 127 | # local_bucket: "bucket-in-2" 128 | # public_key: "" 129 | # 130 | # # We're interested in any NATS message from this particular node for the 131 | # # given NATS subject (wildcards are allowed). 132 | # # 133 | # # NOTE: NATS messages get always published towards our local NATS server 134 | # # without any particular JetStream in mind - as soon as we received them. 135 | # # However, we still need to configure a local stream which gives us our 136 | # # current state on this subject, this aids effective syncing of past 137 | # # messages with the other node. 138 | # # 139 | # nats_subjects: 140 | # - subject: "workload.*.energy" 141 | # public_key: "" 142 | # stream: "workload" 143 | # - subject: "data.thehague.meta" 144 | # public_key: "" 145 | # stream: "data" 146 | 147 | # Set log verbosity. 148 | # 149 | # Possible log levels are: ERROR, WARN, INFO, DEBUG, TRACE. They are scoped to "rhio" by 150 | # default. 151 | # 152 | # If you want to adjust the scope for deeper inspection use a filter value, for example 153 | # "=TRACE" for logging _everything_ or "rhio=INFO,async_nats=DEBUG" etc. 154 | 155 | # log_level: =INFO --------------------------------------------------------------------------------