├── lading ├── src │ ├── payload.rs │ ├── observer │ │ ├── linux │ │ │ ├── utils.rs │ │ │ ├── procfs │ │ │ │ ├── memory.rs │ │ │ │ ├── uptime.rs │ │ │ │ └── memory │ │ │ │ │ └── smaps_rollup.rs │ │ │ ├── wss │ │ │ │ └── pfnset.rs │ │ │ ├── cgroup │ │ │ │ └── v2 │ │ │ │ │ └── memory.rs │ │ │ └── utils │ │ │ │ └── tests │ │ │ │ └── create_process_tree.py │ │ └── linux.rs │ ├── bin │ │ ├── simple_target.rs │ │ └── captool │ │ │ └── analyze.rs │ ├── proto.rs │ ├── lib.rs │ ├── common.rs │ ├── target_metrics.rs │ └── codec.rs ├── README.md ├── proptest-regressions │ ├── payload │ │ └── dogstatsd.txt │ ├── generator │ │ ├── container │ │ │ └── state_machine.txt │ │ └── kubernetes │ │ │ └── state_machine.txt │ └── throttle │ │ └── stable.txt ├── build.rs └── proto │ ├── agent_payload.proto │ └── stateful_encoding.proto ├── .gitignore ├── CODEOWNERS ├── ci ├── requirements.txt ├── fingerprints │ ├── json │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── fluent │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── otel_logs │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── otel_metrics │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── otel_traces │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── trace_agent_v04 │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── apache_common │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── ascii │ │ ├── fingerprint.txt │ │ └── lading.yaml │ ├── dogstatsd │ │ ├── fingerprint.txt │ │ └── lading.yaml │ └── datadog_logs │ │ ├── fingerprint.txt │ │ └── lading.yaml ├── fmt ├── test_custom_lints ├── clippy ├── check ├── integration-test ├── custom_lints ├── deny ├── outdated ├── buf ├── test ├── shellcheck ├── kani ├── Dockerfiles │ └── Dockerfile.fuzz ├── config-validation ├── validate ├── fingerprint └── fuzz ├── examples ├── lading-idle.yaml ├── http-to-http.yaml ├── vector.toml ├── lading-logrotatefs.yaml ├── lading-container.yaml ├── dogstatsd-generation.yaml ├── lading.yaml ├── trace-agent-v04.yaml └── lading-otel-metrics.yaml ├── rustfmt.toml ├── lading_payload ├── fuzz │ ├── .gitignore │ └── fuzz_targets │ │ ├── json_serializer_to_bytes.rs │ │ ├── ascii_serializer_to_bytes.rs │ │ ├── fluent_serializer_to_bytes.rs │ │ ├── syslog5424_serializer_to_bytes.rs │ │ ├── apache_common_serializer_to_bytes.rs │ │ ├── datadog_log_serializer_to_bytes.rs │ │ ├── trace_agent_v04_serializer_to_bytes.rs │ │ ├── ascii_cache_fixed_next_block.rs │ │ ├── json_cache_fixed_next_block.rs │ │ ├── fluent_cache_fixed_next_block.rs │ │ ├── datadog_log_cache_fixed_next_block.rs │ │ ├── syslog5424_cache_fixed_next_block.rs │ │ ├── apache_common_cache_fixed_next_block.rs │ │ ├── opentelemetry_metrics_serializer_to_bytes.rs │ │ ├── trace_agent_v04_cache_fixed_next_block.rs │ │ ├── opentelemetry_logs_serializer_to_bytes.rs │ │ ├── dogstatsd_cache_fixed_next_block.rs │ │ ├── opentelemetry_metrics_cache_fixed_next_block.rs │ │ ├── opentelemetry_logs_cache_fixed_next_block.rs │ │ └── dogstatsd_serializer_to_bytes.rs ├── README.md ├── src │ ├── common.rs │ ├── opentelemetry.rs │ ├── dogstatsd │ │ └── metric │ │ │ └── template.rs │ ├── opentelemetry │ │ └── metric │ │ │ └── unit.rs │ ├── trace_agent.rs │ ├── ascii.rs │ ├── common │ │ └── config.rs │ └── statik.rs ├── proptest-regressions │ ├── trace_agent │ │ └── v04.txt │ ├── block.txt │ ├── dogstatsd │ │ └── common │ │ │ └── tags.txt │ ├── opentelemetry │ │ ├── log │ │ │ └── mod.txt │ │ └── log.txt │ ├── dogstatsd.txt │ ├── opentelemetry_log.txt │ └── common │ │ └── strings.txt ├── benches │ ├── default.rs │ ├── fluent.rs │ ├── ascii.rs │ ├── apache_common.rs │ ├── opentelemetry_log.rs │ ├── trace_agent.rs │ ├── dogstatsd.rs │ ├── opentelemetry_traces.rs │ └── opentelemetry_metric.rs └── Cargo.toml ├── .github ├── actionlint.yml ├── actions │ ├── install-fuse │ │ └── action.yml │ └── install-protobuf │ │ └── action.yml ├── PULL_REQUEST_TEMPLATE.md ├── workflows │ ├── release.yml │ ├── audit.yml │ ├── config-validation.yml │ └── check-changelog.yml └── dependabot.yml ├── lading_signal ├── README.md └── Cargo.toml ├── .dockerignore ├── clippy.toml ├── NOTICE ├── .cargo ├── audit.toml └── config.toml ├── integration ├── shared │ ├── build.rs │ ├── README.md │ ├── Cargo.toml │ ├── proto │ │ └── integration_api.proto │ └── src │ │ └── lib.rs ├── sheepdog │ ├── README.md │ ├── Cargo.toml │ └── Cargo.lock ├── ducks │ ├── README.md │ └── Cargo.toml └── README.md ├── sgconfig.yml ├── rust-toolchain.toml ├── lading_capture ├── src │ ├── test.rs │ ├── metric.rs │ └── test │ │ └── writer.rs ├── proptest-regressions │ ├── line.txt │ ├── json.txt │ ├── accumulator.txt │ ├── manager.txt │ ├── manager │ │ └── state_machine.txt │ └── formats.txt ├── README.md └── Cargo.toml ├── lading_fuzz ├── Cargo.toml └── src │ └── lib.rs ├── .vscode ├── extensions.json └── settings.json ├── lading_throttle ├── README.md └── Cargo.toml ├── buf.yaml ├── lints ├── rules │ ├── no-as-imports.yml │ └── no-long-paths.yml └── tests │ ├── no-as-imports.yml │ ├── no-long-paths.yml │ └── __snapshots__ │ ├── no-as-imports-snapshot.yml │ └── no-long-paths-snapshot.yml ├── .devcontainer ├── Dockerfile └── devcontainer.json ├── LICENSE-3rdparty.sh ├── proptest-regressions ├── payload │ ├── fluent.txt │ ├── splunk_hec.txt │ ├── opentelemetry_metric.txt │ ├── syslog.txt │ ├── datadog_logs.txt │ ├── json.txt │ ├── apache_common.txt │ ├── ascii.txt │ ├── opentelemetry_log.txt │ ├── opentelemetry_trace.txt │ └── dogstatsd.txt └── block.txt ├── .gitlab-ci.yml ├── .gitlab └── fuzz.yml ├── deny.toml ├── LICENSE ├── Cross.toml ├── CONTRIBUTING.md └── Dockerfile /lading/src/payload.rs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .claude 3 | .docs-cache -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @DataDog/single-machine-performance -------------------------------------------------------------------------------- /ci/requirements.txt: -------------------------------------------------------------------------------- 1 | toml>=0.10.0 2 | requests>=2.25.0 3 | -------------------------------------------------------------------------------- /examples/lading-idle.yaml: -------------------------------------------------------------------------------- 1 | generator: [] 2 | blackhole: [] -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2024" 2 | newline_style = "unix" 3 | -------------------------------------------------------------------------------- /lading_payload/fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | coverage 5 | -------------------------------------------------------------------------------- /.github/actionlint.yml: -------------------------------------------------------------------------------- 1 | self-hosted-runner: 2 | labels: 3 | - arm-4core-linux-ubuntu24.04 4 | -------------------------------------------------------------------------------- /lading/src/observer/linux/utils.rs: -------------------------------------------------------------------------------- 1 | pub(in crate::observer::linux) mod process_descendents; 2 | -------------------------------------------------------------------------------- /lading_signal/README.md: -------------------------------------------------------------------------------- 1 | # Lading Signals 2 | 3 | A single-use signal mechanism, one to many. 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | target 2 | amd64.Dockerfile 3 | arm64.Dockerfile 4 | .dockerignore 5 | .git 6 | .gitignore 7 | -------------------------------------------------------------------------------- /ci/fingerprints/json/fingerprint.txt: -------------------------------------------------------------------------------- 1 | json_tcp: 214e592f59988c940c1104412a2f41466cf8897f85b083d36e14d98c9e67376c 2 | -------------------------------------------------------------------------------- /lading/README.md: -------------------------------------------------------------------------------- 1 | # Lading, the binary 2 | 3 | Please see README at the root of this project for more details. 4 | -------------------------------------------------------------------------------- /lading_payload/README.md: -------------------------------------------------------------------------------- 1 | # Payload 2 | 3 | This crate holds the payload generating code for the lading project. 4 | -------------------------------------------------------------------------------- /ci/fingerprints/fluent/fingerprint.txt: -------------------------------------------------------------------------------- 1 | fluent_tcp: dc617db09399e635ffa076c85d6666df87eb1263386585460bec23ad26d67d2d 2 | -------------------------------------------------------------------------------- /ci/fingerprints/otel_logs/fingerprint.txt: -------------------------------------------------------------------------------- 1 | otel_logs_grpc: 43da55d898bd2fd0341519d7545e6c9092b85d9f23ffb65d2f45d48598ef362d -------------------------------------------------------------------------------- /ci/fingerprints/otel_metrics/fingerprint.txt: -------------------------------------------------------------------------------- 1 | otel_metrics_grpc: 1a99c0f4baf2470e547d0038dcaa6e7f134a1a31ae9dea50460e21f5e4e3a562 -------------------------------------------------------------------------------- /ci/fingerprints/otel_traces/fingerprint.txt: -------------------------------------------------------------------------------- 1 | otel_traces_grpc: 7664a22561986337b61f8d807be5ce52688cfdb8b429bac89f32ea9884a17c78 2 | -------------------------------------------------------------------------------- /ci/fingerprints/trace_agent_v04/fingerprint.txt: -------------------------------------------------------------------------------- 1 | trace_agent_v04_tcp: f480ab8b0879901c5422eee62c113981505a90386694b61b87f25b9f1907f3c3 2 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | # https://rust-lang.github.io/rust-clippy/master/index.html#doc_markdown 2 | doc-valid-idents = ["OpenTelemetry", "gRPC", ".."] 3 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Datadog Lading 2 | Copyright 2021-2022 Datadog, Inc. 3 | 4 | This product includes software developed at Datadog (https://www.datadoghq.com/). -------------------------------------------------------------------------------- /.cargo/audit.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | ignore = ["RUSTSEC-2020-0159"] 3 | 4 | [output] 5 | deny = ["unmaintained", "unsound", "yanked"] 6 | quiet = false 7 | -------------------------------------------------------------------------------- /lading_payload/src/common.rs: -------------------------------------------------------------------------------- 1 | //! Common utilities for all lading payloads 2 | 3 | pub mod config; 4 | pub(crate) mod strings; 5 | pub(crate) mod tags; 6 | -------------------------------------------------------------------------------- /integration/shared/build.rs: -------------------------------------------------------------------------------- 1 | fn main() -> Result<(), Box> { 2 | tonic_prost_build::compile_protos("proto/integration_api.proto")?; 3 | Ok(()) 4 | } 5 | -------------------------------------------------------------------------------- /sgconfig.yml: -------------------------------------------------------------------------------- 1 | ruleDirs: 2 | - lints/rules/ 3 | testConfigs: 4 | - testDir: lints/tests/ 5 | globalRules: 6 | ignores: 7 | - "**/fuzz/target/**" 8 | - "**/target/**" 9 | -------------------------------------------------------------------------------- /ci/fmt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | echo "Running cargo fmt..." 7 | cargo fmt --all -- --check 8 | 9 | echo "Cargo fmt passed!" -------------------------------------------------------------------------------- /ci/test_custom_lints: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # The purpose of this script is to test our utility ci/custom_lints. 7 | sg test 8 | -------------------------------------------------------------------------------- /ci/clippy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | echo "Running cargo clippy..." 7 | cargo clippy --all-features 8 | 9 | echo "Cargo clippy passed!" -------------------------------------------------------------------------------- /ci/fingerprints/apache_common/fingerprint.txt: -------------------------------------------------------------------------------- 1 | apache_tcp: 6808d129cca3ad97a818a2e7d38ac4cc7f6ce57365b9f78c99c38f1cd6bba893 2 | apache_http: 29e0d445aafe5bc081ba4027a69c91748b82bd306ef42be9b111606ee9bcca42 -------------------------------------------------------------------------------- /ci/fingerprints/ascii/fingerprint.txt: -------------------------------------------------------------------------------- 1 | ascii_tcp: 4d8342868d4366dc30768d2aeced1f1bb57cdc9c08581aa35a984a61e6753153 2 | ascii_unix_stream: d8d48ae6eeb2c6683601445159dab04e2ab8ce0433021fba6c37dec0b7de8f6b -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.'cfg(target_os = "macos")'] 2 | rustflags = ["-C", "force-frame-pointers=yes"] 3 | 4 | [target.'cfg(target_os = "linux")'] 5 | rustflags = ["-C", "force-frame-pointers=yes"] 6 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | # Update the rust version in-sync with the version in Dockerfile 3 | channel = "1.90.0" 4 | components = ["rustfmt", "clippy", "rust-analyzer"] 5 | profile = "minimal" 6 | -------------------------------------------------------------------------------- /ci/fingerprints/dogstatsd/fingerprint.txt: -------------------------------------------------------------------------------- 1 | dogstatsd_tcp: 005bcbd25db52cff850bc4b6db16b733258ab30b47f9ec57ce8dbae656d95bb4 2 | dogstatsd_udp: bd09515cc51d0f1bc140cfa3fce9786b4851a99a2f38ff3d3c91977396eb8793 3 | -------------------------------------------------------------------------------- /ci/fingerprints/datadog_logs/fingerprint.txt: -------------------------------------------------------------------------------- 1 | datadog_logs_tcp: 39599492ec0f51584a44158519b61cf4db8427996c90df79ef75c4d1aa398c3e 2 | datadog_logs_http: f36c4a768a8292dcf8422ddce022e6efd0bf7a651d61e4a02388060cab6e2991 -------------------------------------------------------------------------------- /lading_capture/src/test.rs: -------------------------------------------------------------------------------- 1 | //! Test utilities for `lading_capture` 2 | //! 3 | //! Common test infrastructure used across tests and fuzzing. 4 | 5 | #![cfg(any(test, feature = "fuzz"))] 6 | 7 | pub mod writer; 8 | -------------------------------------------------------------------------------- /lading_fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lading-fuzz" 3 | version = "0.1.0" 4 | edition = "2021" 5 | license = "MIT" 6 | publish = false 7 | 8 | [dependencies] 9 | 10 | [lints] 11 | workspace = true 12 | -------------------------------------------------------------------------------- /ci/check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | echo "Running cargo check..." 7 | cargo check --locked --all-features --tests 8 | 9 | echo "Cargo check passed!" 10 | -------------------------------------------------------------------------------- /ci/integration-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | echo "Running integration tests..." 7 | cargo test --locked -p sheepdog 8 | 9 | echo "Integration tests passed!" -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "rust-lang.rust-analyzer", 4 | "dustypomerleau.rust-syntax", 5 | "tamasfe.even-better-toml", 6 | "ms-azuretools.vscode-docker" 7 | ] 8 | } -------------------------------------------------------------------------------- /lading/src/bin/simple_target.rs: -------------------------------------------------------------------------------- 1 | //! A simple target for lading that runs forever 2 | 3 | use std::{thread, time}; 4 | 5 | fn main() { 6 | loop { 7 | thread::sleep(time::Duration::from_secs(60)); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /integration/shared/README.md: -------------------------------------------------------------------------------- 1 | # Shared Testing Code 2 | 3 | The `shared` crate contains integration testing code that is shared between 4 | `sheepdog` and `ducks`. This currently consists of RPC definitions and 5 | configuration structures. -------------------------------------------------------------------------------- /lading_throttle/README.md: -------------------------------------------------------------------------------- 1 | # Throttle Mechanisms 2 | 3 | This crate holds the throttle mechanisms for the lading project. They are not 4 | complex, the throttles, but they do need to be correct and much of the code here 5 | is test or proof code. 6 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | breaking: 3 | use: 4 | - WIRE 5 | lint: 6 | use: 7 | - DEFAULT 8 | - COMMENTS 9 | ignore: 10 | - integration 11 | - lading/proto/agent_payload.proto 12 | - lading/proto/stateful_encoding.proto 13 | -------------------------------------------------------------------------------- /integration/sheepdog/README.md: -------------------------------------------------------------------------------- 1 | # Sheepdog 2 | 3 | Sheepdog is the integration testing harness for `lading`. It's implemented using 4 | plain async rust tests that launch a `ducks` process and a `lading` process and 5 | assert on measurements of `lading's` operation. -------------------------------------------------------------------------------- /ci/custom_lints: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | if ! command -v sg &> /dev/null; then 7 | echo "ast-grep (sg) is not installed. Install with: brew install ast-grep" 8 | exit 1 9 | fi 10 | 11 | sg scan --threads 1 12 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[rust]": { 3 | "editor.defaultFormatter": "rust-lang.rust-analyzer", 4 | "editor.formatOnSave": true 5 | }, 6 | "[toml]": { 7 | "editor.defaultFormatter": "tamasfe.even-better-toml", 8 | "editor.formatOnSave": false 9 | } 10 | } -------------------------------------------------------------------------------- /integration/ducks/README.md: -------------------------------------------------------------------------------- 1 | # Ducks 2 | 3 | `ducks` is the integration testing target for `lading`. It has the ability to 4 | accept load from `lading` while maintaining key measurements on that load that 5 | enable integration and correctness testing. It also contains an RPC server that 6 | allows it to be controlled by `sheepdog`. -------------------------------------------------------------------------------- /lading_payload/src/opentelemetry.rs: -------------------------------------------------------------------------------- 1 | //! OpenTelemetry payload generation 2 | //! 3 | //! This module contains payload generators for OpenTelemetry formats. 4 | 5 | pub mod common; 6 | pub mod log; 7 | pub mod metric; 8 | pub mod trace; 9 | 10 | pub use log::OpentelemetryLogs; 11 | pub use metric::OpentelemetryMetrics; 12 | pub use trace::OpentelemetryTraces; 13 | -------------------------------------------------------------------------------- /ci/deny: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # Check if cargo-deny is installed 7 | if ! command -v cargo-deny &> /dev/null; then 8 | echo "cargo-deny is not installed. Installing it now..." 9 | cargo install cargo-deny 10 | fi 11 | 12 | echo "Running cargo deny..." 13 | cargo deny check 14 | 15 | echo "Cargo deny passed!" -------------------------------------------------------------------------------- /lints/rules/no-as-imports.yml: -------------------------------------------------------------------------------- 1 | id: no-as-imports 2 | language: Rust 3 | rule: 4 | all: 5 | - kind: use_declaration 6 | - regex: '\bas\b' 7 | 8 | message: "Found an 'as' import" 9 | severity: error 10 | note: | 11 | In lading project we prefer that imports retain the names from their 12 | origin. Else, it become difficult to track changing context as one reads the 13 | source code. 14 | -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/base:1-bookworm 2 | 3 | RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 4 | && apt-get -y install git build-essential protobuf-compiler pkg-config libssl-dev fuse3 libfuse3-dev \ 5 | # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 6 | && apt-get purge -y imagemagick imagemagick-6-common -------------------------------------------------------------------------------- /LICENSE-3rdparty.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | # set -o xtrace 7 | 8 | rm -f cargo.lock > /dev/null 9 | cargo install cargo-license > /dev/null 10 | cargo license --all-features -a -j --no-deps -d | jq -r '(["Component","Origin","License","Copyright"]) as $cols | map(. as $row | ["name", "repository", "license", "authors"] | map($row[.])) as $rows | $cols, $rows[] | @csv' 11 | -------------------------------------------------------------------------------- /ci/outdated: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # Check if cargo-outdated is installed 7 | if ! command -v cargo-outdated &> /dev/null; then 8 | echo "cargo-outdated is not installed. Installing it now..." 9 | cargo install cargo-outdated 10 | fi 11 | 12 | echo "Checking for outdated dependencies..." 13 | cargo outdated --root-deps-only 14 | 15 | echo "Outdated check complete!" -------------------------------------------------------------------------------- /.github/actions/install-fuse/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Install FUSE' 2 | description: 'Installs FUSE on the runner OS' 3 | runs: 4 | using: 'composite' 5 | steps: 6 | - run: | 7 | if [ "${{ runner.os }}" == "Linux" ]; then 8 | sudo apt-get update && sudo apt-get install -y fuse3 libfuse3-dev 9 | elif [ "${{ runner.os }}" == "macOS" ]; then 10 | brew install macfuse 11 | fi 12 | shell: bash 13 | -------------------------------------------------------------------------------- /proptest-regressions/payload/fluent.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 63ddc030bf36dea9f8e02369a64167b46bbea97bdd85f1528cbae2c3f0980a5a # shrinks to seed = 0, max_bytes = 16 8 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### What does this PR do? 2 | 3 | A brief description of the change being made with this pull request. 4 | 5 | ### Motivation 6 | 7 | What inspired you to submit this pull request? 8 | 9 | ### Related issues 10 | 11 | A list of issues either fixed, containing architectural discussions, otherwise relevant 12 | for this Pull Request. 13 | 14 | ### Additional Notes 15 | 16 | Anything else we should know when reviewing? 17 | -------------------------------------------------------------------------------- /lading_capture/proptest-regressions/line.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc d074554ea3c94e8f553cc717f508bbb7b7542e30e60d930c1827ba04f2d6f7a7 # shrinks to num_series = 2, seed = 0 8 | -------------------------------------------------------------------------------- /proptest-regressions/payload/splunk_hec.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 31a1934e2d37aa4ab25b75e1be8fad33e2ae3fec22c1df882479c1e68cc9e182 # shrinks to seed = 0, max_bytes = 0 8 | -------------------------------------------------------------------------------- /lading/proptest-regressions/payload/dogstatsd.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc d9a4fb20b7328d6983ce4d71a10a02073955cae69a9f75c6d6dd60a959a6e756 # shrinks to seed = 0, max_bytes = 0 8 | -------------------------------------------------------------------------------- /.github/actions/install-protobuf/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Install Protobuf' 2 | description: 'Installs Protobuf compiler based on the runner OS' 3 | runs: 4 | using: 'composite' 5 | steps: 6 | - run: | 7 | if [ "${{ runner.os }}" == "Linux" ]; then 8 | sudo apt-get update && sudo apt-get install -y protobuf-compiler 9 | elif [ "${{ runner.os }}" == "macOS" ]; then 10 | brew install protobuf 11 | fi 12 | shell: bash 13 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/trace_agent/v04.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc f9f0cbae44380779d5511b2cf15398393687d40dc0d6869cd1b80c795baba664 # shrinks to seed = 0, max_bytes = 0 8 | -------------------------------------------------------------------------------- /lading/proptest-regressions/generator/container/state_machine.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 1956f4ac13e0c2a05eb32ec5c7309b3c9eb58489592406137189e89153329669 # shrinks to containers = 1 8 | -------------------------------------------------------------------------------- /lading/proptest-regressions/generator/kubernetes/state_machine.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 39c642d8cf4be057bde8906da24f695d003b618fba4aad60aae6116ca9891921 # shrinks to instances = 1 8 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/block.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 2b0d6d175b1488dc4c09caff469d54a43517f51a61c45ff965a99165da98401a # shrinks to seed = 2494543245988303026, num_chunks = 1 -------------------------------------------------------------------------------- /proptest-regressions/block.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 5d4e7b2574547350cad0dd0039ab9042fa1493f2e92e69e5d12d64cdf71d573e # shrinks to seed = 0, total_bytes = 0, block_bytes_sizes = [] 8 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/dogstatsd/common/tags.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 9c7455b354ddc3d731190002edf0877dd19bf97e09f4bb0450ec2306334b07ff # shrinks to seed = 0, desired_num_tagsets = 1 8 | -------------------------------------------------------------------------------- /proptest-regressions/payload/opentelemetry_metric.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 39803632d5c6543a1f796ab17d44b86bfadb85fdc0a0d1f851c66c75365959f1 # shrinks to seed = 11374604432742766758, max_bytes = 214 8 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/opentelemetry/log/mod.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc c936baccbc78a4d1a4c4e3f22ba82ca3c8b3c8e74894e7d64167d8ec10c3eb7c # shrinks to seed = 0, total_contexts = 1, steps = 1 8 | -------------------------------------------------------------------------------- /ci/fingerprints/json/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "json_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5] 5 | addr: "127.0.0.1:8080" 6 | variant: 7 | json: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | 13 | blackhole: 14 | - http: 15 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /lading/proptest-regressions/throttle/stable.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 08df0e78646a65c4d6f6d21db705b6cd3e675060117d2045a79c9488f39a8ff0 # shrinks to maximum_capacity = 1, last_tick = 0, mut ticks_requests = [(0, 1)] 8 | -------------------------------------------------------------------------------- /ci/fingerprints/fluent/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "fluent_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3] 5 | addr: "127.0.0.1:24224" 6 | variant: 7 | fluent: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | 13 | blackhole: 14 | - http: 15 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /lading_payload/benches/default.rs: -------------------------------------------------------------------------------- 1 | use criterion::criterion_main; 2 | 3 | mod apache_common; 4 | mod ascii; 5 | mod dogstatsd; 6 | mod opentelemetry_log; 7 | mod opentelemetry_metric; 8 | mod opentelemetry_traces; 9 | mod trace_agent; 10 | 11 | criterion_main!( 12 | apache_common::benches, 13 | ascii::benches, 14 | dogstatsd::benches, 15 | opentelemetry_log::benches, 16 | opentelemetry_metric::benches, 17 | opentelemetry_traces::benches, 18 | trace_agent::benches, 19 | ); 20 | -------------------------------------------------------------------------------- /ci/buf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # Check if buf is installed 7 | if ! command -v buf &> /dev/null; then 8 | echo "buf is not installed. Please install it first." 9 | echo "See: https://docs.buf.build/installation" 10 | exit 1 11 | fi 12 | 13 | echo "Running buf lint..." 14 | buf lint 15 | 16 | echo "Running buf breaking check..." 17 | buf breaking --against 'https://github.com/datadog/lading.git#branch=main' 18 | 19 | echo "Buf checks passed!" -------------------------------------------------------------------------------- /examples/http-to-http.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - http: 3 | seed: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32] 4 | headers: {} 5 | target_uri: "http://127.0.0.1:8888/" 6 | bytes_per_second: "1 MiB" 7 | parallel_connections: 1 8 | method: 9 | post: 10 | maximum_prebuild_cache_size_bytes: "10 MiB" 11 | variant: "ascii" 12 | 13 | blackhole: 14 | - http: 15 | binding_addr: "127.0.0.1:8888" 16 | -------------------------------------------------------------------------------- /ci/fingerprints/otel_logs/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "otel_logs_grpc" 3 | grpc: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4] 5 | target_uri: "http://127.0.0.1:4317" 6 | variant: 7 | opentelemetry_logs: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | 13 | blackhole: 14 | - http: 15 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /proptest-regressions/payload/syslog.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc a58f2cd0ace3517aad8878967f0f9eb4709c9df758d2e669753de1cb83c52975 8 | cc 4f02fd6df4c1fc13104afac9dc248871a2bab63febf2938c6f9cadc051467bcf # shrinks to seed = 53466259000180281, max_bytes = 128 9 | -------------------------------------------------------------------------------- /ci/fingerprints/otel_metrics/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "otel_metrics_grpc" 3 | grpc: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5] 5 | target_uri: "http://127.0.0.1:4317" 6 | variant: 7 | opentelemetry_metrics: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | 13 | blackhole: 14 | - http: 15 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /ci/fingerprints/otel_traces/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "otel_traces_grpc" 3 | grpc: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6] 5 | target_uri: "http://127.0.0.1:4317" 6 | variant: 7 | opentelemetry_traces: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | 13 | blackhole: 14 | - http: 15 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /proptest-regressions/payload/datadog_logs.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 237654a2d47f2d0ca3576bc91e02ab0d1bae26f942f1d5c75e168dff786f9323 # shrinks to seed = 0, max_bytes = 2 8 | cc 8455adf2fdc4067489a60567d743c318cb62231d270e421e33d439ed973e28e5 # shrinks to seed = 0, max_bytes = 2 9 | -------------------------------------------------------------------------------- /examples/vector.toml: -------------------------------------------------------------------------------- 1 | data_dir = "/var/lib/vector" 2 | 3 | ## 4 | ## Sources 5 | ## 6 | 7 | [sources.internal_metrics] 8 | type = "internal_metrics" 9 | 10 | [sources.socket_source] 11 | type = "socket" 12 | address = "0.0.0.0:8282" 13 | mode = "tcp" 14 | 15 | ## 16 | ## Sinks 17 | ## 18 | 19 | [sinks.prometheus] 20 | type = "prometheus_exporter" 21 | inputs = ["internal_metrics"] 22 | address = "0.0.0.0:9598" 23 | 24 | [sinks.socket_sink] 25 | type = "socket" 26 | inputs = ["socket_source"] 27 | mode = "tcp" 28 | address = "localhost:8080" 29 | encoding = "json" 30 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | permissions: 4 | contents: write 5 | 6 | on: 7 | push: 8 | tags: 9 | - v[0-9]+.* 10 | 11 | jobs: 12 | create-release: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 16 | 17 | - name: Create GitHub release 18 | uses: taiki-e/create-gh-release-action@b7abb0cf5e72cb5500307b577f9ca3fd4c5be9d2 # v1.8.4 19 | with: 20 | changelog: CHANGELOG.md 21 | token: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /lints/rules/no-long-paths.yml: -------------------------------------------------------------------------------- 1 | id: no-long-paths 2 | language: Rust 3 | rule: 4 | all: 5 | - pattern: $A::$B::$C::$$$REST 6 | - not: 7 | inside: 8 | stopBy: end 9 | pattern: use $$$_; 10 | 11 | message: "Found long path (4+ segments)" 12 | severity: error 13 | note: | 14 | In lading project we prefer that full paths not be used for types and function 15 | calls if three or more segments are in the path. That is, foo::bar is okay but 16 | foo::bar::baz is not. Import a prefix to satisfy this lint. 17 | ignores: 18 | - "**/proto/*.rs" 19 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/dogstatsd.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 97158e48115dce6b53c08d3f3744ebd92cb711273b19aee225daa774f71f6e23 # shrinks to seed = 0, max_bytes = 0 8 | cc 077dc27bd44ddd568537d3742556e1a422619e126a2fbf86c7e7f54374780baf # shrinks to seed = 11798065272331789525, max_bytes = 3627 9 | -------------------------------------------------------------------------------- /proptest-regressions/payload/json.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc f53443df99cdd05984359aabbe67d19f5ae91490302add7c38368f5f12f74aba # shrinks to seed = 8662357858841375442, max_bytes = 0 8 | cc 551c4421c84a666f1da44f00beeee7a6c23b5b351625600dfe8469eaf1b563d3 # shrinks to seed = 12381002064624564645, max_bytes = 2048 9 | -------------------------------------------------------------------------------- /proptest-regressions/payload/apache_common.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 96c3a7b525a3422fff341f3c39ff1f05138ee728d3c247dc78d29828332286db # shrinks to seed = 1560147066498530607, max_bytes = 0 8 | cc 92c09f166a2052779629961dd534db522d17c661a71a87e1b1c54e458f43582d # shrinks to seed = 1318125441911699760, max_bytes = 128 9 | -------------------------------------------------------------------------------- /lading_capture/proptest-regressions/json.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 2018daaf39aec0b0be403f21375a7bd6ff38932e4824927d605f1e490c43c07c # shrinks to run_id = 00000000-0000-0000-0000-000000000000, time = 0, recorded_at = 0, fetch_index = 0, metric_name = "a", metric_kind = Counter, value = Float(-3.8800685012272795e184), labels = {} 8 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/opentelemetry/log.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 8ee584a001de4f7e45893569111c8192d410a82e17c534d1e87dc6b20701cbd7 # shrinks to min_body = 1, max_body = 1, min_trace = 0, max_trace = 0, min_attr_res = 127, max_attr_res = 0, min_attr_scope = 0, max_attr_scope = 0, min_attr_log = 0, max_attr_log = 0 8 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/opentelemetry_log.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 48f21c68f6a47b5f04cbc25d9d9907b9db94c3c1352438d61a636a1e819f5aa1 # shrinks to seed = 0, total_contexts = 1, steps = 1 8 | cc 07dda2b662c8e5b8971ea35f997615743e8be3c2e6604e7fe3ad676ff62a9de4 # shrinks to seed = 6751211249958810060, steps = 2, budget = 1460 9 | -------------------------------------------------------------------------------- /examples/lading-logrotatefs.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - file_gen: 3 | logrotate_fs: 4 | seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 5 | 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] 6 | concurrent_logs: 8 7 | maximum_bytes_per_log: 100MiB 8 | total_rotations: 4 9 | max_depth: 0 10 | variant: "ascii" 11 | load_profile: 12 | constant: 1.3MiB 13 | maximum_prebuild_cache_size_bytes: 1GiB 14 | mount_point: /tmp/logrotate 15 | 16 | blackhole: 17 | - tcp: 18 | binding_addr: "0.0.0.0:8080" 19 | -------------------------------------------------------------------------------- /lading/build.rs: -------------------------------------------------------------------------------- 1 | //! Build script for `lading` crate. 2 | 3 | fn main() -> std::io::Result<()> { 4 | println!("cargo:rerun-if-changed=proto/"); 5 | 6 | let includes = ["proto/"]; 7 | 8 | prost_build::Config::new() 9 | .out_dir("src/proto/") 10 | .protoc_arg("--experimental_allow_proto3_optional") 11 | .compile_protos(&["proto/agent_payload.proto"], &includes)?; 12 | 13 | // Compile stateful_encoding.proto with gRPC services 14 | tonic_prost_build::configure() 15 | .out_dir("src/proto/") 16 | .compile_protos(&["proto/stateful_encoding.proto"], &includes)?; 17 | 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/audit.yml: -------------------------------------------------------------------------------- 1 | name: Security Audit 2 | 3 | 4 | # The clippy-check job requires this permission to properly surface failures 5 | permissions: 6 | checks: write 7 | 8 | on: 9 | pull_request: 10 | push: 11 | branches: 12 | - main 13 | paths: 14 | - '**/Cargo.toml' 15 | - '**/Cargo.lock' 16 | 17 | jobs: 18 | security_audit: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 22 | - uses: actions-rust-lang/audit@410bbe6de17ca06c0a60070cca18c88b485ca5a1 # v1.2.6 23 | with: 24 | TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | include: 2 | - .gitlab/fuzz.yml 3 | 4 | stages: 5 | - fuzz 6 | 7 | variables: 8 | DOCKER_RUNNER_REGISTRY: "registry.ddbuild.io/docker" 9 | DOCKER_RUNNER_IMAGE_TAG: "24.0.4-gbi-focal" 10 | REPO_CI_REGISTRY: "registry.ddbuild.io/images/ci/lading-fuzz" 11 | 12 | ci-container-image: 13 | stage: .pre 14 | image: "$DOCKER_RUNNER_REGISTRY:$DOCKER_RUNNER_IMAGE_TAG" 15 | script: | 16 | IMAGE_TAG="${REPO_CI_REGISTRY}:$(date --utc +%Y-%m-%d)-${CI_COMMIT_SHORT_SHA}" 17 | docker buildx build -t $IMAGE_TAG --push . -f ci/Dockerfiles/Dockerfile.fuzz 18 | when: manual 19 | except: 20 | - tags 21 | - schedules 22 | tags: 23 | - "arch:amd64" 24 | -------------------------------------------------------------------------------- /examples/lading-container.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - container: 3 | repository: busybox 4 | tag: latest 5 | args: 6 | - /bin/sh 7 | - -c 8 | - | 9 | while true; do 10 | nc -l -s 0.0.0.0 -p 5000 11 | done 12 | env: 13 | - FOO=env_foo 14 | - BAR=env_bar 15 | labels: 16 | com.datadoghq.ad.checks: '{"tcp_check": {"instances": [{"name": "foo", "host": "%%host%%", "port": "%%port%%"}]}}' 17 | foo: label_foo 18 | bar: label_bar 19 | network_disabled: false 20 | exposed_ports: 21 | - 5000/tcp 22 | max_lifetime_seconds: 20 23 | number_of_containers: 10 24 | -------------------------------------------------------------------------------- /.gitlab/fuzz.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | # Update this base image by clicking on the `ci-container-image` job in gitlab (pick the latest pipeline on main) 3 | BASE_CI_IMAGE: registry.ddbuild.io/images/ci/lading-fuzz:2025-08-14-b39a488a@sha256:2d7d6d9a445e2048f454a3e5d4175e7600700cac54ba1fc47801376803109bd1 4 | 5 | # Full fuzzing run for main branch and scheduled pipelines 6 | run-fuzzing: 7 | stage: fuzz 8 | tags: ["arch:amd64"] 9 | image: $BASE_CI_IMAGE 10 | timeout: 30m 11 | rules: 12 | - if: '$CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "schedule"' 13 | - if: '$CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push"' 14 | - when: manual 15 | script: ./ci/fuzz_infra.py 16 | -------------------------------------------------------------------------------- /integration/shared/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "shared" 3 | version = "0.1.0" 4 | edition = "2024" 5 | authors = ["Single Machine Performance Team"] 6 | license = "MIT" 7 | description = "Shared code for lading integration tests" 8 | publish = false 9 | 10 | [package.metadata.cargo-machete] 11 | ignored = ["prost", "tonic-prost"] 12 | 13 | [dependencies] 14 | serde = { workspace = true } 15 | tonic = { workspace = true, default-features = false, features = [ 16 | "codegen", 17 | "transport", 18 | "router", 19 | ] } 20 | tonic-prost = { workspace = true } 21 | prost = { workspace = true } 22 | serde_json = { workspace = true } 23 | 24 | [build-dependencies] 25 | tonic-prost-build = { workspace = true } 26 | -------------------------------------------------------------------------------- /lading_capture/proptest-regressions/accumulator.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc aca6fe62d3bfef96674cb7361b052c5e6a76d2921a00cf5add7a457137b232e5 # shrinks to samples_tick_0 = [0.0], samples_tick_1 = [100.0] 8 | cc a10150d46b9a007020db0cebfdb9bb4821355662780227b1e1354117009708c2 # shrinks to samples = [508.88821100089126] 9 | cc 36ff268e64ab014de077fb95d7c25b2d7eb1201d591189a417e9978d5028b29e # shrinks to samples = [833.7830942252561] 10 | -------------------------------------------------------------------------------- /proptest-regressions/payload/ascii.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc b0a5e4db7ec2c4733f43f726a8960b63cd99d8d80e62aa5632b059f7a72106ce # shrinks to seed = 15194034853109340835, max_bytes = 0 8 | cc e2bc9dd7b2802fb4395c3e3efa1158afc6a1a58b769a0a484d71b8d3e2d25c9b # shrinks to seed = 15044724196553407659, max_bytes = 1 9 | cc b60d67e07247984212a3c9eefbee0eb97c7cb7560b4609870c7f943c62e6482a # shrinks to seed = 9286616756472296351, max_bytes = 0 10 | -------------------------------------------------------------------------------- /lading/src/observer/linux/procfs/memory.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod smaps; 2 | pub(crate) mod smaps_rollup; 3 | 4 | const BYTES_PER_KIBIBYTE: u64 = 1024; 5 | 6 | fn next_token<'a>( 7 | source: &'a str, 8 | iter: &'_ mut std::iter::Peekable>, 9 | ) -> Option<&'a str> { 10 | while let Some((_, c)) = iter.peek() { 11 | if !c.is_whitespace() { 12 | break; 13 | } 14 | iter.next(); 15 | } 16 | let start = iter.peek()?.0; 17 | while let Some((_, c)) = iter.peek() { 18 | if c.is_whitespace() { 19 | break; 20 | } 21 | iter.next(); 22 | } 23 | let end = iter.peek().map_or_else(|| source.len(), |&(idx, _)| idx); 24 | Some(&source[start..end]) 25 | } 26 | -------------------------------------------------------------------------------- /proptest-regressions/payload/opentelemetry_log.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 1602a961349d156c1eea7b051fbd4be5a64c1a4275d25865b769e5baad02f8a5 # shrinks to seed = 0, max_bytes = 6 8 | cc 7573a95b625c4411260f4ee28ba9a980424a439a650431a5426b9937d0404a73 # shrinks to seed = 0, max_bytes = 16 9 | cc d85f1ce96214f04957593fc926b7a3fde688c2e5b978e1268e939a2ccb5e2691 # shrinks to seed = 0, max_bytes = 6 10 | cc 1a7e610aa899c23792d70281ba75b2726d5f2660d14dd2720f2a7a712b32728b # shrinks to seed = 0, max_bytes = 0 11 | -------------------------------------------------------------------------------- /proptest-regressions/payload/opentelemetry_trace.txt: -------------------------------------------------------------------------------- 1 | cc 325199fdea8be0042be060879afba97ca248ff0ea9d4c20a085646cd8e10b038 # shrinks to seed = 0, max_bytes = 16 2 | # Seeds for failure cases proptest has generated in the past. It is 3 | # automatically read and these particular cases re-run before any 4 | # novel cases are generated. 5 | # 6 | # It is recommended to check this file in to source control so that 7 | # everyone who runs the test benefits from these saved cases. 8 | cc 90bc9aeffc8a7285d32751ac48b53b5612d27b72782d112eaad9f0a877eec0e4 # shrinks to seed = 0, max_bytes = 6 9 | cc f6ebb8c90afa752f08626702a51a65f419a2e7d8a2e865c1fe189f41ab552acc # shrinks to seed = 0, max_bytes = 6 10 | cc 2e4532dfd3f2a82563ee8941a30a9a94f087c17d8089d964c2eb1a137563cf75 # shrinks to seed = 0, max_bytes = 0 11 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | [licenses] 2 | allow = [ 3 | "MIT", 4 | "CC0-1.0", 5 | "ISC", 6 | "OpenSSL", 7 | "Unlicense", 8 | "BSD-2-Clause", 9 | "BSD-2-Clause-Patent", 10 | "BSD-3-Clause", 11 | "Apache-2.0", 12 | "Apache-2.0 WITH LLVM-exception", 13 | "Zlib", 14 | "Unicode-DFS-2016", 15 | "Unicode-3.0", 16 | ] 17 | unused-allowed-license = "allow" 18 | 19 | [sources] 20 | unknown-git = "deny" 21 | allow-git = ["https://github.com/DataDog/saluki"] 22 | 23 | [advisories] 24 | version = 2 25 | ignore = [ 26 | "RUSTSEC-2024-0370", # proc-macro-error is unmaintained 27 | "RUSTSEC-2024-0384", # instant crate is unmaintained 28 | "RUSTSEC-2024-0387", # opentelemetry_api is unmaintained 29 | "RUSTSEC-2024-0436", # paste is unmaintained 30 | ] 31 | 32 | [bans] 33 | multiple-versions = "allow" 34 | -------------------------------------------------------------------------------- /lading_signal/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lading-signal" 3 | version = "0.1.0" 4 | authors = [ 5 | "Brian L. Troutwine ", 6 | "George Hahn /dev/null; then 14 | echo "cargo-nextest is not installed. Installing it now..." 15 | cargo install cargo-nextest 16 | fi 17 | 18 | echo "Running cargo nextest..." 19 | echo " PROPTEST_CASES=${PROPTEST_CASES}" 20 | echo " PROPTEST_MAX_SHRINK_ITERS=${PROPTEST_MAX_SHRINK_ITERS}" 21 | cargo nextest run --workspace --all-features --exclude sheepdog "$@" 22 | 23 | echo "Tests passed!" -------------------------------------------------------------------------------- /lading_throttle/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lading-throttle" 3 | version = "0.1.0" 4 | authors = [ 5 | "Brian L. Troutwine ", 6 | "George Hahn /dev/null; then 8 | echo "shellcheck is not installed. Please install it first." 9 | echo "On macOS: brew install shellcheck" 10 | echo "On Ubuntu/Debian: apt-get install shellcheck" 11 | exit 1 12 | fi 13 | 14 | # Find all shell scripts in ci/ directory 15 | echo "Running shellcheck on all shell scripts in ci/" 16 | find ci -type f \( -name '*.sh' -o -name '*' \) | while read -r script; do 17 | # Skip if it's not a shell script 18 | if [[ ! -f "$script" ]] || ! head -n1 "$script" | grep -q '^#!/.*sh'; then 19 | continue 20 | fi 21 | 22 | echo "Checking: $script" 23 | shellcheck "$script" 24 | done 25 | 26 | echo "Shellcheck passed!" -------------------------------------------------------------------------------- /ci/fingerprints/dogstatsd/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "dogstatsd_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] 5 | addr: "127.0.0.1:8080" 6 | variant: 7 | dogstatsd: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | - id: "dogstatsd_udp" 13 | udp: 14 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] 15 | addr: "127.0.0.1:8125" 16 | variant: 17 | dogstatsd: {} 18 | bytes_per_second: "100 Mb" 19 | maximum_prebuild_cache_size_bytes: "10 Mb" 20 | parallel_connections: 1 21 | 22 | blackhole: 23 | - http: 24 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "build": { 3 | "dockerfile": "Dockerfile" 4 | }, 5 | "features": { 6 | "ghcr.io/devcontainers/features/common-utils:2": { 7 | "installZsh": "true", 8 | "username": "vscode", 9 | "userUid": "1000", 10 | "userGid": "1000", 11 | "upgradePackages": "true", 12 | "nonFreePackage": "true" 13 | }, 14 | "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { 15 | "installDockerBuildx": "true" 16 | }, 17 | "ghcr.io/devcontainers/features/rust:1": { 18 | "version": "1.85", 19 | "profile": "complete" 20 | }, 21 | "ghcr.io/devcontainers/features/git:1": { 22 | "version": "os-provided", 23 | "ppa": "false" 24 | } 25 | }, 26 | "customizations": { 27 | "vscode": { 28 | "extensions": [ 29 | "rust-lang.rust-analyzer" 30 | ] 31 | } 32 | }, 33 | "postCreateCommand": "cargo clean --quiet && cargo check" 34 | } -------------------------------------------------------------------------------- /ci/fingerprints/ascii/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "ascii_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] 5 | addr: "127.0.0.1:8080" 6 | variant: 7 | ascii: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | - id: "ascii_unix_stream" 13 | unix_stream: 14 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2] 15 | path: "/tmp/lading_ascii.sock" 16 | variant: 17 | ascii: {} 18 | bytes_per_second: "100 Mb" 19 | maximum_block_size: "1 Mb" 20 | maximum_prebuild_cache_size_bytes: "10 Mb" 21 | parallel_connections: 1 22 | 23 | blackhole: 24 | - http: 25 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /integration/sheepdog/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sheepdog" 3 | version = "0.1.0" 4 | edition = "2024" 5 | description = "The integration test harness for lading" 6 | authors = ["Single Machine Performance Team"] 7 | license = "MIT" 8 | publish = false 9 | 10 | [dependencies] 11 | anyhow = { workspace = true } 12 | escargot = { version = "0.5.14" } 13 | shared = { path = "../shared" } 14 | tempfile = { workspace = true } 15 | tokio = { workspace = true, features = [ 16 | "fs", 17 | "io-util", 18 | "macros", 19 | "net", 20 | "process", 21 | "rt", 22 | "signal", 23 | "time", 24 | ] } 25 | tonic = { workspace = true, default-features = false, features = ["transport"] } 26 | hyper-util = { workspace = true } 27 | tower = { workspace = true, features = [ 28 | "limit", 29 | "load-shed", 30 | "timeout", 31 | "util", 32 | ] } 33 | tracing = { workspace = true } 34 | tracing-subscriber = { workspace = true, features = ["env-filter"] } 35 | -------------------------------------------------------------------------------- /proptest-regressions/payload/dogstatsd.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 317db7c7556ceb2d7bd9c3f20595f8239c177f12bf3f7a6d57170f8841895211 # shrinks to seed = 0, max_bytes = 0 8 | cc 91bf80356333fb8e093cbeb07663bae5f3ef4b4403486b95509edb154da7deca # shrinks to seed = 6956949339185811855, max_bytes = 0 9 | cc 4170799d2ddffbaf927b808065048f0d67987ab71e323ac1078a8f5710f18f18 # shrinks to seed = 13960759899723154125, max_bytes = 0 10 | cc d21aae37c9ca177b6de2bf58eac3594b24e47f6536a5803ebaf172c0bd9dbdcd # shrinks to seed = 12697873879551039607, max_bytes = 128 11 | cc 0276c6da085b851ea86a39f027beac42cf4e455d228dad48e268aa835e8f442a # shrinks to seed = 11122760050579860287, max_bytes = 0 12 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/json_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | } 15 | 16 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 17 | 18 | fuzz_target!(|input: Input| { 19 | lading_fuzz::debug_input(&input); 20 | 21 | let budget = input.budget_bytes.get() as usize; 22 | if budget > MAX_BUDGET { 23 | return; 24 | } 25 | 26 | let mut rng = SmallRng::from_seed(input.seed); 27 | let mut bytes = Vec::with_capacity(budget); 28 | 29 | let mut serializer = lading_payload::Json::default(); 30 | 31 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 32 | assert!(bytes.len() <= budget); 33 | } 34 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/ascii_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | } 15 | 16 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 17 | 18 | fuzz_target!(|input: Input| { 19 | lading_fuzz::debug_input(&input); 20 | 21 | let budget = input.budget_bytes.get() as usize; 22 | if budget > MAX_BUDGET { 23 | return; 24 | } 25 | 26 | let mut rng = SmallRng::from_seed(input.seed); 27 | let mut bytes = Vec::with_capacity(budget); 28 | 29 | let mut serializer = lading_payload::Ascii::new(&mut rng); 30 | 31 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 32 | assert!(bytes.len() <= budget); 33 | } 34 | }); -------------------------------------------------------------------------------- /lints/tests/no-as-imports.yml: -------------------------------------------------------------------------------- 1 | id: no-as-imports 2 | valid: 3 | - use foobar; 4 | - >- 5 | use foo::{ 6 | bar::bing::baz::bang, 7 | one::two::three, 8 | four::five::six::seven::eight, 9 | }; 10 | - use self::module::submodule::function; 11 | - use crate::generator::http::Server; 12 | - use foo::{bar::baz, bard::baz::bing, bing::foo}; 13 | - use lading_payload::dogstatsd::metric::Counter; 14 | - use std::collections::{BTreeMap, HashMap}; 15 | - use tokio::sync::broadcast::Sender; 16 | invalid: 17 | - use foo as Foo; 18 | - use foo::bar as FooBar; 19 | - use foo::{ bar as FooBar }; 20 | - use foo::{ bar::baz as FooBaz }; 21 | - use foo::{ bar, bar::baz as BarBaz }; 22 | - use foo::{ bar::baz as BarBaz, bar }; 23 | - use tokio::sync::broadcast::Sender as BroadcastSender; 24 | - >- 25 | use foo::{ 26 | bar::bing::baz::bang, 27 | one::two::three as TwoThree, 28 | four::five::six::seven::eight, 29 | }; 30 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/fluent_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | } 15 | 16 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 17 | 18 | fuzz_target!(|input: Input| { 19 | lading_fuzz::debug_input(&input); 20 | 21 | let budget = input.budget_bytes.get() as usize; 22 | if budget > MAX_BUDGET { 23 | return; 24 | } 25 | 26 | let mut rng = SmallRng::from_seed(input.seed); 27 | let mut bytes = Vec::with_capacity(budget); 28 | 29 | let mut serializer = lading_payload::Fluent::new(&mut rng); 30 | 31 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 32 | assert!(bytes.len() <= budget); 33 | } 34 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/syslog5424_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | } 15 | 16 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 17 | 18 | fuzz_target!(|input: Input| { 19 | lading_fuzz::debug_input(&input); 20 | 21 | let budget = input.budget_bytes.get() as usize; 22 | if budget > MAX_BUDGET { 23 | return; 24 | } 25 | 26 | let mut rng = SmallRng::from_seed(input.seed); 27 | let mut bytes = Vec::with_capacity(budget); 28 | 29 | let mut serializer = lading_payload::Syslog5424::default(); 30 | 31 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 32 | assert!(bytes.len() <= budget); 33 | } 34 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/apache_common_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | } 15 | 16 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 17 | 18 | fuzz_target!(|input: Input| { 19 | lading_fuzz::debug_input(&input); 20 | 21 | let budget = input.budget_bytes.get() as usize; 22 | if budget > MAX_BUDGET { 23 | return; 24 | } 25 | 26 | let mut rng = SmallRng::from_seed(input.seed); 27 | let mut bytes = Vec::with_capacity(budget); 28 | 29 | let mut serializer = lading_payload::ApacheCommon::new(&mut rng); 30 | 31 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 32 | assert!(bytes.len() <= budget); 33 | } 34 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/datadog_log_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | } 15 | 16 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 17 | 18 | fuzz_target!(|input: Input| { 19 | lading_fuzz::debug_input(&input); 20 | 21 | let budget = input.budget_bytes.get() as usize; 22 | if budget > MAX_BUDGET { 23 | return; 24 | } 25 | 26 | let mut rng = SmallRng::from_seed(input.seed); 27 | let mut bytes = Vec::with_capacity(budget); 28 | 29 | let mut serializer = lading_payload::DatadogLog::new(&mut rng); 30 | 31 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 32 | assert!(bytes.len() <= budget); 33 | } 34 | }); 35 | -------------------------------------------------------------------------------- /lading_payload/proptest-regressions/common/strings.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc af61ca38851fafeed4d96b45d1d1aca38e2ab8a1489b51ccae6e0e178640e569 # shrinks to seed = 0, max_bytes = 2, of_size_bytes = 0, alphabet = "a🈐aAaA0𐦀A𐠊A\u{e0100}" 8 | cc e4740adac235cc14baefdb17ab4e7b825a9100e1e140031f72c0eb66783a6bdc # shrinks to seed = 49880598398515969, max_bytes = 3977, of_size_bytes = 0, alphabet = "ힰA\u{11c92}® " 9 | cc 7d00cd4ac860c10fd2a903761b1540638fb95f92bfd352cbf952ec53760bedb4 # shrinks to seed = 170628173656970550, max_bytes = 7313, of_size_bytes = 0, alphabet = " 𒒀" 10 | cc c3bbc5eb64e117c599cfe76df189ae8ec8238c47e76a942970505e5da6dcee1b # shrinks to seed = 0, max_bytes = 43894, of_size_bytes = 43894 11 | -------------------------------------------------------------------------------- /ci/fingerprints/apache_common/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "apache_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3] 5 | addr: "127.0.0.1:8080" 6 | variant: 7 | apache_common: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | - id: "apache_http" 13 | http: 14 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4] 15 | target_uri: "http://127.0.0.1:8080/logs" 16 | headers: {} 17 | method: 18 | post: 19 | variant: 20 | apache_common: {} 21 | maximum_prebuild_cache_size_bytes: "10 Mb" 22 | bytes_per_second: "100 Mb" 23 | maximum_block_size: "1 Mb" 24 | parallel_connections: 1 25 | 26 | blackhole: 27 | - http: 28 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /ci/fingerprints/datadog_logs/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "datadog_logs_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9] 5 | addr: "127.0.0.1:8080" 6 | variant: 7 | datadog_log: {} 8 | bytes_per_second: "100 Mb" 9 | maximum_block_size: "1 Mb" 10 | maximum_prebuild_cache_size_bytes: "10 Mb" 11 | parallel_connections: 1 12 | - id: "datadog_logs_http" 13 | http: 14 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] 15 | target_uri: "http://127.0.0.1:8080/v1/input" 16 | headers: {} 17 | method: 18 | post: 19 | variant: 20 | datadog_log: {} 21 | maximum_prebuild_cache_size_bytes: "10 Mb" 22 | bytes_per_second: "100 Mb" 23 | maximum_block_size: "1 Mb" 24 | parallel_connections: 1 25 | 26 | blackhole: 27 | - http: 28 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /lading/src/proto.rs: -------------------------------------------------------------------------------- 1 | //! Module containing structs generated from `proto/` 2 | 3 | /// Protobuf definitions for our `datadog` blackhole 4 | pub(crate) mod datadog { 5 | /// Related to the [DataDog](https://www.datadoghq.com/) intake API 6 | pub(crate) mod intake { 7 | /// API metrics intake, v2. 8 | pub(crate) mod metrics { 9 | #![allow(clippy::pedantic)] 10 | #![allow(missing_docs)] 11 | #![allow(unreachable_pub)] 12 | #![allow(dead_code)] 13 | include!("proto/datadog.agentpayload.rs"); 14 | } 15 | /// Stateful logs intake via gRPC 16 | pub(crate) mod stateful_encoding { 17 | #![allow(clippy::pedantic)] 18 | #![allow(missing_docs)] 19 | #![allow(unreachable_pub)] 20 | #![allow(dead_code)] 21 | #![allow(clippy::unwrap_used)] 22 | #![allow(clippy::enum_variant_names)] 23 | include!("proto/datadog.intake.stateful.rs"); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /ci/kani: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # Check if a crate argument was provided 7 | if [ $# -eq 0 ]; then 8 | echo "Usage: $0 " 9 | echo "Valid crates: lading_throttle, lading_payload, lading_capture" 10 | exit 1 11 | fi 12 | 13 | CRATE="$1" 14 | 15 | # Validate the crate argument 16 | case "$CRATE" in 17 | lading_throttle|lading_payload|lading_capture) 18 | # Valid crate 19 | ;; 20 | *) 21 | echo "Error: Invalid crate '$CRATE'" 22 | echo "Valid crates: lading_throttle, lading_payload, lading_capture" 23 | exit 1 24 | ;; 25 | esac 26 | 27 | # Check if kani is installed 28 | if ! command -v cargo-kani &> /dev/null; then 29 | echo "kani-verifier is not installed. Installing it now..." 30 | cargo install kani-verifier 31 | fi 32 | 33 | echo "Running kani proofs for $CRATE..." 34 | 35 | # Run kani on the specified crate 36 | (cd "$CRATE" && cargo kani --solver cadical) 37 | 38 | echo "Kani proofs passed for $CRATE!" -------------------------------------------------------------------------------- /ci/Dockerfiles/Dockerfile.fuzz: -------------------------------------------------------------------------------- 1 | # Update the rust version in-sync with the version in rust-toolchain.toml 2 | FROM registry.ddbuild.io/images/rust:ubuntu-22.04 3 | 4 | RUN apt-get update && apt-get install -y \ 5 | protobuf-compiler fuse3 libfuse3-dev python3 python3-pip python3-venv curl unzip \ 6 | && rm -rf /var/lib/apt/lists/* 7 | 8 | # Install vault directly from HashiCorp releases 9 | RUN VAULT_VERSION=1.15.4 && \ 10 | curl -fsSL "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip" -o vault.zip && \ 11 | unzip vault.zip && \ 12 | mv vault /usr/local/bin/vault && \ 13 | rm vault.zip && \ 14 | chmod +x /usr/local/bin/vault 15 | 16 | # Create and activate virtual environment 17 | RUN python3 -m venv /opt/venv 18 | ENV PATH="/opt/venv/bin:$PATH" 19 | 20 | WORKDIR /app 21 | COPY . /app 22 | 23 | # Install Python dependencies in virtual environment 24 | RUN pip install -r ci/requirements.txt 25 | RUN cargo install cargo-fuzz 26 | RUN rustup toolchain install nightly-x86_64-unknown-linux-gnu 27 | -------------------------------------------------------------------------------- /lading_capture/proptest-regressions/manager.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 1aeb4f7061779c388cb41f6fbb9b86ad1ece257bd43a435310c140089c02b207 # shrinks to ops = [WriteCounter { name: "c", value: 0 }, WriteGauge { name: "c", value: 0.0 }, WriteCounter { name: "a", value: 0 }, AdvanceTick, WriteCounter { name: "a", value: 0 }, WriteCounter { name: "a", value: 0 }, WriteCounter { name: "a", value: 0 }, WriteCounter { name: "a", value: 0 }, WriteCounter { name: "a", value: 0 }, WriteCounter { name: "a", value: 0 }] 8 | cc 431e1cda40cd8720df08fdbf3b4d88d35a6c2421e5099ca9979d203a91b75337 # shrinks to ops = [WriteCounter("i", 0), WriteGauge("i", 0), WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0), AdvanceTick, WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0)] 9 | -------------------------------------------------------------------------------- /integration/ducks/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ducks" 3 | version = "0.1.0" 4 | edition = "2024" 5 | description = "The integration testing target for lading" 6 | authors = ["Single Machine Performance Team"] 7 | license = "MIT" 8 | publish = false 9 | 10 | [dependencies] 11 | anyhow = { workspace = true } 12 | bytes = { workspace = true } 13 | entropy = { version = "0.4" } 14 | http-body-util = { workspace = true } 15 | hyper = { workspace = true, features = ["server"] } 16 | hyper-util = { workspace = true } 17 | once_cell = { workspace = true } 18 | shared = { path = "../shared" } 19 | ddsketch-agent = { workspace = true } 20 | tokio = { workspace = true, features = [ 21 | "fs", 22 | "io-util", 23 | "macros", 24 | "net", 25 | "process", 26 | "rt", 27 | "signal", 28 | "time", 29 | ] } 30 | tokio-stream = { workspace = true, features = ["net"] } 31 | tonic = { workspace = true, default-features = false, features = [ 32 | "transport", 33 | "router", 34 | ] } 35 | tracing = { workspace = true } 36 | tracing-subscriber = { workspace = true, features = ["env-filter"] } 37 | -------------------------------------------------------------------------------- /lading_fuzz/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Common utilities for fuzzing in lading. 2 | 3 | #![allow(clippy::print_stderr)] 4 | // Allow unused crate dependencies: cargo-fuzz --build-std adds std crates as 5 | // dependencies that we don't use, breaks the build. 6 | #![allow(unused_crate_dependencies)] 7 | #![allow(clippy::cast_possible_truncation)] 8 | #![allow(clippy::cast_precision_loss)] 9 | #![allow(clippy::cast_sign_loss)] 10 | #![allow(clippy::doc_markdown)] 11 | #![allow(clippy::module_name_repetitions)] 12 | #![allow(clippy::must_use_candidate)] 13 | 14 | use std::fmt::Debug; 15 | 16 | /// Print debug information for fuzz input when FUZZ_DEBUG env var is set. 17 | /// 18 | /// This function checks for the `FUZZ_DEBUG` environment variable and if 19 | /// present, prints the input in debug format with header and footer lines for 20 | /// visibility. 21 | pub fn debug_input(input: &T) { 22 | if std::env::var("FUZZ_DEBUG").is_ok() { 23 | eprintln!("=== FUZZ INPUT DEBUG ==="); 24 | eprintln!("{input:#?}"); 25 | eprintln!("========================"); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2021-2022, Datadog, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /ci/config-validation: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # Validates all configuration files in the examples directory 7 | # This ensures all example configs remain valid as lading evolves 8 | 9 | echo "Building lading binary for config validation..." 10 | cargo build --bin=lading --all-features --quiet 11 | 12 | echo "Finding configuration files in examples directory..." 13 | config_files=$(find examples -name "*.yaml" -o -name "*.yml" | sort) 14 | config_count=$(echo "${config_files}" | wc -l | tr -d ' ') 15 | 16 | echo "Found ${config_count} configuration files to validate" 17 | echo 18 | 19 | failed=0 20 | for config in ${config_files}; do 21 | echo "Validating: ${config}" 22 | if ! ./target/debug/lading config-check --config-path="${config}"; then 23 | echo "FAILED: ${config}" 24 | failed=$((failed + 1)) 25 | fi 26 | done 27 | 28 | echo 29 | if [ ${failed} -gt 0 ]; then 30 | echo "FAILED: ${failed}/${config_count} configurations failed validation" 31 | exit 1 32 | fi 33 | 34 | echo "SUCCESS: All ${config_count} configurations are valid" 35 | -------------------------------------------------------------------------------- /examples/dogstatsd-generation.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - passthru_file: 3 | seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 4 | 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] 5 | path: "/tmp/dsd.txt" 6 | variant: 7 | dogstatsd: 8 | contexts: 9 | constant: 10000 10 | name_length: 11 | inclusive: 12 | min: 1 13 | max: 200 14 | tag_length: 15 | inclusive: 16 | min: 3 17 | max: 100 18 | tags_per_msg: 19 | inclusive: 20 | min: 2 21 | max: 50 22 | kind_weights: 23 | metric: 90 24 | event: 5 25 | service_check: 5 26 | metric_weights: 27 | count: 100 28 | gauge: 10 29 | timer: 0 30 | distribution: 0 31 | set: 0 32 | histogram: 0 33 | bytes_per_second: "80 MiB" 34 | maximum_prebuild_cache_size_bytes: "1 GiB" 35 | 36 | blackhole: 37 | - http: 38 | binding_addr: "0.0.0.0:8089" 39 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "cargo" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | labels: 13 | - "dependencies" # needed to preserve default 14 | - "github_actions" # needed to preserve default 15 | - "no-changelog" # CI dependencies should not require changelog entry 16 | - package-ecosystem: "github-actions" 17 | directory: "/" 18 | schedule: 19 | # Check for updates to GitHub Actions every week 20 | interval: "weekly" 21 | labels: 22 | - "dependencies" # needed to preserve default 23 | - "github_actions" # needed to preserve default 24 | - "no-changelog" # CI dependencies should not require changelog entry 25 | -------------------------------------------------------------------------------- /lading/src/bin/captool/analyze.rs: -------------------------------------------------------------------------------- 1 | //! Analysis operations for capture files 2 | //! 3 | //! This module provides metric listing and analysis operations for both 4 | //! JSONL and Parquet capture formats. 5 | 6 | pub(crate) mod jsonl; 7 | pub(crate) mod parquet; 8 | 9 | use std::collections::BTreeSet; 10 | 11 | /// A unique metric (name + kind) 12 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] 13 | pub(crate) struct MetricInfo { 14 | /// Metric kind ("counter" or "gauge") 15 | pub(crate) kind: String, 16 | /// Metric name 17 | pub(crate) name: String, 18 | } 19 | 20 | /// Statistics for a metric series (metric + specific label set) 21 | #[derive(Debug)] 22 | pub(crate) struct SeriesStats { 23 | /// Label key-value pairs for this series 24 | pub(crate) labels: BTreeSet, 25 | /// Minimum value 26 | pub(crate) min: f64, 27 | /// Maximum value 28 | pub(crate) max: f64, 29 | /// Mean value 30 | pub(crate) mean: f64, 31 | /// Whether values are monotonically increasing 32 | pub(crate) is_monotonic: bool, 33 | /// All values (for `dump_values`) 34 | pub(crate) values: Vec<(u64, f64)>, // (fetch_index, value) 35 | } 36 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/trace_agent_v04_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::{Serialize, trace_agent::v04::{Config, V04}}; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | config: Config, 15 | } 16 | 17 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 18 | 19 | fuzz_target!(|input: Input| { 20 | lading_fuzz::debug_input(&input); 21 | 22 | let budget = input.budget_bytes.get() as usize; 23 | if budget > MAX_BUDGET { 24 | return; 25 | } 26 | 27 | if input.config.valid().is_err() { 28 | return; 29 | } 30 | 31 | let mut rng = SmallRng::from_seed(input.seed); 32 | let mut bytes = Vec::with_capacity(budget); 33 | 34 | let mut serializer = match V04::with_config(input.config, &mut rng) { 35 | Ok(s) => s, 36 | Err(_) => return, 37 | }; 38 | 39 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 40 | assert!(bytes.len() <= budget); 41 | } 42 | }); 43 | -------------------------------------------------------------------------------- /lading_capture/src/metric.rs: -------------------------------------------------------------------------------- 1 | use metrics::Key; 2 | use std::time::Instant; 3 | 4 | #[derive(Debug, Clone)] 5 | pub(crate) enum CounterValue { 6 | Increment(u64), 7 | Absolute(u64), 8 | } 9 | 10 | #[derive(Debug, Clone)] 11 | pub(crate) enum GaugeValue { 12 | Increment(f64), 13 | Decrement(f64), 14 | Set(f64), 15 | } 16 | 17 | #[derive(Debug, Clone)] 18 | pub(crate) struct Counter { 19 | pub key: Key, 20 | pub timestamp: Instant, 21 | pub value: CounterValue, 22 | } 23 | 24 | #[derive(Debug, Clone)] 25 | pub(crate) struct Gauge { 26 | pub key: Key, 27 | pub timestamp: Instant, 28 | pub value: GaugeValue, 29 | } 30 | 31 | #[derive(Debug, Clone)] 32 | pub(crate) struct Histogram { 33 | pub key: Key, 34 | pub timestamp: Instant, 35 | pub value: f64, 36 | } 37 | 38 | #[derive(Debug, Clone)] 39 | pub(crate) enum Metric { 40 | /// Counter increment 41 | Counter(Counter), 42 | /// Gauge set 43 | Gauge(Gauge), 44 | /// Histogram sample 45 | /// 46 | /// Constructed by generators via `histogram()` function or by `histogram!` macro 47 | /// through the `CaptureRecorder`. Captured as individual samples with timestamps. 48 | Histogram(Histogram), 49 | } 50 | -------------------------------------------------------------------------------- /integration/README.md: -------------------------------------------------------------------------------- 1 | # Lading Integration Tests 2 | 3 | This directory contains an integration test harness (`sheepdog`) and target 4 | (`ducks`). The two work together to integration test `lading`. 5 | 6 | ## Setup 7 | 8 | ### Macos: 9 | 10 | Install `protoc` with brew: `brew install protobuf` 11 | 12 | Set `PROTOC=/opt/homebrew/bin/protoc` in your shell 13 | 14 | ### Linux 15 | 16 | Install `protoc` with apt: `apt install protobuf-compiler` 17 | 18 | ## Run Tests 19 | 20 | Integration tests are implemented as rust tests in the `sheepdog` project. 21 | 22 | Run them with: `cargo test -p sheepdog` 23 | 24 | ## Test Flow 25 | 26 | Each test's entry point is a normal async rust test in the `sheepdog` crate. The 27 | test implementation defines `lading` and `ducks` configurations and performs 28 | assertions on the test results. Tests start by launching a `ducks` process and 29 | establishing an RPC link between `sheepdog` and `ducks`. This link is used to 30 | configure, control, and receive measurement information from `ducks`. Next, a 31 | `lading` process is started using a configuration that instructs it to send load 32 | to `ducks`. This load is characterized by `ducks` and measurements are sent to 33 | `sheepdog` for validation. 34 | -------------------------------------------------------------------------------- /.github/workflows/config-validation.yml: -------------------------------------------------------------------------------- 1 | name: Config Validation 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - 'examples/**/*.yaml' 7 | - 'examples/**/*.yml' 8 | - 'lading/**' 9 | - '.github/workflows/config-validation.yml' 10 | - 'ci/config-validation' 11 | push: 12 | branches: 13 | - main 14 | paths: 15 | - 'examples/**/*.yaml' 16 | - 'examples/**/*.yml' 17 | - 'lading/**' 18 | - '.github/workflows/config-validation.yml' 19 | - 'ci/config-validation' 20 | 21 | jobs: 22 | validate-configs: 23 | name: Validate Example Configs 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 27 | - uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2 28 | with: 29 | cache: true 30 | - uses: taiki-e/install-action@50708e9ba8d7b6587a2cb575ddaa9a62e927bc06 # v2.44.60 31 | with: 32 | tool: nextest@0.9.72 33 | - name: Install Protobuf 34 | uses: ./.github/actions/install-protobuf 35 | - name: Install FUSE 36 | uses: ./.github/actions/install-fuse 37 | - name: Run config validation 38 | run: ./ci/config-validation 39 | -------------------------------------------------------------------------------- /ci/fingerprints/trace_agent_v04/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "trace_agent_v04_tcp" 3 | tcp: 4 | seed: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] 5 | addr: "127.0.0.1:8126" 6 | variant: 7 | trace_agent: 8 | v0.4: 9 | contexts: 10 | constant: 100 11 | spans_per_trace: 12 | constant: 5 13 | tags_per_span: 14 | constant: 3 15 | metrics_per_span: 16 | constant: 1 17 | error_rate: 0.1 18 | service_name_length: 19 | constant: 10 20 | operation_name_length: 21 | constant: 15 22 | resource_name_length: 23 | constant: 20 24 | span_type_length: 25 | constant: 5 26 | tag_key_length: 27 | constant: 8 28 | tag_value_length: 29 | constant: 12 30 | metric_key_length: 31 | constant: 10 32 | bytes_per_second: "100 Mb" 33 | maximum_block_size: "1 Mb" 34 | maximum_prebuild_cache_size_bytes: "10 Mb" 35 | parallel_connections: 1 36 | 37 | blackhole: 38 | - http: 39 | binding_addr: "127.0.0.1:9999" -------------------------------------------------------------------------------- /integration/shared/proto/integration_api.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package integration_api; 3 | 4 | // Integration test target service. This is hosted by Ducks. 5 | service IntegrationTarget { 6 | // (Not implemented) Get a stream of log messages from the target 7 | // 8 | // this is necessary for modes where ducks is launched by lading 9 | rpc GetLogs (Empty) returns (stream LogMessage) {} 10 | 11 | // Launch a test endpoint 12 | rpc StartTest (TestConfig) returns (ListenInfo) {} 13 | 14 | // Request all recorded metrics 15 | rpc GetMetrics (Empty) returns (Metrics) {} 16 | 17 | // Shut down all operations and exit 18 | rpc Shutdown (Empty) returns (Empty) {} 19 | } 20 | 21 | // Holds a json-serialized [`DucksConfig`] 22 | message TestConfig { 23 | string json_blob = 1; 24 | } 25 | 26 | message ListenInfo { 27 | uint32 port = 1; 28 | } 29 | 30 | message Empty {} 31 | 32 | message LogMessage { 33 | string message = 1; 34 | } 35 | 36 | message HttpMetrics { 37 | uint64 RequestCount = 1; 38 | uint64 TotalBytes = 2; 39 | double MedianEntropy = 3; 40 | double MedianSize = 4; 41 | } 42 | 43 | message SocketMetrics { 44 | uint64 ReadCount = 1; 45 | uint64 TotalBytes = 2; 46 | double MedianEntropy = 3; 47 | } 48 | 49 | message Metrics { 50 | HttpMetrics http = 1; 51 | SocketMetrics tcp = 2; 52 | SocketMetrics udp = 3; 53 | } -------------------------------------------------------------------------------- /lading_payload/src/dogstatsd/metric/template.rs: -------------------------------------------------------------------------------- 1 | use crate::common::strings::Handle; 2 | 3 | #[derive(Clone, Debug)] 4 | /// A metric `Template` is a `super::Metric` that lacks values and has no 5 | /// container ID but has every other attribute of a `super::Metric`. 6 | pub(crate) enum Template { 7 | Count(Count), 8 | Gauge(Gauge), 9 | Timer(Timer), 10 | Histogram(Histogram), 11 | Set(Set), 12 | Distribution(Dist), 13 | } 14 | 15 | #[derive(Clone, Debug)] 16 | /// The count type in `DogStatsD` metric format. Monotonically increasing value. 17 | pub(crate) struct Count { 18 | pub(crate) name: Handle, 19 | } 20 | 21 | #[derive(Clone, Debug)] 22 | /// The gauge type in `DogStatsD` format. 23 | pub(crate) struct Gauge { 24 | pub(crate) name: Handle, 25 | } 26 | 27 | #[derive(Clone, Debug)] 28 | /// The timer type in `DogStatsD` format. 29 | pub(crate) struct Timer { 30 | pub(crate) name: Handle, 31 | } 32 | 33 | #[derive(Clone, Debug)] 34 | /// The distribution type in `DogStatsD` format. 35 | pub(crate) struct Dist { 36 | pub(crate) name: Handle, 37 | } 38 | 39 | #[derive(Clone, Debug)] 40 | /// The set type in `DogStatsD` format. 41 | pub(crate) struct Set { 42 | pub(crate) name: Handle, 43 | } 44 | 45 | #[derive(Clone, Debug)] 46 | /// The histogram type in `DogStatsD` format. 47 | pub(crate) struct Histogram { 48 | pub(crate) name: Handle, 49 | } 50 | -------------------------------------------------------------------------------- /lading_payload/benches/fluent.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{Fluent, Serialize}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn fluent_setup(c: &mut Criterion) { 8 | c.bench_function("fluent_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _f = Fluent::new(&mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn fluent_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("fluent_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let ta = Fluent::new(&mut rng); 26 | let mut writer = Vec::with_capacity(size); 27 | 28 | ta.to_bytes(rng, size, &mut writer) 29 | .expect("failed to convert to bytes"); 30 | }); 31 | }); 32 | } 33 | group.finish(); 34 | } 35 | 36 | criterion_group!( 37 | name = benches; 38 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 39 | targets = fluent_setup, fluent_all, 40 | ); 41 | -------------------------------------------------------------------------------- /lading_payload/benches/ascii.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{Serialize, ascii}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn ascii_setup(c: &mut Criterion) { 8 | c.bench_function("ascii_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _dd = ascii::Ascii::new(&mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn ascii_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("ascii_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let asc = ascii::Ascii::new(&mut rng); 26 | let mut writer = Vec::with_capacity(size); 27 | 28 | asc.to_bytes(rng, size, &mut writer) 29 | .expect("failed to convert to bytes"); 30 | }); 31 | }); 32 | } 33 | group.finish(); 34 | } 35 | 36 | criterion_group!( 37 | name = benches; 38 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 39 | targets = ascii_setup, ascii_all 40 | ); 41 | -------------------------------------------------------------------------------- /lading_capture/README.md: -------------------------------------------------------------------------------- 1 | # Lading Capture 2 | 3 | A crate containing Lading's capture structures and serialization code. 4 | 5 | ## Fuzzing 6 | 7 | This crate includes an AFL++ fuzzing harness that asserts capture file 8 | validity. We do not use ci/fuzz as `CaptureManager` must install a global 9 | singleton -- the Recorder -- and so we need a forking fuzzer. 10 | 11 | ### Setup 12 | 13 | Install cargo-afl for AFL++ instrumentation: 14 | 15 | ```bash 16 | cargo install cargo-afl 17 | ``` 18 | 19 | Build the fuzz harness with cargo-afl for proper instrumentation: 20 | 21 | ```bash 22 | cargo afl build --release -p lading-capture --bin fuzz_capture_harness --features fuzz 23 | ``` 24 | 25 | Create a seeds directory with initial test cases: 26 | 27 | ```bash 28 | mkdir -p seeds 29 | ``` 30 | 31 | Seed format (10 bytes): 32 | - 8 bytes: seed (u64, little-endian) - seeds the RNG for operation generation 33 | - 2 bytes: `runtime_secs` (u16, little-endian, 1-150) - how long to run the test 34 | 35 | Examples: 36 | 37 | ```bash 38 | # 2 second run 39 | printf '\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00' > seeds/2sec.bin 40 | 41 | # 5 second run 42 | printf '\x01\x00\x00\x00\x00\x00\x00\x00\x05\x00' > seeds/5sec.bin 43 | 44 | # 60 second run 45 | printf '\x01\x00\x00\x00\x00\x00\x00\x00\x3c\x00' > seeds/60sec.bin 46 | ``` 47 | 48 | To run the fuzz rig: 49 | 50 | ```bash 51 | cargo afl fuzz -t 5000 -i seeds/ -o findings/ target/release/fuzz_capture_harness 52 | ``` 53 | -------------------------------------------------------------------------------- /lading/src/observer/linux/wss/pfnset.rs: -------------------------------------------------------------------------------- 1 | use procfs::process::Pfn; 2 | use rustc_hash::FxHashMap; 3 | 4 | /// Structure used to represent a set of page frame numbers (PFNs) 5 | /// in a format suitable for use with the Idle Page Tracking API. 6 | /// 7 | #[derive(Debug)] 8 | pub(super) struct PfnSet(FxHashMap); 9 | 10 | impl PfnSet { 11 | pub(super) fn new() -> Self { 12 | Self(FxHashMap::default()) 13 | } 14 | 15 | pub(super) fn insert(&mut self, pfn: Pfn) { 16 | *self.0.entry(pfn.0 / 64).or_default() |= 1 << (pfn.0 % 64); 17 | } 18 | } 19 | 20 | impl IntoIterator for PfnSet { 21 | type Item = (u64, u64); 22 | type IntoIter = std::collections::hash_map::IntoIter; 23 | 24 | fn into_iter(self) -> Self::IntoIter { 25 | self.0.into_iter() 26 | } 27 | } 28 | 29 | #[cfg(test)] 30 | mod tests { 31 | use super::*; 32 | 33 | #[test] 34 | fn pfn_set() { 35 | let input = vec![0, 64, 65, 66, 128, 136, 255]; 36 | let expected = vec![(0, 0x1), (1, 0x7), (2, 0x101), (3, 0x8000_0000_0000_0000)]; 37 | 38 | let mut pfn_set = PfnSet::new(); 39 | for i in input { 40 | pfn_set.insert(Pfn(i)); 41 | } 42 | let mut output: Vec<_> = pfn_set.into_iter().collect(); 43 | output.sort_by_key(|(k, _)| *k); 44 | 45 | assert_eq!(output, expected); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /lading_payload/benches/apache_common.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{Serialize, apache_common}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn apache_common_setup(c: &mut Criterion) { 8 | c.bench_function("apache_common_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _ac = apache_common::ApacheCommon::new(&mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn apache_common_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("apache_common_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let ac = apache_common::ApacheCommon::new(&mut rng); 26 | let mut writer = Vec::with_capacity(size); 27 | 28 | ac.to_bytes(rng, size, &mut writer) 29 | .expect("failed to convert to bytes"); 30 | }); 31 | }); 32 | } 33 | group.finish(); 34 | } 35 | 36 | criterion_group!( 37 | name = benches; 38 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 39 | targets = apache_common_setup, apache_common_all, 40 | ); 41 | -------------------------------------------------------------------------------- /lading_payload/benches/opentelemetry_log.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{OpentelemetryLogs, Serialize}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn opentelemetry_log_setup(c: &mut Criterion) { 8 | c.bench_function("opentelemetry_log_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _ot = OpentelemetryLogs::new(&mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn opentelemetry_log_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("opentelemetry_log_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let ot = OpentelemetryLogs::new(&mut rng); 26 | let mut writer = Vec::with_capacity(size); 27 | 28 | ot.to_bytes(rng, size, &mut writer) 29 | .expect("failed to convert to bytes"); 30 | }); 31 | }); 32 | } 33 | group.finish(); 34 | } 35 | 36 | criterion_group!( 37 | name = benches; 38 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 39 | targets = opentelemetry_log_setup, opentelemetry_log_all 40 | ); 41 | -------------------------------------------------------------------------------- /lading_payload/benches/trace_agent.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{Serialize, trace_agent::v04}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn trace_agent_setup(c: &mut Criterion) { 8 | c.bench_function("trace_agent_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _ta = v04::V04::with_config(v04::Config::default(), &mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn trace_agent_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("trace_agent_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let mut ta = v04::V04::with_config(v04::Config::default(), &mut rng); 26 | let mut writer = Vec::with_capacity(size); 27 | 28 | ta.to_bytes(rng, size, &mut writer) 29 | .expect("failed to convert to bytes"); 30 | }); 31 | }); 32 | } 33 | group.finish(); 34 | } 35 | 36 | criterion_group!( 37 | name = benches; 38 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 39 | targets = trace_agent_setup, trace_agent_all, 40 | ); 41 | -------------------------------------------------------------------------------- /lading_payload/benches/dogstatsd.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{Serialize, dogstatsd}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn dogstatsd_setup(c: &mut Criterion) { 8 | c.bench_function("dogstatsd_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _dd = dogstatsd::DogStatsD::default(&mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn dogstatsd_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("dogstatsd_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let dd = 26 | dogstatsd::DogStatsD::default(&mut rng).expect("failed to create DogStatsD"); 27 | let mut writer = Vec::with_capacity(size); 28 | 29 | dd.to_bytes(rng, size, &mut writer) 30 | .expect("failed to convert to bytes"); 31 | }); 32 | }); 33 | } 34 | group.finish(); 35 | } 36 | 37 | criterion_group!( 38 | name = benches; 39 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 40 | targets = dogstatsd_setup, dogstatsd_all 41 | ); 42 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/ascii_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | } 16 | 17 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 18 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 19 | 20 | fuzz_target!(|input: Input| { 21 | lading_fuzz::debug_input(&input); 22 | 23 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 24 | return; 25 | } 26 | 27 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 28 | return; 29 | } 30 | 31 | if input.max_block_size.get() > input.total_bytes.get() { 32 | return; 33 | } 34 | 35 | let mut rng = SmallRng::from_seed(input.seed); 36 | let payload = lading_payload::Config::Ascii; 37 | 38 | let cache = match Cache::fixed_with_max_overhead( 39 | &mut rng, 40 | input.total_bytes, 41 | u128::from(input.max_block_size.get()), 42 | &payload, 43 | input.total_bytes.get() as usize, 44 | ) { 45 | Ok(c) => c, 46 | Err(_) => return, 47 | }; 48 | 49 | // Call advance 10 times to exercise the cache rotation 50 | let mut handle = cache.handle(); 51 | for _ in 0..10 { 52 | let _block = cache.advance(&mut handle); 53 | } 54 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/json_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | } 16 | 17 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 18 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 19 | 20 | fuzz_target!(|input: Input| { 21 | lading_fuzz::debug_input(&input); 22 | 23 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 24 | return; 25 | } 26 | 27 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 28 | return; 29 | } 30 | 31 | if input.max_block_size.get() > input.total_bytes.get() { 32 | return; 33 | } 34 | 35 | let mut rng = SmallRng::from_seed(input.seed); 36 | let payload = lading_payload::Config::Json; 37 | 38 | let cache = match Cache::fixed_with_max_overhead( 39 | &mut rng, 40 | input.total_bytes, 41 | u128::from(input.max_block_size.get()), 42 | &payload, 43 | input.total_bytes.get() as usize, 44 | ) { 45 | Ok(c) => c, 46 | Err(_) => return, 47 | }; 48 | 49 | // Call advance 10 times to exercise the cache rotation 50 | let mut handle = cache.handle(); 51 | for _ in 0..10 { 52 | let _block = cache.advance(&mut handle); 53 | } 54 | }); -------------------------------------------------------------------------------- /lading_payload/benches/opentelemetry_traces.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group}; 2 | 3 | use lading_payload::{OpentelemetryTraces, Serialize}; 4 | use rand::{SeedableRng, rngs::SmallRng}; 5 | use std::time::Duration; 6 | 7 | fn opentelemetry_traces_setup(c: &mut Criterion) { 8 | c.bench_function("opentelemetry_traces_setup", |b| { 9 | b.iter(|| { 10 | let mut rng = SmallRng::seed_from_u64(19690716); 11 | let _ot = OpentelemetryTraces::new(&mut rng); 12 | }) 13 | }); 14 | } 15 | 16 | fn opentelemetry_traces_all(c: &mut Criterion) { 17 | let mb = 1_000_000; // 1 MiB 18 | 19 | let mut group = c.benchmark_group("opentelemetry_traces_all"); 20 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 21 | group.throughput(Throughput::Bytes(*size as u64)); 22 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 23 | b.iter(|| { 24 | let mut rng = SmallRng::seed_from_u64(19690716); 25 | let ot = OpentelemetryTraces::new(&mut rng); 26 | let mut writer = Vec::with_capacity(size); 27 | 28 | ot.to_bytes(rng, size, &mut writer) 29 | .expect("failed to convert to bytes"); 30 | }); 31 | }); 32 | } 33 | group.finish(); 34 | } 35 | 36 | criterion_group!( 37 | name = benches; 38 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 39 | targets = opentelemetry_traces_setup, opentelemetry_traces_all 40 | ); 41 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/fluent_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | } 16 | 17 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 18 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 19 | 20 | fuzz_target!(|input: Input| { 21 | lading_fuzz::debug_input(&input); 22 | 23 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 24 | return; 25 | } 26 | 27 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 28 | return; 29 | } 30 | 31 | if input.max_block_size.get() > input.total_bytes.get() { 32 | return; 33 | } 34 | 35 | let mut rng = SmallRng::from_seed(input.seed); 36 | let payload = lading_payload::Config::Fluent; 37 | 38 | let cache = match Cache::fixed_with_max_overhead( 39 | &mut rng, 40 | input.total_bytes, 41 | u128::from(input.max_block_size.get()), 42 | &payload, 43 | input.total_bytes.get() as usize, 44 | ) { 45 | Ok(c) => c, 46 | Err(_) => return, 47 | }; 48 | 49 | // Call advance 10 times to exercise the cache rotation 50 | let mut handle = cache.handle(); 51 | for _ in 0..10 { 52 | let _block = cache.advance(&mut handle); 53 | } 54 | }); -------------------------------------------------------------------------------- /examples/lading.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - tcp: 3 | seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] 4 | addr: "0.0.0.0:8282" 5 | variant: "syslog5424" 6 | bytes_per_second: "500 MiB" 7 | maximum_prebuild_cache_size_bytes: "256 MiB" 8 | - file_tree: 9 | seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] 10 | max_depth: 10 11 | max_sub_folders: 5 12 | max_files: 10 13 | max_nodes: 200 14 | name_len: 10 15 | root: /tmp/rusty 16 | open_per_second: 10 17 | rename_per_second: 1 18 | - process_tree: 19 | seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] 20 | max_depth: 3 21 | max_children: 6 22 | max_tree_per_second: 1 23 | process_sleep_ns: 200000000 24 | executables: 25 | - executable: /usr/bin/ls 26 | args: 27 | mode: static 28 | values: ["-al", "/tmp"] 29 | envs: 30 | mode: static 31 | values: ["ENV=staging"] 32 | - executable: /usr/bin/ls 33 | args: 34 | mode: generate 35 | length: 10 36 | count: 20 37 | envs: 38 | mode: generate 39 | length: 10 40 | count: 20 41 | 42 | blackhole: 43 | - tcp: 44 | binding_addr: "0.0.0.0:8080" 45 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/datadog_log_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | } 16 | 17 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 18 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 19 | 20 | fuzz_target!(|input: Input| { 21 | lading_fuzz::debug_input(&input); 22 | 23 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 24 | return; 25 | } 26 | 27 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 28 | return; 29 | } 30 | 31 | if input.max_block_size.get() > input.total_bytes.get() { 32 | return; 33 | } 34 | 35 | let mut rng = SmallRng::from_seed(input.seed); 36 | let payload = lading_payload::Config::DatadogLog; 37 | 38 | let cache = match Cache::fixed_with_max_overhead( 39 | &mut rng, 40 | input.total_bytes, 41 | u128::from(input.max_block_size.get()), 42 | &payload, 43 | input.total_bytes.get() as usize, 44 | ) { 45 | Ok(c) => c, 46 | Err(_) => return, 47 | }; 48 | 49 | // Call advance 10 times to exercise the cache rotation 50 | let mut handle = cache.handle(); 51 | for _ in 0..10 { 52 | let _block = cache.advance(&mut handle); 53 | } 54 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/syslog5424_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | } 16 | 17 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 18 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 19 | 20 | fuzz_target!(|input: Input| { 21 | lading_fuzz::debug_input(&input); 22 | 23 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 24 | return; 25 | } 26 | 27 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 28 | return; 29 | } 30 | 31 | if input.max_block_size.get() > input.total_bytes.get() { 32 | return; 33 | } 34 | 35 | let mut rng = SmallRng::from_seed(input.seed); 36 | let payload = lading_payload::Config::Syslog5424; 37 | 38 | let cache = match Cache::fixed_with_max_overhead( 39 | &mut rng, 40 | input.total_bytes, 41 | u128::from(input.max_block_size.get()), 42 | &payload, 43 | input.total_bytes.get() as usize, 44 | ) { 45 | Ok(c) => c, 46 | Err(_) => return, 47 | }; 48 | 49 | // Call advance 10 times to exercise the cache rotation 50 | let mut handle = cache.handle(); 51 | for _ in 0..10 { 52 | let _block = cache.advance(&mut handle); 53 | } 54 | }); -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/apache_common_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | } 16 | 17 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 18 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 19 | 20 | fuzz_target!(|input: Input| { 21 | lading_fuzz::debug_input(&input); 22 | 23 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 24 | return; 25 | } 26 | 27 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 28 | return; 29 | } 30 | 31 | if input.max_block_size.get() > input.total_bytes.get() { 32 | return; 33 | } 34 | 35 | let mut rng = SmallRng::from_seed(input.seed); 36 | let payload = lading_payload::Config::ApacheCommon; 37 | 38 | let cache = match Cache::fixed_with_max_overhead( 39 | &mut rng, 40 | input.total_bytes, 41 | u128::from(input.max_block_size.get()), 42 | &payload, 43 | input.total_bytes.get() as usize, 44 | ) { 45 | Ok(c) => c, 46 | Err(_) => return, 47 | }; 48 | 49 | // Call advance 10 times to exercise the cache rotation 50 | let mut handle = cache.handle(); 51 | for _ in 0..10 { 52 | let _block = cache.advance(&mut handle); 53 | } 54 | }); -------------------------------------------------------------------------------- /lints/tests/no-long-paths.yml: -------------------------------------------------------------------------------- 1 | id: no-long-paths 2 | valid: 3 | - use self::module::submodule::function; 4 | - use crate::generator::http::Server; 5 | - use foo::{bar::baz, bard::baz::bing, bing::foo}; 6 | - use lading_payload::dogstatsd::metric::Counter; 7 | - use std::collections::{BTreeMap, HashMap}; 8 | - use tokio::sync::broadcast::Sender; 9 | - let string = "some::long::path::in::string"; 10 | - let raw_string = r#"another::long::path::here"#; 11 | - println!("valid"); 12 | - format!("string with {}", some_variable); 13 | - >- 14 | fn example_two_segments() { 15 | let config = Config::new(); 16 | let server = Server::start(); 17 | let file = File::open("path"); 18 | let result = Response::ok(); 19 | } 20 | - >- 21 | use foo::{ 22 | bar::bing::baz::bang, 23 | one::two::three, 24 | four::five::six::seven::eight, 25 | }; 26 | - let config = target::Config::Pid; 27 | - let server = generator::Server::new(); 28 | - let metrics = json::MetricKind::Counter; 29 | - let parsed = some::parser::parse(); 30 | invalid: 31 | - let metrics = json::MetricKind::private::Counter; 32 | - let very_long = foo::bar::baz::qux::deeply::nested::call(); 33 | - let another = project::module::submodule::function::call::chain(); 34 | - let parsed = some::parser::inner::parse(); 35 | - let value = outer::function(inner::long::path::call()); 36 | - >- 37 | impl SomeStruct { 38 | fn method(&self) { 39 | some::long::path::here().field; // should trigger 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lading_payload/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lading-payload" 3 | version = "0.1.0" 4 | authors = [ 5 | "Brian L. Troutwine ", 6 | "George Hahn MAX_BUDGET { 25 | return; 26 | } 27 | 28 | if input.config.valid().is_err() { 29 | return; 30 | } 31 | 32 | let max_contexts = match input.config.contexts.total_contexts { 33 | lading_payload::common::config::ConfRange::Constant(n) => n, 34 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 35 | }; 36 | if max_contexts > MAX_CONTEXTS { 37 | return; 38 | } 39 | 40 | let mut rng = SmallRng::from_seed(input.seed); 41 | let mut bytes = Vec::with_capacity(budget); 42 | 43 | let mut serializer = match lading_payload::opentelemetry::metric::OpentelemetryMetrics::new(input.config, budget, &mut rng) { 44 | Ok(s) => s, 45 | Err(_) => return, 46 | }; 47 | 48 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 49 | assert!(bytes.len() <= budget); 50 | } 51 | }); -------------------------------------------------------------------------------- /Cross.toml: -------------------------------------------------------------------------------- 1 | # Ref https://github.com/cross-rs/cross/blob/main/docs/custom_images.md#adding-dependencies-to-existing-images 2 | # `cross` is installed in CI via a gh action for speed, which can end up using an old version of cross. 3 | # The version of the cross-image is within each release of cross, so for example I saw the 4 | # 'aarch64-unknown-linux-gnu' image using ubuntu xenial which has an ancient version of protobuf 5 | # Each target has an override to use the latest image off `main` 6 | [target.aarch64-unknown-linux-gnu] 7 | image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" 8 | pre-build = [ 9 | "dpkg --add-architecture $CROSS_DEB_ARCH", 10 | "apt-get update && apt-get --assume-yes install protobuf-compiler", 11 | ] 12 | [target.aarch64-unknown-linux-musl] 13 | image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:main" 14 | pre-build = [ 15 | "dpkg --add-architecture $CROSS_DEB_ARCH", 16 | "apt-get update && apt-get --assume-yes install protobuf-compiler", 17 | ] 18 | [target.x86_64-unknown-linux-gnu] 19 | image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main" 20 | pre-build = [ 21 | "dpkg --add-architecture $CROSS_DEB_ARCH", 22 | "apt-get update && apt-get --assume-yes install protobuf-compiler", 23 | ] 24 | [target.x86_64-unknown-linux-musl] 25 | image = "ghcr.io/cross-rs/x86_64-unknown-linux-musl:main" 26 | pre-build = [ 27 | "dpkg --add-architecture $CROSS_DEB_ARCH", 28 | "apt-get update && apt-get --assume-yes install protobuf-compiler", 29 | ] 30 | [target.x86_64-apple-darwin] 31 | pre-build = ["brew install protobuf"] 32 | [target.aarch64-apple-darwin] 33 | pre-build = ["brew install protobuf"] 34 | -------------------------------------------------------------------------------- /.github/workflows/check-changelog.yml: -------------------------------------------------------------------------------- 1 | name: Changelog Check 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened, labeled, unlabeled] 6 | 7 | jobs: 8 | changelog-check: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout code 12 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 13 | with: 14 | fetch-depth: 2 # Only need preceeding commit 15 | 16 | - name: Check for no-changelog label 17 | id: label-check 18 | uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 19 | with: 20 | script: | 21 | const payload = context.payload; 22 | const label = !!payload.pull_request.labels.find(l => l.name === 'no-changelog'); 23 | core.setOutput('has_label', label.toString()); 24 | 25 | - name: Check if CHANGELOG.md was modified 26 | id: changelog-check 27 | if: steps.label-check.outputs.has_label == 'false' 28 | run: | 29 | if git diff --quiet HEAD^ -- CHANGELOG.md; then 30 | echo "any_changed=false" >> $GITHUB_OUTPUT 31 | else 32 | echo "any_changed=true" >> $GITHUB_OUTPUT 33 | fi 34 | 35 | - name: Assert CHANGELOG.md is modified 36 | uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 37 | if: steps.label-check.outputs.has_label == 'false' && steps.changelog-check.outputs.any_changed != 'true' 38 | with: 39 | script: | 40 | core.setFailed("No changes to CHANGELOG.md detected. Add 'no-changelog' label if this is intentional.") 41 | 42 | -------------------------------------------------------------------------------- /lading_payload/src/opentelemetry/metric/unit.rs: -------------------------------------------------------------------------------- 1 | //! Code to generate units according to 2 | //! . This may be generally 3 | //! useful in the project and is not specifically tied to the OpenTelemetry metrics 4 | //! implementation. 5 | 6 | // The spec defines a fairly elaborate grammar. It's unclear that we need _all_ 7 | // that for our purposes, so for now we rely on a simple table method. I imagine 8 | // we can just keep stuffing the table for a while as seems desirable. 9 | 10 | use crate::opentelemetry::common::GeneratorError; 11 | 12 | const UNITS: &[&str] = &[ 13 | "bit", "Kbit", "Mbit", "Gbit", // data size, bits, decimal 14 | "Kibit", "Mibit", "Gibit", // data size, bits, binary 15 | "By", "KBy", "MBy", "GBy", // data size, bytes, decimal 16 | "KiBy", "MiBy", "GiBy", // data size, bytes, binary 17 | "By/s", "KiBy/s", "bit/s", "Mbit/s", // transfer rate 18 | "s", "ms", "us", "ns", // time 19 | "W", "kW", "kWh", // power 20 | ]; 21 | 22 | /// Errors related to generation 23 | #[derive(thiserror::Error, Debug, Clone, Copy)] 24 | pub enum Error {} 25 | 26 | #[derive(Debug, Clone, Copy)] 27 | pub(crate) struct UnitGenerator {} 28 | 29 | impl UnitGenerator { 30 | pub(crate) fn new() -> Self { 31 | Self {} 32 | } 33 | } 34 | 35 | impl crate::Generator<'_> for UnitGenerator { 36 | type Output = &'static str; 37 | type Error = GeneratorError; 38 | 39 | fn generate(&self, rng: &mut R) -> Result 40 | where 41 | R: rand::Rng + ?Sized, 42 | { 43 | Ok(UNITS[rng.random_range(0..UNITS.len())]) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /lading_capture/proptest-regressions/manager/state_machine.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 7a98abac6c52fb38119dc40c8e051cec5af02a5c17d20490f9c3ab44fa4ddd04 # shrinks to ops = [HistoricalCounterAbs("q", 3836061114605990454, tick_offset=0), WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0), WriteCounter("a", 0), HistoricalCounterIncr("q", 14610682959103561162, tick_offset=0), WriteCounter("a", 0), WriteCounter("a", 0)] 8 | cc 3da9ebcd9c155b7c17dd00eb3601cf3e7a63bc6e1b5686ea84013d52c466fe30 # shrinks to ops = [FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, WriteCounter("a", 1), WriteCounter("a", 1), FlushTick, FlushTick, WriteCounter("a", 1), FlushTick, WriteCounter("a", 1), WriteCounter("a", 1), WriteCounter("a", 1), WriteCounter("a", 1), FlushTick, FlushTick, FlushTick, FlushTick, FlushTick, WriteCounter("a", 1), FlushTick, FlushTick, FlushTick, WriteCounter("a", 1), WriteCounter("a", 1), FlushTick, FlushTick] 9 | -------------------------------------------------------------------------------- /lints/tests/__snapshots__/no-as-imports-snapshot.yml: -------------------------------------------------------------------------------- 1 | id: no-as-imports 2 | snapshots: 3 | use foo as Foo;: 4 | labels: 5 | - source: use foo as Foo; 6 | style: primary 7 | start: 0 8 | end: 15 9 | use foo::bar as FooBar;: 10 | labels: 11 | - source: use foo::bar as FooBar; 12 | style: primary 13 | start: 0 14 | end: 23 15 | ? |- 16 | use foo::{ 17 | bar::bing::baz::bang, 18 | one::two::three as TwoThree, 19 | four::five::six::seven::eight, 20 | }; 21 | : labels: 22 | - source: |- 23 | use foo::{ 24 | bar::bing::baz::bang, 25 | one::two::three as TwoThree, 26 | four::five::six::seven::eight, 27 | }; 28 | style: primary 29 | start: 0 30 | end: 101 31 | use foo::{ bar as FooBar };: 32 | labels: 33 | - source: use foo::{ bar as FooBar }; 34 | style: primary 35 | start: 0 36 | end: 27 37 | use foo::{ bar, bar::baz as BarBaz };: 38 | labels: 39 | - source: use foo::{ bar, bar::baz as BarBaz }; 40 | style: primary 41 | start: 0 42 | end: 37 43 | use foo::{ bar::baz as BarBaz, bar };: 44 | labels: 45 | - source: use foo::{ bar::baz as BarBaz, bar }; 46 | style: primary 47 | start: 0 48 | end: 37 49 | use foo::{ bar::baz as FooBaz };: 50 | labels: 51 | - source: use foo::{ bar::baz as FooBaz }; 52 | style: primary 53 | start: 0 54 | end: 32 55 | use tokio::sync::broadcast::Sender as BroadcastSender;: 56 | labels: 57 | - source: use tokio::sync::broadcast::Sender as BroadcastSender; 58 | style: primary 59 | start: 0 60 | end: 54 61 | -------------------------------------------------------------------------------- /lading/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The lading daemon load generation and introspection tool. 2 | //! 3 | //! This library support the lading binary found elsewhere in this project. The 4 | //! bits and pieces here are not intended to be used outside of supporting 5 | //! lading, although if they are helpful in other domains that's a nice 6 | //! surprise. 7 | 8 | #![deny(clippy::cargo)] 9 | #![allow(clippy::cast_precision_loss)] 10 | #![allow(clippy::multiple_crate_versions)] 11 | 12 | use http_body_util::BodyExt; 13 | 14 | pub mod blackhole; 15 | pub(crate) mod codec; 16 | mod common; 17 | pub mod config; 18 | pub mod generator; 19 | pub mod inspector; 20 | pub mod observer; 21 | pub(crate) mod proto; 22 | pub mod target; 23 | pub mod target_metrics; 24 | 25 | use byte_unit::Byte; 26 | use sysinfo::System; 27 | 28 | #[inline] 29 | pub(crate) fn full>( 30 | chunk: T, 31 | ) -> http_body_util::combinators::BoxBody { 32 | http_body_util::Full::new(chunk.into()) 33 | .map_err(|never| match never {}) 34 | .boxed() 35 | } 36 | 37 | /// Get available memory for the process, checking cgroup v2 limits first, 38 | /// then falling back to system memory. 39 | #[must_use] 40 | pub fn get_available_memory() -> Byte { 41 | if let Ok(content) = std::fs::read_to_string("/sys/fs/cgroup/memory.max") { 42 | let content = content.trim(); 43 | if content == "max" { 44 | return Byte::from_u64(u64::MAX); 45 | } 46 | let ignore_case = true; 47 | if let Ok(limit) = Byte::parse_str(content.trim(), ignore_case) { 48 | return limit; 49 | } 50 | } 51 | 52 | let sys = System::new_all(); 53 | Byte::from_u64(sys.total_memory()) 54 | } 55 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/trace_agent_v04_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::{ 9 | self, 10 | block::Cache, 11 | trace_agent::{self, v04::Config}, 12 | }; 13 | 14 | #[derive(arbitrary::Arbitrary, Debug)] 15 | struct Input { 16 | seed: [u8; 32], 17 | total_bytes: NonZeroU32, 18 | max_block_size: NonZeroU32, 19 | config: Config, 20 | } 21 | 22 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 23 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 24 | 25 | fuzz_target!(|input: Input| { 26 | lading_fuzz::debug_input(&input); 27 | 28 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 29 | return; 30 | } 31 | 32 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 33 | return; 34 | } 35 | 36 | if input.max_block_size.get() > input.total_bytes.get() { 37 | return; 38 | } 39 | 40 | if input.config.valid().is_err() { 41 | return; 42 | } 43 | 44 | let mut rng = SmallRng::from_seed(input.seed); 45 | let payload = lading_payload::Config::TraceAgent(trace_agent::Config::V04(input.config)); 46 | 47 | let cache = match Cache::fixed_with_max_overhead( 48 | &mut rng, 49 | input.total_bytes, 50 | u128::from(input.max_block_size.get()), 51 | &payload, 52 | input.total_bytes.get() as usize, 53 | ) { 54 | Ok(c) => c, 55 | Err(_) => return, 56 | }; 57 | 58 | // Call advance 10 times to exercise the cache rotation 59 | let mut handle = cache.handle(); 60 | for _ in 0..10 { 61 | let _block = cache.advance(&mut handle); 62 | } 63 | }); 64 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for contributing to `lading`! We find `lading` very useful for our 4 | work and hope you have as well and are interested in incorporating your changes 5 | in a timely fashion. To help us help you we ask that you follow a handful of 6 | guidelines when crafting your change. 7 | 8 | ## Guidelines 9 | 10 | 0. Please send us changes as PRs. We don't have any constraints on branch names 11 | but do please add descriptive commit messages to your PR. We provide a 12 | template for your convenience. 13 | 14 | 0. Please include a changelog entry in `CHANGELOG.md` with all non-trivial changes. 15 | If a change is trivial, label the PR as `no-changelog` to avoid a CI ding. 16 | 17 | 0. Do consider tagging reviewers only after CI checks are green. 18 | 19 | 0. Do consider that the smaller your change the easier it is to review and the 20 | more likely we are to incorporate it. If you have a big change in mind, let's 21 | hash it out in an issue first, just so we agree on your idea before you put 22 | the time into it. 23 | 24 | 0. Test your changes, both in test code and by running `lading` with your 25 | changes in place. Preferentially we make use of property tests in this 26 | project. We can help you write those if you've never done property testing. 27 | 28 | 0. We require signed commits. Github's 29 | [documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) 30 | is a big help at getting this set up. 31 | 32 | 0. If you add a dependency please be aware that we require licenses to conform 33 | to the set listed in our 34 | [cargo-deny](https://github.com/EmbarkStudios/cargo-deny) 35 | configuration. License check is part of the CI workflow. 36 | -------------------------------------------------------------------------------- /lading/src/observer/linux/procfs/uptime.rs: -------------------------------------------------------------------------------- 1 | #[derive(thiserror::Error, Debug)] 2 | /// Errors produced by functions in this module 3 | pub enum Error { 4 | /// Wrapper for [`std::io::Error`] 5 | #[error("IO error: {0}")] 6 | Io(#[from] std::io::Error), 7 | /// Unable to parse /proc/uptime 8 | #[error("/proc/uptime malformed: {0}")] 9 | Malformed(&'static str), 10 | /// Unable to parse floating point 11 | #[error("Float Parsing: {0}")] 12 | ParseFloat(#[from] std::num::ParseFloatError), 13 | } 14 | 15 | /// Read `/proc/uptime` 16 | /// 17 | /// Only the first field is used, which is the total uptime in seconds. 18 | pub(crate) async fn poll() -> Result { 19 | let buf = tokio::fs::read_to_string("/proc/uptime").await?; 20 | let uptime_secs = proc_uptime_inner(&buf)?; 21 | Ok(uptime_secs) 22 | } 23 | 24 | /// Parse `/proc/uptime` to extract total uptime in seconds. 25 | /// 26 | /// # Errors 27 | /// 28 | /// Function errors if the file is malformed. 29 | #[inline] 30 | fn proc_uptime_inner(contents: &str) -> Result { 31 | // TODO this should probably be scooted up to procfs.rs. Implies the 32 | // `proc_*` functions there need a test component, making this an inner 33 | // function eventually. 34 | 35 | let fields: Vec<&str> = contents.split_whitespace().collect(); 36 | if fields.is_empty() { 37 | return Err(Error::Malformed("/proc/uptime empty")); 38 | } 39 | let uptime_secs = fields[0].parse::()?; 40 | Ok(uptime_secs) 41 | } 42 | 43 | #[cfg(test)] 44 | mod test { 45 | use super::proc_uptime_inner; 46 | 47 | #[test] 48 | fn parse_uptime_basic() { 49 | let line = "12345.67 4321.00\n"; 50 | let uptime = proc_uptime_inner(line).unwrap(); 51 | assert!((uptime - 12345.67).abs() < f64::EPSILON); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /examples/trace-agent-v04.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - id: "trace_agent_v04" 3 | trace_agent: 4 | seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] 5 | target_uri: "http://localhost:8126" 6 | backoff_behavior: 7 | obey: 8 | max_retries: 3 9 | variant: 10 | v0.4: 11 | contexts: 12 | inclusive: 13 | min: 1000 14 | max: 5000 15 | spans_per_trace: 16 | inclusive: 17 | min: 1 18 | max: 20 19 | tags_per_span: 20 | inclusive: 21 | min: 0 22 | max: 10 23 | metrics_per_span: 24 | inclusive: 25 | min: 0 26 | max: 3 27 | error_rate: 0.05 28 | service_name_length: 29 | inclusive: 30 | min: 5 31 | max: 20 32 | operation_name_length: 33 | inclusive: 34 | min: 5 35 | max: 30 36 | resource_name_length: 37 | inclusive: 38 | min: 10 39 | max: 50 40 | span_type_length: 41 | inclusive: 42 | min: 3 43 | max: 15 44 | tag_key_length: 45 | inclusive: 46 | min: 3 47 | max: 25 48 | tag_value_length: 49 | inclusive: 50 | min: 5 51 | max: 50 52 | metric_key_length: 53 | inclusive: 54 | min: 3 55 | max: 20 56 | bytes_per_second: "50 MiB" 57 | maximum_block_size: "1 MiB" 58 | maximum_prebuild_cache_size_bytes: "256 MiB" 59 | parallel_connections: 1 60 | 61 | blackhole: 62 | - http: 63 | binding_addr: "0.0.0.0:8126" -------------------------------------------------------------------------------- /integration/shared/src/lib.rs: -------------------------------------------------------------------------------- 1 | use integration_api::TestConfig; 2 | use serde::{Deserialize, Serialize}; 3 | use tonic::{IntoRequest, Request}; 4 | 5 | #[allow(clippy::derive_partial_eq_without_eq)] 6 | pub mod integration_api { 7 | use tonic::IntoRequest; 8 | 9 | tonic::include_proto!("integration_api"); 10 | 11 | impl IntoRequest for () { 12 | fn into_request(self) -> tonic::Request { 13 | tonic::Request::new(Empty {}) 14 | } 15 | } 16 | } 17 | 18 | #[derive(Debug, Serialize, Deserialize)] 19 | pub enum ListenConfig { 20 | /// Do not listen on any port 21 | None, 22 | /// Listen on a random port for HTTP messages 23 | Http, 24 | /// Listen on a random port for TCP messages 25 | Tcp, 26 | /// Listen on a random port for UDP messages 27 | Udp, 28 | } 29 | 30 | #[derive(Debug, Serialize, Deserialize)] 31 | pub enum EmitConfig { 32 | /// Do not emit any messages 33 | None, 34 | /// Emit HTTP messages 35 | Http, 36 | /// Emit TCP messages 37 | Tcp, 38 | /// Emit UDP messages 39 | Udp, 40 | } 41 | 42 | #[derive(Debug, Serialize, Deserialize)] 43 | pub enum AssertionConfig { 44 | /// Do not assert on anything 45 | None, 46 | } 47 | 48 | #[derive(Debug, Serialize, Deserialize)] 49 | pub struct DucksConfig { 50 | pub listen: ListenConfig, 51 | pub emit: EmitConfig, 52 | pub assertions: AssertionConfig, 53 | } 54 | 55 | impl IntoRequest for DucksConfig { 56 | fn into_request(self) -> tonic::Request { 57 | Request::new(TestConfig { 58 | json_blob: serde_json::to_string(&self).expect("Failed to convert config to JSON"), 59 | }) 60 | } 61 | } 62 | impl From for DucksConfig { 63 | fn from(val: TestConfig) -> Self { 64 | serde_json::from_str(&val.json_blob).expect("Failed to convert to JSON") 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/opentelemetry_logs_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::{Serialize, common::config::ConfRange, opentelemetry::log::OpentelemetryLogs}; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | config: lading_payload::opentelemetry::log::Config, 15 | } 16 | 17 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 18 | const MAX_CONTEXTS: u32 = 5_000; 19 | const MAX_TRACE_CARDINALITY: u32 = 10_000; // Limit trace IDs to prevent OOM 20 | 21 | fuzz_target!(|input: Input| { 22 | lading_fuzz::debug_input(&input); 23 | 24 | let budget = input.budget_bytes.get() as usize; 25 | if budget > MAX_BUDGET { 26 | return; 27 | } 28 | 29 | if input.config.valid().is_err() { 30 | return; 31 | } 32 | 33 | let max_contexts = match input.config.contexts.total_contexts { 34 | ConfRange::Constant(n) => n, 35 | ConfRange::Inclusive { max, .. } => max, 36 | }; 37 | if max_contexts > MAX_CONTEXTS { 38 | return; 39 | } 40 | 41 | let max_trace_cardinality = match input.config.trace_cardinality { 42 | ConfRange::Constant(n) => n, 43 | ConfRange::Inclusive { max, .. } => max, 44 | }; 45 | if max_trace_cardinality > MAX_TRACE_CARDINALITY { 46 | return; 47 | } 48 | 49 | let mut rng = SmallRng::from_seed(input.seed); 50 | let mut bytes = Vec::with_capacity(budget); 51 | 52 | let mut serializer = match OpentelemetryLogs::new(input.config, budget, &mut rng) { 53 | Ok(s) => s, 54 | Err(_) => return, 55 | }; 56 | 57 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 58 | assert!(bytes.len() <= budget); 59 | } 60 | }); 61 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/dogstatsd_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::{block::Cache, common::config::ConfRange}; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | config: lading_payload::dogstatsd::Config, 16 | } 17 | 18 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 19 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 20 | const MAX_CONTEXTS: u32 = 1_000; 21 | 22 | fuzz_target!(|input: Input| { 23 | lading_fuzz::debug_input(&input); 24 | 25 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 26 | return; 27 | } 28 | 29 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 30 | return; 31 | } 32 | 33 | if input.max_block_size.get() > input.total_bytes.get() { 34 | return; 35 | } 36 | 37 | if input.config.valid().is_err() { 38 | return; 39 | } 40 | 41 | let max_contexts = match input.config.contexts { 42 | ConfRange::Constant(n) => n, 43 | ConfRange::Inclusive { max, .. } => max, 44 | }; 45 | if max_contexts > MAX_CONTEXTS { 46 | return; 47 | } 48 | 49 | let mut rng = SmallRng::from_seed(input.seed); 50 | let payload = lading_payload::Config::DogStatsD(input.config); 51 | 52 | let cache = match Cache::fixed_with_max_overhead( 53 | &mut rng, 54 | input.total_bytes, 55 | u128::from(input.max_block_size.get()), 56 | &payload, 57 | input.total_bytes.get() as usize, 58 | ) { 59 | Ok(c) => c, 60 | Err(_) => return, 61 | }; 62 | 63 | // Call advance 10 times to exercise the cache rotation 64 | let mut handle = cache.handle(); 65 | for _ in 0..10 { 66 | let _block = cache.advance(&mut handle); 67 | } 68 | }); 69 | -------------------------------------------------------------------------------- /lints/tests/__snapshots__/no-long-paths-snapshot.yml: -------------------------------------------------------------------------------- 1 | id: no-long-paths 2 | snapshots: 3 | ? |- 4 | impl SomeStruct { 5 | fn method(&self) { 6 | some::long::path::here().field; // should trigger 7 | } 8 | } 9 | : labels: 10 | - source: some::long::path::here 11 | style: primary 12 | start: 49 13 | end: 71 14 | let another = project::module::submodule::function::call::chain();: 15 | labels: 16 | - source: project::module::submodule::function::call::chain 17 | style: primary 18 | start: 14 19 | end: 63 20 | let config = target::Config::Pid;: 21 | labels: 22 | - source: target::Config::Pid 23 | style: primary 24 | start: 13 25 | end: 32 26 | let metrics = json::MetricKind::Counter;: 27 | labels: 28 | - source: json::MetricKind::Counter 29 | style: primary 30 | start: 14 31 | end: 39 32 | let metrics = json::MetricKind::private::Counter;: 33 | labels: 34 | - source: json::MetricKind::private::Counter 35 | style: primary 36 | start: 14 37 | end: 48 38 | let parsed = some::parser::inner::parse();: 39 | labels: 40 | - source: some::parser::inner::parse 41 | style: primary 42 | start: 13 43 | end: 39 44 | let parsed = some::parser::module::::parse();: 45 | labels: 46 | - source: some::parser::module 47 | style: primary 48 | start: 13 49 | end: 33 50 | let server = generator::Server::new();: 51 | labels: 52 | - source: generator::Server::new 53 | style: primary 54 | start: 13 55 | end: 35 56 | let value = outer::function(inner::long::path::call());: 57 | labels: 58 | - source: inner::long::path::call 59 | style: primary 60 | start: 28 61 | end: 51 62 | let very_long = foo::bar::baz::qux::deeply::nested::call();: 63 | labels: 64 | - source: foo::bar::baz::qux::deeply::nested::call 65 | style: primary 66 | start: 16 67 | end: 56 68 | -------------------------------------------------------------------------------- /lading/src/common.rs: -------------------------------------------------------------------------------- 1 | use std::{fmt, fs, path::PathBuf, process::Stdio, str}; 2 | 3 | use serde::Deserialize; 4 | 5 | #[derive(Debug, Deserialize, PartialEq, Eq, Clone)] 6 | #[serde(deny_unknown_fields)] 7 | /// Defines how sub-process stderr and stdout are handled. 8 | pub struct Output { 9 | #[serde(default)] 10 | /// Determines how stderr is routed. 11 | pub stderr: Behavior, 12 | #[serde(default)] 13 | /// Determines how stderr is routed. 14 | pub stdout: Behavior, 15 | } 16 | 17 | #[derive(Debug, Deserialize, Clone, PartialEq, Eq)] 18 | #[serde(deny_unknown_fields)] 19 | #[serde(untagged)] 20 | /// Defines the [`Output`] behavior for stderr and stdout. 21 | pub enum Behavior { 22 | /// Redirect stdout, stderr to /dev/null 23 | Quiet, 24 | /// Write to a location on-disk. 25 | Log(PathBuf), 26 | } 27 | 28 | impl Default for Behavior { 29 | fn default() -> Self { 30 | Self::Quiet 31 | } 32 | } 33 | 34 | impl fmt::Display for Behavior { 35 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { 36 | match self { 37 | Behavior::Quiet => write!(f, "/dev/null")?, 38 | Behavior::Log(path) => write!(f, "{}", path.display())?, 39 | } 40 | Ok(()) 41 | } 42 | } 43 | 44 | impl str::FromStr for Behavior { 45 | type Err = &'static str; 46 | 47 | fn from_str(input: &str) -> Result { 48 | let mut path = PathBuf::new(); 49 | path.push(input); 50 | Ok(Behavior::Log(path)) 51 | } 52 | } 53 | 54 | pub(crate) fn stdio(behavior: &Behavior) -> Stdio { 55 | match behavior { 56 | Behavior::Quiet => Stdio::null(), 57 | Behavior::Log(path) => { 58 | let fp = fs::File::create(path).unwrap_or_else(|_| { 59 | panic!( 60 | "Full directory path does not exist: {path}", 61 | path = path.display() 62 | ); 63 | }); 64 | Stdio::from(fp) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/opentelemetry_metrics_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::{block::Cache, common::config::ConfRange}; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | config: lading_payload::opentelemetry::metric::Config, 16 | } 17 | 18 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 19 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 20 | const MAX_CONTEXTS: u32 = 5_000; 21 | 22 | fuzz_target!(|input: Input| { 23 | lading_fuzz::debug_input(&input); 24 | 25 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 26 | return; 27 | } 28 | 29 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 30 | return; 31 | } 32 | 33 | if input.max_block_size.get() > input.total_bytes.get() { 34 | return; 35 | } 36 | 37 | if input.config.valid().is_err() { 38 | return; 39 | } 40 | 41 | let max_contexts = match input.config.contexts.total_contexts { 42 | ConfRange::Constant(n) => n, 43 | ConfRange::Inclusive { max, .. } => max, 44 | }; 45 | if max_contexts > MAX_CONTEXTS { 46 | return; 47 | } 48 | 49 | let mut rng = SmallRng::from_seed(input.seed); 50 | let payload = lading_payload::Config::OpentelemetryMetrics(input.config); 51 | 52 | let cache = match Cache::fixed_with_max_overhead( 53 | &mut rng, 54 | input.total_bytes, 55 | u128::from(input.max_block_size.get()), 56 | &payload, 57 | input.total_bytes.get() as usize, 58 | ) { 59 | Ok(c) => c, 60 | Err(_) => return, 61 | }; 62 | 63 | // Call advance 10 times to exercise the cache rotation 64 | let mut handle = cache.handle(); 65 | for _ in 0..10 { 66 | let _block = cache.advance(&mut handle); 67 | } 68 | }); 69 | -------------------------------------------------------------------------------- /lading_capture/proptest-regressions/formats.txt: -------------------------------------------------------------------------------- 1 | # Seeds for failure cases proptest has generated in the past. It is 2 | # automatically read and these particular cases re-run before any 3 | # novel cases are generated. 4 | # 5 | # It is recommended to check this file in to source control so that 6 | # everyone who runs the test benefits from these saved cases. 7 | cc 69095d8270e92a7bb6a6603c2debeae356bbe0448e6161e83716fd015ceeb3c0 # shrinks to lines = [(0, 0, "a", Counter, Int(0), {"h0rq500_2ib1io__": "a", "agcl2ir__1_1_f_": "a"})] 8 | cc 1d29086c14fc312153f4efb9a3d1af38bb3182f39647dca3900c69fa745e0bf7 # shrinks to lines = [(0, 0, "a", Histogram, Float(0.0), {}, Some([123, 34, 98, 105, 110, 115, 34, 58, 91, 123, 34, 107, 34, 58, 48, 44, 34, 110, 34, 58, 57, 56, 125, 44, 123, 34, 107, 34, 58, 51, 50, 55, 54, 55, 44, 34, 110, 34, 58, 50, 125, 93, 44, 34, 99, 111, 117, 110, 116, 34, 58, 49, 48, 48, 44, 34, 109, 105, 110, 34, 58, 48, 46, 48, 44, 34, 109, 97, 120, 34, 58, 49, 46, 53, 50, 49, 50, 55, 56, 57, 56, 50, 51, 56, 49, 48, 50, 53, 53, 101, 51, 48, 56, 44, 34, 115, 117, 109, 34, 58, 110, 117, 108, 108, 44, 34, 97, 118, 103, 34, 58, 50, 46, 50, 48, 49, 53, 52, 54, 48, 51, 55, 48, 48, 48, 57, 48, 50, 52, 101, 51, 48, 54, 125]))] 9 | cc 36661e1611d353487b47416524fefc88aabcafbb70c3a044b41a33a1ed075887 # shrinks to lines = [(0, 0, "a", Histogram, Float(0.0), {}, Some(DDSketch { bins: [Bin { k: 32767, n: 2 }], count: 2, min: 1.7976931348623157e308, max: 1.7976931348623157e308, sum: inf, avg: 1.7976931348623157e308 }))] 10 | cc 5bc9f6f4b31cbd9261bba30f0450185b65de02e3798a507502f670defd57eeac # shrinks to lines = [(0, 0, "a", Histogram, Float(0.0), {}, Some(DDSketch { bins: [Bin { k: -32767, n: 1 }, Bin { k: 32767, n: 1 }], count: 2, min: -1.7976931348623157e308, max: 1.7976931348623157e308, sum: 0.0, avg: inf }))] 11 | cc 3ec7a86e9eaa5004cbb9652c5d93991f64986d4b8d2a591d19b6934c96a3f87e # shrinks to lines = [(0, 0, "a", Histogram, Float(0.0), {}, Some(DDSketch { bins: [Bin { k: -32767, n: 1 }, Bin { k: 0, n: 99 }], count: 100, min: -1.1407692428502447e303, max: 0.0, sum: -1.1407692428502447e303, avg: -1.1407692428502443e301 }))] 12 | -------------------------------------------------------------------------------- /lading_capture/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lading-capture" 3 | version = "0.2.0" 4 | authors = [ 5 | "Brian L. Troutwine ", 6 | "George Hahn () else { 56 | warn!("[{metric_prefix}] Failed to parse {key}: {content}"); 57 | continue; 58 | }; 59 | counter!(format!("{metric_prefix}.{key}"), labels).absolute(value); 60 | } else { 61 | let Ok(value) = value_str.parse::() else { 62 | warn!("[{metric_prefix}] Failed to parse {key}: {content}"); 63 | continue; 64 | }; 65 | gauge!(format!("{metric_prefix}.{key}"), labels).set(value); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lading_payload/src/trace_agent.rs: -------------------------------------------------------------------------------- 1 | //! Datadog Trace Agent payload generators. 2 | //! 3 | //! This module provides payload generators for different versions of the Datadog Trace Agent 4 | //! protocol. Each version has specific format requirements and encoding schemes. 5 | //! 6 | //! The implementation follows lading's core principles: 7 | //! - Pre-computation of all strings at initialization 8 | //! - Dynamic string pools instead of static hardcoded data 9 | //! - Performance-optimized generation suitable for load testing 10 | //! - Deterministic output for reproducible testing 11 | //! 12 | //! See the individual module documentation for version-specific details. 13 | 14 | use serde::{Deserialize, Serialize}; 15 | 16 | pub mod v04; 17 | 18 | /// Configuration for trace agent payload generation by version. 19 | /// 20 | /// Each version has different format requirements and performance characteristics. 21 | /// This allows users to specify which trace agent protocol version to target. 22 | #[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)] 23 | #[serde(rename_all = "snake_case")] 24 | #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] 25 | pub enum Config { 26 | /// Version 0.4: msgpack array of arrays of spans 27 | #[serde(rename = "v0.4")] 28 | V04(v04::Config), 29 | } 30 | 31 | impl Config { 32 | #[must_use] 33 | /// Return the stringy version number of the config. 34 | pub fn version(self) -> &'static str { 35 | match self { 36 | Self::V04(_) => "v0.4", 37 | } 38 | } 39 | 40 | #[must_use] 41 | /// Return the API endpoint path for this trace agent version. 42 | pub fn endpoint_path(self) -> &'static str { 43 | match self { 44 | Self::V04(_) => "/v0.4/traces", 45 | } 46 | } 47 | } 48 | 49 | impl Default for Config { 50 | fn default() -> Self { 51 | Config::V04(v04::Config::default()) 52 | } 53 | } 54 | 55 | impl Config { 56 | /// Validate the configuration 57 | /// # Errors 58 | /// Returns an error if the configuration is invalid 59 | pub fn valid(&self) -> Result<(), String> { 60 | match self { 61 | Config::V04(config) => config 62 | .valid() 63 | .map_err(|_| "invalid configuration".to_string()), 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /examples/lading-otel-metrics.yaml: -------------------------------------------------------------------------------- 1 | generator: 2 | - http: 3 | seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 4 | 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] 5 | headers: 6 | content-type: "application/x-protobuf" 7 | target_uri: "http://127.0.0.1:4318/v1/metrics" 8 | bytes_per_second: "50 MiB" 9 | parallel_connections: 5 10 | method: 11 | post: 12 | maximum_prebuild_cache_size_bytes: "1 GiB" 13 | variant: 14 | # all numbers arbitraray 15 | opentelemetry_metrics: 16 | metric_weights: 17 | gauge: 50 18 | sum_delta: 25 19 | sum_cumulative: 25 20 | contexts: 21 | total_contexts: 22 | constant: 10000 23 | # host., service. etc 24 | attributes_per_resource: 25 | inclusive: 26 | min: 1 27 | max: 64 28 | # auto-instrumentation in client libraries, DB connection etc 29 | scopes_per_resource: 30 | inclusive: 31 | min: 1 32 | max: 32 33 | # rare? build info possibly 34 | attributes_per_scope: 35 | inclusive: 36 | min: 0 37 | max: 4 38 | # exported instruments / telemetry by libraries and custom code 39 | metrics_per_scope: 40 | inclusive: 41 | min: 1 42 | max: 128 43 | # stuff like exit code, user id, cgroup id 44 | attributes_per_metric: 45 | inclusive: 46 | min: 0 47 | max: 255 48 | 49 | blackhole: 50 | - http: 51 | binding_addr: "127.0.0.1:9091" 52 | - http: 53 | binding_addr: "127.0.0.1:9092" 54 | - otlp: 55 | grpc_addr: "127.0.0.1:4317" 56 | http_addr: "127.0.0.1:4318" 57 | concurrent_requests_max: 100 58 | response_delay_millis: 0 59 | 60 | target_metrics: 61 | - prometheus: 62 | uri: "http://127.0.0.1:8888/metrics" # OTel collector metrics endpoint 63 | tags: 64 | component: "otel-collector" 65 | -------------------------------------------------------------------------------- /ci/validate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # This script runs all CI checks in order from simplest to most complex. Exits 7 | # on the first failure. Exit on first failure 8 | 9 | echo "=== Running CI validation suite ===" 10 | echo 11 | 12 | # Function to run a script and handle errors 13 | run_check() { 14 | local script="$1" 15 | local name="$2" 16 | 17 | echo ">>> Running $name..." 18 | if ! "$script"; then 19 | echo "FAILED: $name failed!" 20 | exit 1 21 | fi 22 | echo "PASSED: $name passed!" 23 | echo 24 | } 25 | 26 | # Get the directory where this script is located 27 | CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 28 | 29 | # Run checks in order from simplest to most complex 30 | run_check "$CI_DIR/shellcheck" "shellcheck" 31 | run_check "$CI_DIR/fmt" "cargo fmt" 32 | run_check "$CI_DIR/check" "cargo check" 33 | run_check "$CI_DIR/custom_lints" "project-specific lints" 34 | run_check "$CI_DIR/clippy" "cargo clippy" 35 | run_check "$CI_DIR/deny" "cargo deny" 36 | run_check "$CI_DIR/config-validation" "example config validation" 37 | run_check "$CI_DIR/test" "cargo nextest" 38 | 39 | # NOTE: These we run only in CI, leaving this note here to allow for as-needed 40 | # local validation. 41 | 42 | # run_check "$CI_DIR/integration-test" "integration tests" 43 | # run_check "$CI_DIR/fingerprint" "payload fingerprint verification" 44 | 45 | # Optional checks (don't fail if tools aren't available) 46 | echo ">>> Running optional checks..." 47 | 48 | # Kani proofs (optional) 49 | if command -v cargo-kani &> /dev/null; then 50 | echo ">>> Running kani proofs for each crate..." 51 | # shellcheck disable=SC2043 52 | for crate in lading_throttle; do 53 | echo ">>> Running kani proofs for $crate..." 54 | if ! "$CI_DIR/kani" "$crate"; then 55 | echo "FAILED: kani proofs for $crate failed!" 56 | exit 1 57 | fi 58 | echo "PASSED: kani proofs for $crate passed!" 59 | echo 60 | done 61 | else 62 | echo "WARNING: Skipping kani proofs (kani not installed)" 63 | fi 64 | 65 | # Buf checks (optional) 66 | if command -v buf &> /dev/null; then 67 | run_check "$CI_DIR/buf" "buf checks" 68 | else 69 | echo "WARNING: Skipping buf checks (buf not installed)" 70 | fi 71 | 72 | echo 73 | echo "=== SUCCESS: All CI checks passed! ===" 74 | -------------------------------------------------------------------------------- /ci/fingerprint: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # This script runs fingerprint verification for payload determinism. 7 | echo "=== Running fingerprint verification ===" 8 | echo 9 | 10 | CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 11 | 12 | failed=0 13 | total=0 14 | passed=0 15 | 16 | echo "Building payloadtool..." 17 | cargo build --release --bin payloadtool --quiet 2>/dev/null || cargo build --release --bin payloadtool 18 | 19 | echo 20 | echo "Checking fingerprints..." 21 | echo 22 | 23 | for dir in "$CI_DIR"/fingerprints/*/; do 24 | if [ ! -d "$dir" ]; then 25 | continue 26 | fi 27 | 28 | payload_type=$(basename "$dir") 29 | config_file="$dir/lading.yaml" 30 | fingerprint_file="$dir/fingerprint.txt" 31 | 32 | if [ ! -f "$config_file" ]; then 33 | echo "WARNING: No config found for $payload_type" 34 | continue 35 | fi 36 | 37 | total=$((total + 1)) 38 | 39 | echo -n "Checking $payload_type... " 40 | 41 | if ! actual=$(cargo run --release --bin payloadtool --quiet -- "$config_file" --fingerprint 2>/dev/null); then 42 | echo "FAILED (could not generate fingerprint)" 43 | failed=$((failed + 1)) 44 | continue 45 | fi 46 | 47 | # Check if golden fingerprint exists 48 | if [ ! -f "$fingerprint_file" ]; then 49 | echo "WARNING: No golden fingerprint found" 50 | echo " Generated fingerprint:" 51 | echo "$actual" | while IFS= read -r line; do echo " $line"; done 52 | echo " To save as golden, run:" 53 | echo " echo '$actual' > $fingerprint_file" 54 | failed=$((failed + 1)) 55 | continue 56 | fi 57 | 58 | expected=$(cat "$fingerprint_file") 59 | if [ "$actual" = "$expected" ]; then 60 | echo "PASSED" 61 | passed=$((passed + 1)) 62 | else 63 | echo "FAILED" 64 | echo " Expected:" 65 | echo "$expected" | while IFS= read -r line; do echo " $line"; done 66 | echo " Actual:" 67 | echo "$actual" | while IFS= read -r line; do echo " $line"; done 68 | failed=$((failed + 1)) 69 | fi 70 | done 71 | 72 | echo 73 | echo "=== Summary ===" 74 | echo "Total: $total" 75 | echo "Passed: $passed" 76 | echo "Failed: $failed" 77 | 78 | if [ $failed -gt 0 ]; then 79 | echo 80 | echo "Fingerprint verification FAILED" 81 | exit 1 82 | else 83 | echo 84 | echo "All fingerprints verified successfully!" 85 | fi 86 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/opentelemetry_logs_cache_fixed_next_block.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::block::Cache; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | total_bytes: NonZeroU32, 14 | max_block_size: NonZeroU32, 15 | config: lading_payload::opentelemetry::log::Config, 16 | } 17 | 18 | const MAX_TOTAL_BYTES: u32 = 10 * 1024 * 1024; // 10 MiB 19 | const MAX_BLOCK_SIZE: u32 = 1 * 1024 * 1024; // 1 MiB 20 | const MAX_CONTEXTS: u32 = 5_000; 21 | const MAX_TRACE_CARDINALITY: u32 = 10_000; // Limit trace IDs to prevent OOM 22 | 23 | fuzz_target!(|input: Input| { 24 | lading_fuzz::debug_input(&input); 25 | 26 | if input.total_bytes.get() > MAX_TOTAL_BYTES { 27 | return; 28 | } 29 | 30 | if input.max_block_size.get() > MAX_BLOCK_SIZE { 31 | return; 32 | } 33 | 34 | if input.max_block_size.get() > input.total_bytes.get() { 35 | return; 36 | } 37 | 38 | if input.config.valid().is_err() { 39 | return; 40 | } 41 | 42 | let max_contexts = match input.config.contexts.total_contexts { 43 | lading_payload::common::config::ConfRange::Constant(n) => n, 44 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 45 | }; 46 | if max_contexts > MAX_CONTEXTS { 47 | return; 48 | } 49 | 50 | let max_trace_cardinality = match input.config.trace_cardinality { 51 | lading_payload::common::config::ConfRange::Constant(n) => n, 52 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 53 | }; 54 | if max_trace_cardinality > MAX_TRACE_CARDINALITY { 55 | return; 56 | } 57 | 58 | let mut rng = SmallRng::from_seed(input.seed); 59 | let payload = lading_payload::Config::OpentelemetryLogs(input.config); 60 | 61 | let cache = match Cache::fixed_with_max_overhead( 62 | &mut rng, 63 | input.total_bytes, 64 | u128::from(input.max_block_size.get()), 65 | &payload, 66 | input.total_bytes.get() as usize, 67 | ) { 68 | Ok(c) => c, 69 | Err(_) => return, 70 | }; 71 | 72 | // Call advance 10 times to exercise the cache rotation 73 | let mut handle = cache.handle(); 74 | for _ in 0..10 { 75 | let _block = cache.advance(&mut handle); 76 | } 77 | }); -------------------------------------------------------------------------------- /lading_capture/src/test/writer.rs: -------------------------------------------------------------------------------- 1 | //! In-memory writer for testing 2 | //! 3 | //! Provides a thread-safe in-memory buffer that implements Write for use in 4 | //! tests and fuzzing. 5 | 6 | use std::io::Write; 7 | use std::sync::{Arc, Mutex}; 8 | 9 | /// In-memory writer for testing 10 | /// 11 | /// This writer accumulates all written data in an in-memory buffer that can be 12 | /// retrieved for assertions. Thread-safe through Arc. 13 | #[derive(Clone, Debug)] 14 | pub struct InMemoryWriter { 15 | buffer: Arc>>, 16 | } 17 | 18 | impl InMemoryWriter { 19 | /// Create a new in-memory writer 20 | #[must_use] 21 | pub fn new() -> Self { 22 | Self { 23 | buffer: Arc::new(Mutex::new(Vec::new())), 24 | } 25 | } 26 | 27 | /// Get the raw bytes written to the buffer 28 | /// 29 | /// # Panics 30 | /// 31 | /// Panics if the mutex is poisoned 32 | #[must_use] 33 | pub fn get_bytes(&self) -> Vec { 34 | self.buffer.lock().expect("mutex poisoned").clone() 35 | } 36 | 37 | /// Get the buffer contents as a UTF-8 string 38 | /// 39 | /// # Panics 40 | /// 41 | /// Panics if the buffer contains invalid UTF-8 42 | #[must_use] 43 | pub fn get_string(&self) -> String { 44 | String::from_utf8(self.get_bytes()).expect("buffer contains invalid UTF-8") 45 | } 46 | 47 | /// Parse the buffer contents as JSON lines 48 | /// 49 | /// # Errors 50 | /// 51 | /// Returns an error if any line cannot be parsed as JSON 52 | /// 53 | /// # Panics 54 | /// 55 | /// Panics if the mutex is poisoned 56 | pub fn parse_lines(&self) -> Result, serde_json::Error> { 57 | let buffer = self.buffer.lock().expect("mutex poisoned"); 58 | let content_str = String::from_utf8_lossy(&buffer); 59 | content_str 60 | .lines() 61 | .filter(|line| !line.is_empty()) 62 | .map(serde_json::from_str) 63 | .collect() 64 | } 65 | } 66 | 67 | impl Default for InMemoryWriter { 68 | fn default() -> Self { 69 | Self::new() 70 | } 71 | } 72 | 73 | impl Write for InMemoryWriter { 74 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 75 | self.buffer 76 | .lock() 77 | .expect("mutex poisoned") 78 | .extend_from_slice(buf); 79 | Ok(buf.len()) 80 | } 81 | 82 | fn flush(&mut self) -> std::io::Result<()> { 83 | Ok(()) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /lading_payload/src/ascii.rs: -------------------------------------------------------------------------------- 1 | //! ASCII payload. 2 | 3 | use std::io::Write; 4 | 5 | use rand::Rng; 6 | 7 | use crate::{Error, common::strings}; 8 | 9 | const MAX_LENGTH: u16 = 6_144; // 6 KiB 10 | 11 | #[derive(Debug, Clone)] 12 | /// ASCII text payload 13 | pub struct Ascii { 14 | pool: strings::Pool, 15 | } 16 | 17 | impl Ascii { 18 | /// Construct a new instance of `Ascii` 19 | pub fn new(rng: &mut R) -> Self 20 | where 21 | R: rand::Rng + ?Sized, 22 | { 23 | Self { 24 | // SAFETY: Do not adjust this downward below MAX_LENGTH without also 25 | // adjusting the input to `self.pool.of_size` below. 26 | pool: strings::Pool::with_size(rng, usize::from(MAX_LENGTH * 4)), 27 | } 28 | } 29 | } 30 | 31 | impl crate::Serialize for Ascii { 32 | fn to_bytes(&mut self, mut rng: R, max_bytes: usize, writer: &mut W) -> Result<(), Error> 33 | where 34 | R: Rng + Sized, 35 | W: Write, 36 | { 37 | let mut bytes_remaining = max_bytes; 38 | loop { 39 | let bytes = rng.random_range(1..MAX_LENGTH); 40 | // SAFETY: the maximum request is always less than the size of the 41 | // pool, per our constructor. 42 | let encoding: &str = self 43 | .pool 44 | .of_size(&mut rng, usize::from(bytes)) 45 | .ok_or(Error::StringGenerate)?; 46 | let line_length = encoding.len() + 1; // add one for the newline 47 | match bytes_remaining.checked_sub(line_length) { 48 | Some(remainder) => { 49 | writeln!(writer, "{encoding}")?; 50 | bytes_remaining = remainder; 51 | } 52 | None => break, 53 | } 54 | } 55 | Ok(()) 56 | } 57 | } 58 | 59 | #[cfg(test)] 60 | mod test { 61 | use proptest::prelude::*; 62 | use rand::{SeedableRng, rngs::SmallRng}; 63 | 64 | use crate::{Ascii, Serialize}; 65 | 66 | // The serialized size of the payload must not exceed `max_bytes`. 67 | proptest! { 68 | #[test] 69 | fn payload_not_exceed_max_bytes(seed: u64, max_bytes: u16) { 70 | let max_bytes = max_bytes as usize; 71 | let mut rng = SmallRng::seed_from_u64(seed); 72 | let mut ascii = Ascii::new(&mut rng); 73 | 74 | let mut bytes = Vec::with_capacity(max_bytes); 75 | ascii.to_bytes(rng, max_bytes, &mut bytes)?; 76 | prop_assert!(bytes.len() <= max_bytes); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /lading_payload/src/common/config.rs: -------------------------------------------------------------------------------- 1 | //! Common configuration for all lading payloads 2 | 3 | use rand::distr::uniform::SampleUniform; 4 | use serde::Deserialize; 5 | use std::{cmp, fmt}; 6 | 7 | /// Range expression for configuration 8 | #[derive(Debug, Deserialize, serde::Serialize, Clone, PartialEq, Copy)] 9 | #[serde(deny_unknown_fields)] 10 | #[serde(rename_all = "snake_case")] 11 | #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] 12 | pub enum ConfRange 13 | where 14 | T: PartialEq + cmp::PartialOrd + Clone + Copy, 15 | { 16 | /// A constant T 17 | Constant(T), 18 | /// In which a T is chosen between `min` and `max`, inclusive of `max`. 19 | Inclusive { 20 | /// The minimum of the range. 21 | min: T, 22 | /// The maximum of the range. 23 | max: T, 24 | }, 25 | } 26 | 27 | impl ConfRange 28 | where 29 | T: PartialEq + cmp::PartialOrd + Clone + Copy, 30 | { 31 | /// Returns true if the range provided by the user is valid, false 32 | /// otherwise. 33 | pub(crate) fn valid(&self) -> (bool, &'static str) { 34 | match self { 35 | Self::Constant(_) => (true, ""), 36 | Self::Inclusive { min, max } => (min <= max, "min must be less than or equal to max"), 37 | } 38 | } 39 | 40 | pub(crate) fn start(&self) -> T { 41 | match self { 42 | ConfRange::Constant(c) => *c, 43 | ConfRange::Inclusive { min, .. } => *min, 44 | } 45 | } 46 | 47 | pub(crate) fn end(&self) -> T { 48 | match self { 49 | ConfRange::Constant(c) => *c, 50 | ConfRange::Inclusive { max, .. } => *max, 51 | } 52 | } 53 | } 54 | 55 | impl ConfRange 56 | where 57 | T: PartialEq + cmp::PartialOrd + Clone + Copy + SampleUniform, 58 | { 59 | pub(crate) fn sample(&self, rng: &mut R) -> T 60 | where 61 | R: rand::Rng + ?Sized, 62 | { 63 | match self { 64 | ConfRange::Constant(c) => *c, 65 | ConfRange::Inclusive { min, max } => rng.random_range(*min..=*max), 66 | } 67 | } 68 | } 69 | 70 | impl fmt::Display for ConfRange 71 | where 72 | T: PartialEq + cmp::PartialOrd + Clone + Copy + fmt::Display, 73 | { 74 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 75 | match self { 76 | ConfRange::Constant(c) => write!(f, "{c}"), 77 | ConfRange::Inclusive { min, max } => { 78 | if min == max { 79 | write!(f, "{min}") 80 | } else { 81 | write!(f, "{min}..={max}") 82 | } 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /ci/fuzz: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | set -o nounset 4 | set -o pipefail 5 | 6 | # Default values 7 | MODE="run" 8 | CRATE="" 9 | TARGET="" 10 | 11 | # Parse arguments 12 | while [[ $# -gt 0 ]]; do 13 | case $1 in 14 | --check) 15 | MODE="check" 16 | shift 17 | ;; 18 | --list) 19 | MODE="list" 20 | shift 21 | ;; 22 | *) 23 | if [ -z "$CRATE" ]; then 24 | CRATE="$1" 25 | elif [ -z "$TARGET" ]; then 26 | TARGET="$1" 27 | else 28 | echo "Error: Unexpected argument '$1'" 29 | exit 1 30 | fi 31 | shift 32 | ;; 33 | esac 34 | done 35 | 36 | # Validate crate argument 37 | if [ -z "$CRATE" ]; then 38 | echo "Usage: $0 [--check|--list] [target]" 39 | echo "Valid crates: lading_payload" 40 | echo "" 41 | echo "Examples:" 42 | echo " $0 --check lading_payload # Check all fuzz targets build" 43 | echo " $0 --list lading_payload # List available fuzz targets" 44 | echo " $0 lading_payload # Run specific fuzz target" 45 | exit 1 46 | fi 47 | 48 | # Validate the crate 49 | case "$CRATE" in 50 | lading_payload) 51 | # Valid crate 52 | ;; 53 | *) 54 | echo "Error: Invalid crate '$CRATE'" 55 | echo "Valid crates: lading_payload" 56 | exit 1 57 | ;; 58 | esac 59 | 60 | # Check if cargo fuzz is installed 61 | if ! command -v cargo-fuzz &> /dev/null; then 62 | echo "cargo-fuzz not installed. Installing it now..." 63 | cargo install cargo-fuzz 64 | fi 65 | 66 | # Execute based on mode 67 | case "$MODE" in 68 | check) 69 | echo "Checking all fuzz targets in ${CRATE} build correctly..." 70 | cd "${CRATE}" 71 | # List all targets and check each one builds 72 | for target in $(rustup run nightly cargo fuzz list); do 73 | echo "Checking fuzz target: $target" 74 | rustup run nightly cargo fuzz check "$target" 75 | done 76 | echo "All fuzz targets build successfully!" 77 | ;; 78 | list) 79 | echo "Available fuzz targets in ${CRATE}:" 80 | cd "${CRATE}" 81 | rustup run nightly cargo fuzz list 82 | ;; 83 | run) 84 | if [ -z "$TARGET" ]; then 85 | echo "Error: Target required for run mode" 86 | echo "Usage: $0 " 87 | exit 1 88 | fi 89 | echo "Running fuzz test: ${CRATE}/${TARGET}..." 90 | cd "${CRATE}" 91 | FUZZ_DEBUG=1 rustup run nightly cargo fuzz run --jobs 8 --build-std --release "${TARGET}" 92 | ;; 93 | esac -------------------------------------------------------------------------------- /lading/src/observer/linux.rs: -------------------------------------------------------------------------------- 1 | mod cgroup; 2 | mod procfs; 3 | mod utils; 4 | mod wss; 5 | 6 | use tracing::{error, warn}; 7 | 8 | #[derive(thiserror::Error, Debug)] 9 | /// Errors produced by functions in this module 10 | pub enum Error { 11 | /// Wrapper for [`cgroup::Error`] 12 | #[error("Cgroup: {0}")] 13 | CGroup(#[from] cgroup::Error), 14 | /// Wrapper for [`procfs::Error`] 15 | #[error("Procfs: {0}")] 16 | Procfs(#[from] procfs::Error), 17 | /// Wrapper for [`wss::Error`] 18 | #[error("WSS: {0}")] 19 | Wss(#[from] wss::Error), 20 | } 21 | 22 | #[derive(Debug)] 23 | pub(crate) struct Sampler { 24 | procfs: procfs::Sampler, 25 | cgroup: cgroup::Sampler, 26 | wss: Option, 27 | tick_counter: u8, 28 | } 29 | 30 | impl Sampler { 31 | pub(crate) fn new(parent_pid: i32, labels: Vec<(String, String)>) -> Result { 32 | let procfs_sampler = procfs::Sampler::new(parent_pid)?; 33 | let cgroup_sampler = cgroup::Sampler::new(parent_pid, labels)?; 34 | let wss_sampler = if wss::Sampler::is_available() { 35 | Some(wss::Sampler::new(parent_pid)?) 36 | } else { 37 | warn!( 38 | r"{} isn’t accessible. 39 | Either the kernel hasn’t been compiled with CONFIG_IDLE_PAGE_TRACKING 40 | or the process doesn’t have access to it. 41 | WSS sampling is not available. 42 | 43 | Kernel support can be checked with 44 | ``` 45 | grep CONFIG_IDLE_PAGE_TRACKING /boot/config-$(uname -r) 46 | ``` 47 | 48 | Permissions can be checked with 49 | ``` 50 | id 51 | ls -l /sys/kernel/mm/page_idle/bitmap 52 | ``` 53 | ", 54 | wss::PAGE_IDLE_BITMAP 55 | ); 56 | None 57 | }; 58 | 59 | Ok(Self { 60 | procfs: procfs_sampler, 61 | cgroup: cgroup_sampler, 62 | wss: wss_sampler, 63 | tick_counter: 0, 64 | }) 65 | } 66 | 67 | pub(crate) async fn sample(&mut self) -> Result<(), Error> { 68 | let sample_smaps = self.tick_counter.is_multiple_of(10); 69 | let sample_wss = self.tick_counter.is_multiple_of(60); 70 | self.tick_counter += 1; 71 | if self.tick_counter == 60 { 72 | self.tick_counter = 0; 73 | } 74 | 75 | self.procfs.poll(sample_smaps).await?; 76 | self.cgroup.poll().await?; 77 | 78 | if let Some(wss) = &mut self.wss { 79 | // WSS measures the amount of memory that has been accessed since the last poll. 80 | // As a consequence, the poll interval impacts the measure. 81 | // That’s why we need to be sure we don’t poll more often than once per minute. 82 | if sample_wss { 83 | wss.poll().await?; 84 | } 85 | } 86 | 87 | Ok(()) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /lading_payload/fuzz/fuzz_targets/dogstatsd_serializer_to_bytes.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use arbitrary; 4 | use libfuzzer_sys::fuzz_target; 5 | use rand::{SeedableRng, rngs::SmallRng}; 6 | use std::num::NonZeroU32; 7 | 8 | use lading_payload::Serialize; 9 | 10 | #[derive(arbitrary::Arbitrary, Debug)] 11 | struct Input { 12 | seed: [u8; 32], 13 | budget_bytes: NonZeroU32, 14 | config: lading_payload::dogstatsd::Config, 15 | } 16 | 17 | const MAX_BUDGET: usize = 1 * 1024 * 1024; // 1 MiB 18 | const MAX_CONTEXTS: u32 = 4_100_000; 19 | const MAX_TAG_LENGTH: u16 = 200; 20 | const MAX_TAGS_PER_MSG: u8 = 50; 21 | const MAX_SERVICE_CHECK_NAMES: u16 = 1_000; 22 | const MAX_NAME_LENGTH: u16 = 200; 23 | 24 | fuzz_target!(|input: Input| { 25 | lading_fuzz::debug_input(&input); 26 | 27 | let budget = input.budget_bytes.get() as usize; 28 | if budget > MAX_BUDGET { 29 | return; 30 | } 31 | 32 | if input.config.valid().is_err() { 33 | return; 34 | } 35 | 36 | let max_contexts = match input.config.contexts { 37 | lading_payload::common::config::ConfRange::Constant(n) => n, 38 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 39 | }; 40 | if max_contexts > MAX_CONTEXTS { 41 | return; 42 | } 43 | 44 | let max_tag_length = match input.config.tag_length { 45 | lading_payload::common::config::ConfRange::Constant(n) => n, 46 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 47 | }; 48 | if max_tag_length > MAX_TAG_LENGTH { 49 | return; 50 | } 51 | 52 | let max_tags_per_msg = match input.config.tags_per_msg { 53 | lading_payload::common::config::ConfRange::Constant(n) => n, 54 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 55 | }; 56 | if max_tags_per_msg > MAX_TAGS_PER_MSG { 57 | return; 58 | } 59 | 60 | let max_service_check_names = match input.config.service_check_names { 61 | lading_payload::common::config::ConfRange::Constant(n) => n, 62 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 63 | }; 64 | if max_service_check_names > MAX_SERVICE_CHECK_NAMES { 65 | return; 66 | } 67 | 68 | let max_name_length = match input.config.name_length { 69 | lading_payload::common::config::ConfRange::Constant(n) => n, 70 | lading_payload::common::config::ConfRange::Inclusive { max, .. } => max, 71 | }; 72 | if max_name_length > MAX_NAME_LENGTH { 73 | return; 74 | } 75 | 76 | let mut rng = SmallRng::from_seed(input.seed); 77 | let mut bytes = Vec::with_capacity(budget); 78 | 79 | let mut serializer = match lading_payload::DogStatsD::new(input.config, &mut rng) { 80 | Ok(s) => s, 81 | Err(_) => return, 82 | }; 83 | 84 | if serializer.to_bytes(&mut rng, budget, &mut bytes).is_ok() { 85 | assert!(bytes.len() <= budget); 86 | } 87 | }); 88 | -------------------------------------------------------------------------------- /lading/src/target_metrics.rs: -------------------------------------------------------------------------------- 1 | //! Fetch metrics from the target process 2 | //! 3 | //! This module allows lading to fetch metrics from the target process and 4 | //! include them in the captures file. 5 | //! 6 | 7 | use serde::Deserialize; 8 | use tokio::time::Duration; 9 | 10 | pub mod expvar; 11 | pub mod prometheus; 12 | 13 | #[derive(Debug, Clone, Copy)] 14 | /// Errors produced by [`Server`] 15 | pub enum Error { 16 | /// See [`crate::target_metrics::expvar::Error`] for details. 17 | Expvar(expvar::Error), 18 | /// See [`crate::target_metrics::prometheus::Error`] for details. 19 | Prometheus(prometheus::Error), 20 | } 21 | 22 | #[derive(Debug, Deserialize, PartialEq, Eq, Clone)] 23 | #[serde(rename_all = "snake_case")] 24 | #[serde(deny_unknown_fields)] 25 | /// Configuration for [`Server`] 26 | pub enum Config { 27 | /// See [`crate::target_metrics::expvar::Config`] for details. 28 | Expvar(expvar::Config), 29 | /// See [`crate::target_metrics::prometheus::Config`] for details. 30 | Prometheus(prometheus::Config), 31 | } 32 | 33 | /// The `target_metrics` server. 34 | #[derive(Debug)] 35 | pub enum Server { 36 | /// See [`crate::target_metrics::expvar::Expvar`] for details. 37 | Expvar(expvar::Expvar), 38 | /// See [`crate::target_metrics::prometheus::Prometheus`] for details. 39 | Prometheus(prometheus::Prometheus), 40 | } 41 | 42 | impl Server { 43 | /// Create a new [`Server`] instance 44 | /// 45 | /// The `target_metrics::Server` is responsible for scraping metrics from 46 | /// the target process. 47 | /// 48 | #[must_use] 49 | pub fn new( 50 | config: Config, 51 | shutdown: lading_signal::Watcher, 52 | experiment_started: lading_signal::Watcher, 53 | sample_period: Duration, 54 | ) -> Self { 55 | match config { 56 | Config::Expvar(conf) => Self::Expvar(expvar::Expvar::new( 57 | conf, 58 | shutdown, 59 | experiment_started, 60 | sample_period, 61 | )), 62 | Config::Prometheus(conf) => Self::Prometheus(prometheus::Prometheus::new( 63 | conf, 64 | shutdown, 65 | experiment_started, 66 | sample_period, 67 | )), 68 | } 69 | } 70 | 71 | /// Run this [`Server`] to completion 72 | /// 73 | /// The `target_metrics` server is responsible for fetching metrics directly 74 | /// from the target software. 75 | /// 76 | /// # Errors 77 | /// 78 | /// Function will return an error if the underlying metrics collector 79 | /// returns an error. 80 | /// 81 | /// # Panics 82 | /// 83 | /// None are known. 84 | pub async fn run(self) -> Result<(), Error> { 85 | match self { 86 | Server::Expvar(inner) => inner.run().await.map_err(Error::Expvar), 87 | Server::Prometheus(inner) => inner.run().await.map_err(Error::Prometheus), 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /lading/src/observer/linux/procfs/memory/smaps_rollup.rs: -------------------------------------------------------------------------------- 1 | use heck::ToSnakeCase; 2 | use metrics::gauge; 3 | use tokio::fs; 4 | 5 | use super::{BYTES_PER_KIBIBYTE, next_token}; 6 | 7 | #[derive(thiserror::Error, Debug)] 8 | /// Errors produced by functions in this module 9 | pub(crate) enum Error { 10 | /// Wrapper for [`std::io::Error`] 11 | #[error("IO error: {0}")] 12 | Io(#[from] std::io::Error), 13 | #[error("Number Parsing: {0}")] 14 | ParseInt(#[from] std::num::ParseIntError), 15 | #[error("Parsing: {0}")] 16 | Parsing(String), 17 | } 18 | 19 | #[derive(Debug, Clone, Copy, Default)] 20 | pub(crate) struct Aggregator { 21 | pub(crate) rss: u64, 22 | pub(crate) pss: u64, 23 | } 24 | 25 | // Read `/proc/{pid}/smaps_rollup` and parse it directly into metrics. 26 | pub(crate) async fn poll( 27 | pid: i32, 28 | labels: &[(&'static str, String)], 29 | aggr: &mut Aggregator, 30 | ) -> Result<(), Error> { 31 | let path = format!("/proc/{pid}/smaps_rollup"); 32 | // NOTE `read_to_string` uses as few IO operations as possible in its 33 | // implementation, so we might get the contents here in one go. 34 | let contents: String = fs::read_to_string(path).await?; 35 | let mut lines = contents.lines(); 36 | 37 | lines.next(); // skip header, doesn't have any useful information 38 | // looks like this: 39 | // 00400000-7fff03d61000 ---p 00000000 00:00 0 [rollup] 40 | 41 | for line in lines { 42 | let mut chars = line.char_indices().peekable(); 43 | let Some(name) = next_token(line, &mut chars) else { 44 | // if there is no token on the line, that means empty line, that's fine 45 | continue; 46 | }; 47 | 48 | let value_bytes = { 49 | let value_token = next_token(line, &mut chars).ok_or(Error::Parsing(format!( 50 | "Could not parse numeric value from line: {line}" 51 | )))?; 52 | let unit = next_token(line, &mut chars).ok_or(Error::Parsing(format!( 53 | "Could not parse unit from line: {line}" 54 | )))?; 55 | let numeric = value_token.parse::()?; 56 | 57 | match unit { 58 | "kB" => Ok(numeric.saturating_mul(BYTES_PER_KIBIBYTE)), 59 | unknown => Err(Error::Parsing(format!( 60 | "Unknown unit: {unknown} in line: {line}" 61 | ))), 62 | } 63 | }?; 64 | 65 | let name_len = name.len(); 66 | // Last character is a :, skip it. 67 | let field = name[..name_len - 1].to_snake_case(); 68 | match field.as_str() { 69 | "rss" => aggr.rss = aggr.rss.saturating_add(value_bytes), 70 | "pss" => aggr.pss = aggr.pss.saturating_add(value_bytes), 71 | _ => { /* ignore other fields */ } 72 | } 73 | let metric_name = format!("smaps_rollup.{field}"); 74 | gauge!(metric_name, labels).set(value_bytes as f64); 75 | } 76 | 77 | Ok(()) 78 | } 79 | -------------------------------------------------------------------------------- /lading/src/observer/linux/utils/tests/create_process_tree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | This script creates a tree of idle processes with a specified depth and breadth. 6 | The processes are spawned using fork and will wait for a signal to terminate. 7 | 8 | Usage: 9 | create_process_tree.py 10 | 11 | Example: 12 | create_process_tree.py 3 2 13 | 14 | This will create a process tree with 3 processes at each level and a depth of 2. 15 | It would look like: 16 | 17 | ``` 18 | $ pstree -a -pg 30881 19 | python3,30881,30881 ./create_process_tree.py 3 2 20 | ├─python3,30899,30881 ./create_process_tree.py 3 2 21 | │ ├─python3,30902,30881 ./create_process_tree.py 3 2 22 | │ ├─python3,30903,30881 ./create_process_tree.py 3 2 23 | │ └─python3,30906,30881 ./create_process_tree.py 3 2 24 | ├─python3,30900,30881 ./create_process_tree.py 3 2 25 | │ ├─python3,30904,30881 ./create_process_tree.py 3 2 26 | │ ├─python3,30907,30881 ./create_process_tree.py 3 2 27 | │ └─python3,30909,30881 ./create_process_tree.py 3 2 28 | └─python3,30901,30881 ./create_process_tree.py 3 2 29 | ├─python3,30905,30881 ./create_process_tree.py 3 2 30 | ├─python3,30908,30881 ./create_process_tree.py 3 2 31 | └─python3,30910,30881 ./create_process_tree.py 3 2 32 | ``` 33 | 34 | Each process will print its PID. 35 | 36 | The script also includes a signal handler to terminate the whole process group 37 | when a SIGTERM signal is received. 38 | """ 39 | 40 | import argparse 41 | import logging 42 | import os 43 | import signal 44 | 45 | 46 | def spawn_processes(nb: int, depth: int) -> None: 47 | """ 48 | Spawn a number of processes in a tree structure. 49 | 50 | Args: 51 | nb (int): Number of processes to spawn. 52 | depth (int): Depth of the process tree. 53 | """ 54 | if depth == 0 or nb <= 0: 55 | return 56 | 57 | for i in range(nb): 58 | pid = os.fork() 59 | if pid == 0: # Child process 60 | print(os.getpid(), flush=True) 61 | logging.info(f"Child PID: {os.getpid()}, Parent PID: {os.getppid()}") 62 | spawn_processes(nb, depth - 1) 63 | signal.pause() 64 | 65 | 66 | def handler(signum: int, frame) -> None: 67 | """ 68 | Signal handler to terminate the process group. 69 | 70 | Args: 71 | signum (int): Signal number. 72 | frame (signal.FrameType): Current stack frame. 73 | """ 74 | signal.signal(signal.SIGTERM, signal.SIG_DFL) 75 | os.killpg(os.getpid(), signal.SIGTERM) 76 | 77 | 78 | if __name__ == "__main__": 79 | parser = argparse.ArgumentParser(description="Create a process tree.") 80 | parser.add_argument("nb", type=int, help="Number of processes to spawn per level") 81 | parser.add_argument("depth", type=int, help="Depth of the process tree") 82 | parser.add_argument( 83 | "--log-level", 84 | type=str, 85 | default="warning", 86 | help="Logging level", 87 | choices=["debug", "info", "warning", "error", "critical"], 88 | ) 89 | args = parser.parse_args() 90 | logging.basicConfig(level=args.log_level.upper()) 91 | 92 | os.setpgid(0, 0) 93 | 94 | spawn_processes(args.nb, args.depth) 95 | 96 | signal.signal(signal.SIGTERM, handler) 97 | signal.pause() 98 | -------------------------------------------------------------------------------- /lading_payload/src/statik.rs: -------------------------------------------------------------------------------- 1 | //! Static file/directory payload. 2 | 3 | use std::{ 4 | fs::{self, OpenOptions}, 5 | io::{BufRead, BufReader, Write}, 6 | path::{Path, PathBuf}, 7 | }; 8 | 9 | use rand::{Rng, prelude::IteratorRandom}; 10 | use tracing::debug; 11 | 12 | #[derive(Debug)] 13 | struct Source { 14 | byte_size: u64, 15 | path: PathBuf, 16 | } 17 | 18 | #[derive(Debug)] 19 | /// Static payload 20 | pub struct Static { 21 | sources: Vec, 22 | } 23 | 24 | #[derive(thiserror::Error, Debug)] 25 | /// Errors produced by [`Static`]. 26 | pub enum Error { 27 | /// IO error 28 | #[error(transparent)] 29 | Io(#[from] std::io::Error), 30 | } 31 | 32 | impl Static { 33 | /// Create a new instance of `Static` 34 | /// 35 | /// # Errors 36 | /// 37 | /// See documentation for [`Error`] 38 | pub fn new(path: &Path) -> Result { 39 | let mut sources = Vec::with_capacity(16); 40 | 41 | // Attempt to open the path, if this fails we assume that it is a directory. 42 | let metadata = fs::metadata(path)?; 43 | if metadata.is_file() { 44 | debug!("Static path {} is a file.", path.display()); 45 | let byte_size = metadata.len(); 46 | sources.push(Source { 47 | byte_size, 48 | path: path.to_owned(), 49 | }); 50 | } else if metadata.is_dir() { 51 | debug!("Static path {} is a directory.", path.display()); 52 | for entry in fs::read_dir(path)? { 53 | let entry = entry?; 54 | let entry_pth = entry.path(); 55 | debug!("Attempting to open {} as file.", entry_pth.display()); 56 | if let Ok(file) = OpenOptions::new().read(true).open(&entry_pth) { 57 | let byte_size = file.metadata()?.len(); 58 | sources.push(Source { 59 | byte_size, 60 | path: entry_pth.clone(), 61 | }); 62 | } 63 | } 64 | } 65 | 66 | Ok(Self { sources }) 67 | } 68 | } 69 | 70 | impl crate::Serialize for Static { 71 | fn to_bytes( 72 | &mut self, 73 | mut rng: R, 74 | max_bytes: usize, 75 | writer: &mut W, 76 | ) -> Result<(), crate::Error> 77 | where 78 | R: Rng + Sized, 79 | W: Write, 80 | { 81 | // Filter available static files to those with size less than 82 | // max_bytes. Of the remaining, randomly choose one and write it out. We 83 | // do not change the structure of the file in any respect; it is 84 | // faithfully transmitted. 85 | 86 | let subset = self 87 | .sources 88 | .iter() 89 | .filter(|src| src.byte_size < max_bytes as u64); 90 | if let Some(source) = subset.choose(&mut rng) { 91 | debug!("Opening {} static file.", &source.path.display()); 92 | let file = OpenOptions::new().read(true).open(&source.path)?; 93 | 94 | let mut reader = BufReader::new(file); 95 | let buffer = reader.fill_buf()?; 96 | let buffer_length = buffer.len(); 97 | writer.write_all(buffer)?; 98 | reader.consume(buffer_length); 99 | } 100 | 101 | Ok(()) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /lading/proto/agent_payload.proto: -------------------------------------------------------------------------------- 1 | // Vendored from 2 | // https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto 3 | // SHA 9e27b06757db73581484382d8c0772b3e7f1e009. 4 | 5 | syntax = "proto3"; 6 | 7 | package datadog.agentpayload; 8 | 9 | message CommonMetadata { 10 | string agent_version = 1; 11 | string timezone = 2; 12 | double current_epoch = 3; 13 | string internal_ip = 4; 14 | string public_ip = 5; 15 | string api_key = 6; 16 | } 17 | 18 | message Origin { 19 | reserved 1,2,3; 20 | uint32 origin_product = 4; 21 | uint32 origin_category = 5; 22 | uint32 origin_service = 6; 23 | } 24 | 25 | // Metadata is used in both the MetricSeries and Sketch messages defined below. 26 | message Metadata { 27 | Origin origin = 1; 28 | } 29 | 30 | message MetricPayload { 31 | enum MetricType { 32 | UNSPECIFIED = 0; 33 | COUNT = 1; 34 | RATE = 2; 35 | GAUGE = 3; 36 | } 37 | 38 | message MetricPoint { 39 | // metric value 40 | double value = 1; 41 | // timestamp for this value in seconds since the UNIX epoch 42 | int64 timestamp = 2; 43 | } 44 | 45 | message Resource { 46 | string type = 1; 47 | string name = 2; 48 | } 49 | 50 | message MetricSeries { 51 | // Resources this series applies to; include at least 52 | // { type="host", name= } 53 | repeated Resource resources = 1; 54 | // metric name 55 | string metric = 2; 56 | // tags for this metric 57 | repeated string tags = 3; 58 | // data points for this metric 59 | repeated MetricPoint points = 4; 60 | // type of metric 61 | MetricType type = 5; 62 | // metric unit name 63 | string unit = 6; 64 | // source of this metric (check name, etc.) 65 | string source_type_name = 7; 66 | // interval, in seconds, between samples of this metric 67 | int64 interval = 8; 68 | // Metrics origin metadata 69 | Metadata metadata = 9; 70 | } 71 | repeated MetricSeries series = 1; 72 | } 73 | 74 | message EventsPayload { 75 | message Event { 76 | string title = 1; 77 | string text = 2; 78 | int64 ts = 3; 79 | string priority = 4; 80 | string host = 5; 81 | repeated string tags = 6; 82 | string alert_type = 7; 83 | string aggregation_key = 8; 84 | string source_type_name = 9; 85 | } 86 | repeated Event events = 1; 87 | CommonMetadata metadata = 2; 88 | } 89 | 90 | message SketchPayload { 91 | message Sketch { 92 | message Distribution { 93 | int64 ts = 1; 94 | int64 cnt = 2; 95 | double min = 3; 96 | double max = 4; 97 | double avg = 5; 98 | double sum = 6; 99 | repeated double v = 7; 100 | repeated uint32 g = 8; 101 | repeated uint32 delta = 9; 102 | repeated double buf = 10; 103 | } 104 | message Dogsketch { 105 | int64 ts = 1; 106 | int64 cnt = 2; 107 | double min = 3; 108 | double max = 4; 109 | double avg = 5; 110 | double sum = 6; 111 | repeated sint32 k = 7; 112 | repeated uint32 n = 8; 113 | } 114 | string metric = 1; 115 | string host = 2; 116 | repeated Distribution distributions = 3; 117 | repeated string tags = 4; 118 | reserved 5, 6; 119 | reserved "distributionsK", "distributionsC"; 120 | repeated Dogsketch dogsketches = 7; 121 | Metadata metadata = 8; 122 | } 123 | repeated Sketch sketches = 1; 124 | CommonMetadata metadata = 2; 125 | } 126 | -------------------------------------------------------------------------------- /lading_payload/benches/opentelemetry_metric.rs: -------------------------------------------------------------------------------- 1 | use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; 2 | 3 | use lading_payload::common::config::ConfRange; 4 | use lading_payload::{ 5 | OpentelemetryMetrics, Serialize, 6 | opentelemetry::metric::{Config, Contexts, MetricWeights}, 7 | }; 8 | use rand::{SeedableRng, rngs::SmallRng}; 9 | use std::time::Duration; 10 | 11 | fn opentelemetry_metric_setup(c: &mut Criterion) { 12 | c.bench_function("opentelemetry_metric_setup", |b| { 13 | b.iter(|| { 14 | let mut rng = SmallRng::seed_from_u64(19690716); 15 | let config = Config { 16 | metric_weights: MetricWeights { 17 | gauge: 50, 18 | sum_delta: 25, 19 | sum_cumulative: 25, 20 | }, 21 | contexts: Contexts { 22 | total_contexts: ConfRange::Constant(100), 23 | attributes_per_resource: ConfRange::Inclusive { min: 1, max: 64 }, 24 | scopes_per_resource: ConfRange::Inclusive { min: 1, max: 32 }, 25 | attributes_per_scope: ConfRange::Inclusive { min: 0, max: 4 }, 26 | metrics_per_scope: ConfRange::Inclusive { min: 1, max: 128 }, 27 | attributes_per_metric: ConfRange::Inclusive { min: 0, max: 255 }, 28 | }, 29 | }; 30 | let _ot = OpentelemetryMetrics::new(config, &mut rng) 31 | .expect("failed to create metrics generator"); 32 | }) 33 | }); 34 | } 35 | 36 | fn opentelemetry_metric_all(c: &mut Criterion) { 37 | let mb = 1_000_000; // 1 MiB 38 | 39 | let mut group = c.benchmark_group("opentelemetry_metric_all"); 40 | for size in &[mb, 10 * mb, 100 * mb, 1_000 * mb] { 41 | group.throughput(Throughput::Bytes(*size as u64)); 42 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &size| { 43 | b.iter(|| { 44 | let mut rng = SmallRng::seed_from_u64(19690716); 45 | let config = Config { 46 | metric_weights: MetricWeights { 47 | gauge: 50, 48 | sum_delta: 25, 49 | sum_cumulative: 25, 50 | }, 51 | contexts: Contexts { 52 | total_contexts: ConfRange::Constant(100), 53 | attributes_per_resource: ConfRange::Inclusive { min: 1, max: 64 }, 54 | scopes_per_resource: ConfRange::Inclusive { min: 1, max: 32 }, 55 | attributes_per_scope: ConfRange::Inclusive { min: 0, max: 4 }, 56 | metrics_per_scope: ConfRange::Inclusive { min: 1, max: 128 }, 57 | attributes_per_metric: ConfRange::Inclusive { min: 0, max: 255 }, 58 | }, 59 | }; 60 | let ot = OpentelemetryMetrics::new(config, &mut rng) 61 | .expect("failed to create metrics generator"); 62 | let mut writer = Vec::with_capacity(size); 63 | 64 | ot.to_bytes(rng, size, &mut writer) 65 | .expect("failed to convert to bytes"); 66 | }); 67 | }); 68 | } 69 | group.finish(); 70 | } 71 | 72 | criterion_group!( 73 | name = benches; 74 | config = Criterion::default().measurement_time(Duration::from_secs(90)); 75 | targets = opentelemetry_metric_setup, opentelemetry_metric_all 76 | ); 77 | 78 | criterion_main!(benches); 79 | -------------------------------------------------------------------------------- /lading/src/codec.rs: -------------------------------------------------------------------------------- 1 | use std::io::Read; 2 | 3 | use bytes::{Buf, Bytes}; 4 | use flate2::read::{MultiGzDecoder, ZlibDecoder}; 5 | use http_body_util::combinators::BoxBody; 6 | use hyper::StatusCode; 7 | 8 | /// decode decodes a HTTP request body based on its Content-Encoding header. 9 | /// Only identity, gzip, and deflate are currently supported content encodings. 10 | /// 11 | /// It supports multiple content encodings joined by ,s. They are decoded in the 12 | /// order provided. 13 | /// 14 | /// See [RFC7231](https://httpwg.org/specs/rfc7231.html#header.content-encoding) for more details 15 | /// on this header value. 16 | /// 17 | /// # Errors 18 | /// 19 | /// Will return an `Err` including a `hyper` response body if: 20 | /// 21 | /// * The passed `content_encoding` is unknown 22 | /// * The body cannot be decoded as the specified content type 23 | /// 24 | /// This response body can be passed back as the HTTP response to the client 25 | pub(crate) fn decode( 26 | content_encoding: Option<&hyper::header::HeaderValue>, 27 | mut body: Bytes, 28 | ) -> Result>>> { 29 | if let Some(content_encoding) = content_encoding { 30 | let content_encoding = String::from_utf8_lossy(content_encoding.as_bytes()); 31 | 32 | for encoding in content_encoding 33 | .rsplit(',') 34 | .map(str::trim) 35 | .map(str::to_lowercase) 36 | { 37 | body = match encoding.as_ref() { 38 | "identity" => body, 39 | "gzip" => { 40 | let mut decoded = Vec::new(); 41 | MultiGzDecoder::new(body.reader()) 42 | .read_to_end(&mut decoded) 43 | .map_err(|error| Box::new(encoding_error_to_response(&encoding, error)))?; 44 | decoded.into() 45 | } 46 | "deflate" => { 47 | let mut decoded = Vec::new(); 48 | ZlibDecoder::new(body.reader()) 49 | .read_to_end(&mut decoded) 50 | .map_err(|error| Box::new(encoding_error_to_response(&encoding, error)))?; 51 | decoded.into() 52 | } 53 | "zstd" => { 54 | let mut decoded = Vec::new(); 55 | zstd::Decoder::new(body.reader()) 56 | .map_err(|error| encoding_error_to_response(&encoding, error))? 57 | .read_to_end(&mut decoded) 58 | .map_err(|error| Box::new(encoding_error_to_response(&encoding, error)))?; 59 | 60 | decoded.into() 61 | } 62 | encoding => { 63 | return Err(Box::new( 64 | hyper::Response::builder() 65 | .status(StatusCode::UNSUPPORTED_MEDIA_TYPE) 66 | .body(crate::full(format!( 67 | "Unsupported encoding type: {encoding}" 68 | ))) 69 | .expect("failed to build response"), 70 | )); 71 | } 72 | } 73 | } 74 | } 75 | 76 | Ok(body) 77 | } 78 | 79 | fn encoding_error_to_response( 80 | encoding: &str, 81 | error: impl std::error::Error, 82 | ) -> hyper::Response> { 83 | hyper::Response::builder() 84 | .status(StatusCode::UNSUPPORTED_MEDIA_TYPE) 85 | .body(crate::full(format!( 86 | "failed to decode input as {encoding}: {error}" 87 | ))) 88 | .expect("failed to build response") 89 | } 90 | -------------------------------------------------------------------------------- /integration/sheepdog/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "cfg-if" 7 | version = "1.0.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 10 | 11 | [[package]] 12 | name = "escargot" 13 | version = "0.5.7" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "f5584ba17d7ab26a8a7284f13e5bd196294dd2f2d79773cff29b9e9edef601a6" 16 | dependencies = [ 17 | "log", 18 | "once_cell", 19 | "serde", 20 | "serde_json", 21 | ] 22 | 23 | [[package]] 24 | name = "itoa" 25 | version = "1.0.3" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" 28 | 29 | [[package]] 30 | name = "log" 31 | version = "0.4.17" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" 34 | dependencies = [ 35 | "cfg-if", 36 | ] 37 | 38 | [[package]] 39 | name = "once_cell" 40 | version = "1.13.0" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" 43 | 44 | [[package]] 45 | name = "proc-macro2" 46 | version = "1.0.43" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" 49 | dependencies = [ 50 | "unicode-ident", 51 | ] 52 | 53 | [[package]] 54 | name = "quote" 55 | version = "1.0.21" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" 58 | dependencies = [ 59 | "proc-macro2", 60 | ] 61 | 62 | [[package]] 63 | name = "ryu" 64 | version = "1.0.11" 65 | source = "registry+https://github.com/rust-lang/crates.io-index" 66 | checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" 67 | 68 | [[package]] 69 | name = "serde" 70 | version = "1.0.142" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "e590c437916fb6b221e1d00df6e3294f3fccd70ca7e92541c475d6ed6ef5fee2" 73 | dependencies = [ 74 | "serde_derive", 75 | ] 76 | 77 | [[package]] 78 | name = "serde_derive" 79 | version = "1.0.142" 80 | source = "registry+https://github.com/rust-lang/crates.io-index" 81 | checksum = "34b5b8d809babe02f538c2cfec6f2c1ed10804c0e5a6a041a049a4f5588ccc2e" 82 | dependencies = [ 83 | "proc-macro2", 84 | "quote", 85 | "syn", 86 | ] 87 | 88 | [[package]] 89 | name = "serde_json" 90 | version = "1.0.83" 91 | source = "registry+https://github.com/rust-lang/crates.io-index" 92 | checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7" 93 | dependencies = [ 94 | "itoa", 95 | "ryu", 96 | "serde", 97 | ] 98 | 99 | [[package]] 100 | name = "shared" 101 | version = "0.1.0" 102 | 103 | [[package]] 104 | name = "sheepdog" 105 | version = "0.1.0" 106 | dependencies = [ 107 | "escargot", 108 | "shared", 109 | ] 110 | 111 | [[package]] 112 | name = "syn" 113 | version = "1.0.99" 114 | source = "registry+https://github.com/rust-lang/crates.io-index" 115 | checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" 116 | dependencies = [ 117 | "proc-macro2", 118 | "quote", 119 | "unicode-ident", 120 | ] 121 | 122 | [[package]] 123 | name = "unicode-ident" 124 | version = "1.0.3" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" 127 | -------------------------------------------------------------------------------- /lading/proto/stateful_encoding.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package datadog.intake.stateful; 4 | 5 | // --------------------------------------------------------------------------- 6 | // Dictionary-encoded 7 | // --------------------------------------------------------------------------- 8 | 9 | message DictEntryDefine { 10 | uint64 id = 1; 11 | string value = 2; 12 | } 13 | 14 | message DictEntryDelete { 15 | uint64 id = 1; 16 | } 17 | 18 | // --------------------------------------------------------------------------- 19 | // Pattern dictionary 20 | // --------------------------------------------------------------------------- 21 | 22 | // pos_list is used to indicate where dynamic values should be inserted 23 | // it's more accurate than a marker 24 | message PatternDefine { 25 | uint64 pattern_id = 1; 26 | string template = 2; 27 | uint32 param_count = 3; 28 | repeated uint32 pos_list = 4; 29 | } 30 | 31 | message PatternDelete { 32 | uint64 pattern_id = 1; 33 | } 34 | 35 | // --------------------------------------------------------------------------- 36 | // Log payload 37 | // --------------------------------------------------------------------------- 38 | 39 | message Tag { 40 | DynamicValue key = 1; 41 | DynamicValue value = 2; 42 | } 43 | 44 | message Log { 45 | uint64 timestamp = 1; 46 | oneof content { 47 | StructuredLog structured = 2; 48 | string raw = 3; 49 | } 50 | // TODO: right now we are assuming logs are attached per tag - in the future we may have common tags in the stream 51 | // state and auto-populate them downstream. 52 | // Required tags: `service`, `hostname`, 53 | // Other tags on agent payload: `status`, `source` 54 | // All other tags are sent as `ddtags` 55 | repeated Tag tags = 4; 56 | } 57 | 58 | message StructuredLog { 59 | uint64 pattern_id = 1; 60 | repeated DynamicValue dynamic_values = 2; 61 | } 62 | 63 | // TODO not sure we need numeric type 64 | message DynamicValue { 65 | oneof value { 66 | int64 int_value = 1; 67 | double float_value = 2; 68 | string string_value = 3; 69 | uint64 dict_index = 4; 70 | } 71 | } 72 | 73 | // --------------------------------------------------------------------------- 74 | // Streaming envelope 75 | // --------------------------------------------------------------------------- 76 | 77 | message Datum { 78 | oneof data { 79 | PatternDefine pattern_define = 1; 80 | PatternDelete pattern_delete = 2; 81 | DictEntryDefine dict_entry_define = 3; 82 | DictEntryDelete dict_entry_delete = 4; 83 | Log logs = 5; 84 | } 85 | } 86 | 87 | // DatumSequence wraps a sequence of Datum messages 88 | // Used for serialization in application-level compression 89 | message DatumSequence { 90 | repeated Datum data = 1; 91 | } 92 | 93 | // data is sequence of pattern/dictionary changes + logs 94 | // the ordering is significant, must be processed in order 95 | message StatefulBatch { 96 | uint32 batch_id = 1; 97 | 98 | // Bytes of a serialized DatumSequence. Eventually this will also be compressed. 99 | // This allows for Datums to be compressed while they are buffered in memory before being acked by the server. 100 | bytes data = 2; 101 | } 102 | 103 | message BatchStatus { 104 | uint32 batch_id = 1; 105 | 106 | // TODO: only OK is used right now - should we just remove this enum? 107 | enum Status { 108 | UNKNOWN = 0; 109 | OK = 1; 110 | } 111 | Status status = 2; 112 | } 113 | 114 | // --------------------------------------------------------------------------- 115 | // gRPC service definition (bi-directional streaming) 116 | // --------------------------------------------------------------------------- 117 | 118 | service StatefulLogsService { 119 | rpc LogsStream(stream StatefulBatch) returns (stream BatchStatus); 120 | } 121 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Update the rust version in-sync with the version in rust-toolchain.toml 2 | 3 | # Stage 0: Planner - Extract dependency metadata 4 | FROM docker.io/rust:1.90.0-slim-bookworm AS planner 5 | WORKDIR /app 6 | RUN cargo install cargo-chef --version 0.1.73 7 | COPY . . 8 | RUN cargo chef prepare --recipe-path recipe.json 9 | 10 | # Stage 1: Cacher - Build dependencies only 11 | FROM docker.io/rust:1.90.0-slim-bookworm AS cacher 12 | ARG SCCACHE_BUCKET 13 | ARG SCCACHE_REGION 14 | ARG AWS_ACCESS_KEY_ID 15 | ARG AWS_SECRET_ACCESS_KEY 16 | ARG AWS_SESSION_TOKEN 17 | ENV CARGO_INCREMENTAL=0 18 | WORKDIR /app 19 | RUN apt-get update && apt-get install -y \ 20 | pkg-config=1.8.1-1 \ 21 | libssl-dev=3.0.17-1~deb12u3 \ 22 | protobuf-compiler=3.21.12-3 \ 23 | fuse3=3.14.0-4 \ 24 | libfuse3-dev=3.14.0-4 \ 25 | curl \ 26 | && rm -rf /var/lib/apt/lists/* 27 | # Download pre-built sccache binary 28 | RUN case "$(uname -m)" in \ 29 | x86_64) ARCH=x86_64-unknown-linux-musl ;; \ 30 | aarch64) ARCH=aarch64-unknown-linux-musl ;; \ 31 | *) echo "Unsupported architecture" && exit 1 ;; \ 32 | esac && \ 33 | curl -L https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-${ARCH}.tar.gz | tar xz && \ 34 | mv sccache-v0.8.2-${ARCH}/sccache /usr/local/cargo/bin/ && \ 35 | rm -rf sccache-v0.8.2-${ARCH} 36 | RUN cargo install cargo-chef --version 0.1.73 37 | COPY --from=planner /app/recipe.json recipe.json 38 | # This layer is cached until Cargo.toml/Cargo.lock change 39 | # Use BuildKit secrets to pass AWS credentials securely (not exposed in image metadata) 40 | RUN --mount=type=secret,id=aws_access_key_id \ 41 | --mount=type=secret,id=aws_secret_access_key \ 42 | --mount=type=secret,id=aws_session_token \ 43 | export AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) && \ 44 | export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) && \ 45 | export AWS_SESSION_TOKEN=$(cat /run/secrets/aws_session_token) && \ 46 | export RUSTC_WRAPPER=sccache && \ 47 | cargo chef cook --release --locked --features logrotate_fs --recipe-path recipe.json 48 | 49 | # Stage 2: Builder - Build source code 50 | FROM docker.io/rust:1.90.0-slim-bookworm AS builder 51 | ARG SCCACHE_BUCKET 52 | ARG SCCACHE_REGION 53 | ENV CARGO_INCREMENTAL=0 54 | ENV SCCACHE_BUCKET=${SCCACHE_BUCKET} 55 | ENV SCCACHE_REGION=${SCCACHE_REGION} 56 | WORKDIR /app 57 | RUN apt-get update && apt-get install -y \ 58 | pkg-config=1.8.1-1 \ 59 | libssl-dev=3.0.17-1~deb12u3 \ 60 | protobuf-compiler=3.21.12-3 \ 61 | fuse3=3.14.0-4 \ 62 | libfuse3-dev=3.14.0-4 \ 63 | && rm -rf /var/lib/apt/lists/* 64 | # Copy cached dependencies and sccache from cacher 65 | COPY --from=cacher /app/target target 66 | COPY --from=cacher /usr/local/cargo /usr/local/cargo 67 | # Copy source code (frequently changes) 68 | COPY . . 69 | # Build binary - reuses cached dependencies + sccache 70 | # Use BuildKit secrets to pass AWS credentials securely (not exposed in image metadata) 71 | RUN --mount=type=secret,id=aws_access_key_id \ 72 | --mount=type=secret,id=aws_secret_access_key \ 73 | --mount=type=secret,id=aws_session_token \ 74 | export AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) && \ 75 | export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) && \ 76 | export AWS_SESSION_TOKEN=$(cat /run/secrets/aws_session_token) && \ 77 | export RUSTC_WRAPPER=sccache && \ 78 | cargo build --release --locked --bin lading --features logrotate_fs 79 | 80 | # Stage 3: Runtime 81 | FROM docker.io/debian:bookworm-20241202-slim 82 | RUN apt-get update && apt-get install -y \ 83 | libfuse3-dev=3.14.0-4 \ 84 | fuse3=3.14.0-4 \ 85 | && rm -rf /var/lib/apt/lists/* 86 | COPY --from=builder /app/target/release/lading /usr/bin/lading 87 | 88 | # Smoke test 89 | RUN ["/usr/bin/lading", "--help"] 90 | ENTRYPOINT ["/usr/bin/lading"] 91 | --------------------------------------------------------------------------------