├── .devcontainer └── devcontainer.json ├── .github ├── actions │ └── consul-start │ │ └── action.yml ├── pull_request_template.md └── workflows │ ├── lint-pr.yaml │ ├── semantic-release.yml │ ├── swift-benchmark-delta.yml │ ├── swift-check-api-breaks.yml │ ├── swift-code-coverage.yml │ ├── swift-lint.yml │ ├── swift-linux-build.yml │ ├── swift-linux-previous.yml │ ├── swift-macos-build.yml │ ├── swift-macos-previous.yml │ ├── swift-outdated-dependencies.yml │ ├── swift-sanitizer-address.yml │ └── swift-sanitizer-thread.yml ├── .gitignore ├── .spi.yml ├── .swift-version ├── .swiftformat ├── .swiftlint.yml ├── .swiftlint_base.yml ├── .swiftlint_refinement.yml ├── Benchmarks ├── Benchmarks │ ├── Basic │ │ ├── Basic+SetupTeardown.swift │ │ └── BenchmarkRunner+Basic.swift │ ├── DateTime │ │ └── DateTime.swift │ ├── Histogram │ │ └── Histogram.swift │ └── P90AbsoluteThresholds │ │ └── P90AbsoluteThresholds.swift ├── Package.resolved └── Package.swift ├── LICENSE ├── Package.resolved ├── Package.swift ├── Platform ├── CDarwinOperatingSystemStats │ ├── CDarwinOperatingSystemStats.c │ └── include │ │ └── CDarwinOperatingSystemStats.h └── CLinuxOperatingSystemStats │ ├── CLinuxOperatingSystemStats.c │ └── include │ └── CLinuxOperatingSystemStats.h ├── Plugins ├── BenchmarkBoilerplateGenerator │ └── BenchmarkBoilerplateGenerator.swift ├── BenchmarkCommandPlugin │ ├── ArgumentExtractor+Extensions.swift │ ├── BenchmarkCommandPlugin.swift │ ├── BenchmarkPlugin+Help.swift │ └── Command+Helpers.swift ├── BenchmarkHelpGenerator │ └── BenchmarkHelpGenerator.swift ├── BenchmarkPlugin │ └── BenchmarkSupportPlugin.swift └── BenchmarkTool │ ├── BenchmarkTool+Baselines.swift │ ├── BenchmarkTool+CreateBenchmark.swift │ ├── BenchmarkTool+Export+InfluxCSVFormatter.swift │ ├── BenchmarkTool+Export+JMHElement.swift │ ├── BenchmarkTool+Export+JMHFormatter.swift │ ├── BenchmarkTool+Export.swift │ ├── BenchmarkTool+JSON.swift │ ├── BenchmarkTool+Machine.swift │ ├── BenchmarkTool+Operations.swift │ ├── BenchmarkTool+PrettyPrinting.swift │ ├── BenchmarkTool+ReadP90AbsoluteThresholds.swift │ ├── BenchmarkTool+Thresholds.swift │ ├── BenchmarkTool.swift │ ├── FilePath+Additions.swift │ ├── FilePath+DirectoryView.swift │ └── String+Additions.swift ├── README.md ├── Sources ├── .swiftlint.yml ├── Benchmark │ ├── ARCStats │ │ ├── ARCStats.swift │ │ └── ARCStatsProducer.swift │ ├── Benchmark+ConvenienceInitializers.swift │ ├── Benchmark.swift │ ├── BenchmarkClock.swift │ ├── BenchmarkExecutor+Extensions.swift │ ├── BenchmarkExecutor.swift │ ├── BenchmarkInternals.swift │ ├── BenchmarkMetric+Defaults.swift │ ├── BenchmarkMetric.swift │ ├── BenchmarkResult.swift │ ├── BenchmarkRunner+ReadWrite.swift │ ├── BenchmarkRunner.swift │ ├── BenchmarkThresholds+Defaults.swift │ ├── BenchmarkThresholds.swift │ ├── Blackhole.swift │ ├── Documentation.docc │ │ ├── AboutPercentiles.md │ │ ├── Benchmark.md │ │ ├── BenchmarkMetric.md │ │ ├── BenchmarkMetric_Polarity.md │ │ ├── BenchmarkScalingFactor.md │ │ ├── BenchmarkTimeUnits.md │ │ ├── BenchmarkUnits.md │ │ ├── Benchmark_Configuration.md │ │ ├── ComparingBenchmarksCI.md │ │ ├── CreatingAndComparingBaselines.md │ │ ├── Documentation.md │ │ ├── ExportingBenchmarks.md │ │ ├── GettingStarted.md │ │ ├── Metrics.md │ │ ├── Resources │ │ │ └── Images │ │ │ │ ├── PercentileHistogramExample.png │ │ │ │ └── uplot.png │ │ ├── RunningBenchmarks.md │ │ └── WritingBenchmarks.md │ ├── Int+Extensions.swift │ ├── MallocStats │ │ ├── MallocStats+jemalloc-support.swift │ │ ├── MallocStats.swift │ │ └── MallocStatsProducer+jemalloc.swift │ ├── NIOConcurrencyHelpers │ │ ├── NIOLock.swift │ │ └── lock.swift │ ├── OperatingSystemStats │ │ ├── OperatingSystemStats.swift │ │ ├── OperatingSystemStatsProducer+Darwin.swift │ │ └── OperatingSystemStatsProducer+Linux.swift │ ├── OutputSuppressor.swift │ ├── Progress │ │ ├── Progress.swift │ │ ├── ProgressElements.swift │ │ └── Utilities.swift │ └── Statistics.swift ├── BenchmarkShared │ └── Command+Helpers.swift └── SwiftRuntimeHooks │ ├── include │ └── SwiftRuntimeHooks.h │ └── shims.c ├── Tests └── BenchmarkTests │ ├── AdditionalTests.swift │ ├── BenchmarkMetricsTests.swift │ ├── BenchmarkResultTests.swift │ ├── BenchmarkRunnerTests.swift │ ├── BenchmarkTests.swift │ ├── OperatingSystemAndMallocTests.swift │ └── StatisticsTests.swift ├── Thresholds ├── P90AbsoluteThresholdsBenchmark.P90Date.p90.json └── P90AbsoluteThresholdsBenchmark.P90Malloc.p90.json └── codecov.yml /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Swift", 3 | "image": "swift:5.7", 4 | "features": { 5 | "ghcr.io/devcontainers/features/common-utils:2": { 6 | "installZsh": "false", 7 | "username": "vscode", 8 | "userUid": "1000", 9 | "userGid": "1000", 10 | "upgradePackages": "false" 11 | }, 12 | "ghcr.io/devcontainers/features/git:1": { 13 | "version": "os-provided", 14 | "ppa": "false" 15 | }, 16 | "ghcr.io/heckj/devcontainer-swift-additions/jemalloc:1": { 17 | } 18 | }, 19 | "runArgs": [ 20 | "--cap-add=SYS_PTRACE", 21 | "--security-opt", 22 | "seccomp=unconfined" 23 | ], 24 | // Configure tool-specific properties. 25 | "customizations": { 26 | // Configure properties specific to VS Code. 27 | "vscode": { 28 | // Set *default* container specific settings.json values on container create. 29 | "settings": { 30 | "lldb.library": "/usr/lib/liblldb.so" 31 | }, 32 | // Add the IDs of extensions you want installed when the container is created. 33 | "extensions": [ 34 | "sswg.swift-lang" 35 | ] 36 | } 37 | }, 38 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 39 | // "forwardPorts": [], 40 | 41 | // Use 'postCreateCommand' to run commands after the container is created. 42 | "postCreateCommand": "swift --version", 43 | 44 | // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. 45 | "remoteUser": "vscode" 46 | } 47 | -------------------------------------------------------------------------------- /.github/actions/consul-start/action.yml: -------------------------------------------------------------------------------- 1 | name: "Start Consul" 2 | runs: 3 | using: "composite" 4 | steps: 5 | - uses: nahsi/setup-hashi-tool@v1 6 | if: github.repository == 'ordo-one/package-consul' || github.repository == 'ordo-one/package-distributed-system' 7 | with: 8 | name: consul 9 | - name: Start consul 10 | if: github.repository == 'ordo-one/package-consul' || github.repository == 'ordo-one/package-distributed-system' 11 | shell: bash 12 | run: | 13 | consul agent -dev -log-level=warn & 14 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 4 | 5 | ## How Has This Been Tested? 6 | 7 | Please describe the tests that you ran to verify your changes. 8 | 9 | ## Minimal checklist: 10 | 11 | - [ ] I have performed a self-review of my own code 12 | - [ ] I have added `DocC` code-level documentation for any public interfaces exported by the package 13 | - [ ] I have added unit and/or integration tests that prove my fix is effective or that my feature works 14 | -------------------------------------------------------------------------------- /.github/workflows/lint-pr.yaml: -------------------------------------------------------------------------------- 1 | name: "PR title validation" 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | main: 12 | name: Validate PR title 13 | # runs-on: [self-hosted, ubuntu-latest] 14 | runs-on: ubuntu-latest 15 | timeout-minutes: 30 16 | steps: 17 | - uses: amannn/action-semantic-pull-request@v5 18 | env: 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | with: 21 | # Configure which types are allowed. 22 | # Default: https://github.com/commitizen/conventional-commit-types 23 | types: | 24 | fix 25 | feat 26 | docs 27 | style 28 | refactor 29 | perf 30 | test 31 | build 32 | ci 33 | chore 34 | revert 35 | # Configure which scopes are allowed. 36 | scopes: | 37 | patch 38 | hotfix 39 | minor 40 | major 41 | # Configure that a scope must always be provided. 42 | requireScope: false 43 | # Configure which scopes are disallowed in PR titles. For instance by setting 44 | # the value below, `chore(release): ...` and `ci(e2e,release): ...` will be rejected. 45 | disallowScopes: | 46 | release 47 | # Configure additional validation for the subject based on a regex. 48 | # This example ensures the subject doesn't start with an uppercase character. 49 | #subjectPattern: ^.*\[sc-[0-9]+\].*$ 50 | #subjectPatternError: | 51 | # The pull request title should contain Shortcut case number like '[sc-123]' 52 | # If the PR contains one of these labels, the validation is skipped. 53 | # Multiple labels can be separated by newlines. 54 | # If you want to rerun the validation when labels change, you might want 55 | # to use the `labeled` and `unlabeled` event triggers in your workflow. 56 | ignoreLabels: | 57 | bot 58 | ignore-semantic-pull-request 59 | # For work-in-progress PRs you can typically use draft pull requests 60 | # from GitHub. However, private repositories on the free plan don't have 61 | # this option and therefore this action allows you to opt-in to using the 62 | # special "[WIP]" prefix to indicate this state. This will avoid the 63 | # validation of the PR title and the pull request checks remain pending. 64 | # Note that a second check will be reported if this is enabled. 65 | wip: true 66 | -------------------------------------------------------------------------------- /.github/workflows/semantic-release.yml: -------------------------------------------------------------------------------- 1 | name: Semantic Release 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main, next ] 7 | 8 | jobs: 9 | semantic-release: 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: Semantic Release Config 17 | run: | 18 | echo " 19 | branches: 20 | - main 21 | - name: 'next' 22 | prerelease: true 23 | preset: 'conventionalcommits' 24 | tagFormat: '\${version}' 25 | plugins: 26 | - - '@semantic-release/commit-analyzer' 27 | - releaseRules: 28 | - breaking: true 29 | release: major 30 | - type: build 31 | release: patch 32 | - type: chore 33 | release: false 34 | - type: feat 35 | release: false 36 | - type: fix 37 | release: false 38 | 39 | - scope: 'no-release' 40 | release: false 41 | - scope: 'hotfix' 42 | release: patch 43 | - scope: 'patch' 44 | release: patch 45 | - scope: 'minor' 46 | release: minor 47 | - scope: 'major' 48 | release: major 49 | - - '@semantic-release/release-notes-generator' 50 | - - '@semantic-release/github' 51 | - successComment: false 52 | failTitle: false" > .releaserc.yml 53 | 54 | - name: Setup Node.js 55 | uses: actions/setup-node@v4 56 | with: 57 | node-version: '20' 58 | 59 | - name: Install semantic-release 60 | run: | 61 | npm install semantic-release@v24 conventional-changelog-conventionalcommits@v8 -D 62 | npm list 63 | - name: Release 64 | env: 65 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 66 | run: npx semantic-release 67 | -------------------------------------------------------------------------------- /.github/workflows/swift-benchmark-delta.yml: -------------------------------------------------------------------------------- 1 | name: Benchmark PR vs main 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: [ main ] 7 | 8 | jobs: 9 | benchmark-delta: 10 | timeout-minutes: 30 11 | runs-on: ${{ matrix.os }} 12 | 13 | strategy: 14 | matrix: 15 | os: [ubuntu-latest] 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 21 | - name: Homebrew Mac 22 | if: ${{ runner.os == 'Macos' }} 23 | run: | 24 | echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH 25 | brew install jemalloc 26 | - name: Ubuntu deps 27 | if: ${{ runner.os == 'Linux' }} 28 | run: | 29 | sudo apt-get install -y libjemalloc-dev 30 | - name: Start consul 31 | uses: ./.github/actions/consul-start 32 | - name: Git URL token override and misc 33 | run: | 34 | #git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com" 35 | #/usr/bin/ordo-performance 36 | [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV 37 | [ -f Benchmarks/Package.swift ] && echo "BENCHMARK_PACKAGE_PATH=--package-path Benchmarks" >> $GITHUB_ENV 38 | echo "BENCHMARK_RUN_URL=https://github.com/ordo-one/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV 39 | echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH 40 | - name: Run benchmarks for PR branch 41 | if: ${{ env.hasBenchmark == '1' }} 42 | run: | 43 | echo "exitStatus=1" >> $GITHUB_ENV 44 | swift package ${BENCHMARK_PACKAGE_PATH} --disable-sandbox benchmark baseline update pull_request --no-progress 45 | - name: Switch to branch 'main' 46 | if: ${{ env.hasBenchmark == '1' }} 47 | run: | 48 | git stash 49 | git checkout main 50 | - name: Run benchmarks for branch 'main' 51 | if: ${{ env.hasBenchmark == '1' }} 52 | run: | 53 | swift package ${BENCHMARK_PACKAGE_PATH} --disable-sandbox benchmark baseline update main --no-progress 54 | - name: Compare PR and main 55 | if: ${{ env.hasBenchmark == '1' }} 56 | id: benchmark 57 | continue-on-error: true 58 | run: | 59 | echo $(date) >> $GITHUB_STEP_SUMMARY 60 | swift package ${BENCHMARK_PACKAGE_PATH} benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY 61 | echo "exitStatus=$?" >> $GITHUB_ENV 62 | - if: ${{ env.exitStatus == '0' }} 63 | name: Pull request comment text same 64 | run: | 65 | { 66 | echo "_Pull request is the same as baseline_" 67 | echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)]($BENCHMARK_RUN_URL)" 68 | } > benchmark_comment 69 | - if: ${{ env.exitStatus == '1' }} 70 | name: Pull request comment text failure 71 | run: | 72 | { 73 | echo "_Pull request had an unknown failure_" 74 | echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)]($BENCHMARK_RUN_URL)" 75 | } > benchmark_comment 76 | - if: ${{ env.exitStatus == '2' }} 77 | name: Pull request comment text regression 78 | run: | 79 | { 80 | echo "_Pull request had a regression_" 81 | echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)]($BENCHMARK_RUN_URL)" 82 | } > benchmark_comment 83 | - if: ${{ env.exitStatus == '4' }} 84 | name: Pull request comment text improvement 85 | run: | 86 | { 87 | echo "_Pull request had a performance improvement_" 88 | echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)]($BENCHMARK_RUN_URL)" 89 | } > benchmark_comment 90 | - name: Comment PR 91 | if: ${{ env.hasBenchmark == '1' }} 92 | uses: thollander/actions-comment-pull-request@v3 93 | with: 94 | github-token: ${{ secrets.GITHUB_TOKEN }} 95 | file-path: benchmark_comment 96 | comment-tag: 'Pull request benchmark comparison [${{ matrix.os }}] with' 97 | - name: Exit with correct status 98 | if: ${{ success() || failure() }} 99 | run: | 100 | #/usr/bin/ordo-performance powersave 101 | exit ${{ env.exitStatus }} 102 | -------------------------------------------------------------------------------- /.github/workflows/swift-check-api-breaks.yml: -------------------------------------------------------------------------------- 1 | name: Swift check API breaks 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | types: [opened, synchronize] 7 | 8 | jobs: 9 | analyze-api-breakage: 10 | 11 | runs-on: [ubuntu-latest] 12 | timeout-minutes: 30 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | - name: Ubuntu deps 19 | if: ${{ runner.os == 'Linux' }} 20 | run: | 21 | sudo apt-get install -y libjemalloc-dev 22 | - name: Extract default SPM library target 23 | id: spm_target 24 | run: | 25 | SPM_DEFAULT_TARGET=$(swift package dump-package | jq -r '.products | .[] | select(.type | has("library")) | .name' | head -1) 26 | echo "spmlibrarytarget=${SPM_DEFAULT_TARGET}" >> $GITHUB_ENV 27 | - name: Build 28 | if: ${{ env.spmlibrarytarget }} 29 | run: swift build 30 | - name: Analyze API breakage ((workaround compile issue on first run) 31 | if: ${{ env.spmlibrarytarget }} 32 | continue-on-error: true 33 | run: swift package diagnose-api-breaking-changes origin/main --targets ${{ env.spmlibrarytarget }} 34 | - name: Analyze API breakage 35 | if: ${{ env.spmlibrarytarget }} 36 | run: swift package diagnose-api-breaking-changes origin/main --targets ${{ env.spmlibrarytarget }} 37 | -------------------------------------------------------------------------------- /.github/workflows/swift-code-coverage.yml: -------------------------------------------------------------------------------- 1 | name: Swift code coverage 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | jobs: 10 | test-code-coverage: 11 | runs-on: [ubuntu-22.04] 12 | timeout-minutes: 60 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Ubuntu deps 17 | if: ${{ runner.os == 'Linux' }} 18 | run: | 19 | sudo apt-get install -y libjemalloc-dev 20 | 21 | - uses: khlopko/setup-swift@bfd61cbd14eeef55a27afc45138b61ced7174839 22 | 23 | - name: Swift version 24 | run: swift --version 25 | 26 | - name: Start consul 27 | uses: ./.github/actions/consul-start 28 | 29 | - name: Run tests 30 | continue-on-error: true 31 | run: | 32 | [ -d Tests ] && swift test --parallel --enable-code-coverage 33 | 34 | - name: Export code coverage 35 | run: | 36 | xctest_binary=".build/debug/${{ github.event.repository.name }}PackageTests.xctest" 37 | if [ ! -f ${xctest_binary} ]; then 38 | xctest_binary=$(find .build/debug/ -type f -name "*.xctest" | tail -1) 39 | echo "Will llvm-cov '${xctest_binary}'" 40 | fi 41 | 42 | if [ -f ${xctest_binary} ]; then 43 | llvm-cov export -format="lcov" ${xctest_binary} -instr-profile .build/debug/codecov/default.profdata > info.lcov 44 | fi 45 | 46 | - name: Upload codecov 47 | uses: codecov/codecov-action@v4 48 | with: 49 | token: ${{ secrets.CODECOV_REPO_TOKEN }} 50 | files: info.lcov 51 | fail_ci_if_error: true 52 | -------------------------------------------------------------------------------- /.github/workflows/swift-lint.yml: -------------------------------------------------------------------------------- 1 | name: Swift lint 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | paths: 7 | - '.github/workflows/swiftlint.yml' 8 | - '.swiftlint.yml' 9 | - '**/*.swift' 10 | 11 | jobs: 12 | SwiftLint: 13 | timeout-minutes: 60 14 | runs-on: [ubuntu-latest] 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: GitHub Action for SwiftLint with --strict 18 | uses: norio-nomura/action-swiftlint@3.2.1 19 | with: 20 | args: --strict 21 | -------------------------------------------------------------------------------- /.github/workflows/swift-linux-build.yml: -------------------------------------------------------------------------------- 1 | name: Linux build 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | 10 | jobs: 11 | build-linux: 12 | timeout-minutes: 60 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [ubuntu-latest] 17 | # swift: [ "5.10", "6.0" ] 18 | 19 | runs-on: ${{ matrix.os }} 20 | 21 | steps: 22 | - uses: swift-actions/setup-swift@v2 23 | if: ${{ false }} 24 | with: 25 | swift-version: ${{ matrix.swift }} 26 | 27 | - uses: actions/checkout@v4 28 | 29 | - name: Start consul 30 | uses: ./.github/actions/consul-start 31 | 32 | - name: Ubuntu deps 33 | if: ${{ runner.os == 'Linux' }} 34 | run: | 35 | sudo apt-get install -y libjemalloc-dev 36 | 37 | - name: Swift version 38 | run: swift --version 39 | 40 | - name: Build 41 | run: swift build 42 | 43 | - name: Run tests 44 | run: | 45 | if [ -d Tests ]; then 46 | swift test --parallel 47 | fi 48 | -------------------------------------------------------------------------------- /.github/workflows/swift-linux-previous.yml: -------------------------------------------------------------------------------- 1 | name: Swift Linux build older versions 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | 10 | jobs: 11 | build-linux-previous: 12 | timeout-minutes: 60 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [ubuntu-22.04] 17 | swift: ["5.9", "5.10"] 18 | 19 | 20 | runs-on: ${{ matrix.os }} 21 | 22 | steps: 23 | - uses: swift-actions/setup-swift@v2 24 | with: 25 | swift-version: ${{ matrix.swift }} 26 | 27 | - name: Swift version 28 | run: swift --version 29 | 30 | - uses: actions/checkout@v3 31 | 32 | - name: Start consul 33 | uses: ./.github/actions/consul-start 34 | 35 | - name: Ubuntu deps 36 | if: ${{ runner.os == 'Linux' }} 37 | run: | 38 | sudo apt-get install -y libjemalloc-dev 39 | 40 | - name: Swift version 41 | run: swift --version 42 | 43 | - name: Build 44 | run: swift build 45 | 46 | - name: Run tests 47 | run: | 48 | if [ -d Tests ]; then 49 | swift test --parallel 50 | fi 51 | -------------------------------------------------------------------------------- /.github/workflows/swift-macos-build.yml: -------------------------------------------------------------------------------- 1 | name: macOS build 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | 10 | jobs: 11 | build-macos: 12 | timeout-minutes: 60 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [macos-15] 17 | # swift: [ "5.10", "6.0" ] 18 | 19 | runs-on: ${{ matrix.os }} 20 | # env: 21 | # DEVELOPER_DIR: /Applications/Xcode_16.1.app/Contents/Developer 22 | 23 | steps: 24 | - uses: swift-actions/setup-swift@v2 25 | if: ${{ false }} 26 | with: 27 | swift-version: ${{ matrix.swift }} 28 | 29 | - name: Homebrew Mac 30 | if: ${{ runner.os == 'Macos' }} 31 | run: | 32 | echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH 33 | echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV 34 | brew install jemalloc 35 | 36 | - uses: actions/checkout@v4 37 | 38 | - name: Start consul 39 | uses: ./.github/actions/consul-start 40 | 41 | - name: GH auth 42 | run: | 43 | echo "machine api.github.com login ${{ secrets.GITHUB_TOKEN }} password x-oauth-basic" > $HOME/.netrc 44 | cat ~/.netrc 45 | - name: Swift version 46 | run: swift --version 47 | - name: Build 48 | run: swift build 49 | - name: Run tests 50 | run: | 51 | if [ -d Tests ]; then 52 | swift test --parallel 53 | fi 54 | - name: Run tests (release) 55 | run: | 56 | if [ -d Tests ]; then 57 | swift test -c release --parallel 58 | fi 59 | -------------------------------------------------------------------------------- /.github/workflows/swift-macos-previous.yml: -------------------------------------------------------------------------------- 1 | name: Swift macOS build older versions 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | 10 | jobs: 11 | build-macos-previous: 12 | timeout-minutes: 60 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [macos-14] 17 | xcode: ["15.2", "15.3"] 18 | #swift: ["5.9", "5.10"] 19 | 20 | runs-on: ${{ matrix.os }} 21 | 22 | steps: 23 | - uses: swift-actions/setup-swift@v2 24 | if: ${{ false }} 25 | with: 26 | swift-version: ${{ matrix.swift }} 27 | - uses: maxim-lobanov/setup-xcode@v1 28 | with: 29 | xcode-version: ${{ matrix.xcode }} 30 | 31 | - name: Homebrew Mac 32 | if: ${{ runner.os == 'Macos' }} 33 | run: | 34 | echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH 35 | echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV 36 | brew install jemalloc 37 | 38 | - uses: actions/checkout@v3 39 | 40 | - name: Start consul 41 | uses: ./.github/actions/consul-start 42 | 43 | - name: Swift version 44 | run: swift --version 45 | - name: Build 46 | run: swift build 47 | - name: Run tests 48 | run: | 49 | if [ -d Tests ]; then 50 | swift test --parallel 51 | fi 52 | - name: Run tests (release) 53 | run: | 54 | if [ -d Tests ]; then 55 | swift test -c release --parallel 56 | fi 57 | - name: Setup tmate session 58 | if: false && failure() 59 | uses: mxschmitt/action-tmate@v3 -------------------------------------------------------------------------------- /.github/workflows/swift-outdated-dependencies.yml: -------------------------------------------------------------------------------- 1 | name: Swift outdated dependencies 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '0 8 */100,1-7 * MON' # First Monday of the month 7 | 8 | jobs: 9 | spm-dep-check: 10 | runs-on: [ubuntu-latest] 11 | timeout-minutes: 60 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Check Swift package dependencies 15 | id: spm-dep-check 16 | uses: MarcoEidinger/swift-package-dependencies-check@2.5.0 17 | with: 18 | isMutating: true 19 | failWhenOutdated: false 20 | - name: Create Pull Request 21 | if: steps.spm-dep-check.outputs.outdatedDependencies == 'true' 22 | uses: peter-evans/create-pull-request@v7 23 | with: 24 | commit-message: 'chore: update package dependencies' 25 | branch: updatePackageDepedencies 26 | delete-branch: true 27 | title: 'chore: update package dependencies' 28 | body: ${{ steps.spm-dep-check.outputs.releaseNotes }} 29 | -------------------------------------------------------------------------------- /.github/workflows/swift-sanitizer-address.yml: -------------------------------------------------------------------------------- 1 | name: Address sanitizer 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | 10 | jobs: 11 | address-sanitizer: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | os: [macos-15] 16 | 17 | runs-on: ${{ matrix.os }} 18 | timeout-minutes: 60 19 | steps: 20 | - name: Homebrew Mac 21 | if: ${{ runner.os == 'Macos' }} 22 | run: | 23 | echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH 24 | echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV 25 | brew install jemalloc 26 | 27 | - name: Ubuntu deps 28 | if: ${{ runner.os == 'Linux' }} 29 | run: | 30 | sudo apt-get install -y libjemalloc-dev 31 | 32 | - uses: actions/checkout@v4 33 | 34 | - name: Start consul 35 | uses: ./.github/actions/consul-start 36 | 37 | - name: Swift version 38 | run: swift --version 39 | 40 | # Required to clean build directory before sanitizer! 41 | - name: Clean before debug build sanitizier 42 | run: swift package clean 43 | 44 | - name: Run address sanitizer 45 | run: swift test --sanitize=address 46 | 47 | - name: Clean before release build sanitizier 48 | run: swift package clean 49 | 50 | - name: Run address sanitizer on release build 51 | run: swift test --sanitize=address -c release -Xswiftc -enable-testing 52 | -------------------------------------------------------------------------------- /.github/workflows/swift-sanitizer-thread.yml: -------------------------------------------------------------------------------- 1 | name: Thread sanitizer 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main, next ] 9 | 10 | jobs: 11 | thread-sanitizer: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | os: [ubuntu-latest] 16 | 17 | runs-on: ${{ matrix.os }} 18 | timeout-minutes: 60 19 | steps: 20 | - name: Homebrew Mac 21 | if: ${{ runner.os == 'Macos' }} 22 | run: | 23 | echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH 24 | echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV 25 | brew install jemalloc 26 | 27 | - name: Ubuntu deps 28 | if: ${{ runner.os == 'Linux' }} 29 | run: | 30 | sudo apt-get install -y libjemalloc-dev 31 | echo BENCHMARK_DISABLE_JEMALLOC=true >> $GITHUB_ENV 32 | 33 | - uses: actions/checkout@v4 34 | 35 | - name: Start consul 36 | uses: ./.github/actions/consul-start 37 | 38 | - name: Swift version 39 | run: swift --version 40 | 41 | # Required to clean build directory before sanitizer! 42 | - name: Clean before debug build thread sanitizier 43 | run: swift package clean 44 | 45 | - name: Run thread sanitizer 46 | run: swift test --sanitize=thread 47 | 48 | - name: Clean before release build sanitizier 49 | run: swift package clean 50 | 51 | - name: Run thread sanitizer on release build 52 | run: swift test --sanitize=thread -c release -Xswiftc -enable-testing 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #################################################################### 2 | # DO NOT EDIT THIS FILE MANUALLY 3 | # This is a master file maintained in https://github.com/ordo-one/repository-templates 4 | 5 | # Xcode 6 | # 7 | # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore 8 | 9 | ## User settings 10 | xcuserdata/ 11 | 12 | ## Benchmark data 13 | .benchmarkBaselines/ 14 | 15 | ## DocC build directories 16 | **/.docc-build/ 17 | 18 | ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) 19 | *.xcscmblueprint 20 | *.xccheckout 21 | 22 | ## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) 23 | build/ 24 | DerivedData/ 25 | *.moved-aside 26 | *.pbxuser 27 | !default.pbxuser 28 | *.mode1v3 29 | !default.mode1v3 30 | *.mode2v3 31 | !default.mode2v3 32 | *.perspectivev3 33 | !default.perspectivev3 34 | 35 | ## Obj-C/Swift specific 36 | *.hmap 37 | 38 | ## App packaging 39 | *.ipa 40 | *.dSYM.zip 41 | *.dSYM 42 | 43 | ## Playgrounds 44 | timeline.xctimeline 45 | playground.xcworkspace 46 | 47 | # Swift Package Manager 48 | # 49 | # Add this line if you want to avoid checking in source code from Swift Package Manager dependencies. 50 | # Packages/ 51 | # Package.pins 52 | # Package.resolved 53 | # *.xcodeproj 54 | # 55 | # Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata 56 | # hence it is not needed unless you have added a package configuration file to your project 57 | .swiftpm 58 | .DS_Store 59 | .build/ 60 | 61 | # CocoaPods 62 | # 63 | # We recommend against adding the Pods directory to your .gitignore. However 64 | # you should judge for yourself, the pros and cons are mentioned at: 65 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 66 | # 67 | # Pods/ 68 | # 69 | # Add this line if you want to avoid checking in source code from the Xcode workspace 70 | # *.xcworkspace 71 | 72 | # Carthage 73 | # 74 | # Add this line if you want to avoid checking in source code from Carthage dependencies. 75 | # Carthage/Checkouts 76 | 77 | Carthage/Build/ 78 | 79 | # Accio dependency management 80 | Dependencies/ 81 | .accio/ 82 | 83 | # fastlane 84 | # 85 | # It is recommended to not store the screenshots in the git repo. 86 | # Instead, use fastlane to re-generate the screenshots whenever they are needed. 87 | # For more information about the recommended setup visit: 88 | # https://docs.fastlane.tools/best-practices/source-control/#source-control 89 | 90 | fastlane/report.xml 91 | fastlane/Preview.html 92 | fastlane/screenshots/**/*.png 93 | fastlane/test_output 94 | 95 | # Code Injection 96 | # 97 | # After new code Injection tools there's a generated folder /iOSInjectionProject 98 | # https://github.com/johnno1962/injectionforxcode 99 | 100 | iOSInjectionProject/ 101 | -------------------------------------------------------------------------------- /.spi.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | builder: 3 | configs: 4 | - documentation_targets: [Benchmark] 5 | platform: linux 6 | swift_version: '6.0' 7 | -------------------------------------------------------------------------------- /.swift-version: -------------------------------------------------------------------------------- 1 | 5.7.1 2 | -------------------------------------------------------------------------------- /.swiftformat: -------------------------------------------------------------------------------- 1 | #################################################################### 2 | # DO NOT EDIT THIS FILE 3 | --decimalgrouping 3,4 4 | --disable wrapMultilineStatementBraces 5 | --disable trailingCommas 6 | -------------------------------------------------------------------------------- /.swiftlint.yml: -------------------------------------------------------------------------------- 1 | child_config: .swiftlint_refinement.yml 2 | parent_config: .swiftlint_base.yml 3 | -------------------------------------------------------------------------------- /.swiftlint_base.yml: -------------------------------------------------------------------------------- 1 | #################################################################### 2 | # DO NOT EDIT THIS FILE 3 | # This is a master file maintained in https://github.com/ordo-one/public-repository-templates 4 | # 5 | # Add overrides into `swiftlint_refinement.yml` in the directory it self or 6 | # .swiftlint.yml under the specific directory where the code is located 7 | # 8 | # Documentation is under https://github.com/realm/SwiftLint 9 | #################################################################### 10 | 11 | included: 12 | - Benchmarks 13 | - Sources 14 | - Tests 15 | excluded: 16 | analyzer_rules: 17 | - unused_import 18 | opt_in_rules: 19 | - array_init 20 | - attributes 21 | - closure_end_indentation 22 | - closure_spacing 23 | - collection_alignment 24 | - contains_over_filter_count 25 | - contains_over_filter_is_empty 26 | - contains_over_first_not_nil 27 | - contains_over_range_nil_comparison 28 | - discouraged_none_name 29 | - discouraged_object_literal 30 | - empty_collection_literal 31 | - empty_count 32 | - empty_string 33 | - empty_xctest_method 34 | - enum_case_associated_values_count 35 | - explicit_init 36 | - extension_access_modifier 37 | - fallthrough 38 | - fatal_error_message 39 | - file_name 40 | - first_where 41 | - flatmap_over_map_reduce 42 | - identical_operands 43 | - joined_default_parameter 44 | - last_where 45 | - legacy_multiple 46 | - literal_expression_end_indentation 47 | - lower_acl_than_parent 48 | - modifier_order 49 | - nimble_operator 50 | - nslocalizedstring_key 51 | - number_separator 52 | - object_literal 53 | - operator_usage_whitespace 54 | - overridden_super_call 55 | - override_in_extension 56 | - pattern_matching_keywords 57 | - prefer_self_in_static_references 58 | - prefer_self_type_over_type_of_self 59 | - private_action 60 | - private_outlet 61 | - prohibited_interface_builder 62 | - prohibited_super_call 63 | - quick_discouraged_call 64 | - quick_discouraged_focused_test 65 | - quick_discouraged_pending_test 66 | - reduce_into 67 | - redundant_nil_coalescing 68 | - redundant_type_annotation 69 | - single_test_class 70 | - sorted_first_last 71 | - sorted_imports 72 | - static_operator 73 | - strong_iboutlet 74 | - test_case_accessibility 75 | - toggle_bool 76 | - unavailable_function 77 | - unneeded_parentheses_in_closure_argument 78 | - unowned_variable_capture 79 | - untyped_error_in_catch 80 | - vertical_parameter_alignment_on_call 81 | - vertical_whitespace_closing_braces 82 | - vertical_whitespace_opening_braces 83 | - xct_specific_matcher 84 | - yoda_condition 85 | line_length: 86 | warning: 140 87 | error: 140 88 | ignores_comments: true 89 | ignores_urls: true 90 | ignores_function_declarations: true 91 | ignores_interpolated_strings: true 92 | identifier_name: 93 | excluded: [id, i, j, k] 94 | -------------------------------------------------------------------------------- /.swiftlint_refinement.yml: -------------------------------------------------------------------------------- 1 | excluded: 2 | - Sources/Benchmark/Progress 3 | - Sources/Benchmark/NIOConcurrencyHelpers 4 | -------------------------------------------------------------------------------- /Benchmarks/Benchmarks/Basic/Basic+SetupTeardown.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Basic+SetupTeardown.swift 3 | // 4 | // 5 | // Created by Joakim Hassila on 2023-04-21. 6 | // 7 | 8 | import Benchmark 9 | 10 | func sharedSetup() {} 11 | 12 | // func sharedSetup() -> [Int] { 13 | // [1, 2, 3] 14 | // } 15 | 16 | func sharedTeardown() { 17 | // print("Shared teardown hook") 18 | } 19 | 20 | func testSetUpTearDown() { 21 | // Benchmark.setup = { print("Global setup hook")} 22 | // Benchmark.setup = { 123 } 23 | // Benchmark.teardown = { print("Global teardown hook") } 24 | 25 | Benchmark("SetupTeardown", 26 | configuration: .init(setup: sharedSetup, teardown: sharedTeardown)) { _ in 27 | } setup: { 28 | // print("Local setup hook") 29 | } teardown: { 30 | // print("Local teardown hook") 31 | } 32 | 33 | Benchmark("SetupTeardown2", 34 | configuration: .init(setup: sharedSetup, teardown: sharedTeardown)) { _ in 35 | } 36 | 37 | Benchmark("SetupTeardown3", 38 | configuration: .init(setup: sharedSetup)) { _ in 39 | // let x = benchmark.setupState as! [Int] 40 | // print("\(x)") 41 | } teardown: { 42 | // print("Local teardown hook") 43 | } 44 | 45 | Benchmark("SetupTeardown4", 46 | configuration: .init(setup: sharedSetup)) { _ in 47 | // print("\(benchmark.setupState)") 48 | } setup: { 49 | // return 7 50 | // print("Local setup hook") 51 | } 52 | 53 | Benchmark("SetupTeardown5") { _ in 54 | // print("\(benchmark.setupState)") 55 | } 56 | 57 | Benchmark("SetupTeardown6") { _, _ in 58 | // print("\(setupState)") 59 | } setup: { 60 | [1, 2, 3] 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Benchmarks/Benchmarks/Basic/BenchmarkRunner+Basic.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | import Benchmark 11 | import Foundation 12 | 13 | #if canImport(Darwin) 14 | import Darwin 15 | #elseif canImport(Glibc) 16 | import Glibc 17 | #else 18 | #error("Unsupported Platform") 19 | #endif 20 | 21 | // quiet swiftlint for now 22 | extension BenchmarkRunner {} 23 | 24 | let benchmarks: @Sendable () -> Void = { 25 | var thresholdTolerances: [BenchmarkMetric: BenchmarkThresholds] 26 | 27 | thresholdTolerances = [.wallClock: .relaxed] 28 | 29 | Benchmark.defaultConfiguration = .init(warmupIterations: 0, 30 | maxDuration: .seconds(1), 31 | maxIterations: Int.max, 32 | thresholds: thresholdTolerances) 33 | 34 | testSetUpTearDown() 35 | 36 | // A way to define custom metrics fairly compact 37 | enum CustomMetrics { 38 | static var one: BenchmarkMetric { .custom("CustomMetricOne") } 39 | static var two: BenchmarkMetric { .custom("CustomMetricTwo", polarity: .prefersLarger, useScalingFactor: true) } 40 | static var three: BenchmarkMetric { .custom("CustomMetricThree", polarity: .prefersLarger, useScalingFactor: false) } 41 | } 42 | 43 | Benchmark("Basic", 44 | configuration: .init(metrics: [.wallClock, .throughput, .instructions])) { _ in 45 | } 46 | 47 | Benchmark("Noop", configuration: .init(metrics: [.wallClock, .mallocCountTotal, .instructions])) { _ in 48 | } 49 | 50 | Benchmark("Noop2", configuration: .init(metrics: [.wallClock, .instructions] + .arc)) { _ in 51 | } 52 | 53 | Benchmark("Scaled metrics One", 54 | configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one], 55 | scalingFactor: .one)) { benchmark in 56 | for _ in benchmark.scaledIterations { 57 | blackHole(Int.random(in: 1 ... 1_000)) 58 | } 59 | benchmark.measurement(CustomMetrics.two, Int.random(in: 1 ... 1_000_000)) 60 | benchmark.measurement(CustomMetrics.one, Int.random(in: 1 ... 1_000)) 61 | } 62 | 63 | Benchmark("Scaled metrics K", 64 | configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one], 65 | scalingFactor: .kilo)) { benchmark in 66 | for _ in benchmark.scaledIterations { 67 | blackHole(Int.random(in: 1 ... 1_000)) 68 | } 69 | benchmark.measurement(CustomMetrics.two, Int.random(in: 1 ... 1_000_000)) 70 | benchmark.measurement(CustomMetrics.one, Int.random(in: 1 ... 1_000)) 71 | } 72 | 73 | Benchmark("Scaled metrics M", 74 | configuration: .init(metrics: .all + [CustomMetrics.two, CustomMetrics.one, CustomMetrics.three], 75 | scalingFactor: .mega)) { benchmark in 76 | for _ in benchmark.scaledIterations { 77 | blackHole(Int.random(in: benchmark.scaledIterations)) 78 | } 79 | benchmark.measurement(CustomMetrics.three, Int.random(in: 1 ... 1_000_000_000)) 80 | benchmark.measurement(CustomMetrics.two, Int.random(in: 1 ... 1_000_000)) 81 | benchmark.measurement(CustomMetrics.one, Int.random(in: 1 ... 1_000)) 82 | } 83 | 84 | Benchmark("All metrics", 85 | configuration: .init(metrics: .all, skip: true)) { _ in 86 | } 87 | 88 | let stats = Statistics(numberOfSignificantDigits: .four) 89 | let measurementCount = 8_340 90 | 91 | for measurement in (0 ..< measurementCount).reversed() { 92 | stats.add(measurement) 93 | } 94 | 95 | Benchmark("Statistics", 96 | configuration: .init(metrics: .arc + [.wallClock], 97 | scalingFactor: .kilo, maxDuration: .seconds(1))) { benchmark in 98 | for _ in benchmark.scaledIterations { 99 | blackHole(stats.percentiles()) 100 | } 101 | } 102 | 103 | let parameterization = (0...5).map { 1 << $0 } // 1, 2, 4, ... 104 | 105 | parameterization.forEach { count in 106 | Benchmark("Parameterized", configuration: .init(tags: ["count": count.description])) { benchmark in 107 | for _ in 0 ..< count { 108 | blackHole(Int.random(in: benchmark.scaledIterations)) 109 | } 110 | } 111 | } 112 | 113 | @Sendable 114 | func defaultCounter() -> Int { 10 } 115 | 116 | @Sendable 117 | func dummyCounter(_ count: Int) { 118 | for index in 0 ..< count { 119 | blackHole(index) 120 | } 121 | } 122 | 123 | func concurrentWork(tasks: Int = 4, mallocs: Int = 0) async { 124 | _ = await withTaskGroup(of: Void.self, returning: Void.self, body: { taskGroup in 125 | for _ in 0 ..< tasks { 126 | taskGroup.addTask { 127 | dummyCounter(defaultCounter() * 1_000) 128 | for _ in 0 ..< mallocs { 129 | let something = malloc(1_024 * 1_024) 130 | blackHole(something) 131 | free(something) 132 | } 133 | if let fileHandle = FileHandle(forWritingAtPath: "/dev/null") { 134 | let data = Data("Data to discard".utf8) 135 | fileHandle.write(data) 136 | fileHandle.closeFile() 137 | } 138 | } 139 | } 140 | 141 | for await _ in taskGroup {} 142 | }) 143 | } 144 | 145 | Benchmark("InstructionCount", configuration: .init(metrics: [.instructions], 146 | warmupIterations: 0, 147 | scalingFactor: .kilo, 148 | thresholds: [.instructions: .relaxed])) { _ in 149 | await concurrentWork(tasks: 15, mallocs: 1_000) 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /Benchmarks/Benchmarks/DateTime/DateTime.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | import Benchmark 11 | import DateTime 12 | 13 | let benchmarks: @Sendable () -> Void = { 14 | Benchmark.defaultConfiguration = .init(metrics: [.throughput, .wallClock, .instructions] + .arc, 15 | warmupIterations: 10, 16 | scalingFactor: .kilo, 17 | maxDuration: .seconds(1), 18 | maxIterations: .kilo(10)) 19 | 20 | Benchmark("InternalUTCClock-now") { benchmark in 21 | for _ in benchmark.scaledIterations { 22 | blackHole(InternalUTCClock.now) 23 | } 24 | } 25 | 26 | Benchmark("BenchmarkClock-now") { benchmark in 27 | for _ in benchmark.scaledIterations { 28 | blackHole(DateTime.BenchmarkClock.now) 29 | } 30 | } 31 | 32 | Benchmark("Foundation-Date") { benchmark in 33 | for _ in benchmark.scaledIterations { 34 | blackHole(Foundation.Date()) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Benchmarks/Benchmarks/Histogram/Histogram.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | import Benchmark 11 | import Histogram 12 | 13 | let benchmarks: @Sendable () -> Void = { 14 | let metrics: [BenchmarkMetric] = [ 15 | .wallClock, 16 | .throughput, 17 | .peakMemoryResident, 18 | .contextSwitches, 19 | .syscalls, 20 | .cpuTotal, 21 | .mallocCountTotal, 22 | .allocatedResidentMemory, 23 | .threads, 24 | .threadsRunning, 25 | .instructions 26 | ] 27 | Benchmark.defaultConfiguration = .init(metrics: metrics, 28 | scalingFactor: .mega, 29 | maxDuration: .seconds(1), 30 | maxIterations: .kilo(1)) 31 | Benchmark("Record") { benchmark in 32 | let maxValue: UInt64 = 1_000_000 33 | 34 | var histogram = Histogram(highestTrackableValue: maxValue, numberOfSignificantValueDigits: .three) 35 | 36 | let numValues = 1_024 // so compiler can optimize modulo below 37 | let values = [UInt64]((0 ..< numValues).map { _ in UInt64.random(in: 100 ... 1_000) }) 38 | 39 | benchmark.startMeasurement() 40 | 41 | for i in benchmark.scaledIterations { 42 | blackHole(histogram.record(values[i % numValues])) 43 | } 44 | 45 | benchmark.stopMeasurement() 46 | } 47 | 48 | Benchmark("Record to autoresizing") { benchmark in 49 | benchmark.startMeasurement() 50 | var histogram = Histogram(numberOfSignificantValueDigits: .three) 51 | 52 | let numValues = 1_024 // so compiler can optimize modulo below 53 | let values = [UInt64]((0 ..< numValues).map { _ in UInt64.random(in: 100 ... 10_000) }) 54 | 55 | for i in benchmark.scaledIterations { 56 | blackHole(histogram.record(values[i % numValues])) 57 | } 58 | 59 | benchmark.stopMeasurement() 60 | } 61 | 62 | Benchmark("ValueAtPercentile", 63 | configuration: .init(scalingFactor: .kilo)) { benchmark in 64 | let maxValue: UInt64 = 1_000_000 65 | 66 | var histogram = Histogram(highestTrackableValue: maxValue, numberOfSignificantValueDigits: .three) 67 | 68 | // fill histogram with some data 69 | for _ in 0 ..< 10_000 { 70 | blackHole(histogram.record(UInt64.random(in: 10 ... 1_000))) 71 | } 72 | 73 | let percentiles = [0.0, 25.0, 50.0, 75.0, 80.0, 90.0, 99.0, 100.0] 74 | 75 | benchmark.startMeasurement() 76 | 77 | for i in benchmark.scaledIterations { 78 | blackHole(histogram.valueAtPercentile(percentiles[i % percentiles.count])) 79 | } 80 | 81 | benchmark.stopMeasurement() 82 | } 83 | 84 | Benchmark("Mean", 85 | configuration: .init(scalingFactor: .kilo)) { benchmark in 86 | let maxValue: UInt64 = 1_000_000 87 | 88 | var histogram = Histogram(highestTrackableValue: maxValue, numberOfSignificantValueDigits: .three) 89 | 90 | // fill histogram with some data 91 | for _ in 0 ..< 10_000 { 92 | blackHole(histogram.record(UInt64.random(in: 10 ... 1_000))) 93 | } 94 | 95 | benchmark.startMeasurement() 96 | 97 | for _ in benchmark.scaledIterations { 98 | blackHole(histogram.mean) 99 | } 100 | 101 | benchmark.stopMeasurement() 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Benchmarks/Benchmarks/P90AbsoluteThresholds/P90AbsoluteThresholds.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | import Benchmark 11 | import Foundation 12 | 13 | #if canImport(Darwin) 14 | import Darwin 15 | #elseif canImport(Glibc) 16 | import Glibc 17 | #else 18 | #error("Unsupported Platform") 19 | #endif 20 | 21 | let benchmarks: @Sendable () -> Void = { 22 | var thresholdTolerances: [BenchmarkMetric: BenchmarkThresholds] 23 | let relative: BenchmarkThresholds.RelativeThresholds = [.p25: 25.0, .p50: 50.0, .p75: 75.0, .p90: 100.0, .p99: 101.0, .p100: 201.0] 24 | let absolute: BenchmarkThresholds.AbsoluteThresholds = [.p75: 999, .p90: 1_000, .p99: 1_001, .p100: 2_001] 25 | thresholdTolerances = [.mallocCountTotal: .init(relative: relative, absolute: absolute), 26 | .syscalls: .init(relative: [.p90: 23.0], absolute: [.p90: 123])] 27 | 28 | Benchmark.defaultConfiguration = .init(metrics: [.mallocCountTotal, .syscalls] + .arc, 29 | warmupIterations: 1, 30 | scalingFactor: .kilo, 31 | maxDuration: .seconds(2), 32 | maxIterations: .kilo(100), 33 | thresholds: thresholdTolerances) 34 | 35 | Benchmark("P90Date") { benchmark in 36 | for _ in benchmark.scaledIterations { 37 | blackHole(Foundation.Date()) 38 | } 39 | } 40 | 41 | Benchmark("P90Malloc") { benchmark in 42 | var array: [Int] = [] 43 | 44 | for _ in benchmark.scaledIterations { 45 | var temporaryAllocation = malloc(1) 46 | blackHole(temporaryAllocation) 47 | free(temporaryAllocation) 48 | array.append(contentsOf: 1 ... 1_000) 49 | blackHole(array) 50 | } 51 | } 52 | 53 | func concurrentWork(tasks: Int) async { 54 | _ = await withTaskGroup(of: Void.self, returning: Void.self, body: { taskGroup in 55 | for _ in 0 ..< tasks { 56 | taskGroup.addTask {} 57 | } 58 | 59 | for await _ in taskGroup {} 60 | }) 61 | } 62 | 63 | Benchmark("Retain/release deviation") { _ in 64 | await concurrentWork(tasks: 789) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Benchmarks/Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "pins" : [ 3 | { 4 | "identity" : "hdrhistogram-swift", 5 | "kind" : "remoteSourceControl", 6 | "location" : "https://github.com/HdrHistogram/hdrhistogram-swift", 7 | "state" : { 8 | "revision" : "a69fa24d7b70421870cafa86340ece900489e17e", 9 | "version" : "0.1.2" 10 | } 11 | }, 12 | { 13 | "identity" : "package-datetime", 14 | "kind" : "remoteSourceControl", 15 | "location" : "https://github.com/ordo-one/package-datetime", 16 | "state" : { 17 | "revision" : "d1242188c9f48aad297e6ca9b717776f8660bc31", 18 | "version" : "1.0.2" 19 | } 20 | }, 21 | { 22 | "identity" : "package-jemalloc", 23 | "kind" : "remoteSourceControl", 24 | "location" : "https://github.com/ordo-one/package-jemalloc", 25 | "state" : { 26 | "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", 27 | "version" : "1.0.0" 28 | } 29 | }, 30 | { 31 | "identity" : "swift-argument-parser", 32 | "kind" : "remoteSourceControl", 33 | "location" : "https://github.com/apple/swift-argument-parser", 34 | "state" : { 35 | "revision" : "c8ed701b513cf5177118a175d85fbbbcd707ab41", 36 | "version" : "1.3.0" 37 | } 38 | }, 39 | { 40 | "identity" : "swift-atomics", 41 | "kind" : "remoteSourceControl", 42 | "location" : "https://github.com/apple/swift-atomics", 43 | "state" : { 44 | "revision" : "cd142fd2f64be2100422d658e7411e39489da985", 45 | "version" : "1.2.0" 46 | } 47 | }, 48 | { 49 | "identity" : "swift-numerics", 50 | "kind" : "remoteSourceControl", 51 | "location" : "https://github.com/apple/swift-numerics", 52 | "state" : { 53 | "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", 54 | "version" : "1.0.2" 55 | } 56 | }, 57 | { 58 | "identity" : "swift-system", 59 | "kind" : "remoteSourceControl", 60 | "location" : "https://github.com/apple/swift-system", 61 | "state" : { 62 | "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", 63 | "version" : "1.2.1" 64 | } 65 | }, 66 | { 67 | "identity" : "texttable", 68 | "kind" : "remoteSourceControl", 69 | "location" : "https://github.com/ordo-one/TextTable", 70 | "state" : { 71 | "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", 72 | "version" : "0.0.2" 73 | } 74 | } 75 | ], 76 | "version" : 2 77 | } 78 | -------------------------------------------------------------------------------- /Benchmarks/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 6.0 2 | 3 | import class Foundation.ProcessInfo 4 | import PackageDescription 5 | 6 | // If the environment variable BENCHMARK_DISABLE_JEMALLOC is set, we'll build the package without Jemalloc support 7 | let disableJemalloc = ProcessInfo.processInfo.environment["BENCHMARK_DISABLE_JEMALLOC"] 8 | 9 | let package = Package( 10 | name: "Benchmarks", 11 | platforms: [.macOS(.v15), .iOS(.v17)], 12 | dependencies: [ 13 | .package(path: "../"), 14 | .package(url: "https://github.com/ordo-one/package-datetime", .upToNextMajor(from: "1.0.1")), 15 | .package(url: "https://github.com/HdrHistogram/hdrhistogram-swift", .upToNextMajor(from: "0.1.0")) 16 | ], 17 | targets: [] 18 | ) 19 | 20 | // Add benchmark targets separately 21 | 22 | // Benchmark of the DateTime package (which can't depend on Benchmark as we'll get a circular dependency) 23 | package.targets += [ 24 | .executableTarget( 25 | name: "BenchmarkDateTime", 26 | dependencies: [ 27 | .product(name: "Benchmark", package: "package-benchmark"), 28 | .product(name: "DateTime", package: "package-datetime") 29 | ], 30 | path: "Benchmarks/DateTime", 31 | plugins: [ 32 | .plugin(name: "BenchmarkPlugin", package: "package-benchmark") 33 | ] 34 | ) 35 | ] 36 | 37 | // Benchmark of the benchmark package 38 | package.targets += [ 39 | .executableTarget( 40 | name: "Basic", 41 | dependencies: [ 42 | .product(name: "Benchmark", package: "package-benchmark") 43 | ], 44 | path: "Benchmarks/Basic", 45 | plugins: [ 46 | .plugin(name: "BenchmarkPlugin", package: "package-benchmark") 47 | ] 48 | ) 49 | ] 50 | 51 | // Benchmark of the Histogram package 52 | package.targets += [ 53 | .executableTarget( 54 | name: "HistogramBenchmark", 55 | dependencies: [ 56 | .product(name: "Benchmark", package: "package-benchmark"), 57 | .product(name: "Histogram", package: "hdrhistogram-swift") 58 | ], 59 | path: "Benchmarks/Histogram", 60 | plugins: [ 61 | .plugin(name: "BenchmarkPlugin", package: "package-benchmark") 62 | ] 63 | ) 64 | ] 65 | 66 | // Benchmark testing loading of p90 absolute thresholds 67 | package.targets += [ 68 | .executableTarget( 69 | name: "P90AbsoluteThresholdsBenchmark", 70 | dependencies: [ 71 | .product(name: "Benchmark", package: "package-benchmark") 72 | ], 73 | path: "Benchmarks/P90AbsoluteThresholds", 74 | plugins: [ 75 | .plugin(name: "BenchmarkPlugin", package: "package-benchmark") 76 | ] 77 | ) 78 | ] 79 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "pins" : [ 3 | { 4 | "identity" : "hdrhistogram-swift", 5 | "kind" : "remoteSourceControl", 6 | "location" : "https://github.com/HdrHistogram/hdrhistogram-swift.git", 7 | "state" : { 8 | "revision" : "93a1618c8aa20f6a521a9da656a3e0591889e9dc", 9 | "version" : "0.1.3" 10 | } 11 | }, 12 | { 13 | "identity" : "package-jemalloc", 14 | "kind" : "remoteSourceControl", 15 | "location" : "https://github.com/ordo-one/package-jemalloc.git", 16 | "state" : { 17 | "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", 18 | "version" : "1.0.0" 19 | } 20 | }, 21 | { 22 | "identity" : "swift-argument-parser", 23 | "kind" : "remoteSourceControl", 24 | "location" : "https://github.com/apple/swift-argument-parser.git", 25 | "state" : { 26 | "revision" : "41982a3656a71c768319979febd796c6fd111d5c", 27 | "version" : "1.5.0" 28 | } 29 | }, 30 | { 31 | "identity" : "swift-atomics", 32 | "kind" : "remoteSourceControl", 33 | "location" : "https://github.com/apple/swift-atomics.git", 34 | "state" : { 35 | "revision" : "cd142fd2f64be2100422d658e7411e39489da985", 36 | "version" : "1.2.0" 37 | } 38 | }, 39 | { 40 | "identity" : "swift-numerics", 41 | "kind" : "remoteSourceControl", 42 | "location" : "https://github.com/apple/swift-numerics", 43 | "state" : { 44 | "revision" : "e0ec0f5f3af6f3e4d5e7a19d2af26b481acb6ba8", 45 | "version" : "1.0.3" 46 | } 47 | }, 48 | { 49 | "identity" : "swift-system", 50 | "kind" : "remoteSourceControl", 51 | "location" : "https://github.com/apple/swift-system.git", 52 | "state" : { 53 | "revision" : "a34201439c74b53f0fd71ef11741af7e7caf01e1", 54 | "version" : "1.4.2" 55 | } 56 | }, 57 | { 58 | "identity" : "texttable", 59 | "kind" : "remoteSourceControl", 60 | "location" : "https://github.com/ordo-one/TextTable.git", 61 | "state" : { 62 | "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", 63 | "version" : "0.0.2" 64 | } 65 | } 66 | ], 67 | "version" : 2 68 | } 69 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | 3 | import class Foundation.ProcessInfo 4 | import PackageDescription 5 | 6 | // If the environment variable BENCHMARK_DISABLE_JEMALLOC is set, we'll build the package without Jemalloc support 7 | let disableJemalloc = ProcessInfo.processInfo.environment["BENCHMARK_DISABLE_JEMALLOC"] 8 | 9 | let package = Package( 10 | name: "Benchmark", 11 | platforms: [ 12 | .macOS(.v13), 13 | .iOS(.v16) 14 | ], 15 | products: [ 16 | .plugin(name: "BenchmarkCommandPlugin", targets: ["BenchmarkCommandPlugin"]), 17 | .plugin(name: "BenchmarkPlugin", targets: ["BenchmarkPlugin"]), 18 | .library( 19 | name: "Benchmark", 20 | targets: ["Benchmark"] 21 | ) 22 | ], 23 | dependencies: [ 24 | .package(url: "https://github.com/apple/swift-system.git", .upToNextMajor(from: "1.1.0")), 25 | .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMajor(from: "1.1.0")), 26 | .package(url: "https://github.com/ordo-one/TextTable.git", .upToNextMajor(from: "0.0.1")), 27 | .package(url: "https://github.com/HdrHistogram/hdrhistogram-swift.git", .upToNextMajor(from: "0.1.0")), 28 | .package(url: "https://github.com/apple/swift-atomics.git", .upToNextMajor(from: "1.0.0")) 29 | ], 30 | targets: [ 31 | // Plugins used by users of the package 32 | 33 | // The actual 'benchmark' command plugin 34 | .plugin( 35 | name: "BenchmarkCommandPlugin", 36 | capability: .command( 37 | intent: .custom( 38 | verb: "benchmark", 39 | description: "Run the Benchmark performance test suite." 40 | ) 41 | ), 42 | dependencies: [ 43 | "BenchmarkTool" 44 | ], 45 | path: "Plugins/BenchmarkCommandPlugin" 46 | ), 47 | 48 | // Plugin that generates the boilerplate needed to interface with the Benchmark infrastructure 49 | .plugin( 50 | name: "BenchmarkPlugin", 51 | capability: .buildTool(), 52 | dependencies: [ 53 | "BenchmarkBoilerplateGenerator" 54 | ], 55 | path: "Plugins/BenchmarkPlugin" 56 | ), 57 | 58 | // Tool that the plugin executes to perform the actual work, the real benchmark driver 59 | .executableTarget( 60 | name: "BenchmarkTool", 61 | dependencies: [ 62 | .product(name: "ArgumentParser", package: "swift-argument-parser"), 63 | .product(name: "SystemPackage", package: "swift-system"), 64 | .product(name: "TextTable", package: "TextTable"), 65 | "Benchmark", 66 | "BenchmarkShared" 67 | ], 68 | path: "Plugins/BenchmarkTool" 69 | ), 70 | 71 | // Tool that generates the boilerplate 72 | .executableTarget( 73 | name: "BenchmarkBoilerplateGenerator", 74 | dependencies: [ 75 | .product(name: "ArgumentParser", package: "swift-argument-parser"), 76 | .product(name: "SystemPackage", package: "swift-system") 77 | ], 78 | path: "Plugins/BenchmarkBoilerplateGenerator" 79 | ), 80 | 81 | // Tool that simply generates the man page for the BenchmarkPlugin as we can't use SAP in it... :-/ 82 | .executableTarget( 83 | name: "BenchmarkHelpGenerator", 84 | dependencies: [ 85 | .product(name: "ArgumentParser", package: "swift-argument-parser"), 86 | "BenchmarkShared" 87 | ], 88 | path: "Plugins/BenchmarkHelpGenerator" 89 | ), 90 | 91 | // Getting OS specific information 92 | .target( 93 | name: "CDarwinOperatingSystemStats", 94 | dependencies: [ 95 | ], 96 | path: "Platform/CDarwinOperatingSystemStats" 97 | ), 98 | 99 | // Getting OS specific information 100 | .target( 101 | name: "CLinuxOperatingSystemStats", 102 | dependencies: [ 103 | ], 104 | path: "Platform/CLinuxOperatingSystemStats" 105 | ), 106 | 107 | // Hooks for ARC 108 | .target(name: "SwiftRuntimeHooks"), 109 | 110 | // Shared definitions 111 | .target(name: "BenchmarkShared"), 112 | 113 | .testTarget( 114 | name: "BenchmarkTests", 115 | dependencies: ["Benchmark"] 116 | ), 117 | ] 118 | ) 119 | // Check if this is a SPI build, then we need to disable jemalloc for macOS 120 | 121 | let macOSSPIBuild: Bool // Disables jemalloc for macOS SPI builds as the infrastructure doesn't have jemalloc there 122 | 123 | #if canImport(Darwin) 124 | if let spiBuildEnvironment = ProcessInfo.processInfo.environment["SPI_BUILD"], spiBuildEnvironment == "1" { 125 | macOSSPIBuild = true 126 | print("Building for SPI@macOS, disabling Jemalloc") 127 | } else { 128 | macOSSPIBuild = false 129 | } 130 | #else 131 | macOSSPIBuild = false 132 | #endif 133 | 134 | // Add Benchmark target dynamically 135 | 136 | // Shared dependencies 137 | var dependencies: [PackageDescription.Target.Dependency] = [ 138 | .product(name: "Histogram", package: "hdrhistogram-swift"), 139 | .product(name: "ArgumentParser", package: "swift-argument-parser"), 140 | .product(name: "SystemPackage", package: "swift-system"), 141 | .byNameItem(name: "CDarwinOperatingSystemStats", condition: .when(platforms: [.macOS, .iOS])), 142 | .byNameItem(name: "CLinuxOperatingSystemStats", condition: .when(platforms: [.linux])), 143 | .product(name: "Atomics", package: "swift-atomics"), 144 | "SwiftRuntimeHooks", 145 | "BenchmarkShared", 146 | ] 147 | 148 | if macOSSPIBuild == false { // jemalloc always disable for macOSSPIBuild 149 | if let disableJemalloc, disableJemalloc != "false", disableJemalloc != "0" { 150 | print("Jemalloc disabled through environment variable.") 151 | } else { 152 | package.dependencies += [.package(url: "https://github.com/ordo-one/package-jemalloc.git", .upToNextMajor(from: "1.0.0"))] 153 | dependencies += [.product(name: "jemalloc", package: "package-jemalloc", condition: .when(platforms: [.macOS, .linux]))] 154 | } 155 | } 156 | 157 | package.targets += [.target(name: "Benchmark", dependencies: dependencies)] 158 | -------------------------------------------------------------------------------- /Platform/CDarwinOperatingSystemStats/CDarwinOperatingSystemStats.c: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // This file exists just to keep SwiftPM quiet 12 | 13 | -------------------------------------------------------------------------------- /Platform/CDarwinOperatingSystemStats/include/CDarwinOperatingSystemStats.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // We need this to get at the libproc header 12 | 13 | #ifndef CDarwinOperatingSystemStats_h 14 | #define CDarwinOperatingSystemStats_h 15 | 16 | #if __has_include() 17 | #include 18 | #endif 19 | 20 | #endif /* CDarwinOperatingSystemStats_h */ 21 | -------------------------------------------------------------------------------- /Platform/CLinuxOperatingSystemStats/include/CLinuxOperatingSystemStats.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | #ifndef CLinuxOperatingSystemStats_h 12 | #define CLinuxOperatingSystemStats_h 13 | 14 | struct ioStats { 15 | long long readSyscalls; 16 | long long writeSyscalls; 17 | long long readBytesLogical; 18 | long long writeBytesLogical; 19 | long long readBytesPhysical; 20 | long long writeBytesPhysical; 21 | } ioStats; 22 | 23 | void CLinuxIOStats(const char *s, struct ioStats *ioStats); 24 | 25 | struct processStats { 26 | long cpuUser; 27 | long cpuSystem; 28 | long cpuTotal; 29 | long threads; 30 | long peakMemoryVirtual; 31 | long peakMemoryResident; 32 | } processStats; 33 | 34 | void CLinuxProcessStats(const char *s, struct processStats *processStats); 35 | 36 | struct performanceCounters { 37 | unsigned long long instructions; 38 | } performanceCounters; 39 | 40 | void CLinuxPerformanceCountersCurrent(struct performanceCounters *performanceCounters); // return current counters 41 | void CLinuxPerformanceCountersEnable(); 42 | void CLinuxPerformanceCountersDisable(); 43 | void CLinuxPerformanceCountersReset(); 44 | 45 | #endif /* CLinuxOperatingSystemStats_h */ 46 | -------------------------------------------------------------------------------- /Plugins/BenchmarkBoilerplateGenerator/BenchmarkBoilerplateGenerator.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import ArgumentParser 12 | import SystemPackage 13 | 14 | @main 15 | struct Benchmark: AsyncParsableCommand { 16 | @Option(name: .long, help: "Name of the target") 17 | var target: String 18 | 19 | @Option(name: .long, help: "Output file path") 20 | var output: String 21 | 22 | mutating func run() async throws { 23 | let outputPath = FilePath(output) // package 24 | var boilerplate = """ 25 | import Benchmark 26 | 27 | @main 28 | struct BenchmarkRunner: BenchmarkRunnerHooks { 29 | static func registerBenchmarks() { 30 | _ = benchmarks() 31 | } 32 | } 33 | """ 34 | do { 35 | let fd = try FileDescriptor.open(outputPath, .writeOnly, options: [.truncate, .create], permissions: .ownerReadWrite) 36 | do { 37 | try fd.closeAfter { 38 | do { 39 | try boilerplate.withUTF8 { 40 | _ = try fd.write(UnsafeRawBufferPointer($0)) 41 | } 42 | } catch { 43 | print("Failed to write to file \(outputPath)") 44 | } 45 | } 46 | } catch { 47 | print("Failed to close fd for \(outputPath) after write.") 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Plugins/BenchmarkCommandPlugin/ArgumentExtractor+Extensions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import PackagePlugin 12 | 13 | enum ArgumentParsingError: Error, CustomStringConvertible { 14 | case noMatchingTargetsForRegex 15 | 16 | var description: String { 17 | "no target matching regex for target/skip-target" 18 | } 19 | 20 | var errorDescription: String? { 21 | description 22 | } 23 | } 24 | 25 | @available(macOS 13.0, *) 26 | extension ArgumentExtractor { 27 | mutating func extractSpecifiedTargets(in package: Package, 28 | withOption option: String) throws -> [SwiftSourceModuleTarget] { 29 | let specifiedTargets = extractOption(named: option) 30 | var targets: [SwiftSourceModuleTarget] = [] 31 | var anyMatching = false 32 | 33 | try package.targets.forEach { target in 34 | let path = target.directory.removingLastComponent() 35 | if path.lastComponent == "Benchmarks" { 36 | for specifiedTarget in specifiedTargets { 37 | let regex = try Regex(specifiedTarget) 38 | 39 | if target.name.wholeMatch(of: regex) != nil { 40 | if let swiftSourceModuleTarget = target as? SwiftSourceModuleTarget { 41 | if swiftSourceModuleTarget.kind != .test { 42 | targets.append(swiftSourceModuleTarget) 43 | anyMatching = true 44 | break 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } 51 | 52 | if !specifiedTargets.isEmpty, !anyMatching { 53 | throw ArgumentParsingError.noMatchingTargetsForRegex 54 | } 55 | 56 | return targets 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Plugins/BenchmarkCommandPlugin/BenchmarkPlugin+Help.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | let help = 12 | """ 13 | OVERVIEW: Run benchmarks or update, compare or check performance baselines 14 | 15 | Performs operations on benchmarks (running or listing them), as well as storing, comparing baselines as well as checking them for threshold deviations. 16 | 17 | The init command will create a skeleton benchmark suite for you and add it to Package.swift. 18 | 19 | The `thresholds` commands reads/updates/checks benchmark runs vs. static thresholds. 20 | 21 | For the 'text' default format, the output is implicitly 'stdout' unless otherwise specified. 22 | For all other formats, the output is to a file in either the current working directory, or 23 | the directory specified by the '--path' option, unless the special 'stdout' path is specified 24 | in which case output will go to stdout (useful for e.g. baseline 'tsv' format export piped to youplot). 25 | 26 | To allow writing to the package directory, you may need to pass the appropriate option to swift package: 27 | swift package --allow-writing-to-package-directory benchmark 28 | 29 | USAGE: swift package benchmark 30 | 31 | swift package benchmark [run] 32 | swift package benchmark init 33 | swift package benchmark list 34 | swift package benchmark baseline list 35 | swift package benchmark baseline read [ ... ] [] 36 | swift package benchmark baseline update [] 37 | swift package benchmark baseline delete [ ... ] [] 38 | swift package benchmark baseline check [] [] 39 | swift package benchmark baseline compare [] [] 40 | swift package benchmark thresholds read [] 41 | swift package benchmark thresholds update [] [] 42 | swift package benchmark thresholds check [] [] 43 | swift package benchmark help 44 | 45 | ARGUMENTS: 46 | The benchmark command to perform. If not specified, 'run' is implied. (values: run, list, baseline, thresholds, help, init) 47 | 48 | OPTIONS: 49 | --filter Benchmarks matching the regexp filter that should be run 50 | --skip Benchmarks matching the regexp filter that should be skipped 51 | --target Benchmark targets matching the regexp filter that should be run 52 | --skip-target 53 | Benchmark targets matching the regexp filter that should be skipped 54 | --format The output format to use, default is 'text' (values: text, markdown, influx, jmh, histogramEncoded, histogram, histogramSamples, histogramPercentiles, metricP90AbsoluteThresholds) 55 | --metric Specifies that the benchmark run should use one or more specific metrics instead of the ones defined by the benchmarks. (values: cpuUser, cpuSystem, cpuTotal, wallClock, throughput, 56 | peakMemoryResident, peakMemoryResidentDelta, peakMemoryVirtual, mallocCountSmall, mallocCountLarge, mallocCountTotal, allocatedResidentMemory, memoryLeaked, syscalls, contextSwitches, threads, 57 | threadsRunning, readSyscalls, writeSyscalls, readBytesLogical, writeBytesLogical, readBytesPhysical, writeBytesPhysical, instructions, retainCount, releaseCount, retainReleaseDelta, custom) 58 | --path The path to operate on for data export or threshold operations, default is the current directory (".") for exports and the ("./Thresholds") directory for thresholds. 59 | --quiet Specifies that output should be suppressed (useful for if you just want to check return code) 60 | --scale Specifies that some of the text output should be scaled using the scalingFactor (denoted by '*' in output) 61 | --time-units 62 | Specifies that time related metrics output should be specified units (values: nanoseconds, microseconds, milliseconds, seconds, kiloseconds, megaseconds) 63 | --check-absolute 64 | Set to true if thresholds should be checked against an absolute reference point rather than delta between baselines. 65 | This is used for CI workflows when you want to validate the thresholds vs. a persisted benchmark baseline 66 | rather than comparing PR vs main or vs a current run. This is useful to cut down the build matrix needed 67 | for those wanting to validate performance of e.g. toolchains or OS:s as well (or have other reasons for wanting 68 | a specific check against a given absolute reference.). 69 | If this is enabled, zero or one baselines should be specified for the check operation. 70 | By default, thresholds are checked comparing two baselines, or a baseline and a benchmark run. 71 | --check-absolute-path 72 | The path from which p90 thresholds will be loaded for absolute threshold checks. 73 | This implicitly sets --check-absolute to true as well. 74 | --no-progress Specifies that benchmark progress information should not be displayed 75 | --grouping The grouping to use, one of: ["metric", "benchmark"]. default is 'benchmark' (values: metric, benchmark) 76 | --xswiftc Pass an argument to the Swift compiler when building the benchmark 77 | -h, --help Show help information. 78 | """ 79 | -------------------------------------------------------------------------------- /Plugins/BenchmarkCommandPlugin/Command+Helpers.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // This file need to be manually copied between the Benchmark plugin and the BenchmarkTool when updated, 12 | // as no external dependencies are allowed for SwiftPM command tools. 13 | 14 | // ************************************************************************************************************************* 15 | // The source file is in BenchmarkCommandPlugin and should be edited there, then manually copied to Shared/Command+Helpers. 16 | // ************************************************************************************************************************* 17 | 18 | public enum Command: String, CaseIterable { 19 | case run 20 | case list 21 | case baseline 22 | case thresholds 23 | case help 24 | case `init` 25 | } 26 | 27 | /// The benchmark data output format. 28 | public enum OutputFormat: String, CaseIterable { 29 | /// Text output formatted into a visual table suitable for console output 30 | case text 31 | /// The text output format, formatted in markdown, suitable for GitHub workflows 32 | case markdown 33 | /// Influx data format 34 | case influx 35 | /// JMH format consumable by http://jmh.morethan.io 36 | case jmh 37 | /// The encoded representation of the underlying histograms capturing the benchmark data, for programmatic use (Codable). 38 | case histogramEncoded 39 | /// The histogram percentiles, average, deviation, sample count etc in standard HDR Histogram text format consumable by http://hdrhistogram.github.io/HdrHistogram/plotFiles.html 40 | case histogram 41 | /// The raw histogram samples in TSV format for processing by external tools (e.g. Youplot) 42 | case histogramSamples 43 | /// The percentiles values betwen (0-100) in TSV format for processing by external tools (e.g. Youplot) 44 | case histogramPercentiles 45 | /// The p90 percentile values per metric as a `[BenchmarkMetric: BenchmarkThresholds]` in JSON format, suitable for static thresholds 46 | case metricP90AbsoluteThresholds 47 | } 48 | 49 | public enum Grouping: String, CaseIterable { 50 | case metric 51 | case benchmark 52 | } 53 | 54 | @_documentation(visibility: internal) 55 | public enum TimeUnits: String, CaseIterable { 56 | case nanoseconds 57 | case microseconds 58 | case milliseconds 59 | case seconds 60 | case kiloseconds 61 | case megaseconds 62 | } 63 | 64 | public enum ThresholdsOperation: String, CaseIterable { 65 | case read 66 | case update 67 | case check 68 | } 69 | 70 | public enum BaselineOperation: String, CaseIterable { 71 | case read 72 | case update 73 | case list 74 | case delete 75 | case compare 76 | case check 77 | } 78 | 79 | public enum ExitCode: Int32 { 80 | case success = 0 81 | case genericFailure = 1 82 | case thresholdRegression = 2 83 | case benchmarkJobFailed = 3 84 | case thresholdImprovement = 4 85 | case baselineNotFound = 5 86 | case noPermissions = 6 87 | } 88 | -------------------------------------------------------------------------------- /Plugins/BenchmarkPlugin/BenchmarkSupportPlugin.swift: -------------------------------------------------------------------------------- 1 | import PackagePlugin 2 | 3 | @main 4 | struct PluginFactory: BuildToolPlugin { 5 | func createBuildCommands(context: PackagePlugin.PluginContext, target: PackagePlugin.Target) async throws -> [PackagePlugin.Command] { 6 | guard let target = target as? SwiftSourceModuleTarget else { return [] } 7 | guard target.kind == .executable else { return [] } 8 | let path = target.directory.removingLastComponent() 9 | guard path.lastComponent == "Benchmarks" else { return [] } 10 | 11 | let tool = try context.tool(named: "BenchmarkBoilerplateGenerator") 12 | let outputDirectory = context.pluginWorkDirectory 13 | let swiftFile = outputDirectory.appending("__BenchmarkBoilerplate.swift") 14 | let inputFiles = target.sourceFiles.filter { $0.path.extension == "swift" }.map(\.path) 15 | let outputFiles: [Path] = [swiftFile] 16 | 17 | let commandArgs: [String] = [ 18 | "--target", target.name, 19 | "--output", swiftFile.string 20 | ] 21 | 22 | let command: Command = .buildCommand( 23 | displayName: "Generating plugin support files", 24 | executable: tool.path, 25 | arguments: commandArgs, 26 | inputFiles: inputFiles, 27 | outputFiles: outputFiles 28 | ) 29 | 30 | return [command] 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+CreateBenchmark.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import SystemPackage 12 | 13 | #if canImport(Darwin) 14 | import Darwin 15 | #elseif canImport(Glibc) 16 | import Glibc 17 | #else 18 | #error("Unsupported Platform") 19 | #endif 20 | 21 | extension BenchmarkTool { 22 | var benchmarksDirectory: String { "Benchmarks" } 23 | 24 | mutating func appendBenchmarkTargetToPackageSwift() { 25 | guard let targetName else { 26 | failBenchmark("targetName not specified for appendBenchmarkTargetToPackageSwift()") 27 | return 28 | } 29 | 30 | var boilerplate = """ 31 | 32 | // Benchmark of \(targetName) 33 | package.targets += [ 34 | .executableTarget( 35 | name: "\(targetName)", 36 | dependencies: [ 37 | .product(name: "Benchmark", package: "package-benchmark"), 38 | ], 39 | path: "Benchmarks/\(targetName)", 40 | plugins: [ 41 | .plugin(name: "BenchmarkPlugin", package: "package-benchmark") 42 | ] 43 | ), 44 | ] 45 | """ 46 | 47 | var outputPath = FilePath(baselineStoragePath) // package 48 | var subPath = FilePath() // subpath rooted in package used for directory creation 49 | subPath.append("Package.swift") // package/Benchmarks/targetName 50 | outputPath.append(subPath.components) 51 | 52 | print("Adding new executable target \(targetName) to \(outputPath.description)") 53 | 54 | // Write out benchmark boilerplate 55 | do { 56 | let fd = try FileDescriptor.open( 57 | outputPath, .writeOnly, options: [.append], permissions: .ownerReadWrite 58 | ) 59 | 60 | do { 61 | try fd.closeAfter { 62 | do { 63 | try boilerplate.withUTF8 { 64 | _ = try fd.write(UnsafeRawBufferPointer($0)) 65 | } 66 | } catch { 67 | print("Failed to write to file \(outputPath)") 68 | } 69 | } 70 | } catch { 71 | print("Failed to close fd for \(outputPath) after write.") 72 | } 73 | 74 | } catch { 75 | if errno == EPERM { 76 | print("Lacking permissions to write to \(outputPath)") 77 | print("Give benchmark plugin permissions by running with e.g.:") 78 | print("") 79 | print("swift package --allow-writing-to-package-directory benchmark init \(targetName)") 80 | print("") 81 | } else { 82 | print("Failed to open file \(outputPath), errno = [\(errno)]") 83 | } 84 | } 85 | } 86 | 87 | mutating func createBenchmarkTarget() { 88 | guard let targetName else { 89 | failBenchmark("targetName not specified for createBenchmarkTarget()") 90 | return 91 | } 92 | 93 | var boilerplate = """ 94 | // Benchmark boilerplate generated by Benchmark 95 | 96 | import Benchmark 97 | import Foundation 98 | 99 | let benchmarks = { 100 | Benchmark("SomeBenchmark") { benchmark in 101 | for _ in benchmark.scaledIterations { 102 | blackHole(Date()) // replace this line with your own benchmark 103 | } 104 | } 105 | // Add additional benchmarks here 106 | } 107 | 108 | """ 109 | 110 | var outputPath = FilePath(baselineStoragePath) // package 111 | var subPath = FilePath() // subpath rooted in package used for directory creation 112 | 113 | subPath.append(benchmarksDirectory) // package/Benchmarks 114 | subPath.append("\(targetName)") // package/Benchmarks/targetName 115 | 116 | outputPath.createSubPath(subPath) // Create destination subpath if needed 117 | outputPath.append(subPath.components) 118 | outputPath.append("\(targetName).swift") 119 | 120 | print("Creating benchmark in \(outputPath.description)") 121 | 122 | // Write out benchmark boilerplate, open with .exclusiveCreate to abort if file exists 123 | do { 124 | let fd = try FileDescriptor.open( 125 | outputPath, .writeOnly, options: [.create, .exclusiveCreate], permissions: .ownerReadWrite 126 | ) 127 | 128 | do { 129 | try fd.closeAfter { 130 | do { 131 | try boilerplate.withUTF8 { 132 | _ = try fd.write(UnsafeRawBufferPointer($0)) 133 | } 134 | } catch { 135 | print("Failed to write to file \(outputPath)") 136 | return 137 | } 138 | } 139 | } catch { 140 | print("Failed to close fd for \(outputPath) after write.") 141 | } 142 | 143 | } catch { 144 | switch errno { 145 | case EPERM: 146 | print("Lacking permissions to write to \(outputPath)") 147 | print("Give benchmark plugin permissions by running with e.g.:") 148 | print("") 149 | print("swift package --allow-writing-to-package-directory benchmark init \(targetName)") 150 | print("") 151 | case EEXIST: 152 | print("File already exists at \(outputPath), aborting benchmark target generation.") 153 | print("Consider reverting repository state and add benchmark target manually instead.") 154 | default: 155 | print("Failed to open file \(outputPath), errno = [\(errno)]") 156 | } 157 | return 158 | } 159 | 160 | appendBenchmarkTargetToPackageSwift() 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+Export+JMHElement.swift: -------------------------------------------------------------------------------- 1 | // swiftlint:disable all 2 | // 3 | // Copyright (c) 2022 Ordo One AB. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // 8 | // You may obtain a copy of the License at 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | 12 | // This file was generated from JSON Schema using quicktype, do not modify it directly. 13 | // To parse the JSON, add this file to your project and do: 14 | // 15 | // let jMHJmh = try? JSONDecoder().decode(JMHJmh.self, from: jsonData) 16 | 17 | import Foundation 18 | 19 | // MARK: - JMHElement 20 | 21 | struct JMHElement: Codable { 22 | var benchmark: String 23 | var mode: String 24 | var threads: Int 25 | var forks: Int 26 | var warmupIterations: Int 27 | var warmupTime: String 28 | var warmupBatchSize: Int 29 | var measurementIterations: Int 30 | var measurementTime: String 31 | var measurementBatchSize: Int 32 | var primaryMetric: JMHPrimaryMetric 33 | var secondaryMetrics: [String: JMHPrimaryMetric]? 34 | 35 | enum CodingKeys: String, CodingKey { 36 | case benchmark 37 | case mode 38 | case threads 39 | case forks 40 | case warmupIterations 41 | case warmupTime 42 | case warmupBatchSize 43 | case measurementIterations 44 | case measurementTime 45 | case measurementBatchSize 46 | case primaryMetric 47 | case secondaryMetrics 48 | } 49 | } 50 | 51 | // MARK: - JMHPrimaryMetric 52 | 53 | struct JMHPrimaryMetric: Codable { 54 | var score: Double 55 | var scoreError: Double 56 | var scoreConfidence: [Double] 57 | var scorePercentiles: [String: Double] 58 | var scoreUnit: String 59 | var rawData: [[Double]] 60 | 61 | enum CodingKeys: String, CodingKey { 62 | case score 63 | case scoreError 64 | case scoreConfidence 65 | case scorePercentiles 66 | case scoreUnit 67 | case rawData 68 | } 69 | } 70 | 71 | // MARK: - JMHSecondaryMetrics 72 | 73 | struct JMHSecondaryMetrics: Codable { 74 | var gcAllocRate: JMHPrimaryMetric 75 | var gcAllocRateNorm: JMHPrimaryMetric 76 | var gcChurnPsEdenSpace: JMHPrimaryMetric 77 | var gcChurnPsEdenSpaceNorm: JMHPrimaryMetric 78 | var gcChurnPsSurvivorSpace: JMHPrimaryMetric 79 | var gcChurnPsSurvivorSpaceNorm: JMHPrimaryMetric 80 | var gcCount: JMHGc 81 | var gcTime: JMHGc 82 | 83 | enum CodingKeys: String, CodingKey { 84 | case gcAllocRate 85 | case gcAllocRateNorm 86 | case gcChurnPsEdenSpace 87 | case gcChurnPsEdenSpaceNorm 88 | case gcChurnPsSurvivorSpace 89 | case gcChurnPsSurvivorSpaceNorm 90 | case gcCount 91 | case gcTime 92 | } 93 | } 94 | 95 | // MARK: - JMHGc 96 | 97 | struct JMHGc: Codable { 98 | var score: Int 99 | var scoreError: String 100 | var scoreConfidence: [Int] 101 | var scorePercentiles: [String: Double] 102 | var scoreUnit: String 103 | var rawData: [[Int]] 104 | 105 | enum CodingKeys: String, CodingKey { 106 | case score 107 | case scoreError 108 | case scoreConfidence 109 | case scorePercentiles 110 | case scoreUnit 111 | case rawData 112 | } 113 | } 114 | 115 | typealias JMHJmh = [JMHElement] 116 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+Export+JMHFormatter.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // Merge resulting json using jq with 12 | // jq -s add delta-*.json > delta.json 13 | 14 | import Benchmark 15 | import Foundation 16 | import Numerics 17 | 18 | extension JMHPrimaryMetric { 19 | init(_ result: BenchmarkResult) { 20 | let histogram = result.statistics.histogram 21 | 22 | // TODO: Must validate the calculation of scoreError 23 | // below was cobbled together according to https://stackoverflow.com/a/24725075 24 | // and https://www.calculator.net/confidence-interval-calculator.html 25 | let z999 = 3.291 26 | let error = z999 * histogram.stdDeviation / .sqrt(Double(histogram.totalCount)) 27 | 28 | let score = histogram.mean 29 | 30 | let percentiles = [0.0, 50.0, 90.0, 95.0, 99.0, 99.9, 99.99, 99.999, 99.9999, 100.0] 31 | var percentileValues: [String: Double] = [:] 32 | var recordedValues: [Double] = [] 33 | // let factor = 1 // result.metric == .throughput ? 1 : 1_000_000_000 / result.timeUnits.rawValue 34 | let factor = result.metric.countable == false ? 1_000 : 1 35 | 36 | for p in percentiles { 37 | percentileValues[String(p)] = Statistics.roundToDecimalplaces(Double(histogram.valueAtPercentile(p)) / Double(factor), 3) 38 | } 39 | 40 | for value in histogram.recordedValues() { 41 | for _ in 0 ..< value.count { 42 | recordedValues.append(Statistics.roundToDecimalplaces(Double(value.value) / Double(factor), 3)) 43 | } 44 | } 45 | 46 | self.score = Statistics.roundToDecimalplaces(score / Double(factor), 3) 47 | scoreError = Statistics.roundToDecimalplaces(error / Double(factor), 3) 48 | scoreConfidence = [Statistics.roundToDecimalplaces(score - error) / Double(factor), 49 | Statistics.roundToDecimalplaces(score + error) / Double(factor)] 50 | scorePercentiles = percentileValues 51 | if result.metric.countable { 52 | scoreUnit = result.metric == .throughput ? "# / s" : "#" 53 | } else { 54 | scoreUnit = "μs" // result.timeUnits.description 55 | } 56 | rawData = [recordedValues] 57 | } 58 | } 59 | 60 | extension BenchmarkTool { 61 | func convertToJMH(_ baseline: BenchmarkBaseline) throws -> String { 62 | var resultString = "" 63 | var jmhElements: [JMHElement] = [] 64 | var secondaryMetrics: [String: JMHPrimaryMetric] = [:] // could move to OrderedDictionary for consistent output 65 | 66 | baseline.targets.forEach { benchmarkTarget in 67 | 68 | let results = baseline.resultsByTarget(benchmarkTarget) 69 | 70 | results.forEach { key, result in 71 | 72 | guard let primaryResult = result.first(where: { $0.metric == .throughput }) else { 73 | print("Throughput metric must be present for JMH export [\(key)]") 74 | return 75 | } 76 | 77 | let primaryMetrics = JMHPrimaryMetric(primaryResult) 78 | 79 | for secondaryResult in result { 80 | if secondaryResult.metric != .throughput { 81 | let secondaryMetric = JMHPrimaryMetric(secondaryResult) 82 | secondaryMetrics[secondaryResult.metric.description] = secondaryMetric 83 | } 84 | } 85 | 86 | // Some of these are a bit unclear how to map, so to the best of our understanding: 87 | let benchmarkKey = key.replacingOccurrences(of: " ", with: "_") 88 | let jmh = JMHElement(benchmark: "package.benchmark.\(benchmarkTarget).\(benchmarkKey)", 89 | mode: "thrpt", 90 | threads: 1, 91 | forks: 1, 92 | warmupIterations: primaryResult.warmupIterations, 93 | warmupTime: "1 s", 94 | warmupBatchSize: 1, 95 | measurementIterations: primaryResult.statistics.measurementCount, 96 | measurementTime: "1 s", 97 | measurementBatchSize: 1, 98 | primaryMetric: primaryMetrics, 99 | secondaryMetrics: secondaryMetrics) 100 | 101 | jmhElements.append(jmh) 102 | } 103 | } 104 | 105 | let bytesArray = try JSONEncoder().encode(jmhElements) 106 | resultString = String(bytes: bytesArray, encoding: .utf8)! 107 | 108 | return resultString 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+JSON.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // JSON serialization of benchmark request/reply command sent to controlled process 12 | 13 | import Benchmark 14 | import Foundation 15 | import SystemPackage 16 | 17 | extension BenchmarkTool { 18 | func write(_ reply: BenchmarkCommandRequest) throws { 19 | let bytesArray = try JSONEncoder().encode(reply) 20 | let count: Int = bytesArray.count 21 | let output = FileDescriptor(rawValue: outputFD) 22 | 23 | try withUnsafeBytes(of: count) { (intPtr: UnsafeRawBufferPointer) in 24 | _ = try output.write(intPtr) 25 | } 26 | try bytesArray.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in 27 | let written = try output.write(bytes) 28 | if written != count { 29 | fatalError("written != count \(written) ---- \(count)") 30 | } 31 | } 32 | } 33 | 34 | func read() throws -> BenchmarkCommandReply { 35 | let input = FileDescriptor(rawValue: inputFD) 36 | var bufferLength = 0 37 | 38 | try withUnsafeMutableBytes(of: &bufferLength) { (intPtr: UnsafeMutableRawBufferPointer) in 39 | let readBytes = try input.read(into: intPtr) 40 | if readBytes == 0 { 41 | throw RunCommandError.WaitPIDError 42 | } 43 | } 44 | 45 | var readBytes = [UInt8]() 46 | 47 | while readBytes.count < bufferLength { 48 | let nextBytes = try [UInt8](unsafeUninitializedCapacity: bufferLength - readBytes.count) { buf, count in 49 | count = try input.read(into: UnsafeMutableRawBufferPointer(buf)) 50 | } 51 | readBytes.append(contentsOf: nextBytes) 52 | } 53 | 54 | let request = try JSONDecoder().decode(BenchmarkCommandReply.self, from: Data(readBytes)) 55 | 56 | return request 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+Machine.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // Getting running machine configuration information 12 | 13 | import Benchmark 14 | 15 | #if canImport(Darwin) 16 | import Darwin 17 | #elseif canImport(Glibc) 18 | import Glibc 19 | #else 20 | #error("Unsupported Platform") 21 | #endif 22 | 23 | extension BenchmarkTool { 24 | func benchmarkMachine() -> BenchmarkMachine { 25 | let processors = sysconf(Int32(_SC_NPROCESSORS_ONLN)) 26 | let memory = sysconf(Int32(_SC_PHYS_PAGES)) / 1_024 * sysconf(Int32(_SC_PAGESIZE)) / (1_024 * 1_024) // avoid overflow 27 | 28 | var uuname = utsname() 29 | _ = uname(&uuname) 30 | 31 | let sizeNodename = MemoryLayout.size(ofValue: uuname.nodename) 32 | let nodeName = withUnsafePointer(to: &uuname.nodename) { 33 | $0.withMemoryRebound(to: UInt8.self, capacity: sizeNodename) { 34 | String(cString: $0) 35 | } 36 | } 37 | 38 | let sizeMachine = MemoryLayout.size(ofValue: uuname.machine) 39 | let machine = withUnsafePointer(to: &uuname.machine) { 40 | $0.withMemoryRebound(to: UInt8.self, capacity: sizeMachine) { 41 | String(cString: $0) 42 | } 43 | } 44 | /* // We don't use these currently 45 | let sysnameSize = MemoryLayout.size(ofValue: uuname.sysname) 46 | let sysname = withUnsafePointer(to: &uuname.sysname) { 47 | $0.withMemoryRebound(to: UInt8.self, capacity: sysnameSize) { 48 | String(cString: $0) 49 | } 50 | } 51 | 52 | let releaseSize = MemoryLayout.size(ofValue: uuname.release) 53 | let release = withUnsafePointer(to: &uuname.release) { 54 | $0.withMemoryRebound(to: UInt8.self, capacity: releaseSize) { 55 | String(cString: $0) 56 | } 57 | } 58 | */ 59 | // This is the full kernel version string 60 | let versionSize = MemoryLayout.size(ofValue: uuname.version) 61 | let version = withUnsafePointer(to: &uuname.version) { 62 | $0.withMemoryRebound(to: UInt8.self, capacity: versionSize) { 63 | String(cString: $0) 64 | } 65 | } 66 | 67 | return BenchmarkMachine(hostname: nodeName, 68 | processors: processors, 69 | processorType: machine, 70 | memory: memory, 71 | kernelVersion: version) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+ReadP90AbsoluteThresholds.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import Benchmark 12 | import Foundation 13 | import SystemPackage 14 | 15 | #if canImport(Darwin) 16 | import Darwin 17 | #elseif canImport(Glibc) 18 | import Glibc 19 | #else 20 | #error("Unsupported Platform") 21 | #endif 22 | 23 | extension BenchmarkTool { 24 | /// `makeBenchmarkThresholds` is a convenience function for reading p90 static thresholds that previously have been exported with `metricP90AbsoluteThresholds` 25 | /// 26 | /// - Parameters: 27 | /// - path: The path where the `Thresholds` directory should be located, containing static thresholds files using the naming pattern: 28 | /// `moduleName.benchmarkName.p90.json` 29 | /// - moduleName: The name of the benchmark module, can be extracted in the benchmark using: 30 | /// `String("\(#fileID)".prefix(while: { $0 != "/" }))` 31 | /// - benchmarkName: The name of the benchmark 32 | /// - Returns: A dictionary with static benchmark thresholds per metric or nil if the file could not be found or read 33 | static func makeBenchmarkThresholds( 34 | path: String, 35 | benchmarkIdentifier: BenchmarkIdentifier 36 | ) -> [BenchmarkMetric : BenchmarkThresholds.AbsoluteThreshold]? { 37 | var path = FilePath(path) 38 | if path.isAbsolute { 39 | path.append("\(benchmarkIdentifier.target).\(benchmarkIdentifier.name).p90.json") 40 | } else { 41 | var cwdPath = FilePath(FileManager.default.currentDirectoryPath) 42 | cwdPath.append(path.components) 43 | cwdPath.append("\(benchmarkIdentifier.target).\(benchmarkIdentifier.name).p90.json") 44 | path = cwdPath 45 | } 46 | 47 | var p90Thresholds: [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold] = [:] 48 | var p90ThresholdsRaw: [String: BenchmarkThresholds.AbsoluteThreshold]? 49 | 50 | do { 51 | let fileDescriptor = try FileDescriptor.open(path, .readOnly, options: [], permissions: .ownerRead) 52 | 53 | do { 54 | try fileDescriptor.closeAfter { 55 | do { 56 | var readBytes = [UInt8]() 57 | let bufferSize = 16 * 1_024 * 1_024 58 | 59 | while true { 60 | let nextBytes = try [UInt8](unsafeUninitializedCapacity: bufferSize) { buf, count in 61 | count = try fileDescriptor.read(into: UnsafeMutableRawBufferPointer(buf)) 62 | } 63 | if nextBytes.isEmpty { 64 | break 65 | } 66 | readBytes.append(contentsOf: nextBytes) 67 | } 68 | 69 | p90ThresholdsRaw = try JSONDecoder().decode( 70 | [String: BenchmarkThresholds.AbsoluteThreshold].self, 71 | from: Data(readBytes) 72 | ) 73 | 74 | if let p90ThresholdsRaw { 75 | p90ThresholdsRaw.forEach { metric, threshold in 76 | if let metric = BenchmarkMetric(argument: metric) { 77 | p90Thresholds[metric] = threshold 78 | } 79 | } 80 | } 81 | } catch { 82 | print("Failed to read file at \(path) [\(String(reflecting: error))] \(Errno(rawValue: errno).description)") 83 | } 84 | } 85 | } catch { 86 | print("Failed to close fd for \(path) after reading.") 87 | } 88 | } catch { 89 | if errno != ENOENT { // file not found is ok, e.g. no thresholds found, then silently return nil 90 | print("Failed to open file \(path), errno = [\(errno)] \(Errno(rawValue: errno).description)") 91 | } 92 | } 93 | return p90Thresholds.isEmpty ? nil : p90Thresholds 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/BenchmarkTool+Thresholds.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2024 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import Benchmark 12 | import SystemPackage 13 | import TextTable 14 | 15 | private let percentileWidth = 20 16 | private let maxDescriptionWidth = 40 17 | 18 | fileprivate struct ThresholdsTableEntry { 19 | var description: String 20 | var p90: Int 21 | var absolute: Int 22 | var relative: Double 23 | } 24 | 25 | extension BenchmarkTool { 26 | func printThresholds(_ staticThresholdsPerBenchmark: [BenchmarkIdentifier : [BenchmarkMetric: BenchmarkThresholds.AbsoluteThreshold]]) { 27 | 28 | guard !staticThresholdsPerBenchmark.isEmpty else { 29 | print("No thresholds defined.") 30 | return 31 | } 32 | 33 | print("") 34 | 35 | var tableEntries: [ThresholdsTableEntry] = [] 36 | let table = TextTable { 37 | [Column(title: "Metric", value: "\($0.description)", width: maxDescriptionWidth, align: .left), 38 | Column(title: "Threshold .p90", value: $0.p90, width: percentileWidth, align: .right), 39 | Column(title: "Allowed %", value: $0.relative, width: percentileWidth, align: .right), 40 | Column(title: "Allowed Δ", value: $0.absolute, width: percentileWidth, align: .right)] 41 | } 42 | 43 | staticThresholdsPerBenchmark.forEach { benchmarkIdentifier, staticThresholds in 44 | print("\(benchmarkIdentifier.name)") 45 | 46 | let thresholdDeviations = benchmarks.first(where: { benchmarkIdentifier == .init(target: $0.target, 47 | name: $0.name)})?.configuration.thresholds ?? .init() 48 | 49 | staticThresholds.forEach { threshold in 50 | let absoluteThreshold = thresholdDeviations[threshold.key]?.absolute[.p90] ?? 0 51 | let relativeThreshold = thresholdDeviations[threshold.key]?.relative[.p90] ?? 0 52 | 53 | tableEntries.append(.init(description: threshold.key.description, 54 | p90: threshold.value, 55 | absolute: absoluteThreshold, 56 | relative: relativeThreshold)) 57 | } 58 | table.print(tableEntries, style: format.tableStyle) 59 | tableEntries = [] 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/FilePath+Additions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import SystemPackage 12 | 13 | #if canImport(Darwin) 14 | import Darwin 15 | #elseif canImport(Glibc) 16 | import Glibc 17 | #else 18 | #error("Unsupported Platform") 19 | #endif 20 | 21 | public extension FilePath { 22 | func createSubPath(_ subPath: FilePath) { 23 | var creationPath = self 24 | 25 | subPath.components.forEach { c in 26 | creationPath.append(c) 27 | 28 | do { 29 | let fd = try FileDescriptor.open( 30 | creationPath, .readOnly, options: [.directory], permissions: .ownerReadWrite 31 | ) 32 | 33 | do { 34 | try fd.close() 35 | } catch { print("failed close directory") } 36 | } catch { 37 | switch errno { 38 | case ENOENT: // doesn't exist, let's create it 39 | if mkdir(creationPath.string, S_IRWXU | S_IRWXG | S_IRWXO) == -1 { 40 | if errno == EPERM { 41 | print("Lacking permissions to write to \(creationPath)") 42 | print("Give benchmark plugin permissions by running with e.g.:") 43 | print("") 44 | print("swift package --allow-writing-to-package-directory benchmark baseline update") 45 | print("") 46 | } 47 | print("Failed to create directory at [\(creationPath)], errno = [\(errno)]") 48 | return 49 | } 50 | 51 | default: 52 | print("Failed to handle file \(creationPath), errno = [\(errno)]") 53 | } 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/FilePath+DirectoryView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import SystemPackage 12 | 13 | #if canImport(Darwin) 14 | import Darwin 15 | typealias DirectoryStreamPointer = UnsafeMutablePointer? 16 | #elseif canImport(Glibc) 17 | import Glibc 18 | typealias DirectoryStreamPointer = OpaquePointer? 19 | #else 20 | #error("Unsupported Platform") 21 | #endif 22 | 23 | /// Extends FilePath with basic directory iteration capabilities 24 | public extension FilePath { 25 | /// `DirectoryView` provides an iteratable sequence of the contents of a directory referenced by a `FilePath` 26 | struct DirectoryView { 27 | var directoryStreamPointer: DirectoryStreamPointer = nil 28 | var path: FilePath 29 | 30 | /// Initializer 31 | /// - Parameter path: The file system path to provide directory entries for, should reference a directory 32 | init(path pathName: FilePath) { 33 | path = pathName 34 | path.withPlatformString { 35 | directoryStreamPointer = opendir($0) 36 | } 37 | } 38 | } 39 | 40 | var directoryEntries: DirectoryView { DirectoryView(path: self) } 41 | } 42 | 43 | extension FilePath.DirectoryView: IteratorProtocol, Sequence { 44 | public mutating func next() -> FilePath? { 45 | guard let streamPointer = directoryStreamPointer else { 46 | return nil 47 | } 48 | 49 | guard let directoryEntry = readdir(streamPointer) else { 50 | closedir(streamPointer) 51 | directoryStreamPointer = nil 52 | return nil 53 | } 54 | 55 | let fileName = withUnsafePointer(to: &directoryEntry.pointee.d_name) { pointer -> FilePath.Component in 56 | pointer.withMemoryRebound(to: CChar.self, 57 | capacity: MemoryLayout.size(ofValue: directoryEntry.pointee.d_name)) { 58 | guard let fileName = FilePath.Component(platformString: $0) else { 59 | fatalError("Could not initialize FilePath.Component from platformString \(String(cString: $0))") 60 | } 61 | return fileName 62 | } 63 | } 64 | return path.appending(fileName) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Plugins/BenchmarkTool/String+Additions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | extension String { 12 | func printAsHeader(addWhiteSpace: Bool = true) { 13 | let separator = String(repeating: "=", count: count) 14 | if addWhiteSpace { 15 | print("") 16 | } 17 | print(separator) 18 | print(self) 19 | print(separator) 20 | if addWhiteSpace { 21 | print("") 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sources/.swiftlint.yml: -------------------------------------------------------------------------------- 1 | opt_in_rules: 2 | - file_name 3 | file_name: 4 | excluded: 5 | - "BenchmarkInternals.swift" 6 | - "MallocStats+jemalloc-support.swift" 7 | -------------------------------------------------------------------------------- /Sources/Benchmark/ARCStats/ARCStats.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | /// The ARC stats the ARCStatsProducer can provide 12 | @_documentation(visibility: internal) 13 | struct ARCStats { 14 | var objectAllocCount: Int = 0 /// total number allocations, implicit retain 15 | var retainCount: Int = 0 /// total number retains 16 | var releaseCount: Int = 0 /// total number of releases 17 | } 18 | -------------------------------------------------------------------------------- /Sources/Benchmark/ARCStats/ARCStatsProducer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | import Atomics 12 | import SwiftRuntimeHooks 13 | 14 | // swiftlint:disable prefer_self_in_static_references 15 | 16 | final class ARCStatsProducer { 17 | typealias SwiftRuntimeHook = @convention(c) (UnsafeRawPointer?, UnsafeMutableRawPointer?) -> Void 18 | 19 | static var allocCount: UnsafeAtomic = .create(0) 20 | static var retainCount: UnsafeAtomic = .create(0) 21 | static var releaseCount: UnsafeAtomic = .create(0) 22 | 23 | static func hook() { 24 | let allocObjectHook: SwiftRuntimeHook = { _, _ in 25 | ARCStatsProducer.allocCount.wrappingIncrement(ordering: .relaxed) 26 | } 27 | 28 | let retainHook: SwiftRuntimeHook = { _, _ in 29 | ARCStatsProducer.retainCount.wrappingIncrement(ordering: .relaxed) 30 | } 31 | 32 | let releaseHook: SwiftRuntimeHook = { _, _ in 33 | ARCStatsProducer.releaseCount.wrappingIncrement(ordering: .relaxed) 34 | } 35 | 36 | swift_runtime_set_alloc_object_hook(allocObjectHook, nil) 37 | swift_runtime_set_retain_hook(retainHook, nil) 38 | swift_runtime_set_release_hook(releaseHook, nil) 39 | } 40 | 41 | static func unhook() { 42 | swift_runtime_set_release_hook(nil, nil) 43 | swift_runtime_set_retain_hook(nil, nil) 44 | swift_runtime_set_alloc_object_hook(nil, nil) 45 | } 46 | 47 | static func reset() { 48 | allocCount.store(0, ordering: .relaxed) 49 | retainCount.store(0, ordering: .relaxed) 50 | releaseCount.store(0, ordering: .relaxed) 51 | } 52 | 53 | static func makeARCStats() -> ARCStats { 54 | ARCStats(objectAllocCount: allocCount.load(ordering: .relaxed), 55 | retainCount: retainCount.load(ordering: .relaxed), 56 | releaseCount: releaseCount.load(ordering: .relaxed)) 57 | } 58 | } 59 | 60 | // swiftlint:enable prefer_self_in_static_references 61 | -------------------------------------------------------------------------------- /Sources/Benchmark/BenchmarkExecutor+Extensions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // swiftlint:disable cyclomatic_complexity 11 | 12 | extension BenchmarkExecutor { 13 | func performanceCountersNeeded(_ metric: BenchmarkMetric) -> Bool { 14 | switch metric { 15 | case .instructions: 16 | return true 17 | default: 18 | return false 19 | } 20 | } 21 | } 22 | 23 | extension BenchmarkExecutor { 24 | func mallocStatsProducerNeeded(_ metric: BenchmarkMetric) -> Bool { 25 | switch metric { 26 | case .mallocCountLarge: 27 | return true 28 | case .memoryLeaked: 29 | return true 30 | case .mallocCountSmall: 31 | return true 32 | case .mallocCountTotal: 33 | return true 34 | case .allocatedResidentMemory: 35 | return true 36 | default: 37 | return false 38 | } 39 | } 40 | } 41 | 42 | extension BenchmarkExecutor { 43 | func operatingSystemsStatsProducerNeeded(_ metric: BenchmarkMetric) -> Bool { 44 | switch metric { 45 | case .cpuUser: 46 | return true 47 | case .cpuSystem: 48 | return true 49 | case .cpuTotal: 50 | return true 51 | case .peakMemoryResident: 52 | return true 53 | case .peakMemoryResidentDelta: 54 | return true 55 | case .peakMemoryVirtual: 56 | return true 57 | case .syscalls: 58 | return true 59 | case .contextSwitches: 60 | return true 61 | case .threads: 62 | return true 63 | case .threadsRunning: 64 | return true 65 | case .readSyscalls: 66 | return true 67 | case .writeSyscalls: 68 | return true 69 | case .readBytesLogical: 70 | return true 71 | case .writeBytesLogical: 72 | return true 73 | case .readBytesPhysical: 74 | return true 75 | case .writeBytesPhysical: 76 | return true 77 | case .instructions: 78 | return true 79 | default: 80 | return false 81 | } 82 | } 83 | } 84 | 85 | extension BenchmarkExecutor { 86 | func arcStatsProducerNeeded(_ metric: BenchmarkMetric) -> Bool { 87 | switch metric { 88 | case .objectAllocCount, .retainCount, .releaseCount, .retainReleaseDelta: 89 | return true 90 | default: 91 | return false 92 | } 93 | } 94 | } 95 | 96 | // swiftlint:enable cyclomatic_complexity 97 | -------------------------------------------------------------------------------- /Sources/Benchmark/BenchmarkInternals.swift: -------------------------------------------------------------------------------- 1 | // swiftlint:disable all 2 | // 3 | // Copyright (c) 2022 Ordo One AB. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // 8 | // You may obtain a copy of the License at 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | 12 | // Internal Benchmark framework definitions used for communication with host process etc 13 | 14 | // Command sent from benchmark runner to the benchmark under measurement 15 | 16 | @_documentation(visibility: internal) 17 | public enum BenchmarkCommandRequest: Codable { 18 | case list 19 | case run(benchmark: Benchmark) 20 | case end // exit the benchmark 21 | } 22 | 23 | // Replies from benchmark under measure to benchmark runner 24 | @_documentation(visibility: internal) 25 | public enum BenchmarkCommandReply: Codable { 26 | case list(benchmark: Benchmark) 27 | case ready 28 | case result(benchmark: Benchmark, results: [BenchmarkResult]) // receives results from built-in metric collectors 29 | case run 30 | case end // end of query for list/result 31 | case error(_ description: String) // error while performing operation (e.g. 'run') 32 | } 33 | 34 | // swiftlint:enable all 35 | -------------------------------------------------------------------------------- /Sources/Benchmark/BenchmarkMetric+Defaults.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // Convenience sets of metrics 12 | public extension BenchmarkMetric { 13 | /// A small collection of metrics used for microbenchmarks only interested in CPU 14 | /// 15 | /// The defaults include ``wallClock`` and ``throughput``. 16 | /// 17 | /// There is also an convenience extension on Array defined such that you can write just `.microbenchmark` rather than `BenchmarkMetric.microbenchmark` 18 | /// 19 | static var microbenchmark: [BenchmarkMetric] { 20 | [.wallClock, 21 | .throughput] 22 | } 23 | 24 | /// The default collection of metrics used for a benchmark. 25 | /// 26 | /// The defaults include ``wallClock``, ``cpuTotal``, ``mallocCountTotal``, ``throughput``, and ``peakMemoryResident``. 27 | /// 28 | /// There is also an convenience extension on Array defined such that you can write just `.default` rather than `BenchmarkMetric.default` 29 | /// 30 | static var `default`: [BenchmarkMetric] { 31 | [.wallClock, 32 | .cpuTotal, 33 | .mallocCountTotal, 34 | .throughput, 35 | .instructions, 36 | .peakMemoryResident] 37 | } 38 | 39 | /// A collection of extended system benchmarks. 40 | static var extended: [BenchmarkMetric] { 41 | [.wallClock, 42 | .cpuUser, 43 | .cpuTotal, 44 | .mallocCountTotal, 45 | .throughput, 46 | .peakMemoryResident, 47 | .memoryLeaked, 48 | .syscalls, 49 | .instructions] 50 | } 51 | 52 | /// A collection of memory benchmarks. 53 | static var memory: [BenchmarkMetric] { 54 | [.peakMemoryResident, 55 | .peakMemoryResidentDelta, 56 | .peakMemoryVirtual, 57 | .mallocCountSmall, 58 | .mallocCountLarge, 59 | .mallocCountTotal, 60 | .memoryLeaked, 61 | .allocatedResidentMemory] 62 | } 63 | 64 | /// A collection of ARC metrics 65 | static var arc: [BenchmarkMetric] { 66 | [.objectAllocCount, 67 | .retainCount, 68 | .releaseCount, 69 | .retainReleaseDelta] 70 | } 71 | 72 | /// A collection of system benchmarks. 73 | static var system: [BenchmarkMetric] { 74 | [.wallClock, 75 | .syscalls, 76 | .contextSwitches, 77 | .threads, 78 | .threadsRunning, 79 | .cpuSystem] 80 | } 81 | 82 | /// A collection of disk benchmarks. 83 | static var disk: [BenchmarkMetric] { 84 | [.readSyscalls, 85 | .writeSyscalls, 86 | .readBytesLogical, 87 | .writeBytesLogical, 88 | .readBytesPhysical, 89 | .writeBytesPhysical] 90 | } 91 | 92 | /// A collection of all benchmarks supported by this library. 93 | static var all: [BenchmarkMetric] { 94 | [.cpuUser, 95 | .cpuSystem, 96 | .cpuTotal, 97 | .wallClock, 98 | .throughput, 99 | .peakMemoryResident, 100 | .peakMemoryResidentDelta, 101 | .peakMemoryVirtual, 102 | .mallocCountSmall, 103 | .mallocCountLarge, 104 | .mallocCountTotal, 105 | .memoryLeaked, 106 | .syscalls, 107 | .contextSwitches, 108 | .threads, 109 | .threadsRunning, 110 | .readSyscalls, 111 | .writeSyscalls, 112 | .readBytesLogical, 113 | .writeBytesLogical, 114 | .readBytesPhysical, 115 | .writeBytesPhysical, 116 | .instructions, 117 | .allocatedResidentMemory, 118 | .objectAllocCount, 119 | .retainCount, 120 | .releaseCount, 121 | .retainReleaseDelta] 122 | } 123 | } 124 | 125 | // Nicer convenience extension for Array so one can write `.extended` instead of `BenchmarkMetric.extended` 126 | public extension [BenchmarkMetric] { 127 | /// A suitable set of metrics for microbenchmarks that are CPU-oriented only. 128 | /// 129 | /// The defaults include ``BenchmarkMetric/wallClock`` and ``BenchmarkMetric/throughput`` 130 | static var microbenchmark: [BenchmarkMetric] { 131 | BenchmarkMetric.microbenchmark 132 | } 133 | 134 | /// The default collection of metrics used for a benchmark. 135 | /// 136 | /// The defaults include ``BenchmarkMetric/wallClock``, ``BenchmarkMetric/cpuTotal``, ``BenchmarkMetric/mallocCountTotal``, ``BenchmarkMetric/throughput``, and ``BenchmarkMetric/peakMemoryResident``. 137 | static var `default`: [BenchmarkMetric] { 138 | BenchmarkMetric.default 139 | } 140 | 141 | /// A collection of extended system benchmarks. 142 | static var extended: [BenchmarkMetric] { 143 | BenchmarkMetric.extended 144 | } 145 | 146 | /// A collection of memory benchmarks. 147 | static var memory: [BenchmarkMetric] { 148 | BenchmarkMetric.memory 149 | } 150 | 151 | /// A collection of ARC metrics 152 | static var arc: [BenchmarkMetric] { 153 | BenchmarkMetric.arc 154 | } 155 | 156 | /// A collection of system benchmarks. 157 | static var system: [BenchmarkMetric] { 158 | BenchmarkMetric.system 159 | } 160 | 161 | /// A collection of disk benchmarks. 162 | static var disk: [BenchmarkMetric] { 163 | BenchmarkMetric.disk 164 | } 165 | 166 | /// A collection of all benchmarks supported by this library. 167 | static var all: [BenchmarkMetric] { 168 | BenchmarkMetric.all 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /Sources/Benchmark/BenchmarkRunner+ReadWrite.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // swiftlint disable: file_length type_body_length 12 | import ArgumentParser 13 | import Foundation 14 | import SystemPackage 15 | 16 | // For test dependency injection 17 | protocol BenchmarkRunnerReadWrite { 18 | func write(_ reply: BenchmarkCommandReply) throws 19 | func read() throws -> BenchmarkCommandRequest 20 | } 21 | 22 | extension BenchmarkRunner { 23 | func write(_ reply: BenchmarkCommandReply) throws { 24 | guard outputFD != nil else { 25 | return 26 | } 27 | let bytesArray = try JSONEncoder().encode(reply) 28 | let count: Int = bytesArray.count 29 | let output = FileDescriptor(rawValue: outputFD!) 30 | 31 | // Length header 32 | try withUnsafeBytes(of: count) { (intPtr: UnsafeRawBufferPointer) in 33 | _ = try output.write(intPtr) 34 | } 35 | 36 | // JSON serialization 37 | try bytesArray.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in 38 | let written = try output.write(bytes) 39 | if count != written { 40 | fatalError("count != written \(count) ---- \(written)") 41 | } 42 | } 43 | } 44 | 45 | func read() throws -> BenchmarkCommandRequest { 46 | guard inputFD != nil else { 47 | return .end 48 | } 49 | let input = FileDescriptor(rawValue: inputFD!) 50 | var bufferLength = 0 51 | 52 | // Length header 53 | try withUnsafeMutableBytes(of: &bufferLength) { (intPtr: UnsafeMutableRawBufferPointer) in 54 | _ = try input.read(into: intPtr) 55 | } 56 | 57 | // JSON serialization 58 | var readBytes = [UInt8]() 59 | 60 | while readBytes.count < bufferLength { 61 | let nextBytes = try [UInt8](unsafeUninitializedCapacity: bufferLength - readBytes.count) { buf, count in 62 | count = try input.read(into: UnsafeMutableRawBufferPointer(buf)) 63 | } 64 | readBytes.append(contentsOf: nextBytes) 65 | } 66 | 67 | let request = try JSONDecoder().decode(BenchmarkCommandRequest.self, from: Data(readBytes)) 68 | 69 | return request 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Sources/Benchmark/BenchmarkThresholds+Defaults.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // swiftlint: disable discouraged_none_name 12 | 13 | /// Convenience benchmark relative thresholds 14 | public extension BenchmarkThresholds { 15 | enum Relative { 16 | // The allowed regression per percentile in percent (e.g. '0.2% regression ok for .p25') 17 | public static var strict: RelativeThresholds { 18 | [.p25: 0.0, 19 | .p50: 0.0, 20 | .p75: 0.0, 21 | .p90: 0.0, 22 | .p99: 0.0] 23 | } 24 | 25 | public static var `default`: RelativeThresholds { 26 | [.p25: 5.0, 27 | .p50: 5.0, 28 | .p75: 5.0] 29 | } 30 | 31 | public static var relaxed: RelativeThresholds { 32 | [.p50: 25.0] 33 | } 34 | 35 | public static var none: RelativeThresholds { 36 | [:] 37 | } 38 | } 39 | } 40 | 41 | /// Convenience benchmark absolute thresholds 42 | public extension BenchmarkThresholds { 43 | enum Absolute { 44 | // The tolerance for a given percentile in absolute numbers (e.g. '25 regression ok for .p25') 45 | // Useful for e.g. malloc counters 46 | public static var strict: AbsoluteThresholds { 47 | [.p0: 0, 48 | .p25: 0, 49 | .p50: 0, 50 | .p75: 0, 51 | .p90: 0, 52 | .p99: 0] 53 | } 54 | 55 | public static var `default`: AbsoluteThresholds { 56 | [:] 57 | } 58 | 59 | public static var relaxed: AbsoluteThresholds { 60 | [.p0: 10_000, 61 | .p25: 10_000, 62 | .p50: 10_000, 63 | .p75: 10_000, 64 | .p90: 10_000, 65 | .p99: 10_000, 66 | .p100: 10_000] 67 | } 68 | 69 | public static var none: AbsoluteThresholds { 70 | [:] 71 | } 72 | } 73 | } 74 | 75 | public extension BenchmarkThresholds { 76 | static var strict: BenchmarkThresholds { 77 | BenchmarkThresholds(relative: BenchmarkThresholds.Relative.strict, 78 | absolute: BenchmarkThresholds.Absolute.strict) 79 | } 80 | 81 | static var `default`: BenchmarkThresholds { 82 | BenchmarkThresholds(relative: BenchmarkThresholds.Relative.default, 83 | absolute: BenchmarkThresholds.Absolute.default) 84 | } 85 | 86 | static var relaxed: BenchmarkThresholds { 87 | BenchmarkThresholds(relative: BenchmarkThresholds.Relative.relaxed, 88 | absolute: BenchmarkThresholds.Absolute.relaxed) 89 | } 90 | 91 | static var none: BenchmarkThresholds { 92 | BenchmarkThresholds() 93 | } 94 | } 95 | 96 | // swiftlint:enable discouraged_none_name 97 | -------------------------------------------------------------------------------- /Sources/Benchmark/BenchmarkThresholds.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | /// Definitions of benchmark thresholds per metric 12 | public struct BenchmarkThresholds: Codable { 13 | public typealias RelativeThreshold = Double 14 | public typealias AbsoluteThreshold = Int 15 | 16 | public typealias RelativeThresholds = [BenchmarkResult.Percentile: RelativeThreshold] 17 | public typealias AbsoluteThresholds = [BenchmarkResult.Percentile: AbsoluteThreshold] 18 | 19 | /// Initializing BenchmarkThresholds 20 | /// The Benchmark thresholds define the tolerances to use when comparing two baselines/runs or when comparing with static thresholds. 21 | /// - Parameters: 22 | /// - relative: A dictionary with relative thresholds tolerances per percentile (using for delta comparisons) 23 | /// - absolute: A dictionary with absolute thresholds tolerances per percentile (used both for delta and absolute comparisons) 24 | public init(relative: RelativeThresholds = Self.Relative.none, 25 | absolute: AbsoluteThresholds = Self.Absolute.none) { 26 | self.relative = relative 27 | self.absolute = absolute 28 | } 29 | 30 | public let relative: RelativeThresholds 31 | public let absolute: AbsoluteThresholds 32 | } 33 | -------------------------------------------------------------------------------- /Sources/Benchmark/Blackhole.swift: -------------------------------------------------------------------------------- 1 | // ===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift Collections open source project 4 | // 5 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // 10 | // ===----------------------------------------------------------------------===// 11 | 12 | // Borrowed from Swift Collections Benchmark, thanks! 13 | 14 | /// A function to foil compiler optimizations that would otherwise optimize out code you want to benchmark. 15 | /// 16 | /// The function wraps another object or function, does nothing, and returns. 17 | /// If you want to benchmark the time is takes to create an instance and you don't maintain a reference to it, the compiler may optimize it out entirely, thinking it is unused. 18 | /// To prevent the compiler from removing the code you want to measure, wrap the creation of the instance with `blackHole`. 19 | /// For example, the following code benchmarks the time it takes to create an instance of `Date`, and wraps the creation of the instance to prevent the compiler from optimizing it away: 20 | /// 21 | /// ```swift 22 | /// Benchmark("Foundation Date()", 23 | /// configuration: .init( 24 | /// metrics: [.throughput, .wallClock], 25 | /// scalingFactor: .mega) 26 | /// ) { benchmark in 27 | /// for _ in benchmark.scaledIterations { 28 | /// blackHole(Date()) 29 | /// } 30 | /// } 31 | /// ``` 32 | @_optimize(none) // Used after tip here: https://forums.swift.org/t/compiler-swallows-blackhole/64305/10 - see also https://github.com/apple/swift/commit/1fceeab71e79dc96f1b6f560bf745b016d7fcdcf 33 | public func blackHole(_: some Any) {} 34 | 35 | @_optimize(none) // Used after tip here: https://forums.swift.org/t/compiler-swallows-blackhole/64305/10 - see also https://github.com/apple/swift/commit/1fceeab71e79dc96f1b6f560bf745b016d7fcdcf 36 | public func identity(_ value: T) -> T { 37 | value 38 | } 39 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/AboutPercentiles.md: -------------------------------------------------------------------------------- 1 | # About Percentiles 2 | 3 | About percentiles and how to interpret them. 4 | 5 | ## Overview 6 | 7 | The default text output from Benchmark is oriented around [the five-number summary](https://en.wikipedia.org/wiki/Five-number_summary) percentiles, plus the last decile (`p90`) and the last percentile (`p99`) - it's thus a variation of a [seven-figure summary](https://en.wikipedia.org/wiki/Seven-number_summary) with the focus on the 'bad' end of results (as those are what we typically care about addressing). 8 | 9 | We've found that focusing on percentiles rather than average or standard deviations, is more useful for a wider range of benchmark measurements and gives a deeper understanding of the results. 10 | Percentiles allow for a consistent way of expressing benchmark results of both throughput and latency measurements (which typically do **not** have a standardized distribution, being almost always are multi-modal in nature). 11 | This multi-modal nature of the latency measurements leads to the common statistical measures of mean and standard deviation being potentially misleading. 12 | 13 | That being said, some of the export formats do include more traditional mean and standard deviation statistics. 14 | The Benchmark infrastructure captures _all_ samples for a test run, so you can review the raw data points for your own post-run statistical analysis if desired. It's recommended that you explore the default output formats and the existing analytical tools first. 15 | 16 | ### What are Percentiles? 17 | 18 | A percentile N, typically denoted pN, is a score at or below which a given percentage of N scores in its frequency distribution falls ([Wikipedia: Percentile](https://en.wikipedia.org/wiki/Percentile)). 19 | 20 | For instance, a result of value V measured at percentile p25 means that 25% of all samples are V or lower. The p50 percentile is therefore the same as the [Median](https://en.wikipedia.org/wiki/Median) - the result separating the higher half from the lower half in a data sample. 21 | 22 | Two other percentiles are particularly noteworthy: 23 | 24 | - p0 is the minimum 25 | - p100 is the maximum 26 | 27 | of the data sample. 28 | 29 | ### Why Percentiles? 30 | 31 | It is tempting to think of performance benchmarking as a repeat experiment, whose results can be averaged and for which a standard deviation can be computed to estimate an error margin. However, this is almost never the correct approach, because the results almost never follow a [Gaussian distribution](https://en.wikipedia.org/wiki/Normal_distribution) in practice. 32 | 33 | This is very well explained in [Gil Tene's "Understanding Latency" talk](https://www.youtube.com/watch?v=9MKY4KypBzg) where he [dispells the standard deviation](https://www.youtube.com/watch?v=9MKY4KypBzg&t=833s) by showing what actual systems' latency behaviour looks like and how they practically never resemble the normal distribution. Note that while this presentation talks about "latency" this applies not just to latency in the networking sense but to any performance measurement in general. 34 | 35 | They _can_ be normally distributed but in practice rarely or never are. This is why using percentiles to describe the shape of a distribution is more generally applicable. 36 | 37 | Plotting the full range of percentiles for a measurement, which can be done by [exporting the benchmarks](doc:ExportingBenchmarks), shows the distribution: 38 | 39 | ![Percentile plot example](PercentileHistogramExample) 40 | 41 | The default representation of the Seven-number summary in the measurement table pulls out seven points of this distribution in an attempt to create a simple characterization of it. 42 | 43 | ### Interpreting results 44 | 45 | In order to interpret results it is important to consider your requirements. As you move to the right on the x-axis of the percentile distribution, results will occurr increasingly less frequently. However, do not be tempted to ignore the highest percentiles, because even rare events will happen, in particular in systems with frequent transactions, and slow performance can have disastrous impact on the system when they occur. 46 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/Benchmark.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/Benchmark`` 2 | 3 | ## Topics 4 | 5 | ### AllSymbols 6 | 7 | ### Creating Benchmarks 8 | 9 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-5zf4s`` 10 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-6a2yn`` 11 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-8kbjd`` 12 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-51nii`` 13 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-66qcz`` 14 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-959vi`` 15 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-pgtq`` 16 | - ``Benchmark/Benchmark/init(_:configuration:closure:setup:teardown:)-qn2n`` 17 | 18 | ### Configuring Benchmarks 19 | 20 | - ``Benchmark/configuration-swift.property`` 21 | - ``Benchmark/defaultConfiguration`` 22 | - ``Benchmark/Configuration-swift.struct`` 23 | - ``Benchmark/checkAbsoluteThresholds`` 24 | 25 | ### Writing Benchmarks 26 | 27 | - ``Benchmark/scaledIterations`` 28 | - ``Benchmark/startMeasurement()`` 29 | - ``Benchmark/stopMeasurement()`` 30 | 31 | ### Inspecting Benchmarks 32 | 33 | - ``Benchmark/name`` 34 | - ``Benchmark/currentIteration`` 35 | - ``Benchmark/error(_:)`` 36 | - ``Benchmark/failureReason`` 37 | 38 | ### Collecting Custom Metrics 39 | 40 | - ``Benchmark/measurement(_:_:)`` 41 | - ``Benchmark/customMetricMeasurement`` 42 | 43 | ### Decoding Benchmarks 44 | 45 | - ``Benchmark/init(from:)`` 46 | 47 | ### Comparing Benchmarks 48 | 49 | - ``Benchmark/!=(_:_:)`` 50 | 51 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/BenchmarkMetric.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/BenchmarkMetric`` 2 | 3 | ## Topics 4 | 5 | ### Metric Collections 6 | 7 | - ``BenchmarkMetric/default`` 8 | - ``BenchmarkMetric/microbenchmark`` 9 | - ``BenchmarkMetric/system`` 10 | - ``BenchmarkMetric/extended`` 11 | - ``BenchmarkMetric/memory`` 12 | - ``BenchmarkMetric/disk`` 13 | - ``BenchmarkMetric/all`` 14 | 15 | ### System Metrics 16 | 17 | - ``BenchmarkMetric/wallClock`` 18 | - ``BenchmarkMetric/syscalls`` 19 | - ``BenchmarkMetric/contextSwitches`` 20 | - ``BenchmarkMetric/threads`` 21 | - ``BenchmarkMetric/threadsRunning`` 22 | - ``BenchmarkMetric/cpuSystem`` 23 | - ``BenchmarkMetric/cpuUser`` 24 | 25 | ### Extended System Metrics 26 | 27 | - ``BenchmarkMetric/wallClock`` 28 | - ``BenchmarkMetric/cpuTotal`` 29 | - ``BenchmarkMetric/mallocCountTotal`` 30 | - ``BenchmarkMetric/throughput`` 31 | - ``BenchmarkMetric/peakMemoryResident`` 32 | - ``BenchmarkMetric/memoryLeaked`` 33 | - ``BenchmarkMetric/allocatedResidentMemory`` 34 | - ``BenchmarkMetric/instructions`` 35 | 36 | ### Memory Metrics 37 | 38 | - ``BenchmarkMetric/peakMemoryResident`` 39 | - ``BenchmarkMetric/peakMemoryResidentDelta`` 40 | - ``BenchmarkMetric/peakMemoryVirtual`` 41 | - ``BenchmarkMetric/mallocCountSmall`` 42 | - ``BenchmarkMetric/mallocCountLarge`` 43 | - ``BenchmarkMetric/mallocCountTotal`` 44 | - ``BenchmarkMetric/memoryLeaked`` 45 | - ``BenchmarkMetric/allocatedResidentMemory`` 46 | 47 | ### Reference Counting (retain/release) 48 | 49 | - ``BenchmarkMetric/retainCount`` 50 | - ``BenchmarkMetric/releaseCount`` 51 | - ``BenchmarkMetric/retainReleaseDelta`` 52 | 53 | ### Disk Metrics 54 | 55 | - ``BenchmarkMetric/readSyscalls`` 56 | - ``BenchmarkMetric/writeSyscalls`` 57 | - ``BenchmarkMetric/readBytesLogical`` 58 | - ``BenchmarkMetric/writeBytesLogical`` 59 | - ``BenchmarkMetric/readBytesPhysical`` 60 | - ``BenchmarkMetric/writeBytesPhysical`` 61 | 62 | ### Custom Metrics 63 | 64 | - ``BenchmarkMetric/custom(_:polarity:useScalingFactor:)`` 65 | - ``BenchmarkMetric/polarity-swift.property`` 66 | - ``BenchmarkMetric/Polarity-swift.enum`` 67 | 68 | ### Inspecting Metrics 69 | 70 | - ``BenchmarkMetric/description`` 71 | - ``BenchmarkMetric/countable`` 72 | 73 | ### Decoding a Metric 74 | 75 | - ``BenchmarkMetric/init(from:)`` 76 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/BenchmarkMetric_Polarity.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/BenchmarkMetric/Polarity`` 2 | 3 | ## Topics 4 | 5 | ### Types of Polarity 6 | 7 | - ``BenchmarkMetric/Polarity/prefersLarger`` 8 | - ``BenchmarkMetric/Polarity/prefersSmaller`` 9 | 10 | ### Decoding Polarity 11 | 12 | - ``BenchmarkMetric/Polarity/init(from:)`` 13 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/BenchmarkScalingFactor.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/BenchmarkScalingFactor`` 2 | 3 | ## Topics 4 | 5 | ### Selecting Scaling Factors 6 | 7 | - ``BenchmarkScalingFactor/one`` 8 | - ``BenchmarkScalingFactor/kilo`` 9 | - ``BenchmarkScalingFactor/mega`` 10 | - ``BenchmarkScalingFactor/giga`` 11 | - ``BenchmarkScalingFactor/tera`` 12 | - ``BenchmarkScalingFactor/peta`` 13 | - ``BenchmarkScalingFactor/init(_:)`` 14 | - ``BenchmarkScalingFactor/init(rawValue:)`` 15 | 16 | ### Describing the Scaling Factor 17 | 18 | - ``BenchmarkScalingFactor/description`` 19 | 20 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/BenchmarkTimeUnits.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/BenchmarkTimeUnits`` 2 | 3 | ## Topics 4 | 5 | ### Creating Time Units 6 | 7 | - ``BenchmarkTimeUnits/automatic`` 8 | - ``BenchmarkTimeUnits/nanoseconds`` 9 | - ``BenchmarkTimeUnits/microseconds`` 10 | - ``BenchmarkTimeUnits/milliseconds`` 11 | - ``BenchmarkTimeUnits/seconds`` 12 | - ``BenchmarkTimeUnits/kiloseconds`` 13 | - ``BenchmarkTimeUnits/megaseconds`` 14 | - ``BenchmarkTimeUnits/init(_:)-3tc3e`` 15 | - ``BenchmarkTimeUnits/init(rawValue:)`` 16 | 17 | ### Describing Time Units 18 | 19 | - ``BenchmarkTimeUnits/description`` 20 | 21 | ### Encoding and Decoding Time Units 22 | 23 | - ``BenchmarkTimeUnits/encode(to:)`` 24 | - ``BenchmarkTimeUnits/init(from:)`` 25 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/BenchmarkUnits.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/BenchmarkUnits`` 2 | 3 | ## Topics 4 | 5 | ### Creating Units 6 | 7 | - ``BenchmarkUnits/automatic`` 8 | - ``BenchmarkUnits/count`` 9 | - ``BenchmarkUnits/kilo`` 10 | - ``BenchmarkUnits/mega`` 11 | - ``BenchmarkUnits/giga`` 12 | - ``BenchmarkUnits/tera`` 13 | - ``BenchmarkUnits/peta`` 14 | - ``BenchmarkUnits/init(rawValue:)`` 15 | 16 | ### Describing Benchmark Units 17 | 18 | - ``BenchmarkUnits/description`` 19 | 20 | ### Encoding and Decoding Benchmark Units 21 | 22 | - ``BenchmarkUnits/encode(to:)`` 23 | - ``BenchmarkUnits/init(from:)`` 24 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/Benchmark_Configuration.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark/Benchmark/Configuration-swift.struct`` 2 | 3 | ## Topics 4 | 5 | ### Creating Configurations 6 | 7 | - ``Benchmark/Configuration-swift.struct/init(metrics:tags:timeUnits:units:warmupIterations:scalingFactor:maxDuration:maxIterations:skip:thresholds:setup:teardown:)`` 8 | 9 | ### Inspecting Configurations 10 | 11 | - ``Benchmark/Configuration-swift.struct/maxDuration`` 12 | - ``Benchmark/Configuration-swift.struct/maxIterations`` 13 | - ``Benchmark/Configuration-swift.struct/metrics`` 14 | - ``Benchmark/Configuration-swift.struct/skip`` 15 | - ``Benchmark/Configuration-swift.struct/thresholds`` 16 | - ``Benchmark/Configuration-swift.struct/scalingFactor`` 17 | - ``Benchmark/Configuration-swift.struct/units`` 18 | - ``Benchmark/Configuration-swift.struct/timeUnits`` 19 | - ``Benchmark/Configuration-swift.struct/warmupIterations`` 20 | 21 | ### Decoding Configurations 22 | 23 | - ``Benchmark/Configuration-swift.struct/init(from:)`` 24 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/CreatingAndComparingBaselines.md: -------------------------------------------------------------------------------- 1 | # Creating and Comparing Benchmark Baselines 2 | 3 | Benchmark supports storing, and comparing, benchmark baselines as you develop. 4 | 5 | ## Overview 6 | 7 | While developing locally, you can set benchmark baselines, compare a baseline against a benchmark run, or compare two different baselines. 8 | 9 | ### Creating a Baseline 10 | 11 | Typical workflow for a developer who wants to track performance metrics on the local machine while during performance work, would be to store one or more baselines. 12 | 13 | To create or update a baseline named `alpha`, run the following command: 14 | 15 | ```bash 16 | swift package --allow-writing-to-package-directory benchmark baseline update alpha 17 | ``` 18 | 19 | ### Comparing against a Baseline 20 | 21 | As you are making performance updates to your code, compare the current state of your code against a recorded baseline with the following command: 22 | 23 | ```bash 24 | swift package benchmark baseline compare alpha 25 | ``` 26 | 27 | If you have stored multiple baselines (for example for different approaches to solving a given performance issue), you can easily compare the two approaches by using named baselines for each and then compare them. 28 | The following command compares a baseline named `alpha` against baseline named `beta`: 29 | 30 | ```bash 31 | swift package benchmark baseline compare alpha beta 32 | ``` 33 | 34 | ### Comparing a test run against static thresholds 35 | 36 | The following will run all benchmarks and compare them against a previously saved static threshold. 37 | ```bash 38 | swift package benchmark thresholds check 39 | ``` 40 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/Documentation.md: -------------------------------------------------------------------------------- 1 | # ``Benchmark`` 2 | 3 | Benchmark allows you to easily create sophisticated Swift performance benchmarks 4 | 5 | ## Overview 6 | 7 | Performance is a key feature for many apps and frameworks. 8 | Benchmark helps make it easy to measure and track many different metrics that affects performance, such as CPU usage, memory usage and use of operating system resources such as threads and system calls. 9 | Benchmark provides a quick way for validation of performance metrics, while other more specialized tools such as Instruments, DTrace, Heaptrack, Leaks, Sample, etc support finding root causes for any deviations found. 10 | 11 | Benchmark is suitable both for smaller benchmarks focusing on execution time of small code snippets as well as for more extensive benchmarks that care about several additional metrics such as memory allocations, syscalls, thread usage, context switches, ARC traffic, and more. 12 | 13 | Thanks to the use of [Histogram](https://github.com/ordo-one/package-histogram) it's especially suitable for capturing latency statistics for large number of samples. 14 | 15 | ## Topics 16 | 17 | ### Essentials 18 | 19 | - 20 | - 21 | 22 | ### Workflows 23 | 24 | - 25 | - 26 | - 27 | 28 | ### Benchmarks 29 | 30 | - 31 | - 32 | - 33 | - 34 | - 35 | - 36 | 37 | ### Defining a Benchmark 38 | 39 | - ``Benchmark/Benchmark`` 40 | 41 | ### Configuring Benchmarks 42 | 43 | - ``Benchmark/Configuration-swift.struct`` 44 | - ``BenchmarkMetric`` 45 | - ``BenchmarkTimeUnits`` 46 | - ``BenchmarkUnits`` 47 | - ``BenchmarkScalingFactor`` 48 | 49 | ### Supporting Functions 50 | 51 | - ``Benchmark/blackHole(_:)`` 52 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/ExportingBenchmarks.md: -------------------------------------------------------------------------------- 1 | # Exporting Benchmark Results 2 | 3 | Export Benchmarks into other formats to analyze or visualize the data. 4 | 5 | ## Overview 6 | 7 | Benchmark supports exporting its results into a variety of formats including text formats, Java Microbenchmark Harness (JMH), Influx, and as a serialized [HDR Histogram](http://hdrhistogram.org). 8 | This allows for tracking performance over time or analyzing/visualizing with other tools such as [JMH visualizer](https://jmh.morethan.io), [Gnuplot](http://www.gnuplot.info), [YouPlot](https://github.com/red-data-tools/YouPlot), [HDR Histogram analyzer](http://hdrhistogram.github.io/HdrHistogram/plotFiles.html) and more. 9 | 10 | To export the benchmark information, add the desired format with the `--format` option when running the benchmarks. 11 | For example, to export your benchmarks into JMH format, use the command: 12 | 13 | ```bash 14 | swift package --allow-writing-to-package-directory benchmark --format jmh 15 | ``` 16 | 17 | It's also possible to use the output and use it with external plotting tools, e.g.: 18 | 19 | ```bash 20 | swift package benchmark --filter "Sc.*" --path stdout --format histogramPercentiles --no-progress --metric wallClock | uplot lineplot -H -w 80 -h 30 21 | ``` 22 | 23 | ![YouPlot sample](uplot) 24 | 25 | ### Streaming Text formats 26 | 27 | - term `text`: The default output, displaying a textual grid of information for your benchmarks, suitable for use in the console. 28 | - term `markdown`: The same content as `text`, but extended with explicit markdown support, suitable for use as output from e.g. a GitHub workflow action. 29 | 30 | The default text output from Benchmark is oriented around [the five-number summary](https://en.wikipedia.org/wiki/Five-number_summary) percentiles, plus the last decile (`p90`) and the last percentile (`p99`) - it's thus a variation of a [seven-figure summary](https://en.wikipedia.org/wiki/Seven-number_summary) with the focus on the 'bad' end of results (as those are what we typically care about addressing). 31 | The output streams to the terminal, allowing you to easily capture it to write to a file or preserve in an environment variable, which can be useful in continuous integration scenarios. 32 | For more information on using this output within continuous integration, see the examples in . 33 | 34 | ### Saved Formats 35 | 36 | - term `histogram`: Each benchmark and metric combination is written to a file with the file name extension `txt`. Each file contains a sequence of percentiles for that metric combination, as well as statistical summary information. This is the standard HDR Histogram text format usable by [the HDR Histogram plotFiles online tool](http://hdrhistogram.github.io/HdrHistogram/plotFiles.html). 37 | - term `histogramEncoded`: Each benchmark and metric combination is written to a file with the file name extension `json`, containing the serialized [Histogram](https://github.com/ordo-one/package-histogram)) in JSON format (Codable). 38 | - term `histogramSamples`: All samples for each benchmark and metric combination is written to a file with the file name extension `tsv`. 39 | - term `histogramPercentiles`: Each percentiles values between (0-99, 99.9, 99.99, ... 99.99999, 100) inluding a header line for processing by external tools (e.g. Youplot) `tsv`. 40 | - term `influx`: A single file is generated with the file name extension `csv` with the values encoded as metrics using the [Influx Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_reference/). 41 | - term `jmh`: A single file is generated with the file name extension `jmh` encoded in the [java microbenchmark harness](https://openjdk.org/projects/code-tools/jmh/) format. You can quickly compare the contained metrics by dropping the file into the [JMH visualizer](https://jmh.morethan.io) using a browser. 42 | 43 | 44 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/GettingStarted.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | Before creating your own benchmarks, you must install the required prerequisites and add a dependency on Benchmark to your package. 4 | 5 | ## Overview 6 | 7 | There are three steps that needs to be performed to get up and running with your own benchmarks: 8 | 9 | * Install prerequisite dependencies if needed (currently that's only `jemalloc`) 10 | * Add a dependency on Benchmark to your `Package.swift` file 11 | * Add one or more benchmark executable targets to the top level `Benchmarks/` directory for auto discovery 12 | 13 | After having done those, running your benchmarks are as simple as running `swift package benchmark`. 14 | 15 | ### Installing Prerequisites and Platform Support 16 | 17 | Benchmark requires Swift 5.7 support as it uses Regex and Duration types introduced with the `macOS 13` runtime, most versions of Linux will work as long as Swift 5.7+ is used. 18 | 19 | Benchmark also by default depends on and uses the [jemalloc](https://jemalloc.net) memory allocation library, which is used by the Benchmark infrastructure to capture memory allocation statistics. 20 | 21 | For platforms where `jemalloc` isn't available it's possible to build the Benchmark package without a `jemalloc` dependency by setting the environment variable BENCHMARK_DISABLE_JEMALLOC to any value except `false` or `0`. 22 | 23 | E.g. to run the benchmark on the command line without memory allocation stats could look like: 24 | 25 | ```bash 26 | BENCHMARK_DISABLE_JEMALLOC=true swift package benchmark 27 | ``` 28 | 29 | The Benchmark package requires you to install jemalloc on any machine used for benchmarking if you want malloc statistics. 30 | 31 | If you want to avoid adding the `jemalloc` dependency to your main project while still getting malloc statistics when benchmarking, the recommended approach is to embed a separate Swift project in a subdirectory that uses your project, then the dependency on `jemalloc` is contained to that subproject only. 32 | 33 | #### Installing `jemalloc` on macOS 34 | 35 | ``` 36 | brew install jemalloc 37 | ```` 38 | 39 | #### Installing `jemalloc` on Ubuntu 40 | 41 | ``` 42 | sudo apt-get install -y libjemalloc-dev 43 | ``` 44 | 45 | #### Installing `jemalloc` on Amazon Linux 2 46 | For Amazon Linux 2 users have reported that the following works: 47 | 48 | Docker file configuration 49 | ```dockerfile 50 | RUN sudo yum -y install bzip2 make 51 | RUN curl https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 -L -o jemalloc-5.3.0.tar.bz2 52 | RUN tar -xf jemalloc-5.3.0.tar.bz2 53 | RUN cd jemalloc-5.3.0 && ./configure && make && sudo make install 54 | ``` 55 | 56 | `make install` installs the libraries in `/usr/local/lib`, which the plugin can’t find, so you also have to do: 57 | 58 | ``` 59 | $ sudo ldconfig /usr/local/lib 60 | ``` 61 | 62 | Alternatively: 63 | ``` 64 | echo /usr/local/lib > /etc/ld.so.conf.d/local_lib.conf && ldconfig 65 | ``` 66 | 67 | ### Adding dependencies 68 | 69 | To add the dependency on Benchmark, add a dependency to your package: 70 | 71 | ```swift 72 | .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.0.0")), 73 | ``` 74 | 75 | ### Add benchmark exectuable targets using `benchmark init` 76 | The absolutely easiest way to add new benchmark executable targets to your project is by using: 77 | ```bash 78 | swift package --allow-writing-to-package-directory benchmark init MyNewBenchmarkTarget 79 | ``` 80 | 81 | This will perform the following steps for you: 82 | 83 | * Create a `Benchmarks/MyNewBenchmarkTarget` directory 84 | * Create a `Benchmarks/MyNewBenchmarkTarget/MyNewBenchmarkTarget.swift` benchmark target with the required boilerplate 85 | * Add a new executable target for the benchmark to the end of your `Package.swift` file 86 | 87 | The `init` command validates that the name you specify isn't used by any existing target and will not overwrite any existing file with that name in the Benchmarks/ location. 88 | 89 | After you've created the new target, you can directly run it with e.g.: 90 | ```bash 91 | swift package benchmark --target MyNewBenchmarkTarget 92 | ``` 93 | 94 | ### Add benchmark exectuable targets manually 95 | Optionally if you don't want the plugin to modify your project for you, you can do those steps manually. 96 | 97 | First create an executable target in `Package.swift` for each benchmark suite you want to measure. 98 | 99 | The source for all benchmarks *must reside in a directory named `Benchmarks`* in the root of your swift package. 100 | 101 | The benchmark plugin uses this directory combined with the executable target information to automatically discover and run your benchmarks. 102 | 103 | For each executable target, include dependencies on both `Benchmark` (supporting framework) and `BenchmarkPlugin` (boilerplate generator) from `package-benchmark`. 104 | 105 | The following example shows an benchmark suite named `My-Benchmark` with the required dependency on `Benchmark` and the source files for the benchmark that reside in the directory `Benchmarks/My-Benchmark`: 106 | 107 | ``` 108 | .executableTarget( 109 | name: "My-Benchmark", 110 | dependencies: [ 111 | .product(name: "Benchmark", package: "package-benchmark"), 112 | .product(name: "BenchmarkPlugin", package: "package-benchmark"), 113 | ], 114 | path: "Benchmarks/My-Benchmark" 115 | ), 116 | ``` 117 | 118 | ### Run benchmark plugin in release mode 119 | If you find that the runtime for e.g. baseline processing is long, it's recommended to try running the benchmark 120 | plugin in release mode configuration, e.g. 121 | 122 | ``` 123 | swift package -c release benchmark baseline read myBaseLine 124 | ``` 125 | 126 | ### Dedicated GitHub runner instances 127 | 128 | For reproducible and good comparable results, it is *highly* recommended to set up a private GitHub runner that is completely dedicated for performance benchmark runs, as the standard GitHub CI runners are deployed on a shared infrastructure the deviations between runs can be significant and difficult to assess. 129 | 130 | ### Sample Project 131 | 132 | There's a [sample project](https://github.com/ordo-one/package-benchmark-samples) showing usage of the basic API which can be a good starting point if you want to look at how a project can be setup. 133 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/Metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics and Thresholds 2 | 3 | Benchmarks supports a wide range of benchmark metrics and also allows you to create custom benchmark metrics. 4 | 5 | ## Overview 6 | 7 | A fairly wide range of metrics can be captured by the benchmarks - most metrics are avilable on both macOS and Linux, but a few are not easily obtained and will thus not yield results on that platform, even if specified. 8 | 9 | ### Metrics 10 | 11 | Currently supported metrics are: 12 | 13 | - term `cpuUser`: CPU user space time spent for running the test 14 | - term `cpuSystem`: CPU system time spent for running the test 15 | - term `cpuTotal`: CPU total time spent for running the test (system + user) 16 | - term `wallClock`: Wall clock time for running the test 17 | - term `throughput`: The throughput in operations / second 18 | - term `peakMemoryResident`: The resident memory usage - sampled during runtime 19 | - term `peakMemoryResidentDelta`: The resident memory usage - sampled during runtime (excluding start of benchmark baseline) 20 | - term `peakMemoryVirtual`: The virtual memory usage - sampled during runtime 21 | - term `mallocCountSmall`: The number of small malloc calls according to jemalloc 22 | - term `mallocCountLarge`: The number of large malloc calls according to jemalloc 23 | - term `mallocCountTotal`: The total number of mallocs according to jemalloc 24 | - term `allocatedResidentMemory`: The amount of allocated resident memory by the application (not including allocator metadata overhead etc) according to jemalloc 25 | - term `memoryLeaked`: The number of small+large mallocs - small+large frees in resident memory (just a possible leak) 26 | - term `syscalls`: The number of syscalls made during the test -- macOS only 27 | - term `contextSwitches`: The number of context switches made during the test -- macOS only 28 | - term `threads`: The maximum number of threads in the process under the test (not exact, sampled) 29 | - term `threadsRunning`: The maximum number of threads actually running under the test (not exact, sampled) -- macOS only 30 | - term `readSyscalls`: The number of I/O read syscalls performed e.g. read(2) / pread(2) -- Linux only 31 | - term `writeSyscalls`: The number of I/O write syscalls performed e.g. write(2) / pwrite(2) -- Linux only 32 | - term `readBytesLogical`: The number of bytes read from storage (but may be satisfied by pagecache!) -- Linux only 33 | - term `writeBytesLogical`: The number bytes written to storage (but may be cached) -- Linux only 34 | - term `readBytesPhysical`: The number of bytes physically read from a block device (i.e. disk) -- Linux only 35 | - term `writeBytesPhysical`: The number of bytes physicall written to a block device (i.e. disk) -- Linux only 36 | - term `instructions`: The number of instructions executed -- on Linux using perf_events, for macOS using rusage() 37 | - term `retainCount`: The number of retain calls (ARC) 38 | - term `releaseCount`: The number of release calls (ARC) 39 | - term `retainReleaseDelta`: abs(retainCount - releaseCount) - if this is non-zero, it would typically mean the benchmark has a retain cycle (use Memory Graph Debugger to troubleshoot) 40 | 41 | Additionally, _custom metrics_ are supported `custom(_ name: String, polarity: Polarity = .prefersSmaller, useScalingFactor: Bool = true)` as outlined in the writing benchmarks documentation. 42 | 43 | ### Thresholds 44 | 45 | For comparison (`swift package benchmark baseline compare` or `swift package benchmark baseline check`) operations, there's a set of default thresholds that are used which are strict. It is also possible to define both absolute and relative thresholds, _per metric_, that will be used for such comparisons (or that a given metric should be skipped completely). 46 | 47 | In addition to comparing the delta between e.g. a `PR` and `main`, there's also an option to compare against an absolute threshold which is useful for more complex projects that may want to reduce the size of the build matrix required to validate all thresholds. 48 | 49 | Absolute thresholds are usually setup with e.g. `.mega(10)` or `.milliseconds(600)`. 50 | 51 | See or look at the sample code to see how custom thresholds can be set up. 52 | -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/Resources/Images/PercentileHistogramExample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ordo-one/package-benchmark/5c4569a9e7e9acac4db43f31e58252539ca71f0b/Sources/Benchmark/Documentation.docc/Resources/Images/PercentileHistogramExample.png -------------------------------------------------------------------------------- /Sources/Benchmark/Documentation.docc/Resources/Images/uplot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ordo-one/package-benchmark/5c4569a9e7e9acac4db43f31e58252539ca71f0b/Sources/Benchmark/Documentation.docc/Resources/Images/uplot.png -------------------------------------------------------------------------------- /Sources/Benchmark/Int+Extensions.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | public extension Int { 12 | static func hours(_ hours: Int) -> Int { 13 | hours * 1_000_000_000 * 60 * 60 14 | } 15 | 16 | static func minutes(_ minutes: Int) -> Int { 17 | minutes * 1_000_000_000 * 60 18 | } 19 | 20 | static func seconds(_ seconds: Int) -> Int { 21 | seconds * 1_000_000_000 22 | } 23 | 24 | static func milliseconds(_ milliseconds: Int) -> Int { 25 | milliseconds * 1_000_000 26 | } 27 | 28 | static func microseconds(_ microseconds: Int) -> Int { 29 | microseconds * 1_000 30 | } 31 | 32 | static func nanoseconds(_ value: Int) -> Int { 33 | value 34 | } 35 | 36 | static func nanoseconds(_ value: UInt) -> Int { 37 | Int(value) 38 | } 39 | 40 | static func giga(_ value: Int) -> Int { 41 | value * 1_000_000_000 42 | } 43 | 44 | static func mega(_ value: Int) -> Int { 45 | value * 1_000_000 46 | } 47 | 48 | static func kilo(_ value: Int) -> Int { 49 | value * 1_000 50 | } 51 | 52 | static func count(_ value: Int) -> Int { 53 | value 54 | } 55 | 56 | static func count(_ value: UInt) -> Int { 57 | Int(value) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sources/Benchmark/MallocStats/MallocStats.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | /// The memory allocation stats the the MallocStatsProducer can provide 12 | @_documentation(visibility: internal) 13 | struct MallocStats { 14 | var mallocCountTotal: Int = 0 /// total number of mallocs done 15 | var mallocCountSmall: Int = 0 /// number of small mallocs (as defined by jemalloc) 16 | var mallocCountLarge: Int = 0 /// number of large mallocs (as defined by jemalloc) 17 | 18 | /// Maximum number of bytes in physically resident data pages mapped by the allocator, 19 | /// comprising all pages dedicated to allocator metadata, pages backing active allocations 20 | /// , and unused dirty pages. This is a maximum rather than precise because pages may 21 | /// not actually be physically resident if they correspond to demand-zeroed virtual memory 22 | /// that has not yet been touched. This is a multiple of the page size. 23 | var allocatedResidentMemory: Int = 0 // in bytes 24 | } 25 | -------------------------------------------------------------------------------- /Sources/Benchmark/OperatingSystemStats/OperatingSystemStats.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | /// The stats that the OperatingSystemStatsProducer can provide 12 | struct OperatingSystemStats { 13 | /// CPU user space time spent for running the test 14 | var cpuUser: Int = 0 15 | /// CPU system time spent for running the test 16 | var cpuSystem: Int = 0 17 | /// CPU total time spent for running the test (system + user) 18 | var cpuTotal: Int = 0 19 | /// Measure resident memory usage - sampled during runtime 20 | var peakMemoryResident: Int = 0 21 | /// Measure virtual memory usage - sampled during runtime 22 | var peakMemoryVirtual: Int = 0 23 | /// Measure number of syscalls made during the test 24 | var syscalls: Int = 0 25 | /// Measure number of context switches made during the test 26 | var contextSwitches: Int = 0 27 | /// Sample the maximum number of threads in the process under the test (not exact) 28 | var threads: Int = 0 29 | /// Sample the maximum number of threads actually running under the test (not exact) 30 | var threadsRunning: Int = 0 31 | /// The number of I/O read syscalls performed e.g. read(2) / pread(2) -- Linux only 32 | var readSyscalls: Int = 0 33 | /// The number of I/O write syscalls performed e.g. write(2) / pwrite(2), on macOS the logical writes return by rusage 34 | var writeSyscalls: Int = 0 35 | /// The number of bytes read from storage (but may be satisfied by pagecache!) -- Linux only 36 | var readBytesLogical: Int = 0 37 | /// The number bytes written to storage (but may be cached) -- Linux only 38 | var writeBytesLogical: Int = 0 39 | /// The number of bytes physically read from a block device (i.e. disk) -- Linux only 40 | var readBytesPhysical: Int = 0 41 | /// The number of bytes physicall written to a block device (i.e. disk) -- Linux only 42 | var writeBytesPhysical: Int = 0 43 | } 44 | 45 | struct PerformanceCounters { 46 | /// The number instructions executed 47 | var instructions: UInt64 = 0 48 | } 49 | -------------------------------------------------------------------------------- /Sources/Benchmark/OutputSuppressor.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | final class OutputSuppressor { 4 | private var originalStdout: Int32? 5 | private var originalStderr: Int32? 6 | private var nullFile: Int32? 7 | 8 | func suppressOutput() throws { 9 | // Save original file descriptors 10 | originalStdout = dup(FileHandle.standardOutput.fileDescriptor) 11 | originalStderr = dup(FileHandle.standardError.fileDescriptor) 12 | 13 | // Open /dev/null 14 | nullFile = open("/dev/null", O_WRONLY) 15 | guard nullFile != -1 else { 16 | throw NSError(domain: "OutputSuppressor", code: 1, userInfo: [NSLocalizedDescriptionKey: "Failed to open /dev/null"]) 17 | } 18 | 19 | // Redirect stdout and stderr to /dev/null 20 | guard dup2(nullFile!, FileHandle.standardOutput.fileDescriptor) != -1, 21 | dup2(nullFile!, FileHandle.standardError.fileDescriptor) != -1 else { 22 | throw NSError(domain: "OutputSuppressor", code: 2, userInfo: [NSLocalizedDescriptionKey: "Failed to redirect output"]) 23 | } 24 | } 25 | 26 | func restoreOutput() throws { 27 | // Restore original stdout and stderr 28 | guard let stdout = originalStdout, 29 | let stderr = originalStderr else { 30 | throw NSError(domain: "OutputSuppressor", code: 3, userInfo: [NSLocalizedDescriptionKey: "Original file descriptors not found"]) 31 | } 32 | 33 | guard dup2(stdout, FileHandle.standardOutput.fileDescriptor) != -1, 34 | dup2(stderr, FileHandle.standardError.fileDescriptor) != -1 else { 35 | throw NSError(domain: "OutputSuppressor", code: 4, userInfo: [NSLocalizedDescriptionKey: "Failed to restore output"]) 36 | } 37 | 38 | // Close file descriptors 39 | close(stdout) 40 | close(stderr) 41 | if let null = nullFile { 42 | close(null) 43 | } 44 | 45 | // Reset stored descriptors 46 | originalStdout = nil 47 | originalStderr = nil 48 | nullFile = nil 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Sources/Benchmark/Progress/Progress.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Progress.swift 3 | // 4 | // Created by Justus Kandzi on 27/12/15. 5 | // Copyright © 2015 Justus Kandzi. All rights reserved. 6 | // 7 | // The MIT License (MIT) 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a copy 10 | // of this software and associated documentation files (the "Software"), to deal 11 | // in the Software without restriction, including without limitation the rights 12 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | // copies of the Software, and to permit persons to whom the Software is 14 | // furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included in all 17 | // copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | // SOFTWARE. 26 | // 27 | 28 | 29 | // MARK: - ProgressBarDisplayer 30 | 31 | @_documentation(visibility: internal) 32 | public protocol ProgressBarPrinter { 33 | mutating func display(_ progressBar: ProgressBar) 34 | } 35 | 36 | @_documentation(visibility: internal) 37 | struct ProgressBarTerminalPrinter: ProgressBarPrinter { 38 | var lastPrintedTime = 0.0 39 | 40 | init() { 41 | // the cursor is moved up before printing the progress bar. 42 | // have to move the cursor down one line initially. 43 | print("") 44 | } 45 | 46 | mutating func display(_ progressBar: ProgressBar) { 47 | let currentTime = getTimeOfDay() 48 | if (currentTime - lastPrintedTime > 0.1 || progressBar.index == progressBar.count) { 49 | print("\u{1B}[1A\u{1B}[K\(progressBar.value)") 50 | lastPrintedTime = currentTime 51 | } 52 | } 53 | } 54 | 55 | 56 | // MARK: - ProgressBar 57 | 58 | @_documentation(visibility: internal) 59 | public struct ProgressBar { 60 | private(set) public var index = 0 61 | public let startTime = getTimeOfDay() 62 | 63 | public let count: Int 64 | let configuration: [ProgressElementType]? 65 | 66 | public static var defaultConfiguration: [ProgressElementType] = [ProgressIndex(), ProgressBarLine(), ProgressTimeEstimates()] 67 | 68 | var printer: ProgressBarPrinter 69 | 70 | public var value: String { 71 | let configuration = self.configuration ?? ProgressBar.defaultConfiguration 72 | let values = configuration.map { $0.value(self) } 73 | return values.joined(separator: " ") 74 | } 75 | 76 | public init(count: Int, configuration: [ProgressElementType]? = nil, printer: ProgressBarPrinter? = nil) { 77 | self.count = count 78 | self.configuration = configuration 79 | self.printer = printer ?? ProgressBarTerminalPrinter() 80 | } 81 | 82 | public mutating func next() { 83 | guard index <= count else { return } 84 | let anotherSelf = self 85 | printer.display(anotherSelf) 86 | index += 1 87 | } 88 | 89 | public mutating func setValue(_ index: Int) { 90 | guard index <= count && index >= 0 else { return } 91 | self.index = index 92 | let anotherSelf = self 93 | printer.display(anotherSelf) 94 | } 95 | 96 | } 97 | 98 | 99 | // MARK: - GeneratorType 100 | 101 | @_documentation(visibility: internal) 102 | public struct ProgressGenerator: IteratorProtocol { 103 | var source: G 104 | var progressBar: ProgressBar 105 | 106 | init(source: G, count: Int, configuration: [ProgressElementType]? = nil, printer: ProgressBarPrinter? = nil) { 107 | self.source = source 108 | self.progressBar = ProgressBar(count: count, configuration: configuration, printer: printer) 109 | } 110 | 111 | public mutating func next() -> G.Element? { 112 | progressBar.next() 113 | return source.next() 114 | } 115 | } 116 | 117 | 118 | // MARK: - SequenceType 119 | 120 | @_documentation(visibility: internal) 121 | public struct Progress: Sequence { 122 | let generator: G 123 | let configuration: [ProgressElementType]? 124 | let printer: ProgressBarPrinter? 125 | 126 | public init(_ generator: G, configuration: [ProgressElementType]? = nil, printer: ProgressBarPrinter? = nil) { 127 | self.generator = generator 128 | self.configuration = configuration 129 | self.printer = printer 130 | } 131 | 132 | public func makeIterator() -> ProgressGenerator { 133 | let count = generator.underestimatedCount 134 | return ProgressGenerator(source: generator.makeIterator(), count: count, configuration: configuration, printer: printer) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Sources/Benchmark/Progress/ProgressElements.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ProgressElements.swift 3 | // Progress.swift 4 | // 5 | // Created by Justus Kandzi on 04/01/16. 6 | // Copyright © 2016 Justus Kandzi. All rights reserved. 7 | // 8 | // The MIT License (MIT) 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the "Software"), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is 15 | // furnished to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | // 20 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | // SOFTWARE. 27 | // 28 | 29 | 30 | @_documentation(visibility: internal) 31 | public protocol ProgressElementType { 32 | func value(_ progressBar: ProgressBar) -> String 33 | } 34 | 35 | 36 | /// the progress bar element e.g. "[---------------------- ]" 37 | @_documentation(visibility: internal) 38 | public struct ProgressBarLine: ProgressElementType { 39 | let barLength: Int 40 | 41 | public init(barLength: Int = 30) { 42 | self.barLength = barLength 43 | } 44 | 45 | public func value(_ progressBar: ProgressBar) -> String { 46 | var completedBarElements = 0 47 | if progressBar.count == 0 { 48 | completedBarElements = barLength 49 | } else { 50 | completedBarElements = Int(Double(barLength) * (Double(progressBar.index) / Double(progressBar.count))) 51 | } 52 | 53 | var barArray = [String](repeating: "-", count: completedBarElements) 54 | barArray += [String](repeating: " ", count: barLength - completedBarElements) 55 | return "[" + barArray.joined(separator: "") + "]" 56 | } 57 | } 58 | 59 | 60 | /// the index element e.g. "2 of 3" 61 | @_documentation(visibility: internal) 62 | public struct ProgressIndex: ProgressElementType { 63 | public init() {} 64 | 65 | public func value(_ progressBar: ProgressBar) -> String { 66 | return "\(progressBar.index) of \(progressBar.count)" 67 | } 68 | } 69 | 70 | 71 | /// the percentage element e.g. "90.0%" 72 | @_documentation(visibility: internal) 73 | public struct ProgressPercent: ProgressElementType { 74 | let decimalPlaces: Int 75 | 76 | public init(decimalPlaces: Int = 0) { 77 | self.decimalPlaces = decimalPlaces 78 | } 79 | 80 | public func value(_ progressBar: ProgressBar) -> String { 81 | var percentDone = 100.0 82 | if progressBar.count > 0 { 83 | percentDone = Double(progressBar.index) / Double(progressBar.count) * 100 84 | } 85 | 86 | var padded = "\(percentDone.format(decimalPlaces))%" 87 | while padded.count < 4 { 88 | padded = " " + padded 89 | } 90 | return padded // "\(percentDone.format(decimalPlaces))%" 91 | } 92 | } 93 | 94 | 95 | /// the time estimates e.g. "ETA: 00:00:02 (at 1.00 it/s)" 96 | @_documentation(visibility: internal) 97 | public struct ProgressTimeEstimates: ProgressElementType { 98 | public init() {} 99 | 100 | public func value(_ progressBar: ProgressBar) -> String { 101 | let totalTime = getTimeOfDay() - progressBar.startTime 102 | 103 | var itemsPerSecond = 0.0 104 | var estimatedTimeRemaining = 0.0 105 | if progressBar.index > 0 { 106 | itemsPerSecond = Double(progressBar.index) / totalTime 107 | estimatedTimeRemaining = (Double(progressBar.count - progressBar.index) / itemsPerSecond) + 1.0 108 | } 109 | 110 | if progressBar.index >= progressBar.count { 111 | estimatedTimeRemaining = 0.0 112 | } 113 | 114 | let estimatedTimeRemainingString = formatDuration(estimatedTimeRemaining) 115 | 116 | return "ETA: \(estimatedTimeRemainingString)" 117 | } 118 | 119 | fileprivate func formatDuration(_ duration: Double) -> String { 120 | let duration = Int(duration) 121 | let seconds = Double(duration % 60) 122 | let minutes = Double((duration / 60) % 60) 123 | let hours = Double(duration / 3600) 124 | return "\(hours.format(0, minimumIntegerPartLength: 2)):\(minutes.format(0, minimumIntegerPartLength: 2)):\(seconds.format(0, minimumIntegerPartLength: 2))" 125 | } 126 | } 127 | 128 | 129 | /// an arbitrary string that can be added to the progress bar. 130 | @_documentation(visibility: internal) 131 | public struct ProgressString: ProgressElementType { 132 | let string: String 133 | 134 | public init(string: String) { 135 | self.string = string 136 | } 137 | 138 | public func value(_: ProgressBar) -> String { 139 | return string 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /Sources/Benchmark/Progress/Utilities.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Utilities.swift 3 | // Progress.swift 4 | // 5 | // Created by Justus Kandzi on 04/01/16. 6 | // Copyright © 2016 Justus Kandzi. All rights reserved. 7 | // 8 | // The MIT License (MIT) 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the "Software"), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is 15 | // furnished to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | // 20 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | // SOFTWARE. 27 | // 28 | 29 | #if os(Linux) 30 | import Glibc 31 | #else 32 | import Darwin.C 33 | #endif 34 | 35 | @_documentation(visibility: internal) 36 | func getTimeOfDay() -> Double { 37 | var tv = timeval() 38 | gettimeofday(&tv, nil) 39 | return Double(tv.tv_sec) + Double(tv.tv_usec) / 1000000 40 | } 41 | 42 | @_documentation(visibility: internal) 43 | extension Double { 44 | func format(_ decimalPartLength: Int, minimumIntegerPartLength: Int = 0) -> String { 45 | let value = String(self) 46 | let components = value 47 | .split() { $0 == "." } 48 | .map { String($0) } 49 | 50 | var integerPart = components.first ?? "0" 51 | 52 | let missingLeadingZeros = minimumIntegerPartLength - integerPart.count 53 | if missingLeadingZeros > 0 { 54 | integerPart = stringWithZeros(missingLeadingZeros) + integerPart 55 | } 56 | 57 | if decimalPartLength == 0 { 58 | return integerPart 59 | } 60 | 61 | var decimalPlaces = components.last?.substringWithRange(0, end: decimalPartLength) ?? "0" 62 | let missingPlaceCount = decimalPartLength - decimalPlaces.count 63 | decimalPlaces += stringWithZeros(missingPlaceCount) 64 | 65 | return "\(integerPart).\(decimalPlaces)" 66 | } 67 | 68 | fileprivate func stringWithZeros(_ count: Int) -> String { 69 | return Array(repeating: "0", count: count).joined(separator: "") 70 | } 71 | } 72 | 73 | @_documentation(visibility: internal) 74 | extension String { 75 | func substringWithRange(_ start: Int, end: Int) -> String { 76 | var end = end 77 | if start < 0 || start > self.count { 78 | return "" 79 | } 80 | else if end < 0 || end > self.count { 81 | end = self.count 82 | } 83 | let range = self.index(self.startIndex, offsetBy: start) ..< self.index(self.startIndex, offsetBy: end) 84 | return String(self[range]) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Sources/Benchmark/Statistics.swift: -------------------------------------------------------------------------------- 1 | // swiftlint:disable all 2 | // 3 | // Copyright (c) 2022 Ordo One AB. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // 8 | // You may obtain a copy of the License at 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | 12 | import Histogram 13 | import Numerics 14 | 15 | // A type that provides distribution / percentile calculations of latency measurements 16 | @_documentation(visibility: internal) 17 | public final class Statistics: Codable { 18 | public static let defaultMaximumMeasurement = 1_000_000_000 // 1 second in nanoseconds 19 | public static let defaultPercentilesToCalculate = [0.0, 25.0, 50.0, 75.0, 90.0, 99.0, 100.0] 20 | public static let defaultPercentilesToCalculateP90Index = 4 21 | 22 | public enum Units: Int, Codable, CaseIterable { 23 | case count = 1 // e.g. nanoseconds 24 | case kilo = 1_000 // microseconds 25 | case mega = 1_000_000 // milliseconds 26 | case giga = 1_000_000_000 // seconds 27 | case tera = 1_000_000_000_000 // 1K seconds 28 | case peta = 1_000_000_000_000_000 // 1M seconds 29 | case automatic = 0 // will pick time unit above automatically 30 | 31 | public var description: String { 32 | switch self { 33 | case .count: 34 | return "#" 35 | case .kilo: 36 | return "K" 37 | case .mega: 38 | return "M" 39 | case .giga: 40 | return "G" 41 | case .tera: 42 | return "T" 43 | case .peta: 44 | return "P" 45 | case .automatic: 46 | return "#" 47 | } 48 | } 49 | 50 | public var timeDescription: String { 51 | switch self { 52 | case .count: 53 | return "ns" 54 | case .kilo: 55 | return "μs" 56 | case .mega: 57 | return "ms" 58 | case .giga: 59 | return "s" 60 | case .tera: 61 | return "ks" 62 | case .peta: 63 | return "Ms" 64 | case .automatic: 65 | return "#" 66 | } 67 | } 68 | 69 | public init(fromMagnitudeOf value: Double) { 70 | let magnitude = Double.log10(value) 71 | switch magnitude { 72 | case ..<4.0: 73 | self = .count 74 | case 4.0 ..< 7.0: 75 | self = .kilo 76 | case 7.0 ..< 10.0: 77 | self = .mega 78 | case 10.0 ..< 13.0: 79 | self = .giga 80 | case 13.0 ..< 16.0: 81 | self = .tera 82 | case 16.0...: 83 | self = .peta 84 | default: 85 | self = .kilo 86 | } 87 | } 88 | } 89 | 90 | var _cachedPercentiles: [Int] = [] 91 | var _cacheUnits: Statistics.Units = .automatic 92 | var _cachedPercentilesHistogramCount: UInt64 = 0 93 | 94 | public func percentiles(for percentilesToCalculate: [Double] = defaultPercentilesToCalculate) -> [Int] { 95 | if percentilesToCalculate == Self.defaultPercentilesToCalculate { 96 | if _cachedPercentilesHistogramCount == histogram.totalCount, _cachedPercentiles.count > 0 { 97 | return _cachedPercentiles 98 | } 99 | } 100 | 101 | var percentileResults: [Int] = [] 102 | 103 | for var p in percentilesToCalculate { 104 | if prefersLarger { 105 | p = 100.0 - p 106 | } 107 | 108 | let value = histogram.valueAtPercentile(p) 109 | percentileResults.append(Int(value)) 110 | } 111 | 112 | if percentilesToCalculate == Self.defaultPercentilesToCalculate { 113 | _cachedPercentilesHistogramCount = histogram.totalCount 114 | _cachedPercentiles = percentileResults 115 | } 116 | 117 | return percentileResults 118 | } 119 | 120 | // Returns the actual units to use (either specified, or automatic) 121 | public func units() -> Statistics.Units { 122 | if timeUnits != .automatic { 123 | return timeUnits 124 | } 125 | 126 | if onlyZeroMeasurements { 127 | return .count 128 | } 129 | 130 | if _cachedPercentilesHistogramCount != histogram.totalCount || _cacheUnits == .automatic { 131 | _cacheUnits = Statistics.Units(fromMagnitudeOf: histogram.mean) 132 | _cachedPercentilesHistogramCount = histogram.totalCount 133 | } 134 | 135 | return _cacheUnits 136 | } 137 | 138 | public let prefersLarger: Bool 139 | public let timeUnits: Statistics.Units 140 | public var histogram: Histogram 141 | 142 | public var onlyZeroMeasurements: Bool { 143 | histogram.countForValue(0) == histogram.totalCount 144 | } 145 | 146 | public var measurementCount: Int { 147 | Int(histogram.totalCount) 148 | } 149 | 150 | public var average: Double { 151 | histogram.mean 152 | } 153 | 154 | public init(maximumMeasurement: Int = defaultMaximumMeasurement, 155 | numberOfSignificantDigits: SignificantDigits = .three, 156 | units: Statistics.Units = .automatic, 157 | prefersLarger: Bool = false) { 158 | self.prefersLarger = prefersLarger 159 | timeUnits = units 160 | _cacheUnits = timeUnits 161 | histogram = Histogram(highestTrackableValue: UInt64(maximumMeasurement), 162 | numberOfSignificantValueDigits: numberOfSignificantDigits) 163 | histogram.autoResize = true 164 | } 165 | 166 | /// Add a measurement for inclusion in statistics 167 | /// - Parameter measurement: A measurement expressed in nanoseconds 168 | @inlinable 169 | @inline(__always) 170 | public func add(_ measurement: Int) { 171 | guard measurement >= 0 else { 172 | return // We sometimes got a <0 measurement, should run with fatalError and try to see how that could occur 173 | // fatalError() 174 | } 175 | 176 | histogram.record(UInt64(measurement)) 177 | } 178 | 179 | // Rounds decimals for display 180 | public static func roundToDecimalplaces(_ original: Double, _ decimals: Int = 2) -> Double { 181 | let factor: Double = .pow(10.0, Double(decimals)) 182 | var original: Double = original * factor 183 | original.round(.toNearestOrEven) 184 | return original / factor 185 | } 186 | } 187 | 188 | // swiftlint:enable all 189 | -------------------------------------------------------------------------------- /Sources/BenchmarkShared/Command+Helpers.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | // Project wide shared types 12 | 13 | @_documentation(visibility: internal) 14 | public enum Command: String, CaseIterable { 15 | case run 16 | case list 17 | case baseline 18 | case thresholds 19 | case help 20 | case `init` 21 | } 22 | 23 | /// The benchmark data output format. 24 | public enum OutputFormat: String, CaseIterable { 25 | /// Text output formatted into a visual table suitable for console output 26 | case text 27 | /// The text output format, formatted in markdown, suitable for GitHub workflows 28 | case markdown 29 | /// Influx data format 30 | case influx 31 | /// JMH format consumable by http://jmh.morethan.io 32 | case jmh 33 | /// The encoded representation of the underlying histograms capturing the benchmark data, for programmatic use (Codable). 34 | case histogramEncoded 35 | /// The histogram percentiles, average, deviation, sample count etc in standard HDR Histogram text format consumable by http://hdrhistogram.github.io/HdrHistogram/plotFiles.html 36 | case histogram 37 | /// The raw histogram samples in TSV format for processing by external tools (e.g. Youplot) 38 | case histogramSamples 39 | /// The percentiles values betwen (0-100) in TSV format for processing by external tools (e.g. Youplot) 40 | case histogramPercentiles 41 | /// The p90 percentile values per metric as a `[BenchmarkMetric: BenchmarkThresholds]` in JSON format, suitable for static thresholds 42 | case metricP90AbsoluteThresholds 43 | } 44 | 45 | public enum Grouping: String, CaseIterable { 46 | case metric 47 | case benchmark 48 | } 49 | 50 | @_documentation(visibility: internal) 51 | public enum TimeUnits: String, CaseIterable { 52 | case nanoseconds 53 | case microseconds 54 | case milliseconds 55 | case seconds 56 | case kiloseconds 57 | case megaseconds 58 | } 59 | 60 | @_documentation(visibility: internal) 61 | public enum ThresholdsOperation: String, CaseIterable { 62 | case read 63 | case update 64 | case check 65 | } 66 | 67 | @_documentation(visibility: internal) 68 | public enum BaselineOperation: String, CaseIterable { 69 | case read 70 | case update 71 | case list 72 | case delete 73 | case compare 74 | case check 75 | } 76 | 77 | public enum ExitCode: Int32 { 78 | case success = 0 79 | case genericFailure = 1 80 | case thresholdRegression = 2 81 | case benchmarkJobFailed = 3 82 | case thresholdImprovement = 4 83 | case baselineNotFound = 5 84 | case noPermissions = 6 85 | } 86 | -------------------------------------------------------------------------------- /Sources/SwiftRuntimeHooks/include/SwiftRuntimeHooks.h: -------------------------------------------------------------------------------- 1 | #ifndef PACKAGE_BENCHMARK_SWIFT_RUNTIME_HOOKS_H 2 | #define PACKAGE_BENCHMARK_SWIFT_RUNTIME_HOOKS_H 3 | 4 | typedef void (*swift_runtime_hook_t)(const void *, void *); 5 | 6 | void swift_runtime_set_alloc_object_hook(swift_runtime_hook_t hook, void * context); 7 | void swift_runtime_set_retain_hook(swift_runtime_hook_t hook, void * context); 8 | void swift_runtime_set_release_hook(swift_runtime_hook_t hook, void * context); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /Sources/SwiftRuntimeHooks/shims.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "SwiftRuntimeHooks.h" 5 | 6 | typedef struct HeapObject_s HeapObject; 7 | typedef struct HeapMetadata_s HeapMetadata; 8 | 9 | extern HeapObject * (*_swift_allocObject)(HeapMetadata const *metadata, 10 | size_t requiredSize, 11 | size_t requiredAlignmentMask); 12 | 13 | extern HeapObject * (*_swift_retain)(HeapObject*); 14 | extern HeapObject * (*_swift_release)(HeapObject*); 15 | 16 | extern HeapObject * (*_swift_tryRetain)(HeapObject*); 17 | 18 | // unclear if the following two needs hooking and they don't seem to be called on Apple Silicon 19 | extern HeapObject * (*_swift_retain_n)(HeapObject *object, uint32_t n); 20 | extern HeapObject * (*_swift_release_n)(HeapObject *object, uint32_t n); 21 | 22 | struct hook_data_s { 23 | HeapObject * (*orig)(HeapObject*); 24 | HeapObject * (*origTry)(HeapObject*); 25 | HeapObject * (*orig_n)(HeapObject*, uint32_t); 26 | swift_runtime_hook_t hook; 27 | void * context; 28 | }; 29 | 30 | struct hook_data_alloc_s { 31 | HeapObject * (*orig)(HeapMetadata const *, size_t, size_t); 32 | swift_runtime_hook_t hook; 33 | void * context; 34 | }; 35 | 36 | /*===========================================================================*/ 37 | 38 | static struct hook_data_alloc_s _swift_alloc_object_hook_data = {NULL, NULL, NULL}; 39 | 40 | static HeapObject * _swift_alloc_object_hook(HeapMetadata const *metadata, 41 | size_t requiredSize, 42 | size_t requiredAlignmentMask) { 43 | HeapObject * ret = (*_swift_alloc_object_hook_data.orig)(metadata, requiredSize, requiredAlignmentMask); 44 | (*_swift_alloc_object_hook_data.hook)(ret, _swift_alloc_object_hook_data.context); 45 | return ret; 46 | } 47 | 48 | void swift_runtime_set_alloc_object_hook(swift_runtime_hook_t hook, void * context) { 49 | if (hook == NULL) { 50 | _swift_allocObject = _swift_alloc_object_hook_data.orig; 51 | struct hook_data_alloc_s hook_data = {NULL, NULL, NULL}; 52 | _swift_alloc_object_hook_data = hook_data; 53 | } else { 54 | struct hook_data_alloc_s hook_data = {_swift_allocObject, hook, context}; 55 | _swift_alloc_object_hook_data = hook_data; 56 | _swift_allocObject = _swift_alloc_object_hook; 57 | } 58 | } 59 | /*===========================================================================*/ 60 | 61 | static struct hook_data_s _swift_retain_hook_data = {NULL, NULL, NULL, NULL, NULL}; 62 | 63 | static HeapObject * _swift_retain_hook(HeapObject * heapObject) { 64 | HeapObject * ret = (*_swift_retain_hook_data.orig)(heapObject); 65 | (*_swift_retain_hook_data.hook)(heapObject, _swift_retain_hook_data.context); 66 | return ret; 67 | } 68 | 69 | // This doesn't seem to be called for Apple Silicon at least, but keeping it here 70 | static HeapObject * _swift_tryRetain_hook(HeapObject * heapObject) { 71 | HeapObject * ret = (*_swift_retain_hook_data.origTry)(heapObject); 72 | if (ret != NULL) { 73 | (*_swift_retain_hook_data.hook)(heapObject, _swift_retain_hook_data.context); 74 | } 75 | return ret; 76 | } 77 | 78 | // This doesn't seem to be called for Apple Silicon at least, but keeping it here 79 | static HeapObject * _swift_retain_n_hook(HeapObject * heapObject, uint32_t n) { 80 | int i; 81 | HeapObject * ret = (*_swift_retain_hook_data.orig_n)(heapObject, n); 82 | for (i = 0; i < n; i++) { 83 | (*_swift_retain_hook_data.hook)(heapObject, _swift_retain_hook_data.context); 84 | } 85 | return ret; 86 | } 87 | 88 | void swift_runtime_set_retain_hook(swift_runtime_hook_t hook, void * context) { 89 | if (hook == NULL) { 90 | _swift_retain = _swift_retain_hook_data.orig; 91 | _swift_tryRetain = _swift_retain_hook_data.origTry; 92 | _swift_retain_n = _swift_retain_hook_data.orig_n; 93 | struct hook_data_s hook_data = {NULL, NULL, NULL, NULL, NULL}; 94 | _swift_retain_hook_data = hook_data; 95 | } else { 96 | struct hook_data_s hook_data = {_swift_retain, _swift_tryRetain, _swift_retain_n, hook, context}; 97 | _swift_retain_hook_data = hook_data; 98 | _swift_retain = _swift_retain_hook; 99 | _swift_tryRetain = _swift_tryRetain_hook; 100 | _swift_retain_n = _swift_retain_n_hook; 101 | } 102 | } 103 | 104 | /*===========================================================================*/ 105 | 106 | static struct hook_data_s _swift_release_hook_data = {NULL, NULL, NULL, NULL, NULL}; 107 | 108 | static HeapObject * _swift_release_hook(HeapObject * heapObject) { 109 | HeapObject * ret = (*_swift_release_hook_data.orig)(heapObject); 110 | (*_swift_release_hook_data.hook)(heapObject, _swift_release_hook_data.context); 111 | return ret; 112 | } 113 | 114 | // This doesn't seem to be called for Apple Silicon at least, but keeping it here 115 | static HeapObject * _swift_release_n_hook(HeapObject * heapObject, uint32_t n) { 116 | int i; 117 | HeapObject * ret = (*_swift_release_hook_data.orig_n)(heapObject, n); 118 | for (i = 0; i < n; i++) { 119 | (*_swift_release_hook_data.hook)(heapObject, _swift_release_hook_data.context); 120 | } 121 | return ret; 122 | } 123 | 124 | void swift_runtime_set_release_hook(swift_runtime_hook_t hook, void * context) { 125 | if (hook == NULL) { 126 | _swift_release = _swift_release_hook_data.orig; 127 | _swift_release_n = _swift_release_hook_data.orig_n; 128 | struct hook_data_s hook_data = {NULL, NULL, NULL, NULL, NULL}; 129 | _swift_release_hook_data = hook_data; 130 | } else { 131 | struct hook_data_s hook_data = {_swift_release, NULL, _swift_release_n, hook, context}; 132 | _swift_release_hook_data = hook_data; 133 | _swift_release = _swift_release_hook; 134 | _swift_release_n = _swift_release_n_hook; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Tests/BenchmarkTests/AdditionalTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | @testable import Benchmark 12 | import Foundation 13 | import XCTest 14 | 15 | final class AdditionalTests: XCTestCase { 16 | // Disabled for now as it breaks when run on the public CI 17 | /* 18 | func testBlackhole() throws { // due to https://github.com/ordo-one/package-benchmark/issues/178 19 | func runWork(_ testIterations: Int) -> ContinuousClock.Duration { 20 | let clock = ContinuousClock() 21 | return clock.measure { 22 | for idx in 1 ... testIterations { 23 | Benchmark.blackHole(idx) 24 | } 25 | } 26 | } 27 | 28 | var results: [ContinuousClock.Duration] = [] 29 | var testIterations = 100_000 30 | for _ in 0 ..< 3 { 31 | results.append(runWork(testIterations)) 32 | testIterations *= 10 33 | } 34 | 35 | var comparisonValue = 0 36 | results.forEach { result in 37 | let microseconds = result.components.seconds * 1_000_000 + result.components.attoseconds / 1_000_000_000_000 38 | let logValue = log10(Double(microseconds)).rounded() 39 | 40 | XCTAssertFalse(logValue.isNaN, "blackHole seems broken, runtime is too fast") 41 | XCTAssertFalse(logValue.isInfinite, "blackHole seems broken, runtime is too fast") 42 | 43 | let newValue = Int(logValue) 44 | XCTAssert(newValue > comparisonValue, "blackHole should take 10x longer for each iteration") 45 | comparisonValue = newValue 46 | // print("result \(result), microseconds = \(microseconds), log = \(newValue)") 47 | } 48 | } 49 | */ 50 | } 51 | -------------------------------------------------------------------------------- /Tests/BenchmarkTests/BenchmarkMetricsTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2023 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | /// 10 | 11 | @testable import Benchmark 12 | import XCTest 13 | 14 | final class BenchmarkMetricsTests: XCTestCase { 15 | private let metrics: [BenchmarkMetric] = [ 16 | .cpuUser, 17 | .cpuSystem, 18 | .cpuTotal, 19 | .wallClock, 20 | .throughput, 21 | .peakMemoryResident, 22 | .peakMemoryResidentDelta, 23 | .peakMemoryVirtual, 24 | .mallocCountSmall, 25 | .mallocCountLarge, 26 | .mallocCountTotal, 27 | .allocatedResidentMemory, 28 | .memoryLeaked, 29 | .syscalls, 30 | .contextSwitches, 31 | .threads, 32 | .threadsRunning, 33 | .readSyscalls, 34 | .writeSyscalls, 35 | .readBytesLogical, 36 | .writeBytesLogical, 37 | .readBytesPhysical, 38 | .writeBytesPhysical, 39 | .instructions, 40 | .objectAllocCount, 41 | .retainCount, 42 | .releaseCount, 43 | .retainReleaseDelta, 44 | .custom("test", polarity: .prefersSmaller, useScalingFactor: false), 45 | .custom("test2", polarity: .prefersLarger, useScalingFactor: true) 46 | ] 47 | 48 | private let textualMetrics: [String] = [ 49 | "cpuUser", 50 | "cpuSystem", 51 | "cpuTotal", 52 | "wallClock", 53 | "throughput", 54 | "peakMemoryResident", 55 | "peakMemoryResidentDelta", 56 | "peakMemoryVirtual", 57 | "mallocCountSmall", 58 | "mallocCountLarge", 59 | "mallocCountTotal", 60 | "allocatedResidentMemory", 61 | "memoryLeaked", 62 | "syscalls", 63 | "contextSwitches", 64 | "threads", 65 | "threadsRunning", 66 | "readSyscalls", 67 | "writeSyscalls", 68 | "readBytesLogical", 69 | "writeBytesLogical", 70 | "readBytesPhysical", 71 | "writeBytesPhysical", 72 | "instructions", 73 | "objectAllocCount", 74 | "retainCount", 75 | "releaseCount", 76 | "retainReleaseDelta" 77 | ] 78 | 79 | func testBenchmarkMetrics() throws { 80 | var description = "" 81 | var rawValues = 0 82 | metrics.forEach { metric in 83 | description += metric.description 84 | rawValues += metric.useScalingFactor ? 0 : 1 85 | rawValues += metric.countable ? 0 : 1 86 | rawValues += metric.polarity == .prefersLarger ? 0 : 1 87 | } 88 | 89 | XCTAssert(rawValues > 10) 90 | XCTAssert(description.count > 10) 91 | } 92 | 93 | func testBenchmarkTextualMetrics() throws { 94 | var description = "" 95 | 96 | for metricIndex in 0 ..< textualMetrics.count { 97 | if let metric = BenchmarkMetric(argument: textualMetrics[metricIndex]) { 98 | description += metric.description 99 | XCTAssertEqual(metrics[metricIndex], metric) 100 | } else { 101 | XCTFail("Could not extract metric \(textualMetrics[metricIndex])") 102 | } 103 | } 104 | 105 | XCTAssert(description.count > 10) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /Tests/BenchmarkTests/BenchmarkRunnerTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | /// 10 | 11 | @testable import Benchmark 12 | import XCTest 13 | 14 | final class BenchmarkRunnerTests: XCTestCase, BenchmarkRunnerReadWrite { 15 | private var readMessage: Int = 0 16 | private var writeCount: Int = 0 17 | 18 | // swiftlint:disable test_case_accessibility 19 | func write(_: BenchmarkCommandReply) throws { 20 | writeCount += 1 21 | // print("write \(reply)") 22 | } 23 | 24 | func read() throws -> BenchmarkCommandRequest { 25 | // print("read request") 26 | Benchmark.testSkipBenchmarkRegistrations = true 27 | let benchmark = Benchmark("Minimal benchmark") { _ in 28 | } 29 | let benchmark2 = Benchmark("Minimal benchmark 2") { _ in 30 | } 31 | let benchmark3 = Benchmark("Minimal benchmark 3") { _ in 32 | } 33 | let returnValues: [BenchmarkCommandRequest] = [.run(benchmark: benchmark!), 34 | .run(benchmark: benchmark2!), 35 | .run(benchmark: benchmark3!), 36 | .end] 37 | 38 | readMessage += 1 39 | return returnValues[readMessage - 1] 40 | } 41 | 42 | func testBenchmarkRunner() async throws { 43 | BenchmarkRunner.testReadWrite = self 44 | 45 | Benchmark("Minimal benchmark", configuration: .init(metrics: BenchmarkMetric.all, maxIterations: 1)) { _ in } 46 | Benchmark("Minimal benchmark 2", configuration: .init(warmupIterations: 0, maxIterations: 2)) { _ in } 47 | Benchmark("Minimal benchmark 3", configuration: .init(timeUnits: .seconds, maxIterations: 3)) { _ in } 48 | 49 | var runner = BenchmarkRunner() 50 | runner.inputFD = 0 51 | runner.outputFD = 0 52 | runner.debug = false 53 | runner.quiet = false 54 | runner.timeUnits = .nanoseconds 55 | try await runner.run() 56 | XCTAssertEqual(writeCount, 6) // 3 tests results + 3 end markers 57 | } 58 | } 59 | 60 | // swiftlint:enable test_case_accessibility 61 | -------------------------------------------------------------------------------- /Tests/BenchmarkTests/BenchmarkTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | @testable import Benchmark 12 | import XCTest 13 | 14 | final class BenchmarkTests: XCTestCase { 15 | func testBenchmarkRun() throws { 16 | let benchmark = Benchmark("testBenchmarkRun benchmark") { _ in 17 | } 18 | XCTAssertNotNil(benchmark) 19 | benchmark?.run() 20 | } 21 | 22 | func testBenchmarkRunAsync() throws { 23 | func asyncFunc() async {} 24 | let benchmark = Benchmark("testBenchmarkRunAsync benchmark") { _ in 25 | await asyncFunc() 26 | } 27 | XCTAssertNotNil(benchmark) 28 | benchmark?.runAsync() 29 | } 30 | 31 | func testBenchmarkRunCustomMetric() throws { 32 | let benchmark = Benchmark("testBenchmarkRunCustomMetric benchmark", 33 | configuration: .init(metrics: [.custom("customMetric")])) { benchmark in 34 | for measurement in 1 ... 100 { 35 | benchmark.measurement(.custom("customMetric"), measurement) 36 | } 37 | } 38 | XCTAssertNotNil(benchmark) 39 | benchmark?.run() 40 | } 41 | 42 | func testBenchmarkEqualityAndDifference() throws { 43 | let benchmark = Benchmark("testBenchmarkEqualityAndDifference benchmark") { _ in 44 | } 45 | let benchmark2 = Benchmark("testBenchmarkEqualityAndDifference benchmark 2") { _ in 46 | } 47 | XCTAssertNotEqual(benchmark, benchmark2) 48 | } 49 | 50 | func testBenchmarkRunFailure() throws { 51 | let benchmark = Benchmark("testBenchmarkRunFailure benchmark", 52 | configuration: .init(metrics: [.custom("customMetric")])) { benchmark in 53 | benchmark.error("Benchmark failed") 54 | } 55 | XCTAssertNotNil(benchmark) 56 | benchmark?.run() 57 | XCTAssertNotNil(benchmark?.failureReason) 58 | XCTAssertEqual(benchmark?.failureReason, "Benchmark failed") 59 | } 60 | 61 | func testBenchmarkRunMoreParameters() throws { 62 | let benchmark = Benchmark("testBenchmarkRunMoreParameters benchmark", 63 | configuration: .init( 64 | metrics: .all, 65 | timeUnits: .milliseconds, 66 | warmupIterations: 0, 67 | scalingFactor: .mega 68 | )) { benchmark in 69 | for outerloop in benchmark.scaledIterations { 70 | blackHole(outerloop) 71 | } 72 | } 73 | XCTAssertNotNil(benchmark) 74 | benchmark?.run() 75 | } 76 | 77 | func testBenchmarkParameterizedDescription() throws { 78 | let benchmark = Benchmark("testBenchmarkParameterizedDescription benchmark", 79 | configuration: .init( 80 | tags: [ 81 | "foo": "bar", 82 | "bin": String(42), 83 | "pi": String(3.14) 84 | ] 85 | )) { _ in } 86 | XCTAssertNotNil(benchmark) 87 | XCTAssertEqual(benchmark?.name, "testBenchmarkParameterizedDescription benchmark (bin: 42, foo: bar, pi: 3.14)") 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Tests/BenchmarkTests/StatisticsTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2022 Ordo One AB. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // 7 | // You may obtain a copy of the License at 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | 11 | @testable import Benchmark 12 | import XCTest 13 | 14 | final class StatisticsTests: XCTestCase { 15 | func testStatisticsResults() throws { 16 | let stats = Statistics(numberOfSignificantDigits: .four) 17 | let measurementCount = 8_340 18 | 19 | // Add 2*measurementCount measurements, one 0, one max 20 | for measurement in (0 ..< measurementCount).reversed() { 21 | stats.add(measurement) 22 | } 23 | 24 | for measurement in 1 ... measurementCount { 25 | stats.add(measurement) 26 | } 27 | 28 | XCTAssertEqual(Statistics.roundToDecimalplaces(123.4567898972239487234), 123.46) 29 | XCTAssertEqual(stats.measurementCount, measurementCount * 2) 30 | XCTAssertEqual(stats.units(), .count) 31 | XCTAssertEqual(round(stats.histogram.mean), round(Double(measurementCount / 2))) 32 | 33 | let percentiles = stats.percentiles() 34 | 35 | XCTAssertEqual(percentiles[0], 0) 36 | XCTAssertEqual(percentiles[1], Int(round(Double(measurementCount) * 0.25))) 37 | XCTAssertEqual(percentiles[2], Int(round(Double(measurementCount) * 0.5))) 38 | XCTAssertEqual(percentiles[3], Int(round(Double(measurementCount) * 0.75))) 39 | XCTAssertEqual(percentiles[4], Int(round(Double(measurementCount) * 0.9))) 40 | XCTAssertEqual(percentiles[5], Int(round(Double(measurementCount) * 0.99))) 41 | XCTAssertEqual(percentiles[6], Int(measurementCount)) 42 | } 43 | 44 | func testOnlyZeroMeasurements() throws { 45 | let stats = Statistics() 46 | let measurementCount = 100 47 | let range = 0 ..< measurementCount 48 | 49 | for _ in range { 50 | stats.add(0) 51 | } 52 | 53 | XCTAssertEqual(stats.measurementCount, range.count) 54 | XCTAssertEqual(stats.histogram.mean, 0.0) 55 | 56 | XCTAssert(stats.onlyZeroMeasurements) 57 | 58 | let percentiles = stats.percentiles() 59 | 60 | XCTAssert(stats.onlyZeroMeasurements) 61 | XCTAssertEqual(stats.units(), .count) 62 | XCTAssertEqual(percentiles[0], 0) 63 | XCTAssertEqual(percentiles[1], 0) 64 | XCTAssertEqual(percentiles[2], 0) 65 | XCTAssertEqual(percentiles[3], 0) 66 | XCTAssertEqual(percentiles[4], 0) 67 | XCTAssertEqual(percentiles[5], 0) 68 | XCTAssertEqual(percentiles[6], 0) 69 | } 70 | 71 | func testFewerMeasurementsThanPercentiles() throws { 72 | let stats = Statistics() 73 | let measurementCount = 5 74 | let range = 1 ..< measurementCount 75 | var accumulatedMeasurement = 0 76 | 77 | for measurement in range { 78 | stats.add(measurement) 79 | accumulatedMeasurement += measurement 80 | } 81 | 82 | XCTAssertEqual(stats.measurementCount, range.count) 83 | XCTAssertEqual(stats.units(), .count) 84 | XCTAssertEqual(round(stats.histogram.mean), round(Double(accumulatedMeasurement) / Double(range.count))) 85 | 86 | let percentiles = stats.percentiles() 87 | 88 | XCTAssertEqual(percentiles[0], 1) 89 | XCTAssertEqual(percentiles[1], Int(round(Double(range.count) * 0.25))) 90 | XCTAssertEqual(percentiles[2], Int(round(Double(range.count) * 0.5))) 91 | XCTAssertEqual(percentiles[3], Int(round(Double(range.count) * 0.75))) 92 | XCTAssertEqual(percentiles[4], Int(round(Double(range.count) * 0.9))) 93 | XCTAssertEqual(percentiles[5], Int(round(Double(range.count) * 0.99))) 94 | XCTAssertEqual(percentiles[6], Int(range.count)) 95 | } 96 | 97 | func testAutomaticUnits() throws { 98 | typealias Case = (value: Int, units: Statistics.Units) 99 | 100 | let cases = [ 101 | Case(value: 0, units: .count), 102 | Case(value: 1, units: .count), 103 | Case(value: 9_999, units: .count), 104 | Case(value: 10_000, units: .kilo), 105 | Case(value: 100_000, units: .kilo), 106 | Case(value: 1_000_000, units: .kilo), 107 | Case(value: 9_999_999, units: .kilo), 108 | Case(value: 10_000_000, units: .mega), 109 | Case(value: 9_999_999_999, units: .mega), 110 | Case(value: 10_000_000_000, units: .giga) 111 | ] 112 | 113 | for (value, expectedUnits) in cases { 114 | let units = Statistics.Units(fromMagnitudeOf: Double(value)) 115 | XCTAssertEqual(units, expectedUnits, "Expected units for \(value) are \(expectedUnits)") 116 | } 117 | } 118 | 119 | func testHistograms() throws { 120 | let measurementCount = 300 121 | let stats = Statistics(prefersLarger: true) 122 | 123 | for measurement in 1 ... measurementCount { 124 | stats.add(measurement) 125 | } 126 | 127 | XCTAssert(Int(stats.average) > (measurementCount / 3)) 128 | XCTAssertGreaterThan(stats.histogram.totalCount, 100) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /Thresholds/P90AbsoluteThresholdsBenchmark.P90Date.p90.json: -------------------------------------------------------------------------------- 1 | { 2 | "syscalls" : 2, 3 | "mallocCountTotal" : 0 4 | } 5 | -------------------------------------------------------------------------------- /Thresholds/P90AbsoluteThresholdsBenchmark.P90Malloc.p90.json: -------------------------------------------------------------------------------- 1 | { 2 | "syscalls" : 6, 3 | "mallocCountTotal" : 1012 4 | } 5 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "Benchmarks" # ignore Benchmarks 3 | --------------------------------------------------------------------------------