├── .craft.yml ├── .dockerignore ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md ├── actions │ └── changelog │ │ ├── .gitignore │ │ ├── README.md │ │ ├── index.js │ │ ├── package.json │ │ └── prettier.config.js ├── dependabot.yml └── workflows │ ├── CI_mat.yml │ ├── changelog.yaml │ ├── ci.yaml │ ├── codeql.yaml │ ├── release.yaml │ └── validate-pipelines.yaml ├── .gitignore ├── .golangci.yml ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.md ├── Makefile ├── README.md ├── artwork ├── vroom-icon.png └── vroom-logo.png ├── cloudbuild.yaml ├── cmd ├── downloader │ ├── README.md │ └── main.go ├── issuedetection │ └── main.go └── vroom │ ├── chunk.go │ ├── chunk_test.go │ ├── config.go │ ├── flamegraph.go │ ├── kafka.go │ ├── main.go │ ├── metrics.go │ ├── profile.go │ ├── regressed.go │ └── utils.go ├── devservices └── config.yml ├── go.mod ├── go.sum ├── gocd └── templates │ ├── bash │ ├── check-cloudbuild.sh │ ├── check-github.sh │ ├── deploy.sh │ └── wait-canary.sh │ ├── jsonnetfile.json │ ├── jsonnetfile.lock.json │ ├── pipelines │ └── vroom.libsonnet │ └── vroom.jsonnet ├── internal ├── android │ └── display.go ├── chunk │ ├── android.go │ ├── android_utils.go │ ├── android_utils_test.go │ ├── chunk.go │ ├── sample.go │ ├── sample_readjob.go │ ├── sample_test.go │ ├── sample_utils.go │ └── sample_utils_test.go ├── clientsdk │ └── clientsdk.go ├── debugmeta │ └── debugmeta.go ├── errorutil │ └── errorutil.go ├── flamegraph │ ├── flamegraph.go │ ├── flamegraph_test.go │ ├── span_util.go │ └── span_util_test.go ├── frame │ ├── frame.go │ ├── frame_test.go │ └── python_std_lib.go ├── httputil │ ├── decompress.go │ ├── request.go │ └── transaction.go ├── logutil │ └── logutil.go ├── measurements │ └── measurements.go ├── metadata │ └── metadata.go ├── metrics │ ├── metrics.go │ └── metrics_test.go ├── nodetree │ ├── nodetree.go │ └── nodetree_test.go ├── occurrence │ ├── detect_frame.go │ ├── detect_frame_test.go │ ├── find.go │ ├── frame_drop.go │ ├── frame_drop_test.go │ ├── kafka.go │ ├── occurrence.go │ ├── occurrence_test.go │ └── regressed_frame.go ├── packageutil │ └── package.go ├── platform │ └── platform.go ├── profile │ ├── android.go │ ├── android_test.go │ ├── consts.go │ ├── legacy.go │ ├── legacy_test.go │ ├── profile.go │ ├── readjob.go │ ├── trace.go │ └── version.go ├── sample │ ├── sample.go │ └── sample_test.go ├── speedscope │ └── speedscope.go ├── storageutil │ ├── storageutil.go │ └── storageutil_test.go ├── testutil │ └── testutil.go ├── timeutil │ ├── time.go │ └── time_test.go ├── transaction │ ├── metadata.go │ └── transaction.go └── utils │ ├── options.go │ └── structs.go ├── scripts ├── build.sh ├── make_python_stdlib.py └── run.sh └── test ├── data ├── cocoa.json └── node.json └── gcs └── sentry-profiles └── .gitignore /.craft.yml: -------------------------------------------------------------------------------- 1 | minVersion: 1.0.0 2 | changelogPolicy: auto 3 | artifactProvider: 4 | name: none 5 | statusProvider: 6 | name: github 7 | config: 8 | contexts: 9 | - 'build-vroom (sentryio)' 10 | targets: 11 | - name: github 12 | - id: release 13 | name: docker 14 | source: us-central1-docker.pkg.dev/sentryio/vroom/vroom 15 | target: getsentry/vroom 16 | - id: latest 17 | name: docker 18 | source: us-central1-docker.pkg.dev/sentryio/vroom/vroom 19 | target: getsentry/vroom 20 | targetFormat: '{{{target}}}:latest' 21 | preReleaseCommand: "" 22 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | ** 2 | !/.git 3 | !/cmd/vroom 4 | !/internal 5 | !/pkg 6 | !/vendor 7 | !go.mod 8 | !go.sum 9 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @getsentry/profiling 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 14 | ### Legal Boilerplate 15 | 16 | Look, I get it. The entity doing business as "Sentry" was incorporated in the State of Delaware in 2015 as Functional Software, Inc. and is gonna need some rights from me in order to utilize my contributions in this here PR. So here's the deal: I retain all rights, title and interest in and to my contributions, and by keeping this boilerplate intact I confirm that Sentry can use, modify, copy, and redistribute my contributions, under Sentry's choice of terms. 17 | -------------------------------------------------------------------------------- /.github/actions/changelog/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /.github/actions/changelog/README.md: -------------------------------------------------------------------------------- 1 | # Changelog Checker 2 | 3 | This is a custom script to check if the changelog files contain the entry for the current pull request. 4 | 5 | 6 | ### Development 7 | 8 | To make any contributions or changes to this code you must make sure that you have `node` installed. Once you have it, just 9 | run `npm install` in this folder to install all the dependencies. 10 | 11 | The main entry point is `changelog.js` file. This file contain all the supported checks. 12 | -------------------------------------------------------------------------------- /.github/actions/changelog/index.js: -------------------------------------------------------------------------------- 1 | module.exports = async ({github, context, core}) => { 2 | const PR_LINK = `[#${context.payload.pull_request.number}](${context.payload.pull_request.html_url})`; 3 | 4 | function getCleanTitle(title) { 5 | // remove fix(component): prefix 6 | title = title.split(': ').slice(-1)[0].trim(); 7 | // remove links to JIRA tickets, i.e. a suffix like [ISSUE-123] 8 | title = title.split('[')[0].trim(); 9 | // remove trailing dots 10 | title = title.replace(/\.+$/, ''); 11 | 12 | return title; 13 | } 14 | 15 | function getChangelogDetails(title) { 16 | return ` 17 | For changes to the _vroom_, please add an entry to \`CHANGELOG.md\` under the following heading: 18 | 1. **Features**: For new user-visible functionality. 19 | 2. **Bug Fixes**: For user-visible bug fixes. 20 | 3. **Internal**: For features and bug fixes in internal operation, especially processing mode. 21 | To the changelog entry, please add a link to this PR (consider a more descriptive message): 22 | \`\`\`md 23 | - ${title}. (${PR_LINK}) 24 | \`\`\` 25 | If none of the above apply, you can opt out by adding _#skip-changelog_ to the PR description. 26 | `; 27 | } 28 | 29 | function logOutputError(title) { 30 | core.info(''); 31 | core.info('\u001b[1mInstructions and example for changelog'); 32 | core.info(getChangelogDetails(title)); 33 | core.info(''); 34 | core.info('\u001b[1mSee check status:'); 35 | core.info( 36 | `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` 37 | ); 38 | } 39 | 40 | async function containsChangelog(path) { 41 | const {data} = await github.rest.repos.getContent({ 42 | owner: context.repo.owner, 43 | repo: context.repo.repo, 44 | ref: context.ref, 45 | path, 46 | }); 47 | const buf = Buffer.alloc(data.content.length, data.content, data.encoding); 48 | const fileContent = buf.toString(); 49 | return fileContent.includes(PR_LINK); 50 | } 51 | 52 | async function checkChangelog(pr) { 53 | if ((pr.body || '').includes('#skip-changelog')) { 54 | core.info('#skip-changelog is set. Skipping the checks.'); 55 | return; 56 | } 57 | 58 | const hasChangelog = (await containsChangelog('CHANGELOG.md')); 59 | 60 | if (!hasChangelog) { 61 | core.error('Please consider adding a changelog entry for the next release.', { 62 | title: 'Missing changelog entry.', 63 | file: 'CHANGELOG.md', 64 | startLine: 3, 65 | }); 66 | const title = getCleanTitle(pr.title); 67 | core.summary 68 | .addHeading('Instructions and example for changelog') 69 | .addRaw(getChangelogDetails(title)) 70 | .write(); 71 | core.setFailed('CHANGELOG entry is missing.'); 72 | logOutputError(title); 73 | return; 74 | } 75 | 76 | core.summary.clear(); 77 | core.info("CHANGELOG entry is added, we're good to go."); 78 | } 79 | 80 | async function checkAll() { 81 | const {data: pr} = await github.rest.pulls.get({ 82 | owner: context.repo.owner, 83 | repo: context.repo.repo, 84 | pull_number: context.payload.pull_request.number, 85 | }); 86 | 87 | // While in draft mode, skip the check because changelogs often cause merge conflicts. 88 | if (pr.merged || pr.draft) { 89 | return; 90 | } 91 | 92 | await checkChangelog(pr); 93 | } 94 | 95 | await checkAll(); 96 | }; 97 | -------------------------------------------------------------------------------- /.github/actions/changelog/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ChangelogChecker", 3 | "version": "1.0.0", 4 | "description": "Makes sure that the CHANGELOG.md has the entry for the current pull request.", 5 | "main": "index.js", 6 | "scripts": { 7 | "format": "prettier --write index.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/getsentry/relay.git" 12 | }, 13 | "author": "Oleksandr Kylymnychenko ", 14 | "license": "MIT", 15 | "bugs": { 16 | "url": "https://github.com/getsentry/relay/issues" 17 | }, 18 | "homepage": "https://github.com/getsentry/relay#readme", 19 | "dependencies": { 20 | "@actions/core": "^1.10.0", 21 | "@actions/github": "^5.1.1" 22 | }, 23 | "devDependencies": { 24 | "prettier": "2.8.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.github/actions/changelog/prettier.config.js: -------------------------------------------------------------------------------- 1 | /* eslint-env node */ 2 | module.exports = { 3 | bracketSpacing: false, 4 | bracketSameLine: false, 5 | printWidth: 90, 6 | semi: true, 7 | singleQuote: true, 8 | tabWidth: 2, 9 | trailingComma: 'es5', 10 | useTabs: false, 11 | arrowParens: 'avoid', 12 | }; 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | # security updates only 8 | open-pull-requests-limit: 0 9 | 10 | - package-ecosystem: "docker" 11 | directory: "/" 12 | schedule: 13 | interval: "daily" 14 | 15 | - package-ecosystem: "github-actions" 16 | directory: "/" 17 | schedule: 18 | interval: "daily" 19 | -------------------------------------------------------------------------------- /.github/workflows/CI_mat.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getsentry/vroom/a0bbd507ab959fd6e5bd5c4423dbb80fc23ca2fd/.github/workflows/CI_mat.yml -------------------------------------------------------------------------------- /.github/workflows/changelog.yaml: -------------------------------------------------------------------------------- 1 | name: changelog 2 | on: 3 | pull_request: 4 | types: [opened, synchronize, reopened, edited, ready_for_review] 5 | 6 | jobs: 7 | build: 8 | name: changelog 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea 13 | env: 14 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 15 | with: 16 | script: | 17 | const changelog = require('./.github/actions/changelog/index.js') 18 | await changelog({github, context, core}) 19 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: continuous-integration 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - '*' 7 | push: 8 | branches: 9 | - main 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 13 | cancel-in-progress: true 14 | 15 | env: 16 | SHELL: /bin/bash 17 | 18 | defaults: 19 | run: 20 | shell: bash 21 | 22 | jobs: 23 | lint: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v4 27 | with: 28 | fetch-depth: 0 29 | - uses: actions/setup-go@v5 30 | with: 31 | go-version: stable 32 | cache: false 33 | - run: go install golang.org/x/tools/cmd/goimports@latest 34 | - name: golangci-lint 35 | uses: golangci/golangci-lint-action@v8 36 | with: 37 | version: latest 38 | - uses: pre-commit/action@v3.0.1 39 | 40 | test-vroom: 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@v4 44 | with: 45 | fetch-depth: 0 46 | - uses: actions/setup-go@v5 47 | with: 48 | go-version: stable 49 | cache: false 50 | - run: make test 51 | 52 | publish-to-dockerhub: 53 | name: Publish Vroom to DockerHub 54 | runs-on: ubuntu-latest 55 | if: ${{ (github.ref_name == 'main') }} 56 | steps: 57 | - uses: actions/checkout@v4 58 | - timeout-minutes: 20 59 | run: until docker pull "us-central1-docker.pkg.dev/sentryio/vroom/vroom:${{ github.sha }}" 2>/dev/null; do sleep 10; done 60 | - name: Push built docker image 61 | shell: bash 62 | run: | 63 | IMAGE_URL="us-central1-docker.pkg.dev/sentryio/vroom/vroom:${{ github.sha }}" 64 | docker login --username=sentrybuilder --password ${{ secrets.DOCKER_HUB_RW_TOKEN }} 65 | # We push 3 tags to Dockerhub: 66 | # first, the full sha of the commit 67 | docker tag "$IMAGE_URL" getsentry/vroom:${GITHUB_SHA} 68 | docker push getsentry/vroom:${GITHUB_SHA} 69 | # second, the short sha of the commit 70 | SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") 71 | docker tag "$IMAGE_URL" getsentry/vroom:${SHORT_SHA} 72 | docker push getsentry/vroom:${SHORT_SHA} 73 | # finally, nightly 74 | docker tag "$IMAGE_URL" getsentry/vroom:nightly 75 | docker push getsentry/vroom:nightly 76 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main" ] 20 | schedule: 21 | - cron: '32 16 * * 3' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go', 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Use only 'java' to analyze code written in Java, Kotlin or both 38 | # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both 39 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v4 44 | 45 | # Initializes the CodeQL tools for scanning. 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@v3 48 | with: 49 | languages: ${{ matrix.language }} 50 | # If you wish to specify custom queries, you can do so here or in a config file. 51 | # By default, queries listed here will override any specified in a config file. 52 | # Prefix the list here with "+" to use these queries and those in the config file. 53 | 54 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 55 | # queries: security-extended,security-and-quality 56 | 57 | 58 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 59 | # If this step fails, then you should remove it and run the build manually (see below) 60 | - name: Autobuild 61 | uses: github/codeql-action/autobuild@v3 62 | 63 | # ℹ️ Command-line programs to run using the OS shell. 64 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 65 | 66 | # If the Autobuild fails above, remove it and uncomment the following three lines. 67 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 68 | 69 | # - run: | 70 | # echo "Run, Build Application using script" 71 | # ./location_of_script_within_repo/buildscript.sh 72 | 73 | - name: Perform CodeQL Analysis 74 | uses: github/codeql-action/analyze@v3 75 | with: 76 | category: "/language:${{matrix.language}}" 77 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version: 7 | description: Version to release (optional) 8 | required: false 9 | force: 10 | description: Force a release even when there are release-blockers (optional) 11 | required: false 12 | 13 | schedule: 14 | # We want the release to be at 9-10am Pacific Time 15 | # We also want it to be 1 hour before the on-prem release 16 | - cron: "0 17 15 * *" 17 | 18 | jobs: 19 | release: 20 | runs-on: ubuntu-latest 21 | name: "Release a new vroom version" 22 | steps: 23 | - name: Get auth token 24 | id: token 25 | uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 26 | with: 27 | app-id: ${{ vars.SENTRY_RELEASE_BOT_CLIENT_ID }} 28 | private-key: ${{ secrets.SENTRY_RELEASE_BOT_PRIVATE_KEY }} 29 | - uses: actions/checkout@v4 30 | with: 31 | token: ${{ steps.token.outputs.token }} 32 | fetch-depth: 0 33 | - name: Prepare release 34 | uses: getsentry/action-prepare-release@v1 35 | env: 36 | GITHUB_TOKEN: ${{ steps.token.outputs.token }} 37 | with: 38 | version: ${{ github.event.inputs.version }} 39 | force: ${{ github.event.inputs.force }} 40 | calver: true 41 | -------------------------------------------------------------------------------- /.github/workflows/validate-pipelines.yaml: -------------------------------------------------------------------------------- 1 | name: Validate Deployment Pipelines 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [main] 7 | 8 | concurrency: 9 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | files-changed: 14 | name: files-changed 15 | runs-on: ubuntu-latest 16 | # Map a step output to a job output 17 | outputs: 18 | gocd: ${{ steps.changes.outputs.gocd }} 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Check for relevant file changes 22 | uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 23 | id: changes 24 | with: 25 | filters: | 26 | gocd: 27 | - 'gocd/**' 28 | - '.github/workflows/validate-pipelines.yml' 29 | 30 | validate: 31 | if: needs.files-changed.outputs.gocd == 'true' 32 | needs: files-changed 33 | name: Validate GoCD Pipelines 34 | runs-on: ubuntu-latest 35 | 36 | # required for google auth 37 | permissions: 38 | contents: "read" 39 | id-token: "write" 40 | 41 | steps: 42 | - uses: actions/checkout@v4 43 | - id: 'auth' 44 | uses: google-github-actions/auth@v2 45 | with: 46 | workload_identity_provider: 'projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool' 47 | service_account: 'gha-gocd-api@sac-prod-sa.iam.gserviceaccount.com' 48 | token_format: 'id_token' 49 | id_token_audience: '610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com' 50 | id_token_include_email: true 51 | - uses: getsentry/action-gocd-jsonnet@v1 52 | with: 53 | jb-install: true 54 | jsonnet-dir: gocd/templates 55 | generated-dir: gocd/generated-pipelines 56 | - uses: getsentry/action-validate-gocd-pipelines@v1 57 | with: 58 | configrepo: vroom__main 59 | gocd_access_token: ${{ secrets.GOCD_ACCESS_TOKEN }} 60 | google_oidc_token: ${{ steps.auth.outputs.id_token }} 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | /vroom 8 | /downloader 9 | /issuedetection 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | /gocd/templates/vendor/ 18 | /gocd/generated-pipelines/ 19 | 20 | # IDE 21 | .vscode/ 22 | 23 | .DS_Store -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: none 4 | enable: 5 | - bodyclose 6 | - copyloopvar 7 | #- depguard 8 | - dogsled 9 | #- dupl 10 | - errcheck 11 | - forbidigo 12 | - gochecknoinits 13 | #- goconst 14 | #- gocritic 15 | - gocyclo 16 | - godot 17 | #- gosec 18 | #- gosimple 19 | #- govet 20 | - ineffassign 21 | - misspell 22 | - nakedret 23 | - prealloc 24 | - revive 25 | #- staticcheck 26 | - unconvert 27 | - unparam 28 | - unused 29 | - whitespace 30 | exclusions: 31 | generated: lax 32 | presets: 33 | - comments 34 | - common-false-positives 35 | - legacy 36 | - std-error-handling 37 | paths: 38 | - third_party$ 39 | - builtin$ 40 | - examples$ 41 | formatters: 42 | enable: 43 | - gofmt 44 | - goimports 45 | exclusions: 46 | generated: lax 47 | paths: 48 | - third_party$ 49 | - builtin$ 50 | - examples$ 51 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/TekWizely/pre-commit-golang 3 | rev: v1.0.0-rc.1 4 | hooks: 5 | - id: go-build-repo-mod 6 | - id: go-imports 7 | - id: go-mod-tidy-repo 8 | - id: golangci-lint-repo-mod 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG GOVERSION=latest 2 | FROM golang:$GOVERSION AS builder 3 | 4 | WORKDIR /src 5 | COPY . . 6 | 7 | RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o . -ldflags="-s -w -X main.release=$(git rev-parse HEAD)" ./cmd/vroom 8 | 9 | FROM debian:bookworm-slim 10 | 11 | EXPOSE 8080 12 | 13 | ARG PROFILES_DIR=/var/lib/sentry-profiles 14 | 15 | RUN apt-get update \ 16 | && apt-get install -y ca-certificates tzdata --no-install-recommends \ 17 | && apt-get clean \ 18 | && rm -rf /var/lib/apt/lists/* \ 19 | && mkdir -p $PROFILES_DIR 20 | 21 | ENV SENTRY_BUCKET_PROFILES=file://localhost/$PROFILES_DIR 22 | 23 | COPY --from=builder /src/vroom /bin/vroom 24 | 25 | WORKDIR /var/vroom 26 | 27 | ENTRYPOINT ["/bin/vroom"] 28 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # Functional Source License, Version 1.1, Apache 2.0 Future License 2 | 3 | ## Abbreviation 4 | 5 | FSL-1.1-Apache-2.0 6 | 7 | ## Notice 8 | 9 | Copyright 2022-2024 Functional Software, Inc. dba Sentry 10 | 11 | ## Terms and Conditions 12 | 13 | ### Licensor ("We") 14 | 15 | The party offering the Software under these Terms and Conditions. 16 | 17 | ### The Software 18 | 19 | The "Software" is each version of the software that we make available under 20 | these Terms and Conditions, as indicated by our inclusion of these Terms and 21 | Conditions with the Software. 22 | 23 | ### License Grant 24 | 25 | Subject to your compliance with this License Grant and the Patents, 26 | Redistribution and Trademark clauses below, we hereby grant you the right to 27 | use, copy, modify, create derivative works, publicly perform, publicly display 28 | and redistribute the Software for any Permitted Purpose identified below. 29 | 30 | ### Permitted Purpose 31 | 32 | A Permitted Purpose is any purpose other than a Competing Use. A Competing Use 33 | means making the Software available to others in a commercial product or 34 | service that: 35 | 36 | 1. substitutes for the Software; 37 | 38 | 2. substitutes for any other product or service we offer using the Software 39 | that exists as of the date we make the Software available; or 40 | 41 | 3. offers the same or substantially similar functionality as the Software. 42 | 43 | Permitted Purposes specifically include using the Software: 44 | 45 | 1. for your internal use and access; 46 | 47 | 2. for non-commercial education; 48 | 49 | 3. for non-commercial research; and 50 | 51 | 4. in connection with professional services that you provide to a licensee 52 | using the Software in accordance with these Terms and Conditions. 53 | 54 | ### Patents 55 | 56 | To the extent your use for a Permitted Purpose would necessarily infringe our 57 | patents, the license grant above includes a license under our patents. If you 58 | make a claim against any party that the Software infringes or contributes to 59 | the infringement of any patent, then your patent license to the Software ends 60 | immediately. 61 | 62 | ### Redistribution 63 | 64 | The Terms and Conditions apply to all copies, modifications and derivatives of 65 | the Software. 66 | 67 | If you redistribute any copies, modifications or derivatives of the Software, 68 | you must include a copy of or a link to these Terms and Conditions and not 69 | remove any copyright notices provided in or with the Software. 70 | 71 | ### Disclaimer 72 | 73 | THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR 74 | IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR 75 | PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. 76 | 77 | IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE 78 | SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, 79 | EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. 80 | 81 | ### Trademarks 82 | 83 | Except for displaying the License Details and identifying us as the origin of 84 | the Software, you have no right under these Terms and Conditions to use our 85 | trademarks, trade names, service marks or product names. 86 | 87 | ## Grant of Future License 88 | 89 | We hereby irrevocably grant you an additional license to use the Software under 90 | the Apache License, Version 2.0 that is effective on the second anniversary of 91 | the date we make the Software available. On or after that date, you may use the 92 | Software under the Apache License, Version 2.0, in which case the following 93 | will apply: 94 | 95 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 96 | this file except in compliance with the License. 97 | 98 | You may obtain a copy of the License at 99 | 100 | http://www.apache.org/licenses/LICENSE-2.0 101 | 102 | Unless required by applicable law or agreed to in writing, software distributed 103 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 104 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 105 | specific language governing permissions and limitations under the License. 106 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build run test issuedetection downloader python-stdlib gocd 2 | 3 | build: 4 | ./scripts/build.sh 5 | 6 | issuedetection: 7 | go build -o . -ldflags="-s -w" ./cmd/issuedetection 8 | 9 | downloader: 10 | go build -o . -ldflags="-s -w" ./cmd/downloader 11 | 12 | dev: build 13 | ./scripts/run.sh 14 | 15 | docker: 16 | ./build/package/docker/build.sh 17 | ./build/package/docker/publish.sh 18 | 19 | deploy: 20 | ./deployments/deploy.sh 21 | 22 | test: 23 | go test ./... 24 | 25 | format: 26 | gofmt -l -w -s . 27 | 28 | python-stdlib: 29 | python scripts/make_python_stdlib.py 30 | 31 | gocd: 32 | rm -rf ./gocd/generated-pipelines 33 | mkdir -p ./gocd/generated-pipelines 34 | cd ./gocd/templates && jb install && jb update 35 | 36 | # Format 37 | find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnetfmt -i 38 | # Lint 39 | find . -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet-lint -J ./gocd/templates/vendor 40 | # Build 41 | cd ./gocd/templates && find . -type f \( -name '*.jsonnet' \) -print0 | xargs -n 1 -0 jsonnet --ext-code output-files=true -J vendor -m ../generated-pipelines 42 | 43 | # Convert JSON to yaml 44 | cd ./gocd/generated-pipelines && find . -type f \( -name '*.yaml' \) -print0 | xargs -n 1 -0 yq -p json -o yaml -i 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 | 5 | 6 | Sentry 7 | 8 | 9 |

10 | 11 | # Sentry vroom 12 | 13 | [![GitHub Release](https://img.shields.io/github/release/getsentry/vroom.svg)](https://github.com/getsentry/vroom/releases/latest) 14 | 15 |

16 | vroom 17 |

18 | 19 | `vroom` is Sentry's profiling service, processing and deriving data about your profiles. It's written in Go. 20 | 21 | The name was inspired by this [video](https://www.youtube.com/watch?v=t_rzYnXEQlE). 22 | 23 | ## Development 24 | 25 | In order to develop for `vroom`, you will need: 26 | - `golang` >= 1.18 27 | - `make` 28 | - `pre-commit` 29 | 30 | ### pre-commit 31 | 32 | In order to install `pre-commit`, you will need `python` and run: 33 | ```sh 34 | pip install --user pre-commit 35 | ``` 36 | 37 | Once `pre-commit` is installed, you'll have to set up the actual git hook scripts with: 38 | ```sh 39 | pre-commit install 40 | ``` 41 | 42 | ### Build development server 43 | 44 | ```sh 45 | make dev 46 | ``` 47 | 48 | ### Run tests 49 | 50 | ```sh 51 | make test 52 | ``` 53 | 54 | ## Release Management 55 | 56 | We use GitHub actions to release new versions. `vroom` is automatically released using Calendar Versioning on a monthly basis together with sentry (see https://develop.sentry.dev/self-hosted/releases/), so there should be no reason to create a release manually. That said, manual releases are possible with the "Release" action. 57 | -------------------------------------------------------------------------------- /artwork/vroom-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getsentry/vroom/a0bbd507ab959fd6e5bd5c4423dbb80fc23ca2fd/artwork/vroom-icon.png -------------------------------------------------------------------------------- /artwork/vroom-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getsentry/vroom/a0bbd507ab959fd6e5bd5c4423dbb80fc23ca2fd/artwork/vroom-logo.png -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/docker' 3 | args: [ 4 | 'build', 5 | '-t', 'us-central1-docker.pkg.dev/sentryio/vroom/vroom:$COMMIT_SHA', 6 | '-t', 'us-central1-docker.pkg.dev/sentryio/vroom/vroom:latest', 7 | '--cache-from', 'us-central1-docker.pkg.dev/sentryio/vroom/vroom:latest', 8 | '.', 9 | ] 10 | - name: 'gcr.io/cloud-builders/docker' 11 | entrypoint: 'bash' 12 | args: 13 | - '-c' 14 | - | 15 | [ "$BRANCH_NAME" != "main" ] && exit 0 16 | docker push us-central1-docker.pkg.dev/$PROJECT_ID/vroom/vroom:latest 17 | images: [ 18 | 'us-central1-docker.pkg.dev/sentryio/vroom/vroom:$COMMIT_SHA', 19 | ] 20 | -------------------------------------------------------------------------------- /cmd/downloader/README.md: -------------------------------------------------------------------------------- 1 | # Downloader 2 | 3 | Script used to download profiles from the GCS bucket 4 | 5 | ## Prerequisites 6 | 7 | * list and read permission for the `sentryio-profiles` bucket 8 | 9 | ## List of profiles to download 10 | 11 | In order to download the profiles it's necessary to first create a list of `gs` paths of the profiles we want to download. 12 | 13 | ### How To 14 | 15 | 1. authenticate (for gsutil CLI): `gcloud auth login` 16 | 2. set the sentryio project: `gcloud config set project sentryio` 17 | 3. save gs profiles path to a file: `gsutil ls gs://sentryio-profiles/{org_id}/{project_id}/ | head -n {num_of_profiles_we_want} > profiles_list.txt` 18 | 19 | ## Download the profiles 20 | 21 | ### How To 22 | 23 | 1. obtain credentials and put them in a well known location for *Application Default Credential*: `gcloud auth application-default login` 24 | 2. create a folder where the profiles will be stored: `mkdir profiles` 25 | 3. build the downloader: `make downloader` 26 | 4. run the downloader: `./downloader ./profiles_list.txt ./profiles` -------------------------------------------------------------------------------- /cmd/downloader/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | "strings" 12 | "sync" 13 | 14 | "cloud.google.com/go/storage" 15 | ) 16 | 17 | func download( 18 | client *storage.Client, 19 | root string, 20 | objects chan string, 21 | errorsChan chan error, 22 | wg *sync.WaitGroup, 23 | ) { 24 | defer wg.Done() 25 | 26 | b := client.Bucket("sentryio-profiles") 27 | for objectName := range objects { 28 | parts := strings.Split(objectName, "/") 29 | count := len(parts) 30 | dirPath := fmt.Sprintf("%s/%s/%s", root, parts[count-3], parts[count-2]) 31 | 32 | if _, err := os.Stat(dirPath); errors.Is(err, os.ErrNotExist) { 33 | err := os.MkdirAll(dirPath, 0755) 34 | if err != nil { 35 | errorsChan <- err 36 | continue 37 | } 38 | } 39 | 40 | objectName := fmt.Sprintf("%s/%s/%s", parts[count-3], parts[count-2], parts[count-1]) 41 | fileName := fmt.Sprintf("%s.json", objectName) 42 | path := fmt.Sprintf("%s/%s", root, fileName) 43 | 44 | if _, err := os.Stat(path); err == nil { 45 | continue 46 | } 47 | 48 | f, err := os.Create(path) 49 | if err != nil { 50 | errorsChan <- err 51 | continue 52 | } 53 | 54 | ctx := context.Background() 55 | rc, err := b.Object(objectName).NewReader(ctx) 56 | if err != nil { 57 | errorsChan <- err 58 | continue 59 | } 60 | 61 | if _, err := io.Copy(f, rc); err != nil { 62 | errorsChan <- err 63 | continue 64 | } 65 | 66 | err = rc.Close() 67 | if err != nil { 68 | errorsChan <- err 69 | continue 70 | } 71 | 72 | err = f.Close() 73 | if err != nil { 74 | errorsChan <- err 75 | continue 76 | } 77 | 78 | log.Println(objectName) 79 | } 80 | } 81 | 82 | func main() { 83 | args := os.Args[1:] 84 | if len(args) != 2 { 85 | fmt.Println( // nolint 86 | "./downloader ", 87 | ) 88 | return 89 | } 90 | 91 | ctx := context.Background() 92 | storageClient, err := storage.NewClient(ctx) 93 | if err != nil { 94 | log.Fatal(err) 95 | } 96 | defer storageClient.Close() 97 | 98 | objectPathList := args[0] 99 | destination := args[1] 100 | file, err := os.Open(objectPathList) 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | defer file.Close() 105 | 106 | var wg sync.WaitGroup 107 | 108 | objects := make(chan string) 109 | errorsChan := make(chan error) 110 | for i := 0; i < 128; i++ { 111 | wg.Add(1) 112 | go download(storageClient, destination, objects, errorsChan, &wg) 113 | } 114 | 115 | go func() { 116 | for err := range errorsChan { 117 | log.Println(err) 118 | } 119 | }() 120 | 121 | scanner := bufio.NewScanner(file) 122 | for scanner.Scan() { 123 | objects <- scanner.Text() 124 | } 125 | 126 | if err := scanner.Err(); err != nil { 127 | log.Fatal(err) 128 | } 129 | 130 | close(objects) 131 | wg.Wait() 132 | close(errorsChan) 133 | } 134 | -------------------------------------------------------------------------------- /cmd/issuedetection/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "io/fs" 9 | "log" 10 | "log/slog" 11 | "os" 12 | "path/filepath" 13 | "sync" 14 | 15 | gojson "github.com/goccy/go-json" 16 | "github.com/pierrec/lz4" 17 | 18 | "github.com/getsentry/vroom/internal/occurrence" 19 | "github.com/getsentry/vroom/internal/profile" 20 | ) 21 | 22 | const ( 23 | workersCount int = 512 24 | ) 25 | 26 | func main() { 27 | debug := flag.Bool("debug", false, "activate debug logs") 28 | root := flag.String("path", ".", "path to a profile or a directory with profiles") 29 | 30 | flag.Parse() 31 | 32 | if *debug { 33 | opts := slog.HandlerOptions{ 34 | Level: slog.LevelDebug, 35 | } 36 | handler := slog.NewTextHandler(os.Stdout, &opts) 37 | slog.SetDefault(slog.New(handler)) 38 | } 39 | 40 | f, err := os.Open(*root) 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | defer f.Close() 45 | 46 | pathChannel := make(chan string, workersCount) 47 | errChannel := make(chan error) 48 | 49 | go func() { 50 | for err := range errChannel { 51 | log.Println(err) 52 | } 53 | }() 54 | 55 | var wg sync.WaitGroup 56 | 57 | for w := 0; w < workersCount; w++ { 58 | wg.Add(1) 59 | go AnalyzeProfile(pathChannel, errChannel, &wg) 60 | } 61 | 62 | err = filepath.WalkDir(*root, func(path string, d fs.DirEntry, err error) error { 63 | if err != nil { 64 | return err 65 | } 66 | if d.IsDir() { 67 | return nil 68 | } 69 | pathChannel <- path 70 | return nil 71 | }) 72 | if err != nil { 73 | log.Fatal(err) 74 | } 75 | 76 | close(pathChannel) 77 | wg.Wait() 78 | close(errChannel) 79 | } 80 | 81 | func AnalyzeProfile(pathChannel chan string, errChan chan error, wg *sync.WaitGroup) { 82 | defer wg.Done() 83 | 84 | for path := range pathChannel { 85 | f, err := os.Open(path) 86 | if err != nil { 87 | if !errors.Is(err, io.EOF) { 88 | errChan <- err 89 | } 90 | continue 91 | } 92 | zr := lz4.NewReader(f) 93 | var p profile.Profile 94 | err = gojson.NewDecoder(zr).Decode(&p) 95 | if err != nil { 96 | if !errors.Is(err, io.EOF) { 97 | errChan <- err 98 | } 99 | continue 100 | } 101 | callTrees, err := p.CallTrees() 102 | if err != nil { 103 | errChan <- err 104 | continue 105 | } 106 | for _, o := range occurrence.Find(p, callTrees) { 107 | fmt.Println( // nolint 108 | o.Event.Platform, 109 | o.Event.ProjectID, 110 | o.EvidenceData["profile_id"], 111 | o.EvidenceData["frame_duration_ns"], 112 | o.IssueTitle, 113 | o.Subtitle, 114 | ) 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /cmd/vroom/chunk_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "net/http/httptest" 10 | "os" 11 | "testing" 12 | 13 | "github.com/getsentry/vroom/internal/chunk" 14 | "github.com/getsentry/vroom/internal/frame" 15 | "github.com/getsentry/vroom/internal/platform" 16 | "github.com/getsentry/vroom/internal/profile" 17 | "github.com/getsentry/vroom/internal/storageutil" 18 | "github.com/getsentry/vroom/internal/testutil" 19 | "github.com/google/uuid" 20 | "github.com/segmentio/kafka-go" 21 | 22 | "gocloud.dev/blob" 23 | _ "gocloud.dev/blob/fileblob" 24 | ) 25 | 26 | var fileBlobBucket *blob.Bucket 27 | 28 | func TestMain(m *testing.M) { 29 | temporaryDirectory, err := os.MkdirTemp(os.TempDir(), "sentry-profiles-*") 30 | if err != nil { 31 | log.Fatalf("couldn't create a temporary directory: %s", err.Error()) 32 | } 33 | 34 | fileBlobBucket, err = blob.OpenBucket(context.Background(), "file://localhost/"+temporaryDirectory) 35 | if err != nil { 36 | log.Fatalf("couldn't open a local filesystem bucket: %s", err.Error()) 37 | } 38 | 39 | code := m.Run() 40 | 41 | if err := fileBlobBucket.Close(); err != nil { 42 | log.Printf("couldn't close the local filesystem bucket: %s", err.Error()) 43 | } 44 | 45 | err = os.RemoveAll(temporaryDirectory) 46 | if err != nil { 47 | log.Printf("couldn't remove the temporary directory: %s", err.Error()) 48 | } 49 | 50 | os.Exit(code) 51 | } 52 | 53 | func TestPostAndReadSampleChunk(t *testing.T) { 54 | profilerID := uuid.New().String() 55 | chunkID := uuid.New().String() 56 | chunkData := chunk.SampleChunk{ 57 | ID: chunkID, 58 | ProfilerID: profilerID, 59 | Environment: "dev", 60 | Platform: "python", 61 | Release: "1.2", 62 | OrganizationID: 1, 63 | ProjectID: 1, 64 | Version: "2", 65 | Profile: chunk.SampleData{ 66 | Frames: []frame.Frame{ 67 | { 68 | Function: "test", 69 | InApp: &testutil.True, 70 | Platform: platform.Python, 71 | }, 72 | }, 73 | Stacks: [][]int{ 74 | {0}, 75 | }, 76 | Samples: []chunk.Sample{ 77 | {StackID: 0, Timestamp: 1.0}, 78 | }, 79 | }, 80 | Measurements: json.RawMessage("null"), 81 | } 82 | 83 | objectName := fmt.Sprintf( 84 | "%d/%d/%s/%s", 85 | chunkData.OrganizationID, 86 | chunkData.ProjectID, 87 | chunkData.ProfilerID, 88 | chunkData.ID, 89 | ) 90 | 91 | tests := []struct { 92 | name string 93 | blobBucket *blob.Bucket 94 | objectName string 95 | }{ 96 | { 97 | name: "Filesystem", 98 | blobBucket: fileBlobBucket, 99 | objectName: objectName, 100 | }, 101 | } 102 | 103 | for _, test := range tests { 104 | t.Run(test.name, func(t *testing.T) { 105 | env := environment{ 106 | storage: test.blobBucket, 107 | profilingWriter: KafkaWriterMock{}, 108 | config: ServiceConfig{ 109 | ProfileChunksKafkaTopic: "snuba-profile-chunks", 110 | }, 111 | } 112 | jsonValue, err := json.Marshal(chunkData) 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | 117 | req := httptest.NewRequest("POST", "/", bytes.NewBuffer(jsonValue)) 118 | w := httptest.NewRecorder() 119 | 120 | // POST the chunk and check the we get a 204 response status code 121 | env.postChunk(w, req) 122 | resp := w.Result() 123 | defer resp.Body.Close() 124 | if resp.StatusCode != 204 { 125 | t.Fatalf("Expected status code 204. Found: %d", resp.StatusCode) 126 | } 127 | 128 | // read the chunk with UnmarshalCompressed and make sure that we can unmarshal 129 | // the data into the Chunk struct and that it matches the original 130 | var c chunk.Chunk 131 | err = storageutil.UnmarshalCompressed( 132 | context.Background(), 133 | test.blobBucket, 134 | objectName, 135 | &c, 136 | ) 137 | if err != nil { 138 | t.Fatal(err) 139 | } 140 | if diff := testutil.Diff(&chunkData, c.Chunk()); diff != "" { 141 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 142 | } 143 | }) 144 | } 145 | } 146 | 147 | func TestPostAndReadAndroidChunk(t *testing.T) { 148 | profilerID := uuid.New().String() 149 | chunkID := uuid.New().String() 150 | chunkData := chunk.AndroidChunk{ 151 | BuildID: "1234", 152 | ID: chunkID, 153 | ProfilerID: profilerID, 154 | DurationNS: 2_000_000, 155 | Platform: "android", 156 | Timestamp: 1.123, 157 | Profile: profile.Android{ 158 | Clock: "Dual", 159 | Events: []profile.AndroidEvent{ 160 | { 161 | Action: "Enter", 162 | ThreadID: 1, 163 | MethodID: 1, 164 | Time: profile.EventTime{ 165 | Monotonic: profile.EventMonotonic{ 166 | Wall: profile.Duration{ 167 | Secs: 0, 168 | Nanos: 1000, 169 | }, 170 | }, 171 | }, 172 | }, 173 | { 174 | Action: "Enter", 175 | ThreadID: 1, 176 | MethodID: 2, 177 | Time: profile.EventTime{ 178 | Monotonic: profile.EventMonotonic{ 179 | Wall: profile.Duration{ 180 | Secs: 0, 181 | Nanos: 1500, 182 | }, 183 | }, 184 | }, 185 | }, 186 | { 187 | Action: "Exit", 188 | ThreadID: 1, 189 | MethodID: 2, 190 | Time: profile.EventTime{ 191 | Monotonic: profile.EventMonotonic{ 192 | Wall: profile.Duration{ 193 | Secs: 0, 194 | Nanos: 2000, 195 | }, 196 | }, 197 | }, 198 | }, 199 | { 200 | Action: "Exit", 201 | ThreadID: 1, 202 | MethodID: 1, 203 | Time: profile.EventTime{ 204 | Monotonic: profile.EventMonotonic{ 205 | Wall: profile.Duration{ 206 | Secs: 0, 207 | Nanos: 2500, 208 | }, 209 | }, 210 | }, 211 | }, 212 | }, 213 | Methods: []profile.AndroidMethod{ 214 | { 215 | ClassName: "class1", 216 | ID: 1, 217 | Name: "method1", 218 | Signature: "()", 219 | }, 220 | { 221 | ClassName: "class2", 222 | ID: 2, 223 | Name: "method2", 224 | Signature: "()", 225 | }, 226 | }, 227 | StartTime: 0, 228 | Threads: []profile.AndroidThread{ 229 | { 230 | ID: 1, 231 | Name: "main", 232 | }, 233 | }, 234 | }, // end profile 235 | OrganizationID: 1, 236 | ProjectID: 1, 237 | Received: 1.123, 238 | Measurements: json.RawMessage("null"), 239 | } 240 | 241 | objectName := fmt.Sprintf( 242 | "%d/%d/%s/%s", 243 | chunkData.OrganizationID, 244 | chunkData.ProjectID, 245 | chunkData.ProfilerID, 246 | chunkData.ID, 247 | ) 248 | 249 | tests := []struct { 250 | name string 251 | blobBucket *blob.Bucket 252 | objectName string 253 | }{ 254 | { 255 | name: "Filesystem", 256 | blobBucket: fileBlobBucket, 257 | objectName: objectName, 258 | }, 259 | } 260 | 261 | for _, test := range tests { 262 | t.Run(test.name, func(t *testing.T) { 263 | env := environment{ 264 | storage: test.blobBucket, 265 | profilingWriter: KafkaWriterMock{}, 266 | config: ServiceConfig{ 267 | ProfileChunksKafkaTopic: "snuba-profile-chunks", 268 | }, 269 | } 270 | jsonValue, err := json.Marshal(chunkData) 271 | if err != nil { 272 | t.Fatal(err) 273 | } 274 | 275 | req := httptest.NewRequest("POST", "/", bytes.NewBuffer(jsonValue)) 276 | w := httptest.NewRecorder() 277 | 278 | // POST the chunk and check the we get a 204 response status code 279 | env.postChunk(w, req) 280 | resp := w.Result() 281 | defer resp.Body.Close() 282 | if resp.StatusCode != 204 { 283 | t.Fatalf("Expected status code 204. Found: %d", resp.StatusCode) 284 | } 285 | 286 | // read the chunk with UnmarshalCompressed and make sure that we can unmarshal 287 | // the data into the Chunk struct and that it matches the original 288 | var c chunk.Chunk 289 | err = storageutil.UnmarshalCompressed( 290 | context.Background(), 291 | test.blobBucket, 292 | objectName, 293 | &c, 294 | ) 295 | if err != nil { 296 | t.Fatal(err) 297 | } 298 | if diff := testutil.Diff(&chunkData, c.Chunk()); diff != "" { 299 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 300 | } 301 | }) 302 | } 303 | } 304 | 305 | type KafkaWriterMock struct{} 306 | 307 | func (k KafkaWriterMock) WriteMessages(_ context.Context, _ ...kafka.Message) error { 308 | return nil 309 | } 310 | 311 | func (k KafkaWriterMock) Close() error { 312 | return nil 313 | } 314 | -------------------------------------------------------------------------------- /cmd/vroom/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type ( 4 | ServiceConfig struct { 5 | Environment string `env:"SENTRY_ENVIRONMENT" env-default:"development"` 6 | Port int `env:"PORT" env-default:"8085"` 7 | WorkerPoolSize int `env:"WORKER_POOL_SIZE" env-default:"10"` 8 | 9 | SentryDSN string `env:"SENTRY_DSN"` 10 | 11 | KafkaSaslMechanism string `env:"SENTRY_KAFKA_SASL_MECHANISM"` 12 | KafkaSaslUsername string `env:"SENTRY_KAFKA_SASL_USERNAME"` 13 | KafkaSaslPassword string `env:"SENTRY_KAFKA_SASL_PASSWORD"` 14 | KafkaSslCaPath string `env:"SENTRY_KAFKA_SSL_CA_PATH"` 15 | KafkaSslCertPath string `env:"SENTRY_KAFKA_SSL_CERT_PATH"` 16 | KafkaSslKeyPath string `env:"SENTRY_KAFKA_SSL_KEY_PATH"` 17 | 18 | OccurrencesKafkaBrokers []string `env:"SENTRY_KAFKA_BROKERS_OCCURRENCES" env-default:"localhost:9092"` 19 | ProfilingKafkaBrokers []string `env:"SENTRY_KAFKA_BROKERS_PROFILING" env-default:"localhost:9092"` 20 | SpansKafkaBrokers []string `env:"SENTRY_KAFKA_BROKERS_SPANS" env-default:"localhost:9092"` 21 | 22 | CallTreesKafkaTopic string `env:"SENTRY_KAFKA_TOPIC_CALL_TREES" env-default:"profiles-call-tree"` 23 | OccurrencesKafkaTopic string `env:"SENTRY_KAFKA_TOPIC_OCCURRENCES" env-default:"ingest-occurrences"` 24 | ProfileChunksKafkaTopic string `env:"SENTRY_KAFKA_TOPIC_PROFILE_CHUNKS" env-default:"snuba-profile-chunks"` 25 | ProfilesKafkaTopic string `env:"SENTRY_KAKFA_TOPIC_PROFILES" env-default:"processed-profiles"` 26 | 27 | BucketURL string `env:"SENTRY_BUCKET_PROFILES" env-default:"file://./test/gcs/sentry-profiles"` 28 | } 29 | ) 30 | -------------------------------------------------------------------------------- /cmd/vroom/flamegraph.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net/http" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/getsentry/sentry-go" 11 | "github.com/julienschmidt/httprouter" 12 | 13 | "github.com/getsentry/vroom/internal/flamegraph" 14 | "github.com/getsentry/vroom/internal/metrics" 15 | "github.com/getsentry/vroom/internal/utils" 16 | ) 17 | 18 | type ( 19 | postFlamegraphBody struct { 20 | Transaction []utils.TransactionProfileCandidate `json:"transaction"` 21 | Continuous []utils.ContinuousProfileCandidate `json:"continuous"` 22 | GenerateMetrics bool `json:"generate_metrics"` 23 | } 24 | ) 25 | 26 | func (env *environment) postFlamegraph(w http.ResponseWriter, r *http.Request) { 27 | ctx := r.Context() 28 | downloadContext, cancel := context.WithTimeout(ctx, time.Second*10) 29 | defer cancel() 30 | hub := sentry.GetHubFromContext(ctx) 31 | ps := httprouter.ParamsFromContext(ctx) 32 | rawOrganizationID := ps.ByName("organization_id") 33 | organizationID, err := strconv.ParseUint(rawOrganizationID, 10, 64) 34 | if err != nil { 35 | if hub != nil { 36 | hub.CaptureException(err) 37 | } 38 | w.WriteHeader(http.StatusBadRequest) 39 | return 40 | } 41 | 42 | hub.Scope().SetTag("organization_id", rawOrganizationID) 43 | 44 | var body postFlamegraphBody 45 | s := sentry.StartSpan(ctx, "processing") 46 | s.Description = "Decoding data" 47 | err = json.NewDecoder(r.Body).Decode(&body) 48 | s.Finish() 49 | if err != nil { 50 | if hub != nil { 51 | hub.CaptureException(err) 52 | } 53 | w.WriteHeader(http.StatusBadRequest) 54 | return 55 | } 56 | 57 | s = sentry.StartSpan(ctx, "processing") 58 | var ma *metrics.Aggregator 59 | if body.GenerateMetrics { 60 | agg := metrics.NewAggregator(maxUniqueFunctionsPerProfile, 5, minDepth) 61 | ma = &agg 62 | } 63 | speedscope, err := flamegraph.GetFlamegraphFromCandidates( 64 | downloadContext, 65 | env.storage, 66 | organizationID, 67 | body.Transaction, 68 | body.Continuous, 69 | readJobs, 70 | ma, 71 | s, 72 | ) 73 | s.Finish() 74 | if err != nil { 75 | if hub != nil { 76 | hub.CaptureException(err) 77 | } 78 | w.WriteHeader(http.StatusInternalServerError) 79 | return 80 | } 81 | 82 | s = sentry.StartSpan(ctx, "json.marshal") 83 | defer s.Finish() 84 | b, err := json.Marshal(speedscope) 85 | if err != nil { 86 | if hub != nil { 87 | hub.CaptureException(err) 88 | } 89 | w.WriteHeader(http.StatusInternalServerError) 90 | return 91 | } 92 | 93 | w.Header().Set("Content-Type", "application/json") 94 | w.WriteHeader(http.StatusOK) 95 | _, _ = w.Write(b) 96 | } 97 | -------------------------------------------------------------------------------- /cmd/vroom/kafka.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/getsentry/vroom/internal/chunk" 7 | "github.com/getsentry/vroom/internal/nodetree" 8 | "github.com/getsentry/vroom/internal/platform" 9 | "github.com/getsentry/vroom/internal/profile" 10 | "github.com/segmentio/kafka-go" 11 | ) 12 | 13 | type ( 14 | // FunctionsKafkaMessage is representing the struct we send to Kafka to insert functions in ClickHouse. 15 | FunctionsKafkaMessage struct { 16 | Environment string `json:"environment,omitempty"` 17 | Functions []nodetree.CallTreeFunction `json:"functions"` 18 | ID string `json:"profile_id"` 19 | Platform platform.Platform `json:"platform"` 20 | ProjectID uint64 `json:"project_id"` 21 | Received int64 `json:"received"` 22 | Release string `json:"release,omitempty"` 23 | RetentionDays int `json:"retention_days"` 24 | Timestamp int64 `json:"timestamp"` 25 | TransactionName string `json:"transaction_name"` 26 | StartTimestamp float64 `json:"start_timestamp,omitempty"` 27 | EndTimestamp float64 `json:"end_timestamp,omitempty"` 28 | ProfilingType string `json:"profiling_type,omitempty"` 29 | MaterializationVersion uint8 `json:"materialization_version"` 30 | } 31 | 32 | // ProfileKafkaMessage is representing the struct we send to Kafka to insert a profile in ClickHouse. 33 | ProfileKafkaMessage struct { 34 | AndroidAPILevel uint32 `json:"android_api_level,omitempty"` 35 | Architecture string `json:"architecture,omitempty"` 36 | DeviceClassification string `json:"device_classification,omitempty"` 37 | DeviceLocale string `json:"device_locale"` 38 | DeviceManufacturer string `json:"device_manufacturer"` 39 | DeviceModel string `json:"device_model"` 40 | DeviceOSBuildNumber string `json:"device_os_build_number,omitempty"` 41 | DeviceOSName string `json:"device_os_name"` 42 | DeviceOSVersion string `json:"device_os_version"` 43 | DurationNS uint64 `json:"duration_ns"` 44 | Environment string `json:"environment,omitempty"` 45 | ID string `json:"profile_id"` 46 | OrganizationID uint64 `json:"organization_id"` 47 | Platform platform.Platform `json:"platform"` 48 | ProjectID uint64 `json:"project_id"` 49 | Received int64 `json:"received"` 50 | RetentionDays int `json:"retention_days"` 51 | SDKName string `json:"sdk_name,omitempty"` 52 | SDKVersion string `json:"sdk_version,omitempty"` 53 | TraceID string `json:"trace_id"` 54 | TransactionID string `json:"transaction_id"` 55 | TransactionName string `json:"transaction_name"` 56 | VersionCode string `json:"version_code"` 57 | VersionName string `json:"version_name"` 58 | } 59 | ) 60 | 61 | func buildFunctionsKafkaMessage(p profile.Profile, functions []nodetree.CallTreeFunction) FunctionsKafkaMessage { 62 | return FunctionsKafkaMessage{ 63 | Environment: p.Environment(), 64 | Functions: functions, 65 | ID: p.ID(), 66 | Platform: p.Platform(), 67 | ProjectID: p.ProjectID(), 68 | Received: p.Received().Unix(), 69 | Release: p.Release(), 70 | RetentionDays: p.RetentionDays(), 71 | Timestamp: p.Timestamp().Unix(), 72 | TransactionName: p.Transaction().Name, 73 | MaterializationVersion: 1, 74 | } 75 | } 76 | 77 | func buildChunkFunctionsKafkaMessage(c *chunk.Chunk, functions []nodetree.CallTreeFunction) FunctionsKafkaMessage { 78 | return FunctionsKafkaMessage{ 79 | Environment: c.GetEnvironment(), 80 | Functions: functions, 81 | ID: c.GetProfilerID(), 82 | Platform: c.GetPlatform(), 83 | ProjectID: c.GetProjectID(), 84 | Received: int64(c.GetReceived()), 85 | Release: c.GetRelease(), 86 | RetentionDays: c.GetRetentionDays(), 87 | Timestamp: int64(c.StartTimestamp()), 88 | StartTimestamp: c.StartTimestamp(), 89 | EndTimestamp: c.EndTimestamp(), 90 | ProfilingType: "continuous", 91 | MaterializationVersion: 1, 92 | } 93 | } 94 | 95 | func buildProfileKafkaMessage(p profile.Profile) ProfileKafkaMessage { 96 | t := p.Transaction() 97 | m := p.Metadata() 98 | return ProfileKafkaMessage{ 99 | AndroidAPILevel: m.AndroidAPILevel, 100 | Architecture: m.Architecture, 101 | DeviceClassification: m.DeviceClassification, 102 | DeviceLocale: m.DeviceLocale, 103 | DeviceManufacturer: m.DeviceManufacturer, 104 | DeviceModel: m.DeviceModel, 105 | DeviceOSBuildNumber: m.DeviceOSBuildNumber, 106 | DeviceOSName: m.DeviceOSName, 107 | DeviceOSVersion: m.DeviceOSVersion, 108 | DurationNS: p.DurationNS(), 109 | Environment: p.Environment(), 110 | ID: p.ID(), 111 | OrganizationID: p.OrganizationID(), 112 | Platform: p.Platform(), 113 | ProjectID: p.ProjectID(), 114 | Received: p.Received().Unix(), 115 | RetentionDays: p.RetentionDays(), 116 | SDKName: m.SDKName, 117 | SDKVersion: m.SDKVersion, 118 | TraceID: t.TraceID, 119 | TransactionID: t.ID, 120 | TransactionName: t.Name, 121 | VersionCode: m.VersionCode, 122 | VersionName: m.VersionName, 123 | } 124 | } 125 | 126 | type KafkaWriter interface { 127 | WriteMessages(ctx context.Context, msgs ...kafka.Message) error 128 | Close() error 129 | } 130 | -------------------------------------------------------------------------------- /cmd/vroom/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "log/slog" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/CAFxX/httpcompression" 15 | "github.com/getsentry/sentry-go" 16 | sentryhttp "github.com/getsentry/sentry-go/http" 17 | "github.com/ilyakaznacheev/cleanenv" 18 | "github.com/julienschmidt/httprouter" 19 | "github.com/segmentio/kafka-go" 20 | "gocloud.dev/blob" 21 | _ "gocloud.dev/blob/azureblob" 22 | _ "gocloud.dev/blob/fileblob" 23 | _ "gocloud.dev/blob/gcsblob" 24 | _ "gocloud.dev/blob/s3blob" 25 | "gocloud.dev/gcerrors" 26 | 27 | "github.com/getsentry/vroom/internal/httputil" 28 | "github.com/getsentry/vroom/internal/logutil" 29 | "github.com/getsentry/vroom/internal/storageutil" 30 | ) 31 | 32 | type environment struct { 33 | config ServiceConfig 34 | 35 | occurrencesWriter KafkaWriter 36 | profilingWriter KafkaWriter 37 | 38 | storage *blob.Bucket 39 | } 40 | 41 | var ( 42 | release string 43 | readJobs chan storageutil.ReadJob 44 | ) 45 | 46 | const ( 47 | KiB int64 = 1024 48 | MiB = 1024 * KiB 49 | ) 50 | 51 | func newEnvironment() (*environment, error) { 52 | var e environment 53 | err := cleanenv.ReadEnv(&e.config) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | ctx := context.Background() 59 | e.storage, err = blob.OpenBucket(ctx, e.config.BucketURL) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | e.occurrencesWriter = &kafka.Writer{ 65 | Addr: kafka.TCP(e.config.OccurrencesKafkaBrokers...), 66 | Async: true, 67 | Balancer: kafka.CRC32Balancer{}, 68 | BatchSize: 100, 69 | ReadTimeout: 3 * time.Second, 70 | Topic: e.config.OccurrencesKafkaTopic, 71 | WriteTimeout: 3 * time.Second, 72 | Transport: createKafkaRoundTripper(e.config), 73 | } 74 | e.profilingWriter = &kafka.Writer{ 75 | Addr: kafka.TCP(e.config.ProfilingKafkaBrokers...), 76 | Async: true, 77 | Balancer: kafka.CRC32Balancer{}, 78 | BatchBytes: 20 * MiB, 79 | BatchSize: 10, 80 | Compression: kafka.Lz4, 81 | ReadTimeout: 3 * time.Second, 82 | WriteTimeout: 3 * time.Second, 83 | Transport: createKafkaRoundTripper(e.config), 84 | } 85 | return &e, nil 86 | } 87 | 88 | func (e *environment) shutdown() { 89 | err := e.storage.Close() 90 | if err != nil { 91 | sentry.CaptureException(err) 92 | } 93 | err = e.occurrencesWriter.Close() 94 | if err != nil { 95 | sentry.CaptureException(err) 96 | } 97 | err = e.profilingWriter.Close() 98 | if err != nil { 99 | sentry.CaptureException(err) 100 | } 101 | sentry.Flush(5 * time.Second) 102 | } 103 | 104 | func (e *environment) newRouter() (*httprouter.Router, error) { 105 | compress, err := httpcompression.DefaultAdapter() 106 | if err != nil { 107 | return nil, err 108 | } 109 | 110 | routes := []struct { 111 | method string 112 | path string 113 | handler http.HandlerFunc 114 | }{ 115 | { 116 | http.MethodGet, 117 | "/organizations/:organization_id/projects/:project_id/profiles/:profile_id", 118 | e.getProfile, 119 | }, 120 | { 121 | http.MethodGet, 122 | "/organizations/:organization_id/projects/:project_id/raw_profiles/:profile_id", 123 | e.getRawProfile, 124 | }, 125 | { 126 | http.MethodGet, 127 | "/organizations/:organization_id/projects/:project_id/raw_chunks/:profiler_id/:chunk_id", 128 | e.getRawChunk, 129 | }, 130 | { 131 | http.MethodPost, 132 | "/organizations/:organization_id/projects/:project_id/chunks", 133 | e.postProfileFromChunkIDs, 134 | }, 135 | { 136 | http.MethodPost, 137 | "/organizations/:organization_id/flamegraph", 138 | e.postFlamegraph, 139 | }, 140 | { 141 | http.MethodPost, 142 | "/organizations/:organization_id/metrics", 143 | e.postMetrics, 144 | }, 145 | {http.MethodGet, "/health", e.getHealth}, 146 | {http.MethodPost, "/chunk", e.postChunk}, 147 | {http.MethodPost, "/profile", e.postProfile}, 148 | {http.MethodPost, "/regressed", e.postRegressed}, 149 | } 150 | 151 | router := httprouter.New() 152 | 153 | for _, route := range routes { 154 | handlerFunc := httputil.AnonymizeTransactionName(route.handler) 155 | handlerFunc = httputil.DecompressPayload(handlerFunc) 156 | handler := compress(handlerFunc) 157 | 158 | router.Handler(route.method, route.path, handler) 159 | } 160 | 161 | return router, nil 162 | } 163 | 164 | func main() { 165 | logutil.ConfigureLogger() 166 | 167 | env, err := newEnvironment() 168 | if err != nil { 169 | log.Fatal("error setting up environment", err) 170 | } 171 | 172 | err = sentry.Init(sentry.ClientOptions{ 173 | Dsn: env.config.SentryDSN, 174 | EnableTracing: true, 175 | TracesSampleRate: 1.0, 176 | Environment: env.config.Environment, 177 | Release: release, 178 | BeforeSendTransaction: httputil.SetHTTPStatusCodeTag, 179 | BeforeSend: func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { 180 | code := gcerrors.Code(hint.OriginalException) 181 | switch code { 182 | // Ignore unknown or network errors as gcerrors returns a specific GCS error 183 | // in case we have generic network errors, even if it didn't come from the gocloud 184 | // library and we can't check for the specific gocloud error type as it's in 185 | // an internal package. 186 | case gcerrors.Canceled, gcerrors.DeadlineExceeded, gcerrors.Unknown, gcerrors.OK: 187 | default: 188 | event.Fingerprint = []string{"{{ default }}", code.String()} 189 | } 190 | return event 191 | }, 192 | }) 193 | if err != nil { 194 | log.Fatal("can't initialize sentry", err) 195 | } 196 | 197 | router, err := env.newRouter() 198 | if err != nil { 199 | sentry.CaptureException(err) 200 | log.Fatal("error setting up the router", err) 201 | } 202 | 203 | server := http.Server{ 204 | Addr: fmt.Sprintf(":%d", env.config.Port), 205 | ReadHeaderTimeout: time.Second, 206 | Handler: sentryhttp.New(sentryhttp.Options{}).Handle(router), 207 | } 208 | 209 | waitForShutdown := make(chan os.Signal) 210 | go func() { 211 | c := make(chan os.Signal, 1) 212 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 213 | <-c 214 | 215 | cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 216 | defer cancel() 217 | 218 | if err := server.Shutdown(cctx); err != nil { 219 | sentry.CaptureException(err) 220 | slog.Error("error shutting down server", "err", err) 221 | } 222 | 223 | close(waitForShutdown) 224 | }() 225 | 226 | slog.Info("vroom started") 227 | 228 | readJobs = make(chan storageutil.ReadJob, 10*env.config.WorkerPoolSize) 229 | for i := 0; i < env.config.WorkerPoolSize; i++ { 230 | go storageutil.ReadWorker(readJobs) 231 | } 232 | 233 | err = server.ListenAndServe() 234 | if err != nil && err != http.ErrServerClosed { 235 | sentry.CaptureException(err) 236 | slog.Error("server failed", "err", err) 237 | } 238 | 239 | <-waitForShutdown 240 | 241 | // Shutdown the rest of the environment after the HTTP connections are closed 242 | close(readJobs) 243 | env.shutdown() 244 | slog.Info("vroom graceful shutdown") 245 | } 246 | 247 | func (e *environment) getHealth(w http.ResponseWriter, _ *http.Request) { 248 | if _, err := os.Stat("/tmp/vroom.down"); err != nil { 249 | w.WriteHeader(http.StatusOK) 250 | } else { 251 | w.WriteHeader(http.StatusBadGateway) 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /cmd/vroom/metrics.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/getsentry/sentry-go" 9 | "github.com/julienschmidt/httprouter" 10 | 11 | "github.com/getsentry/vroom/internal/metrics" 12 | "github.com/getsentry/vroom/internal/utils" 13 | ) 14 | 15 | type ( 16 | postMetricsRequestBody struct { 17 | Transaction []utils.TransactionProfileCandidate `json:"transaction"` 18 | Continuous []utils.ContinuousProfileCandidate `json:"continuous"` 19 | } 20 | 21 | postMetricsResponse struct { 22 | FunctionsMetrics []utils.FunctionMetrics `json:"functions_metrics"` 23 | } 24 | ) 25 | 26 | func (env *environment) postMetrics(w http.ResponseWriter, r *http.Request) { 27 | ctx := r.Context() 28 | hub := sentry.GetHubFromContext(ctx) 29 | ps := httprouter.ParamsFromContext(ctx) 30 | rawOrganizationID := ps.ByName("organization_id") 31 | organizationID, err := strconv.ParseUint(rawOrganizationID, 10, 64) 32 | if err != nil { 33 | if hub != nil { 34 | hub.CaptureException(err) 35 | } 36 | w.WriteHeader(http.StatusBadRequest) 37 | return 38 | } 39 | 40 | hub.Scope().SetTag("organization_id", rawOrganizationID) 41 | 42 | var body postMetricsRequestBody 43 | s := sentry.StartSpan(ctx, "processing") 44 | s.Description = "Decoding data" 45 | err = json.NewDecoder(r.Body).Decode(&body) 46 | s.Finish() 47 | if err != nil { 48 | if hub != nil { 49 | hub.CaptureException(err) 50 | } 51 | w.WriteHeader(http.StatusBadRequest) 52 | return 53 | } 54 | 55 | s = sentry.StartSpan(ctx, "processing") 56 | ma := metrics.NewAggregator(maxUniqueFunctionsPerProfile, 5, minDepth) 57 | functionsMetrics, err := ma.GetMetricsFromCandidates( 58 | ctx, 59 | env.storage, 60 | organizationID, 61 | body.Transaction, 62 | body.Continuous, 63 | readJobs, 64 | ) 65 | s.Finish() 66 | if err != nil { 67 | if hub != nil { 68 | hub.CaptureException(err) 69 | } 70 | w.WriteHeader(http.StatusInternalServerError) 71 | return 72 | } 73 | 74 | s = sentry.StartSpan(ctx, "json.marshal") 75 | defer s.Finish() 76 | b, err := json.Marshal(postMetricsResponse{ 77 | FunctionsMetrics: functionsMetrics, 78 | }) 79 | if err != nil { 80 | if hub != nil { 81 | hub.CaptureException(err) 82 | } 83 | w.WriteHeader(http.StatusInternalServerError) 84 | return 85 | } 86 | 87 | w.Header().Set("Content-Type", "application/json") 88 | w.WriteHeader(http.StatusOK) 89 | _, _ = w.Write(b) 90 | } 91 | -------------------------------------------------------------------------------- /cmd/vroom/regressed.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net/http" 7 | 8 | "github.com/getsentry/sentry-go" 9 | "github.com/getsentry/vroom/internal/occurrence" 10 | ) 11 | 12 | func (env *environment) postRegressed(w http.ResponseWriter, r *http.Request) { 13 | ctx := r.Context() 14 | hub := sentry.GetHubFromContext(ctx) 15 | 16 | regressedFunctions, err := decodeRegressedFunctionPayload(ctx, r) 17 | if err != nil { 18 | hub.CaptureException(err) 19 | w.WriteHeader(http.StatusBadRequest) 20 | return 21 | } 22 | 23 | emitted := []occurrence.RegressedFunction{} 24 | occurrences := []*occurrence.Occurrence{} 25 | for _, regressedFunction := range regressedFunctions { 26 | s := sentry.StartSpan(ctx, "processing") 27 | s.Description = "Generating occurrence for payload" 28 | occurrence, err := occurrence.ProcessRegressedFunction(ctx, env.storage, regressedFunction, readJobs) 29 | s.Finish() 30 | if err != nil { 31 | hub.CaptureException(err) 32 | continue 33 | } else if occurrence == nil { 34 | continue 35 | } 36 | emitted = append(emitted, regressedFunction) 37 | occurrences = append(occurrences, occurrence) 38 | } 39 | 40 | s := sentry.StartSpan(ctx, "json.marshal") 41 | data := struct { 42 | Occurrences int `json:"occurrences"` 43 | Emitted []occurrence.RegressedFunction `json:"emitted"` 44 | }{Occurrences: len(occurrences), Emitted: emitted} 45 | b, err := json.Marshal(data) 46 | s.Finish() 47 | if err != nil { 48 | hub.CaptureException(err) 49 | w.WriteHeader(http.StatusInternalServerError) 50 | return 51 | } 52 | 53 | occurrenceMessages, err := occurrence.GenerateKafkaMessageBatch(occurrences) 54 | if err != nil { 55 | hub.CaptureException(err) 56 | w.WriteHeader(http.StatusInternalServerError) 57 | return 58 | } 59 | 60 | s = sentry.StartSpan(ctx, "processing") 61 | s.Description = "Send occurrences to Kafka" 62 | err = env.occurrencesWriter.WriteMessages(ctx, occurrenceMessages...) 63 | s.Finish() 64 | if err != nil { 65 | hub.CaptureException(err) 66 | w.WriteHeader(http.StatusInternalServerError) 67 | return 68 | } 69 | 70 | w.Header().Set("Content-Type", "application/json") 71 | w.WriteHeader(http.StatusOK) 72 | _, _ = w.Write(b) 73 | } 74 | 75 | func decodeRegressedFunctionPayload(ctx context.Context, r *http.Request) ([]occurrence.RegressedFunction, error) { 76 | s := sentry.StartSpan(ctx, "processing") 77 | s.Description = "Decoding payload" 78 | defer s.Finish() 79 | 80 | var regressedFunctions []occurrence.RegressedFunction 81 | err := json.NewDecoder(r.Body).Decode(®ressedFunctions) 82 | return regressedFunctions, err 83 | } 84 | -------------------------------------------------------------------------------- /cmd/vroom/utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "log" 7 | "net" 8 | "os" 9 | "strings" 10 | "time" 11 | 12 | "github.com/segmentio/kafka-go" 13 | "github.com/segmentio/kafka-go/sasl" 14 | "github.com/segmentio/kafka-go/sasl/plain" 15 | "github.com/segmentio/kafka-go/sasl/scram" 16 | ) 17 | 18 | type MetricSummary struct { 19 | Min float64 20 | Max float64 21 | Sum float64 22 | Count uint64 23 | } 24 | 25 | func createKafkaRoundTripper(e ServiceConfig) kafka.RoundTripper { 26 | var saslMechanism sasl.Mechanism 27 | var tlsConfig *tls.Config 28 | 29 | switch strings.ToUpper(e.KafkaSaslMechanism) { 30 | case "PLAIN": 31 | saslMechanism = plain.Mechanism{ 32 | Username: e.KafkaSaslUsername, 33 | Password: e.KafkaSaslPassword, 34 | } 35 | case "SCRAM-SHA-256": 36 | mechanism, err := scram.Mechanism(scram.SHA256, e.KafkaSaslUsername, e.KafkaSaslPassword) 37 | if err != nil { 38 | log.Fatal("unable to create scram-sha-256 mechanism", err) 39 | return nil 40 | } 41 | 42 | saslMechanism = mechanism 43 | case "SCRAM-SHA-512": 44 | mechanism, err := scram.Mechanism(scram.SHA512, e.KafkaSaslUsername, e.KafkaSaslPassword) 45 | if err != nil { 46 | log.Fatal("unable to create scram-sha-512 mechanism", err) 47 | return nil 48 | } 49 | 50 | saslMechanism = mechanism 51 | } 52 | 53 | if e.KafkaSslCertPath != "" && e.KafkaSslKeyPath != "" { 54 | certs, err := tls.LoadX509KeyPair(e.KafkaSslCertPath, e.KafkaSslKeyPath) 55 | if err != nil { 56 | log.Fatal("unable to load certificate key pair", err) 57 | return nil 58 | } 59 | 60 | caCertificatePool, err := x509.SystemCertPool() 61 | if err != nil { 62 | caCertificatePool = x509.NewCertPool() 63 | } 64 | if e.KafkaSslCaPath != "" { 65 | caFile, err := os.ReadFile(e.KafkaSslCaPath) 66 | if err != nil { 67 | log.Fatal("unable to read ca file", err) 68 | return nil 69 | } 70 | 71 | if ok := caCertificatePool.AppendCertsFromPEM(caFile); !ok { 72 | log.Fatal("unable to append ca certificate to pool") 73 | return nil 74 | } 75 | } 76 | 77 | tlsConfig = &tls.Config{ 78 | RootCAs: caCertificatePool, 79 | Certificates: []tls.Certificate{certs}, 80 | } 81 | } 82 | 83 | return &kafka.Transport{ 84 | SASL: saslMechanism, 85 | TLS: tlsConfig, 86 | Dial: (&net.Dialer{ 87 | Timeout: 3 * time.Second, 88 | DualStack: true, 89 | }).DialContext, 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /devservices/config.yml: -------------------------------------------------------------------------------- 1 | x-sentry-service-config: 2 | version: 0.1 3 | service_name: vroom 4 | dependencies: 5 | kafka: 6 | description: Shared instance of kafka used by sentry services 7 | remote: 8 | repo_name: sentry-shared-kafka 9 | branch: main 10 | repo_link: git@github.com:getsentry/sentry-shared-kafka.git 11 | vroom: 12 | description: Sentry's profiling service, processing and deriving data about your profiles 13 | modes: 14 | default: [kafka, vroom] 15 | services: 16 | vroom: 17 | image: us-central1-docker.pkg.dev/sentryio/vroom/vroom:latest 18 | ports: 19 | - 127.0.0.1:8085:8085 20 | environment: 21 | SENTRY_KAFKA_BROKERS_PROFILING: kafka-kafka-1:9092 22 | SENTRY_KAFKA_BROKERS_OCCURRENCES: kafka-kafka-1:9092 23 | SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles 24 | SENTRY_SNUBA_HOST: http://127.0.0.1:1218 25 | volumes: 26 | - sentry-vroom:/var/lib/sentry-profiles 27 | networks: 28 | - devservices 29 | extra_hosts: 30 | host.docker.internal: host-gateway 31 | labels: 32 | - orchestrator=devservices 33 | restart: unless-stopped 34 | 35 | volumes: 36 | sentry-vroom: 37 | 38 | networks: 39 | devservices: 40 | name: devservices 41 | external: true 42 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/getsentry/vroom 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.7 6 | 7 | require ( 8 | cloud.google.com/go/storage v1.29.0 9 | github.com/CAFxX/httpcompression v0.0.8 10 | github.com/andybalholm/brotli v1.1.0 11 | github.com/fsouza/fake-gcs-server v1.44.0 12 | github.com/getsentry/sentry-go v0.31.0 13 | github.com/goccy/go-json v0.10.0 14 | github.com/google/go-cmp v0.5.9 15 | github.com/google/uuid v1.6.0 16 | github.com/ilyakaznacheev/cleanenv v1.4.2 17 | github.com/json-iterator/go v1.1.12 18 | github.com/julienschmidt/httprouter v1.3.0 19 | github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 20 | github.com/pierrec/lz4 v2.6.1+incompatible 21 | github.com/pierrec/lz4/v4 v4.1.15 22 | github.com/segmentio/kafka-go v0.4.38 23 | gocloud.dev v0.29.0 24 | google.golang.org/api v0.114.0 25 | ) 26 | 27 | require ( 28 | cloud.google.com/go v0.110.0 // indirect 29 | cloud.google.com/go/compute v1.19.1 // indirect 30 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 31 | cloud.google.com/go/iam v0.13.0 // indirect 32 | cloud.google.com/go/pubsub v1.30.0 // indirect 33 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect 34 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect 35 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect 36 | github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect 37 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 38 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 39 | github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect 40 | github.com/BurntSushi/toml v1.2.1 // indirect 41 | github.com/aws/aws-sdk-go v1.44.200 // indirect 42 | github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect 43 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect 44 | github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect 45 | github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect 46 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect 47 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.51 // indirect 48 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect 49 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect 50 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect 51 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect 52 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect 53 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect 54 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect 55 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect 56 | github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2 // indirect 57 | github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect 58 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect 59 | github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect 60 | github.com/aws/smithy-go v1.13.5 // indirect 61 | github.com/felixge/httpsnoop v1.0.3 // indirect 62 | github.com/frankban/quicktest v1.14.4 // indirect 63 | github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 64 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 65 | github.com/golang/protobuf v1.5.3 // indirect 66 | github.com/google/renameio/v2 v2.0.0 // indirect 67 | github.com/google/wire v0.5.0 // indirect 68 | github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect 69 | github.com/googleapis/gax-go/v2 v2.7.1 // indirect 70 | github.com/gorilla/handlers v1.5.1 // indirect 71 | github.com/gorilla/mux v1.8.0 // indirect 72 | github.com/jmespath/go-jmespath v0.4.0 // indirect 73 | github.com/joho/godotenv v1.4.0 // indirect 74 | github.com/klauspost/compress v1.17.7 // indirect 75 | github.com/kylelemons/godebug v1.1.0 // indirect 76 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 77 | github.com/modern-go/reflect2 v1.0.2 // indirect 78 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect 79 | github.com/pkg/xattr v0.4.9 // indirect 80 | github.com/sirupsen/logrus v1.9.0 // indirect 81 | github.com/xdg/scram v1.0.5 // indirect 82 | github.com/xdg/stringprep v1.0.3 // indirect 83 | go.opencensus.io v0.24.0 // indirect 84 | golang.org/x/crypto v0.36.0 // indirect 85 | golang.org/x/net v0.38.0 // indirect 86 | golang.org/x/oauth2 v0.7.0 // indirect 87 | golang.org/x/sync v0.12.0 // indirect 88 | golang.org/x/sys v0.31.0 // indirect 89 | golang.org/x/text v0.23.0 // indirect 90 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect 91 | google.golang.org/appengine v1.6.7 // indirect 92 | google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect 93 | google.golang.org/grpc v1.56.3 // indirect 94 | google.golang.org/protobuf v1.33.0 // indirect 95 | gopkg.in/yaml.v3 v3.0.1 // indirect 96 | olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect 97 | ) 98 | -------------------------------------------------------------------------------- /gocd/templates/bash/check-cloudbuild.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /devinfra/scripts/checks/googlecloud/check_cloudbuild.py \ 4 | sentryio \ 5 | vroom \ 6 | build-vroom \ 7 | ${GO_REVISION_VROOM_REPO} \ 8 | main 9 | -------------------------------------------------------------------------------- /gocd/templates/bash/check-github.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /devinfra/scripts/checks/githubactions/checkruns.py \ 4 | getsentry/vroom \ 5 | ${GO_REVISION_VROOM_REPO} \ 6 | test-vroom 7 | -------------------------------------------------------------------------------- /gocd/templates/bash/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval $(/devinfra/scripts/regions/project_env_vars.py --region="${SENTRY_REGION}") 4 | 5 | /devinfra/scripts/k8s/k8stunnel 6 | /devinfra/scripts/k8s/k8s-deploy.py \ 7 | --label-selector="${LABEL_SELECTOR}" \ 8 | --image="us-central1-docker.pkg.dev/sentryio/vroom/vroom:${GO_REVISION_VROOM_REPO}" \ 9 | --container-name="vroom" 10 | -------------------------------------------------------------------------------- /gocd/templates/bash/wait-canary.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | /devinfra/scripts/canary/canarychecks.py \ 4 | --skip-check=${SKIP_CANARY_CHECKS} \ 5 | --wait-minutes=5 6 | -------------------------------------------------------------------------------- /gocd/templates/jsonnetfile.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "dependencies": [ 4 | { 5 | "source": { 6 | "git": { 7 | "remote": "https://github.com/getsentry/gocd-jsonnet.git", 8 | "subdir": "libs" 9 | } 10 | }, 11 | "version": "v2.13.0" 12 | } 13 | ], 14 | "legacyImports": true 15 | } 16 | -------------------------------------------------------------------------------- /gocd/templates/jsonnetfile.lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "dependencies": [ 4 | { 5 | "source": { 6 | "git": { 7 | "remote": "https://github.com/getsentry/gocd-jsonnet.git", 8 | "subdir": "libs" 9 | } 10 | }, 11 | "version": "6ddc943ae87444b48e16995639dfe89f33a0f444", 12 | "sum": "NH9U5jQ8oCSPXLuBw27OqAaPLBUDqMGHvRLxfo84hNQ=" 13 | } 14 | ], 15 | "legacyImports": false 16 | } 17 | -------------------------------------------------------------------------------- /gocd/templates/pipelines/vroom.libsonnet: -------------------------------------------------------------------------------- 1 | local gocdtasks = import 'github.com/getsentry/gocd-jsonnet/libs/gocd-tasks.libsonnet'; 2 | 3 | local deploy_canary_stage(region) = 4 | if region == 'us' then 5 | [ 6 | { 7 | 'deploy-canary': { 8 | fetch_materials: true, 9 | jobs: { 10 | deploy: { 11 | timeout: 600, 12 | elastic_profile_id: 'vroom', 13 | environment_variables: { 14 | LABEL_SELECTOR: 'service=vroom,environment=production,env=canary', 15 | WAIT_MINUTES: '5', 16 | }, 17 | tasks: [ 18 | gocdtasks.script(importstr '../bash/deploy.sh'), 19 | gocdtasks.script(importstr '../bash/wait-canary.sh'), 20 | ], 21 | }, 22 | }, 23 | }, 24 | }, 25 | ] 26 | else 27 | []; 28 | 29 | function(region) { 30 | environment_variables: { 31 | GITHUB_TOKEN: '{{SECRET:[devinfra-github][token]}}', 32 | SENTRY_REGION: region, 33 | SKIP_CANARY_CHECKS: false, 34 | }, 35 | materials: { 36 | vroom_repo: { 37 | git: 'git@github.com:getsentry/vroom.git', 38 | shallow_clone: true, 39 | branch: 'main', 40 | destination: 'vroom', 41 | }, 42 | }, 43 | lock_behavior: 'unlockWhenFinished', 44 | stages: [ 45 | { 46 | checks: { 47 | fetch_materials: true, 48 | jobs: { 49 | deploy: { 50 | timeout: 600, 51 | elastic_profile_id: 'vroom', 52 | tasks: [ 53 | gocdtasks.script(importstr '../bash/check-github.sh'), 54 | gocdtasks.script(importstr '../bash/check-cloudbuild.sh'), 55 | ], 56 | }, 57 | }, 58 | }, 59 | }, 60 | ] + deploy_canary_stage(region) + [ 61 | { 62 | 'deploy-primary': { 63 | fetch_materials: true, 64 | jobs: { 65 | deploy: { 66 | timeout: 600, 67 | elastic_profile_id: 'vroom', 68 | environment_variables: { 69 | LABEL_SELECTOR: 'service=vroom,environment=production', 70 | }, 71 | tasks: [ 72 | gocdtasks.script(importstr '../bash/deploy.sh'), 73 | ], 74 | }, 75 | }, 76 | }, 77 | }, 78 | ], 79 | } 80 | -------------------------------------------------------------------------------- /gocd/templates/vroom.jsonnet: -------------------------------------------------------------------------------- 1 | local vroom = import './pipelines/vroom.libsonnet'; 2 | local pipedream = import 'github.com/getsentry/gocd-jsonnet/libs/pipedream.libsonnet'; 3 | 4 | local pipedream_config = { 5 | name: 'vroom', 6 | materials: { 7 | vroom_repo: { 8 | git: 'git@github.com:getsentry/vroom.git', 9 | shallow_clone: true, 10 | branch: 'main', 11 | destination: 'vroom', 12 | }, 13 | }, 14 | rollback: { 15 | material_name: 'vroom_repo', 16 | stage: 'deploy-primary', 17 | elastic_profile_id: 'vroom', 18 | }, 19 | }; 20 | 21 | pipedream.render(pipedream_config, vroom) 22 | -------------------------------------------------------------------------------- /internal/android/display.go: -------------------------------------------------------------------------------- 1 | package android 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func StripPackageNameFromFullMethodName(s, p string) string { 8 | return strings.TrimPrefix(s, p+".") 9 | } 10 | -------------------------------------------------------------------------------- /internal/chunk/android.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "encoding/json" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/getsentry/vroom/internal/clientsdk" 9 | "github.com/getsentry/vroom/internal/debugmeta" 10 | "github.com/getsentry/vroom/internal/frame" 11 | "github.com/getsentry/vroom/internal/nodetree" 12 | "github.com/getsentry/vroom/internal/platform" 13 | "github.com/getsentry/vroom/internal/profile" 14 | "github.com/getsentry/vroom/internal/utils" 15 | ) 16 | 17 | type ( 18 | AndroidChunk struct { 19 | BuildID string `json:"build_id,omitempty"` 20 | ID string `json:"chunk_id"` 21 | ProfilerID string `json:"profiler_id"` 22 | 23 | DebugMeta debugmeta.DebugMeta `json:"debug_meta"` 24 | 25 | ClientSDK clientsdk.ClientSDK `json:"client_sdk"` 26 | DurationNS uint64 `json:"duration_ns"` 27 | Environment string `json:"environment"` 28 | Platform platform.Platform `json:"platform"` 29 | Release string `json:"release"` 30 | Timestamp float64 `json:"timestamp"` 31 | 32 | Profile profile.Android `json:"profile"` 33 | Measurements json.RawMessage `json:"measurements"` 34 | 35 | OrganizationID uint64 `json:"organization_id"` 36 | ProjectID uint64 `json:"project_id"` 37 | Received float64 `json:"received"` 38 | RetentionDays int `json:"retention_days"` 39 | 40 | Options utils.Options `json:"options,omitempty"` 41 | } 42 | ) 43 | 44 | func (c AndroidChunk) StoragePath() string { 45 | return StoragePath( 46 | c.OrganizationID, 47 | c.ProjectID, 48 | c.ProfilerID, 49 | c.ID, 50 | ) 51 | } 52 | 53 | func (c AndroidChunk) DurationMS() uint64 { 54 | return uint64(time.Duration(c.DurationNS).Milliseconds()) 55 | } 56 | 57 | func (c AndroidChunk) CallTrees(_ *string) (map[string][]*nodetree.Node, error) { 58 | c.Profile.SdkStartTime = uint64(c.StartTimestamp() * 1e9) 59 | callTrees := c.Profile.CallTrees() 60 | stringThreadCallTrees := make(map[string][]*nodetree.Node) 61 | for tid, callTree := range callTrees { 62 | threadID := strconv.FormatUint(tid, 10) 63 | stringThreadCallTrees[threadID] = callTree 64 | } 65 | return stringThreadCallTrees, nil 66 | } 67 | 68 | func (c AndroidChunk) SDKName() string { 69 | return c.ClientSDK.Name 70 | } 71 | 72 | func (c AndroidChunk) SDKVersion() string { 73 | return c.ClientSDK.Version 74 | } 75 | 76 | func (c AndroidChunk) EndTimestamp() float64 { 77 | return c.Timestamp + float64(c.DurationNS)*1e-9 78 | } 79 | 80 | func (c AndroidChunk) GetEnvironment() string { 81 | return c.Environment 82 | } 83 | 84 | func (c AndroidChunk) GetID() string { 85 | return c.ID 86 | } 87 | 88 | func (c AndroidChunk) GetPlatform() platform.Platform { 89 | return c.Platform 90 | } 91 | 92 | func (c AndroidChunk) GetProfilerID() string { 93 | return c.ProfilerID 94 | } 95 | 96 | func (c AndroidChunk) GetProjectID() uint64 { 97 | return c.ProjectID 98 | } 99 | 100 | func (c AndroidChunk) GetReceived() float64 { 101 | return c.Received 102 | } 103 | 104 | func (c AndroidChunk) GetRelease() string { 105 | return c.Release 106 | } 107 | 108 | func (c AndroidChunk) GetRetentionDays() int { 109 | return c.RetentionDays 110 | } 111 | 112 | func (c AndroidChunk) StartTimestamp() float64 { 113 | return c.Timestamp 114 | } 115 | 116 | func (c AndroidChunk) GetOrganizationID() uint64 { 117 | return c.OrganizationID 118 | } 119 | 120 | func (c AndroidChunk) GetOptions() utils.Options { 121 | return c.Options 122 | } 123 | 124 | func (c AndroidChunk) GetFrameWithFingerprint(target uint32) (frame.Frame, error) { 125 | for _, m := range c.Profile.Methods { 126 | f := m.Frame() 127 | if f.Fingerprint() == target { 128 | return f, nil 129 | } 130 | } 131 | return frame.Frame{}, frame.ErrFrameNotFound 132 | } 133 | 134 | func (c *AndroidChunk) Normalize() { 135 | } 136 | -------------------------------------------------------------------------------- /internal/chunk/android_utils.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "encoding/json" 5 | "sort" 6 | "time" 7 | 8 | "github.com/getsentry/vroom/internal/measurements" 9 | "github.com/getsentry/vroom/internal/profile" 10 | "github.com/getsentry/vroom/internal/speedscope" 11 | ) 12 | 13 | var member void 14 | 15 | type void struct{} 16 | 17 | func SpeedscopeFromAndroidChunks(chunks []AndroidChunk, startTS, endTS uint64) (speedscope.Output, error) { 18 | if len(chunks) == 0 { 19 | return speedscope.Output{}, nil 20 | } 21 | maxTsNS := uint64(0) 22 | threadSet := make(map[uint64]void) 23 | // fingerprint to method ID 24 | methodToID := make(map[uint32]uint64) 25 | sort.Slice(chunks, func(i, j int) bool { 26 | return chunks[i].EndTimestamp() <= chunks[j].StartTimestamp() 27 | }) 28 | 29 | mergedMeasurement := make(map[string]measurements.MeasurementV2) 30 | 31 | chunk := chunks[0] 32 | firstChunkStartTimestampNS := uint64(chunk.StartTimestamp() * 1e9) 33 | // Initially, adjustedChunkStartTimestampNS will just be the 34 | // chunk timestamp. If the chunk starts before the allowed 35 | // time range though, we only keep events that fall within 36 | // the range, set the startTimestamp to the start value of 37 | // the allowed range and adjust the relative ts of each 38 | // events. 39 | adjustedChunkStartTimestampNS := firstChunkStartTimestampNS 40 | buildTimestamp := chunk.Profile.TimestampGetter() 41 | // clean up the events in the first chunk 42 | events := make([]profile.AndroidEvent, 0, len(chunk.Profile.Events)) 43 | methods := make([]profile.AndroidMethod, 0, len(chunk.Profile.Methods)) 44 | // updates methods ID 45 | tmpMethodsID := make(map[uint64]uint64) 46 | for i, method := range chunk.Profile.Methods { 47 | id := uint64(i + 1) 48 | tmpMethodsID[method.ID] = id 49 | method.ID = id 50 | methodToID[method.Frame().Fingerprint()] = id 51 | methods = append(methods, method) 52 | } 53 | delta := int64(0) 54 | if firstChunkStartTimestampNS < startTS { 55 | delta = -int64(startTS - firstChunkStartTimestampNS) 56 | adjustedChunkStartTimestampNS = startTS 57 | } 58 | addTimeDelta := chunk.Profile.AddTimeDelta(delta) 59 | for _, event := range chunk.Profile.Events { 60 | ts := buildTimestamp(event.Time) + firstChunkStartTimestampNS 61 | if ts < startTS || ts > endTS { 62 | // we filter out events out of range 63 | continue 64 | } 65 | // If the event falls within allowed range, but the first chunk 66 | // begins before the start range (delta != 0), adjust the relative ts 67 | // of each event by subtracting the delta. 68 | if delta != 0 { 69 | err := addTimeDelta(&event) 70 | if err != nil { 71 | return speedscope.Output{}, err 72 | } 73 | // update ts 74 | ts = buildTimestamp(event.Time) + adjustedChunkStartTimestampNS 75 | } 76 | event.MethodID = tmpMethodsID[event.MethodID] 77 | events = append(events, event) 78 | maxTsNS = max(maxTsNS, ts) 79 | } 80 | for _, thread := range chunk.Profile.Threads { 81 | threadSet[thread.ID] = member 82 | } 83 | if len(chunk.Measurements) > 0 { 84 | err := json.Unmarshal(chunk.Measurements, &mergedMeasurement) 85 | if err != nil { 86 | return speedscope.Output{}, err 87 | } 88 | } 89 | 90 | // If chunk started before the allowed time range 91 | // update the chunk timestamp (firstChunkStartTimestampNS) 92 | // since later on, other chunks will use this to compute 93 | // the right offset (relative ts in nanoseconds). 94 | if delta != 0 { 95 | firstChunkStartTimestampNS = adjustedChunkStartTimestampNS 96 | } 97 | 98 | for i := 1; i < len(chunks); i++ { 99 | c := chunks[i] 100 | chunkStartTimestampNs := uint64(c.StartTimestamp() * 1e9) 101 | buildTimestamp := c.Profile.TimestampGetter() 102 | // Delta between the current chunk timestamp and the very first one. 103 | // This will be needed to correctly offset the events relative ts, 104 | // which need to be relative not to the start of this chunk, but to 105 | // the start of the very first one. 106 | delta := chunkStartTimestampNs - firstChunkStartTimestampNS 107 | addTimeDelta := c.Profile.AddTimeDelta(int64(delta)) 108 | // updates methods ID 109 | tmpMethodsID = make(map[uint64]uint64) 110 | for _, method := range c.Profile.Methods { 111 | fingerprint := method.Frame().Fingerprint() 112 | if id, ok := methodToID[fingerprint]; !ok { 113 | newID := uint64(len(methodToID) + 1) 114 | methodToID[fingerprint] = newID 115 | tmpMethodsID[method.ID] = newID 116 | method.ID = newID 117 | methods = append(methods, method) 118 | } else { 119 | tmpMethodsID[method.ID] = id 120 | } 121 | } 122 | 123 | // filter events 124 | for _, event := range c.Profile.Events { 125 | ts := buildTimestamp(event.Time) + chunkStartTimestampNs 126 | if ts < startTS || ts > endTS { 127 | continue 128 | } 129 | event.MethodID = tmpMethodsID[event.MethodID] 130 | // Before adding the event, update its relative timestamp 131 | // which, in this case, should not be relative to the current 132 | // chunk timestamp, but rather relative to the very 1st one. 133 | err := addTimeDelta(&event) 134 | if err != nil { 135 | return speedscope.Output{}, err 136 | } 137 | ts = buildTimestamp(event.Time) + firstChunkStartTimestampNS 138 | events = append(events, event) 139 | maxTsNS = max(maxTsNS, ts) 140 | } 141 | // Update threads. 142 | for _, thread := range c.Profile.Threads { 143 | if _, ok := threadSet[thread.ID]; !ok { 144 | chunk.Profile.Threads = append(c.Profile.Threads, thread) 145 | threadSet[thread.ID] = member 146 | } 147 | } 148 | // In case we have measurements, merge them too. 149 | if len(c.Measurements) > 0 { 150 | var chunkMeasurements map[string]measurements.MeasurementV2 151 | err := json.Unmarshal(c.Measurements, &chunkMeasurements) 152 | if err != nil { 153 | return speedscope.Output{}, err 154 | } 155 | for k, measurement := range chunkMeasurements { 156 | if el, ok := mergedMeasurement[k]; ok { 157 | el.Values = append(el.Values, measurement.Values...) 158 | mergedMeasurement[k] = el 159 | } else { 160 | mergedMeasurement[k] = measurement 161 | } 162 | } 163 | } 164 | } 165 | chunk.Profile.Events = events 166 | chunk.Profile.Methods = methods 167 | chunk.DurationNS = maxTsNS - startTS 168 | 169 | s, err := chunk.Profile.Speedscope() 170 | if err != nil { 171 | return speedscope.Output{}, err 172 | } 173 | s.DurationNS = chunk.DurationNS 174 | s.Metadata.Timestamp = time.Unix(0, int64(firstChunkStartTimestampNS)).UTC() 175 | s.ChunkID = chunk.ID 176 | s.Platform = chunk.Platform 177 | 178 | if len(mergedMeasurement) > 0 { 179 | s.Measurements = mergedMeasurement 180 | } 181 | 182 | return s, nil 183 | } 184 | -------------------------------------------------------------------------------- /internal/chunk/android_utils_test.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/getsentry/vroom/internal/platform" 8 | "github.com/getsentry/vroom/internal/profile" 9 | "github.com/getsentry/vroom/internal/speedscope" 10 | "github.com/getsentry/vroom/internal/testutil" 11 | ) 12 | 13 | var androidChunk1 = AndroidChunk{ 14 | Timestamp: 0.0, 15 | DurationNS: 1500, 16 | ID: "1a009sd87", 17 | Platform: platform.Android, 18 | Profile: profile.Android{ 19 | Clock: "Dual", 20 | Events: []profile.AndroidEvent{ 21 | { 22 | Action: "Enter", 23 | ThreadID: 1, 24 | MethodID: 1, 25 | Time: profile.EventTime{ 26 | Monotonic: profile.EventMonotonic{ 27 | Wall: profile.Duration{ 28 | Secs: 0, 29 | Nanos: 1000, 30 | }, 31 | }, 32 | }, 33 | }, 34 | { 35 | Action: "Enter", 36 | ThreadID: 1, 37 | MethodID: 2, 38 | Time: profile.EventTime{ 39 | Monotonic: profile.EventMonotonic{ 40 | Wall: profile.Duration{ 41 | Secs: 0, 42 | Nanos: 1500, 43 | }, 44 | }, 45 | }, 46 | }, 47 | { 48 | Action: "Enter", 49 | ThreadID: 1, 50 | MethodID: 3, 51 | Time: profile.EventTime{ 52 | Monotonic: profile.EventMonotonic{ 53 | Wall: profile.Duration{ 54 | Secs: 0, 55 | Nanos: 2000, 56 | }, 57 | }, 58 | }, 59 | }, 60 | { 61 | Action: "Exit", 62 | ThreadID: 1, 63 | MethodID: 3, 64 | Time: profile.EventTime{ 65 | Monotonic: profile.EventMonotonic{ 66 | Wall: profile.Duration{ 67 | Secs: 0, 68 | Nanos: 2500, 69 | }, 70 | }, 71 | }, 72 | }, 73 | }, 74 | Methods: []profile.AndroidMethod{ 75 | { 76 | ClassName: "class1", 77 | ID: 1, 78 | Name: "method1", 79 | Signature: "()", 80 | }, 81 | { 82 | ClassName: "class2", 83 | ID: 2, 84 | Name: "method2", 85 | Signature: "()", 86 | }, 87 | { 88 | ClassName: "class3", 89 | ID: 3, 90 | Name: "method3", 91 | Signature: "()", 92 | }, 93 | }, 94 | StartTime: 0, 95 | Threads: []profile.AndroidThread{ 96 | { 97 | ID: 1, 98 | Name: "main", 99 | }, 100 | }, 101 | }, 102 | } 103 | 104 | var androidChunk2 = AndroidChunk{ 105 | Timestamp: 2.5e-6, 106 | DurationNS: 2000, 107 | ID: "ee3409d8", 108 | Platform: platform.Android, 109 | Profile: profile.Android{ 110 | Clock: "Dual", 111 | Events: []profile.AndroidEvent{ 112 | { 113 | Action: "Enter", 114 | ThreadID: 1, 115 | MethodID: 1, 116 | Time: profile.EventTime{ 117 | Monotonic: profile.EventMonotonic{ 118 | Wall: profile.Duration{ 119 | Secs: 0, 120 | Nanos: 500, 121 | }, 122 | }, 123 | }, 124 | }, 125 | { 126 | Action: "Exit", 127 | ThreadID: 1, 128 | MethodID: 1, 129 | Time: profile.EventTime{ 130 | Monotonic: profile.EventMonotonic{ 131 | Wall: profile.Duration{ 132 | Secs: 0, 133 | Nanos: 1000, 134 | }, 135 | }, 136 | }, 137 | }, 138 | { 139 | Action: "Exit", 140 | ThreadID: 1, 141 | MethodID: 3, 142 | Time: profile.EventTime{ 143 | Monotonic: profile.EventMonotonic{ 144 | Wall: profile.Duration{ 145 | Secs: 0, 146 | Nanos: 1500, 147 | }, 148 | }, 149 | }, 150 | }, 151 | { 152 | Action: "Exit", 153 | ThreadID: 1, 154 | MethodID: 2, 155 | Time: profile.EventTime{ 156 | Monotonic: profile.EventMonotonic{ 157 | Wall: profile.Duration{ 158 | Secs: 0, 159 | Nanos: 2000, 160 | }, 161 | }, 162 | }, 163 | }, 164 | }, 165 | Methods: []profile.AndroidMethod{ 166 | { 167 | ClassName: "class4", 168 | ID: 1, 169 | Name: "method4", 170 | Signature: "()", 171 | }, 172 | { 173 | ClassName: "class1", 174 | ID: 2, 175 | Name: "method1", 176 | Signature: "()", 177 | }, 178 | { 179 | ClassName: "class2", 180 | ID: 3, 181 | Name: "method2", 182 | Signature: "()", 183 | }, 184 | }, 185 | StartTime: 0, 186 | Threads: []profile.AndroidThread{ 187 | { 188 | ID: 1, 189 | Name: "main", 190 | }, 191 | }, 192 | }, 193 | } 194 | 195 | func TestSpeedscopeFromAndroidChunks(t *testing.T) { 196 | tests := []struct { 197 | name string 198 | have []AndroidChunk 199 | want speedscope.Output 200 | start uint64 201 | end uint64 202 | }{ 203 | { 204 | name: "All chunks included in the time range", 205 | have: []AndroidChunk{androidChunk1, androidChunk2}, 206 | want: speedscope.Output{ 207 | AndroidClock: "Dual", 208 | DurationNS: 4500, 209 | ChunkID: "1a009sd87", 210 | Platform: platform.Android, 211 | Profiles: []any{ 212 | &speedscope.EventedProfile{ 213 | EndValue: 4500, 214 | Events: []speedscope.Event{ 215 | { 216 | Type: "O", 217 | Frame: 0, 218 | At: 1000, 219 | }, 220 | { 221 | Type: "O", 222 | Frame: 1, 223 | At: 1500, 224 | }, 225 | { 226 | Type: "O", 227 | Frame: 2, 228 | At: 2000, 229 | }, 230 | { 231 | Type: "C", 232 | Frame: 2, 233 | At: 2500, 234 | }, 235 | { 236 | Type: "O", 237 | Frame: 3, 238 | At: 3000, 239 | }, 240 | { 241 | Type: "C", 242 | Frame: 3, 243 | At: 3500, 244 | }, 245 | { 246 | Type: "C", 247 | Frame: 1, 248 | At: 4000, 249 | }, 250 | { 251 | Type: "C", 252 | Frame: 0, 253 | At: 4500, 254 | }, 255 | }, 256 | Name: "main", 257 | StartValue: 1000, 258 | ThreadID: 1, 259 | Type: "evented", 260 | Unit: "nanoseconds", 261 | }, 262 | }, 263 | Shared: speedscope.SharedData{ 264 | Frames: []speedscope.Frame{ 265 | {Image: "class1", IsApplication: true, Name: "class1.method1()"}, 266 | {Image: "class2", IsApplication: true, Name: "class2.method2()"}, 267 | {Image: "class3", IsApplication: true, Name: "class3.method3()"}, 268 | {Image: "class4", IsApplication: true, Name: "class4.method4()"}, 269 | }, 270 | }, 271 | Metadata: speedscope.ProfileMetadata{ 272 | ProfileView: speedscope.ProfileView{ 273 | Timestamp: time.Unix(0, 0).UTC(), 274 | }, 275 | }, 276 | }, 277 | start: 0, 278 | end: 6000, 279 | }, 280 | { 281 | name: "First chunk begins before allowed range (overlap )", 282 | have: []AndroidChunk{androidChunk1, androidChunk2}, 283 | want: speedscope.Output{ 284 | AndroidClock: "Dual", 285 | DurationNS: 3000, 286 | ChunkID: "1a009sd87", 287 | Platform: platform.Android, 288 | Profiles: []any{ 289 | &speedscope.EventedProfile{ 290 | EndValue: 3000, 291 | Events: []speedscope.Event{ 292 | { 293 | Type: "O", 294 | Frame: 1, 295 | At: 0, 296 | }, 297 | { 298 | Type: "O", 299 | Frame: 2, 300 | At: 500, 301 | }, 302 | { 303 | Type: "C", 304 | Frame: 2, 305 | At: 1000, 306 | }, 307 | { 308 | Type: "O", 309 | Frame: 3, 310 | At: 1500, 311 | }, 312 | { 313 | Type: "C", 314 | Frame: 3, 315 | At: 2000, 316 | }, 317 | { 318 | Type: "C", 319 | Frame: 1, 320 | At: 2500, 321 | }, 322 | }, 323 | Name: "main", 324 | StartValue: 0, 325 | ThreadID: 1, 326 | Type: "evented", 327 | Unit: "nanoseconds", 328 | }, 329 | }, 330 | Shared: speedscope.SharedData{ 331 | Frames: []speedscope.Frame{ 332 | {Image: "class1", IsApplication: true, Name: "class1.method1()"}, 333 | {Image: "class2", IsApplication: true, Name: "class2.method2()"}, 334 | {Image: "class3", IsApplication: true, Name: "class3.method3()"}, 335 | {Image: "class4", IsApplication: true, Name: "class4.method4()"}, 336 | }, 337 | }, 338 | Metadata: speedscope.ProfileMetadata{ 339 | ProfileView: speedscope.ProfileView{ 340 | Timestamp: time.Unix(0, 1500).UTC(), 341 | }, 342 | }, 343 | }, 344 | start: 1500, 345 | end: 6000, 346 | }, 347 | } 348 | 349 | for _, test := range tests { 350 | t.Run(test.name, func(t *testing.T) { 351 | s, err := SpeedscopeFromAndroidChunks(test.have, test.start, test.end) 352 | if err != nil { 353 | t.Fatal(err) 354 | } 355 | if diff := testutil.Diff(s, test.want); diff != "" { 356 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 357 | } 358 | }) 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /internal/chunk/chunk.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/getsentry/vroom/internal/frame" 8 | "github.com/getsentry/vroom/internal/nodetree" 9 | "github.com/getsentry/vroom/internal/platform" 10 | "github.com/getsentry/vroom/internal/utils" 11 | ) 12 | 13 | type ( 14 | chunkInterface interface { 15 | GetEnvironment() string 16 | GetID() string 17 | GetOrganizationID() uint64 18 | GetPlatform() platform.Platform 19 | GetProfilerID() string 20 | GetProjectID() uint64 21 | GetReceived() float64 22 | GetRelease() string 23 | GetRetentionDays() int 24 | GetOptions() utils.Options 25 | GetFrameWithFingerprint(uint32) (frame.Frame, error) 26 | CallTrees(activeThreadID *string) (map[string][]*nodetree.Node, error) 27 | 28 | DurationMS() uint64 29 | EndTimestamp() float64 30 | SDKName() string 31 | SDKVersion() string 32 | StartTimestamp() float64 33 | StoragePath() string 34 | 35 | Normalize() 36 | } 37 | 38 | Chunk struct { 39 | chunk chunkInterface 40 | } 41 | ) 42 | 43 | func New(c chunkInterface) Chunk { 44 | return Chunk{ 45 | chunk: c, 46 | } 47 | } 48 | 49 | type version struct { 50 | Version string `json:"version"` 51 | } 52 | 53 | func (c *Chunk) UnmarshalJSON(b []byte) error { 54 | var v version 55 | err := json.Unmarshal(b, &v) 56 | if err != nil { 57 | return err 58 | } 59 | switch v.Version { 60 | case "": 61 | c.chunk = new(AndroidChunk) 62 | default: 63 | c.chunk = new(SampleChunk) 64 | } 65 | return json.Unmarshal(b, &c.chunk) 66 | } 67 | 68 | func (c Chunk) MarshalJSON() ([]byte, error) { 69 | return json.Marshal(c.chunk) 70 | } 71 | 72 | func (c Chunk) Chunk() chunkInterface { 73 | return c.chunk 74 | } 75 | 76 | func StoragePath(OrganizationID uint64, ProjectID uint64, ProfilerID string, ID string) string { 77 | return fmt.Sprintf( 78 | "%d/%d/%s/%s", 79 | OrganizationID, 80 | ProjectID, 81 | ProfilerID, 82 | ID, 83 | ) 84 | } 85 | 86 | func (c Chunk) GetEnvironment() string { 87 | return c.chunk.GetEnvironment() 88 | } 89 | 90 | func (c Chunk) GetID() string { 91 | return c.chunk.GetID() 92 | } 93 | 94 | func (c Chunk) GetOrganizationID() uint64 { 95 | return c.chunk.GetOrganizationID() 96 | } 97 | 98 | func (c Chunk) GetPlatform() platform.Platform { 99 | return c.chunk.GetPlatform() 100 | } 101 | 102 | func (c Chunk) GetProfilerID() string { 103 | return c.chunk.GetProfilerID() 104 | } 105 | 106 | func (c Chunk) GetProjectID() uint64 { 107 | return c.chunk.GetProjectID() 108 | } 109 | 110 | func (c Chunk) GetReceived() float64 { 111 | return c.chunk.GetReceived() 112 | } 113 | 114 | func (c Chunk) GetRelease() string { 115 | return c.chunk.GetRelease() 116 | } 117 | 118 | func (c Chunk) GetRetentionDays() int { 119 | return c.chunk.GetRetentionDays() 120 | } 121 | 122 | func (c Chunk) GetOptions() utils.Options { 123 | return c.chunk.GetOptions() 124 | } 125 | 126 | func (c Chunk) GetFrameWithFingerprint(f uint32) (frame.Frame, error) { 127 | return c.chunk.GetFrameWithFingerprint(f) 128 | } 129 | 130 | func (c Chunk) CallTrees(activeThreadID *string) (map[string][]*nodetree.Node, error) { 131 | return c.chunk.CallTrees(activeThreadID) 132 | } 133 | 134 | func (c Chunk) DurationMS() uint64 { 135 | return c.chunk.DurationMS() 136 | } 137 | func (c Chunk) EndTimestamp() float64 { 138 | return c.chunk.EndTimestamp() 139 | } 140 | func (c Chunk) SDKName() string { 141 | return c.chunk.SDKName() 142 | } 143 | func (c Chunk) SDKVersion() string { 144 | return c.chunk.SDKVersion() 145 | } 146 | func (c Chunk) StartTimestamp() float64 { 147 | return c.chunk.StartTimestamp() 148 | } 149 | func (c Chunk) StoragePath() string { 150 | return c.chunk.StoragePath() 151 | } 152 | 153 | func (c *Chunk) Normalize() { 154 | c.chunk.Normalize() 155 | } 156 | -------------------------------------------------------------------------------- /internal/chunk/sample.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "hash/fnv" 7 | "math" 8 | "sort" 9 | 10 | "github.com/getsentry/vroom/internal/clientsdk" 11 | "github.com/getsentry/vroom/internal/debugmeta" 12 | "github.com/getsentry/vroom/internal/frame" 13 | "github.com/getsentry/vroom/internal/nodetree" 14 | "github.com/getsentry/vroom/internal/platform" 15 | "github.com/getsentry/vroom/internal/sample" 16 | "github.com/getsentry/vroom/internal/utils" 17 | ) 18 | 19 | var ( 20 | ErrInvalidStackID = errors.New("profile contains invalid stack id") 21 | ErrInvalidFrameID = errors.New("profile contains invalid frame id") 22 | ) 23 | 24 | type ( 25 | // Chunk is an implementation of the Sample V2 format. 26 | SampleChunk struct { 27 | ID string `json:"chunk_id"` 28 | ProfilerID string `json:"profiler_id"` 29 | 30 | DebugMeta debugmeta.DebugMeta `json:"debug_meta"` 31 | 32 | ClientSDK clientsdk.ClientSDK `json:"client_sdk"` 33 | Environment string `json:"environment"` 34 | Platform platform.Platform `json:"platform"` 35 | Release string `json:"release"` 36 | 37 | Version string `json:"version"` 38 | 39 | Profile SampleData `json:"profile"` 40 | 41 | OrganizationID uint64 `json:"organization_id"` 42 | ProjectID uint64 `json:"project_id"` 43 | Received float64 `json:"received"` 44 | RetentionDays int `json:"retention_days"` 45 | 46 | Measurements json.RawMessage `json:"measurements"` 47 | 48 | Options utils.Options `json:"options,omitempty"` 49 | } 50 | 51 | SampleData struct { 52 | Frames []frame.Frame `json:"frames"` 53 | Samples []Sample `json:"samples"` 54 | Stacks [][]int `json:"stacks"` 55 | ThreadMetadata map[string]sample.ThreadMetadata `json:"thread_metadata"` 56 | } 57 | 58 | Sample struct { 59 | StackID int `json:"stack_id"` 60 | ThreadID string `json:"thread_id"` 61 | Timestamp float64 `json:"timestamp"` 62 | } 63 | ) 64 | 65 | func (c SampleChunk) StoragePath() string { 66 | return StoragePath( 67 | c.OrganizationID, 68 | c.ProjectID, 69 | c.ProfilerID, 70 | c.ID, 71 | ) 72 | } 73 | 74 | func (c *SampleChunk) Normalize() { 75 | for i := range c.Profile.Frames { 76 | f := c.Profile.Frames[i] 77 | f.Normalize(c.Platform) 78 | c.Profile.Frames[i] = f 79 | } 80 | 81 | if c.Platform == platform.Python { 82 | c.Profile.trimPythonStacks() 83 | } 84 | } 85 | 86 | // CallTrees generates call trees from samples. 87 | func (c SampleChunk) CallTrees(activeThreadID *string) (map[string][]*nodetree.Node, error) { 88 | sort.SliceStable(c.Profile.Samples, func(i, j int) bool { 89 | return c.Profile.Samples[i].Timestamp < c.Profile.Samples[j].Timestamp 90 | }) 91 | 92 | treesByThreadID := make(map[string][]*nodetree.Node) 93 | samplesByThreadID := make(map[string][]Sample) 94 | 95 | for _, s := range c.Profile.Samples { 96 | samplesByThreadID[s.ThreadID] = append(samplesByThreadID[s.ThreadID], s) 97 | } 98 | 99 | var current *nodetree.Node 100 | h := fnv.New64() 101 | for _, samples := range samplesByThreadID { 102 | // The last sample is not represented, only used for its timestamp. 103 | for sampleIndex := 0; sampleIndex < len(samples)-1; sampleIndex++ { 104 | s := samples[sampleIndex] 105 | if activeThreadID != nil && s.ThreadID != *activeThreadID { 106 | continue 107 | } 108 | 109 | if len(c.Profile.Stacks) <= s.StackID { 110 | return nil, ErrInvalidStackID 111 | } 112 | 113 | stack := c.Profile.Stacks[s.StackID] 114 | for i := len(stack) - 1; i >= 0; i-- { 115 | if len(c.Profile.Frames) <= stack[i] { 116 | return nil, ErrInvalidFrameID 117 | } 118 | } 119 | 120 | // here while we save the nextTimestamp val, we convert it to nanosecond 121 | // since the Node struct and utilities use uint64 ns values 122 | nextTimestamp := uint64(samples[sampleIndex+1].Timestamp * 1e9) 123 | sampleTimestamp := uint64(s.Timestamp * 1e9) 124 | 125 | for i := len(stack) - 1; i >= 0; i-- { 126 | f := c.Profile.Frames[stack[i]] 127 | f.WriteToHash(h) 128 | fingerprint := h.Sum64() 129 | if current == nil { 130 | i := len(treesByThreadID[s.ThreadID]) - 1 131 | if i >= 0 && treesByThreadID[s.ThreadID][i].Fingerprint == fingerprint && 132 | treesByThreadID[s.ThreadID][i].EndNS == sampleTimestamp { 133 | current = treesByThreadID[s.ThreadID][i] 134 | current.Update(nextTimestamp) 135 | } else { 136 | n := nodetree.NodeFromFrame(f, sampleTimestamp, nextTimestamp, fingerprint) 137 | treesByThreadID[s.ThreadID] = append(treesByThreadID[s.ThreadID], n) 138 | current = n 139 | } 140 | } else { 141 | i := len(current.Children) - 1 142 | if i >= 0 && current.Children[i].Fingerprint == fingerprint && current.Children[i].EndNS == sampleTimestamp { 143 | current = current.Children[i] 144 | current.Update(nextTimestamp) 145 | } else { 146 | n := nodetree.NodeFromFrame(f, sampleTimestamp, nextTimestamp, fingerprint) 147 | current.Children = append(current.Children, n) 148 | current = n 149 | } 150 | } 151 | } // end stack loop 152 | h.Reset() 153 | current = nil 154 | } 155 | } 156 | 157 | return treesByThreadID, nil 158 | } 159 | 160 | func (d *SampleData) trimPythonStacks() { 161 | // Find the module frame index in frames 162 | mfi := -1 163 | for i, f := range d.Frames { 164 | if f.File == "" && f.Function == "" { 165 | mfi = i 166 | } 167 | } 168 | 169 | // We do nothing if we don't find it 170 | if mfi == -1 { 171 | return 172 | } 173 | 174 | for si, s := range d.Stacks { 175 | l := len(s) 176 | 177 | // ignore empty stacks 178 | if l == 0 { 179 | continue 180 | } 181 | 182 | // found the module frame so trim it 183 | if s[l-1] == mfi { 184 | d.Stacks[si] = d.Stacks[si][:l-1] 185 | } 186 | } 187 | } 188 | 189 | func (c SampleChunk) DurationMS() uint64 { 190 | return uint64(math.Round((c.EndTimestamp() - c.StartTimestamp()) * 1e3)) 191 | } 192 | 193 | func (c SampleChunk) SDKName() string { 194 | return c.ClientSDK.Name 195 | } 196 | 197 | func (c SampleChunk) SDKVersion() string { 198 | return c.ClientSDK.Version 199 | } 200 | 201 | func (c SampleChunk) EndTimestamp() float64 { 202 | count := len(c.Profile.Samples) 203 | if count == 0 { 204 | return 0 205 | } 206 | return c.Profile.Samples[count-1].Timestamp 207 | } 208 | 209 | func (c SampleChunk) GetEnvironment() string { 210 | return c.Environment 211 | } 212 | 213 | func (c SampleChunk) GetID() string { 214 | return c.ID 215 | } 216 | 217 | func (c SampleChunk) GetPlatform() platform.Platform { 218 | return c.Platform 219 | } 220 | 221 | func (c SampleChunk) GetProfilerID() string { 222 | return c.ProfilerID 223 | } 224 | 225 | func (c SampleChunk) GetProjectID() uint64 { 226 | return c.ProjectID 227 | } 228 | 229 | func (c SampleChunk) GetReceived() float64 { 230 | return c.Received 231 | } 232 | 233 | func (c SampleChunk) GetRelease() string { 234 | return c.Release 235 | } 236 | 237 | func (c SampleChunk) GetRetentionDays() int { 238 | return c.RetentionDays 239 | } 240 | 241 | func (c SampleChunk) StartTimestamp() float64 { 242 | if len(c.Profile.Samples) == 0 { 243 | return 0 244 | } 245 | return c.Profile.Samples[0].Timestamp 246 | } 247 | 248 | func (c SampleChunk) GetOrganizationID() uint64 { 249 | return c.OrganizationID 250 | } 251 | 252 | func (c SampleChunk) GetOptions() utils.Options { 253 | return c.Options 254 | } 255 | 256 | func (c SampleChunk) GetFrameWithFingerprint(target uint32) (frame.Frame, error) { 257 | for _, f := range c.Profile.Frames { 258 | if f.Fingerprint() == target { 259 | return f, nil 260 | } 261 | } 262 | return frame.Frame{}, frame.ErrFrameNotFound 263 | } 264 | -------------------------------------------------------------------------------- /internal/chunk/sample_readjob.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/getsentry/vroom/internal/nodetree" 7 | "github.com/getsentry/vroom/internal/storageutil" 8 | "gocloud.dev/blob" 9 | ) 10 | 11 | type ( 12 | ReadJob struct { 13 | Ctx context.Context 14 | Storage *blob.Bucket 15 | OrganizationID uint64 16 | ProjectID uint64 17 | ProfilerID string 18 | ChunkID string 19 | TransactionID string 20 | ThreadID *string 21 | Start uint64 22 | End uint64 23 | Result chan<- storageutil.ReadJobResult 24 | } 25 | 26 | ReadJobResult struct { 27 | Err error 28 | Chunk *Chunk 29 | TransactionID string 30 | ThreadID *string 31 | Start uint64 32 | End uint64 33 | } 34 | ) 35 | 36 | func (job ReadJob) Read() { 37 | var chunk Chunk 38 | 39 | err := storageutil.UnmarshalCompressed( 40 | job.Ctx, 41 | job.Storage, 42 | StoragePath(job.OrganizationID, job.ProjectID, job.ProfilerID, job.ChunkID), 43 | &chunk, 44 | ) 45 | 46 | job.Result <- ReadJobResult{ 47 | Err: err, 48 | Chunk: &chunk, 49 | TransactionID: job.TransactionID, 50 | ThreadID: job.ThreadID, 51 | Start: job.Start, 52 | End: job.End, 53 | } 54 | } 55 | 56 | func (result ReadJobResult) Error() error { 57 | return result.Err 58 | } 59 | 60 | type ( 61 | CallTreesReadJob ReadJob 62 | 63 | CallTreesReadJobResult struct { 64 | Err error 65 | CallTrees map[string][]*nodetree.Node 66 | Chunk *Chunk 67 | TransactionID string 68 | ThreadID *string 69 | Start uint64 70 | End uint64 71 | } 72 | ) 73 | 74 | func (job CallTreesReadJob) Read() { 75 | var chunk Chunk 76 | 77 | err := storageutil.UnmarshalCompressed( 78 | job.Ctx, 79 | job.Storage, 80 | StoragePath(job.OrganizationID, job.ProjectID, job.ProfilerID, job.ChunkID), 81 | &chunk, 82 | ) 83 | if err != nil { 84 | job.Result <- CallTreesReadJobResult{Err: err} 85 | return 86 | } 87 | 88 | callTrees, err := chunk.CallTrees(job.ThreadID) 89 | 90 | job.Result <- CallTreesReadJobResult{ 91 | Err: err, 92 | CallTrees: callTrees, 93 | Chunk: &chunk, 94 | TransactionID: job.TransactionID, 95 | ThreadID: job.ThreadID, 96 | Start: job.Start, 97 | End: job.End, 98 | } 99 | } 100 | 101 | func (result CallTreesReadJobResult) Error() error { 102 | return result.Err 103 | } 104 | -------------------------------------------------------------------------------- /internal/chunk/sample_test.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/getsentry/vroom/internal/frame" 7 | "github.com/getsentry/vroom/internal/nodetree" 8 | "github.com/getsentry/vroom/internal/platform" 9 | "github.com/getsentry/vroom/internal/testutil" 10 | "github.com/getsentry/vroom/internal/utils" 11 | ) 12 | 13 | func TestCallTrees(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | chunk SampleChunk 17 | want map[string][]*nodetree.Node 18 | }{ 19 | { 20 | name: "call tree with multiple samples per frame", 21 | chunk: SampleChunk{ 22 | Profile: SampleData{ 23 | Samples: []Sample{ 24 | {StackID: 0, Timestamp: 0.010, ThreadID: "1"}, 25 | {StackID: 1, Timestamp: 0.040, ThreadID: "1"}, 26 | {StackID: 1, Timestamp: 0.050, ThreadID: "1"}, 27 | }, 28 | Stacks: [][]int{ 29 | {1, 0}, 30 | {2, 1, 0}, 31 | }, 32 | Frames: []frame.Frame{ 33 | {Function: "function0"}, 34 | {Function: "function1"}, 35 | {Function: "function2"}, 36 | }, 37 | }, 38 | }, // end chunk 39 | want: map[string][]*nodetree.Node{ 40 | "1": { 41 | { 42 | DurationNS: 40_000_000, 43 | EndNS: 50_000_000, 44 | Fingerprint: 15444731332182868858, 45 | IsApplication: true, 46 | Name: "function0", 47 | SampleCount: 2, 48 | StartNS: 10_000_000, 49 | Frame: frame.Frame{Function: "function0"}, 50 | ProfileIDs: make(map[string]struct{}), 51 | Profiles: make(map[utils.ExampleMetadata]struct{}), 52 | Children: []*nodetree.Node{ 53 | { 54 | DurationNS: 40_000_000, 55 | EndNS: 50_000_000, 56 | StartNS: 10_000_000, 57 | Fingerprint: 14164357600995800812, 58 | IsApplication: true, 59 | Name: "function1", 60 | SampleCount: 2, 61 | Frame: frame.Frame{Function: "function1"}, 62 | ProfileIDs: make(map[string]struct{}), 63 | Profiles: make(map[utils.ExampleMetadata]struct{}), 64 | Children: []*nodetree.Node{ 65 | { 66 | DurationNS: 10_000_000, 67 | EndNS: 50_000_000, 68 | Fingerprint: 9531802423075301657, 69 | IsApplication: true, 70 | Name: "function2", 71 | SampleCount: 1, 72 | StartNS: 40_000_000, 73 | Frame: frame.Frame{Function: "function2"}, 74 | ProfileIDs: make(map[string]struct{}), 75 | Profiles: make(map[utils.ExampleMetadata]struct{}), 76 | }, 77 | }, 78 | }, 79 | }, 80 | }, 81 | }, 82 | }, 83 | }, // end first test 84 | { 85 | name: "call tree with single sample frames", 86 | chunk: SampleChunk{ 87 | Profile: SampleData{ 88 | Samples: []Sample{ 89 | {StackID: 0, Timestamp: 0.010, ThreadID: "1"}, 90 | {StackID: 1, Timestamp: 0.040, ThreadID: "1"}, 91 | }, 92 | Stacks: [][]int{ 93 | {1, 0}, 94 | {2, 1, 0}, 95 | }, 96 | Frames: []frame.Frame{ 97 | {Function: "function0"}, 98 | {Function: "function1"}, 99 | {Function: "function2"}, 100 | }, 101 | }, 102 | }, 103 | want: map[string][]*nodetree.Node{ 104 | "1": { 105 | { 106 | DurationNS: 30_000_000, 107 | EndNS: 40_000_000, 108 | Fingerprint: 15444731332182868858, 109 | IsApplication: true, 110 | Name: "function0", 111 | SampleCount: 1, 112 | StartNS: 10_000_000, 113 | Frame: frame.Frame{Function: "function0"}, 114 | ProfileIDs: make(map[string]struct{}), 115 | Profiles: make(map[utils.ExampleMetadata]struct{}), 116 | Children: []*nodetree.Node{ 117 | { 118 | DurationNS: 30_000_000, 119 | EndNS: 40_000_000, 120 | Fingerprint: 14164357600995800812, 121 | IsApplication: true, 122 | Name: "function1", 123 | SampleCount: 1, 124 | StartNS: 10_000_000, 125 | Frame: frame.Frame{Function: "function1"}, 126 | ProfileIDs: make(map[string]struct{}), 127 | Profiles: make(map[utils.ExampleMetadata]struct{}), 128 | }, 129 | }, 130 | }, 131 | }, 132 | }, 133 | }, // end first test 134 | { 135 | name: "call tree with single samples", 136 | chunk: SampleChunk{ 137 | Profile: SampleData{ 138 | Samples: []Sample{ 139 | {StackID: 0, Timestamp: 0.010, ThreadID: "1"}, 140 | {StackID: 1, Timestamp: 0.020, ThreadID: "1"}, 141 | {StackID: 2, Timestamp: 0.030, ThreadID: "1"}, 142 | }, 143 | Stacks: [][]int{ 144 | {0}, 145 | {1}, 146 | {2}, 147 | }, 148 | Frames: []frame.Frame{ 149 | {Function: "function0"}, 150 | {Function: "function1"}, 151 | {Function: "function2"}, 152 | }, 153 | }, 154 | }, 155 | want: map[string][]*nodetree.Node{ 156 | "1": { 157 | { 158 | DurationNS: 10_000_000, 159 | EndNS: 20_000_000, 160 | Fingerprint: 15444731332182868858, 161 | IsApplication: true, 162 | Name: "function0", 163 | SampleCount: 1, 164 | StartNS: 10_000_000, 165 | Frame: frame.Frame{Function: "function0"}, 166 | ProfileIDs: make(map[string]struct{}), 167 | Profiles: make(map[utils.ExampleMetadata]struct{}), 168 | }, 169 | { 170 | DurationNS: 10_000_000, 171 | EndNS: 30_000_000, 172 | Fingerprint: 15444731332182868859, 173 | IsApplication: true, 174 | Name: "function1", 175 | SampleCount: 1, 176 | StartNS: 20_000_000, 177 | Frame: frame.Frame{Function: "function1"}, 178 | ProfileIDs: make(map[string]struct{}), 179 | Profiles: make(map[utils.ExampleMetadata]struct{}), 180 | }, 181 | }, 182 | }, 183 | }, // end third test 184 | } 185 | 186 | for _, test := range tests { 187 | t.Run(test.name, func(t *testing.T) { 188 | callTrees, err := test.chunk.CallTrees(nil) 189 | if err != nil { 190 | t.Fatalf("error while generating call trees: %+v\n", err) 191 | } 192 | if diff := testutil.Diff(callTrees, test.want); diff != "" { 193 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 194 | } 195 | }) 196 | } 197 | } 198 | 199 | func TestTrimPythonStacks(t *testing.T) { 200 | tests := []struct { 201 | name string 202 | input SampleChunk 203 | output SampleChunk 204 | }{ 205 | { 206 | name: "Remove module frame at the end of a stack", 207 | input: SampleChunk{ 208 | Platform: platform.Python, 209 | Profile: SampleData{ 210 | Frames: []frame.Frame{ 211 | { 212 | File: "", 213 | Module: "__main__", 214 | InApp: &testutil.True, 215 | Line: 11, 216 | Function: "", 217 | Path: "/usr/src/app/", 218 | Platform: "python", 219 | }, 220 | { 221 | File: "app/util.py", 222 | Module: "app.util", 223 | InApp: &testutil.True, 224 | Line: 98, 225 | Function: "foobar", 226 | Path: "/usr/src/app/util.py", 227 | Platform: "python", 228 | }, 229 | }, 230 | Stacks: [][]int{ 231 | {1, 0}, 232 | }, 233 | }, 234 | }, 235 | output: SampleChunk{ 236 | Platform: platform.Python, 237 | Profile: SampleData{ 238 | Frames: []frame.Frame{ 239 | { 240 | File: "", 241 | Module: "__main__", 242 | InApp: &testutil.True, 243 | Line: 11, 244 | Function: "", 245 | Path: "/usr/src/app/", 246 | Platform: "python", 247 | }, 248 | { 249 | File: "app/util.py", 250 | Module: "app.util", 251 | InApp: &testutil.True, 252 | Line: 98, 253 | Function: "foobar", 254 | Path: "/usr/src/app/util.py", 255 | Platform: "python", 256 | }, 257 | }, 258 | Stacks: [][]int{ 259 | {1}, 260 | }, 261 | }, 262 | }, 263 | }, 264 | } 265 | 266 | for _, test := range tests { 267 | t.Run(test.name, func(t *testing.T) { 268 | test.input.Normalize() 269 | if diff := testutil.Diff(test.input, test.output); diff != "" { 270 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 271 | } 272 | }) 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /internal/chunk/sample_utils.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "sort" 7 | 8 | "github.com/getsentry/vroom/internal/measurements" 9 | "gocloud.dev/blob" 10 | ) 11 | 12 | func MergeSampleChunks(chunks []SampleChunk, startTS, endTS uint64) (SampleChunk, error) { 13 | if len(chunks) == 0 { 14 | return SampleChunk{}, nil 15 | } 16 | sort.Slice(chunks, func(i, j int) bool { 17 | return chunks[i].EndTimestamp() <= chunks[j].StartTimestamp() 18 | }) 19 | 20 | mergedMeasurement := make(map[string]measurements.MeasurementV2) 21 | 22 | start := float64(startTS) / 1e9 23 | end := float64(endTS) / 1e9 24 | 25 | chunk := chunks[0] 26 | if len(chunk.Measurements) > 0 { 27 | err := json.Unmarshal(chunk.Measurements, &mergedMeasurement) 28 | if err != nil { 29 | return SampleChunk{}, err 30 | } 31 | } 32 | 33 | // clean up the samples in the first chunk 34 | samples := make([]Sample, 0, len(chunk.Profile.Samples)) 35 | for _, sample := range chunk.Profile.Samples { 36 | if sample.Timestamp < start || sample.Timestamp > end { 37 | // sample from chunk lies outside start/end range so skip it 38 | continue 39 | } 40 | samples = append(samples, sample) 41 | } 42 | 43 | for i := 1; i < len(chunks); i++ { 44 | c := chunks[i] 45 | // Update all the frame indices of the chunk we're going to add/merge 46 | // to the first one. 47 | // If the first chunk had a couple of frames, and the second chunk too, 48 | // then all the stacks in the second chunk that refers to frames at index 49 | // fr[0] and fr[1], once merged should refer to frames at index fr[2], fr[3]. 50 | for j, stack := range c.Profile.Stacks { 51 | for z, frameID := range stack { 52 | c.Profile.Stacks[j][z] = frameID + len(chunk.Profile.Frames) 53 | } 54 | } 55 | chunk.Profile.Frames = append(chunk.Profile.Frames, c.Profile.Frames...) 56 | // The same goes for chunk samples stack IDs 57 | for j, sample := range c.Profile.Samples { 58 | c.Profile.Samples[j].StackID = sample.StackID + len(chunk.Profile.Stacks) 59 | } 60 | chunk.Profile.Stacks = append(chunk.Profile.Stacks, c.Profile.Stacks...) 61 | for _, sample := range c.Profile.Samples { 62 | if sample.Timestamp < start || sample.Timestamp > end { 63 | // sample from chunk lies outside start/end range so skip it 64 | continue 65 | } 66 | samples = append(samples, sample) 67 | } 68 | 69 | // Update threadMetadata 70 | for k, threadMetadata := range c.Profile.ThreadMetadata { 71 | if _, ok := chunk.Profile.ThreadMetadata[k]; !ok { 72 | chunk.Profile.ThreadMetadata[k] = threadMetadata 73 | } 74 | } 75 | 76 | // In case we have measurements, merge them too 77 | if len(c.Measurements) > 0 { 78 | var chunkMeasurements map[string]measurements.MeasurementV2 79 | err := json.Unmarshal(c.Measurements, &chunkMeasurements) 80 | if err != nil { 81 | return SampleChunk{}, err 82 | } 83 | for k, measurement := range chunkMeasurements { 84 | if el, ok := mergedMeasurement[k]; ok { 85 | el.Values = append(el.Values, measurement.Values...) 86 | mergedMeasurement[k] = el 87 | } else { 88 | mergedMeasurement[k] = measurement 89 | } 90 | } 91 | } 92 | } 93 | 94 | chunk.Profile.Samples = samples 95 | 96 | if len(mergedMeasurement) > 0 { 97 | jsonRawMesaurement, err := json.Marshal(mergedMeasurement) 98 | if err != nil { 99 | return SampleChunk{}, err 100 | } 101 | chunk.Measurements = jsonRawMesaurement 102 | } 103 | 104 | return chunk, nil 105 | } 106 | 107 | // The task the workers expect as input. 108 | // 109 | // Result: the channel used to send back the output. 110 | type TaskInput struct { 111 | Ctx context.Context 112 | ProfilerID string 113 | ChunkID string 114 | OrganizationID uint64 115 | ProjectID uint64 116 | Storage *blob.Bucket 117 | Result chan<- SampleTaskOutput 118 | } 119 | 120 | // The output sent back by the worker. 121 | type SampleTaskOutput struct { 122 | Err error 123 | Chunk SampleChunk 124 | } 125 | -------------------------------------------------------------------------------- /internal/chunk/sample_utils_test.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/getsentry/vroom/internal/frame" 8 | "github.com/getsentry/vroom/internal/sample" 9 | "github.com/getsentry/vroom/internal/testutil" 10 | ) 11 | 12 | func TestMergeSampleChunks(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | have []SampleChunk 16 | want SampleChunk 17 | start uint64 18 | end uint64 19 | }{ 20 | { 21 | name: "contiguous chunks", 22 | have: []SampleChunk{ 23 | { 24 | Profile: SampleData{ 25 | Frames: []frame.Frame{ 26 | {Function: "c"}, 27 | {Function: "d"}, 28 | }, 29 | Samples: []Sample{ 30 | {StackID: 0, Timestamp: 3.0}, 31 | {StackID: 1, Timestamp: 4.0}, 32 | {StackID: 1, Timestamp: 5.0}, // outside range, will be dropped 33 | }, 34 | Stacks: [][]int{ 35 | {0, 1}, 36 | {0, 1}, 37 | }, 38 | ThreadMetadata: map[string]sample.ThreadMetadata{"0x000000016d8fb180": {Name: "com.apple.network.connections"}}, 39 | }, 40 | Measurements: json.RawMessage(`{"first_metric":{"unit":"ms","values":[{"timestamp":2.0,"value":1.2}]}}`), 41 | }, 42 | // other chunk 43 | { 44 | Profile: SampleData{ 45 | Frames: []frame.Frame{ 46 | {Function: "a"}, 47 | {Function: "b"}, 48 | }, 49 | Samples: []Sample{ 50 | {StackID: 0, Timestamp: 0.0}, 51 | {StackID: 0, Timestamp: 1.0}, 52 | {StackID: 1, Timestamp: 2.0}, 53 | }, 54 | Stacks: [][]int{ 55 | {0, 1}, 56 | {0, 1}, 57 | }, 58 | ThreadMetadata: map[string]sample.ThreadMetadata{"0x0000000102adc700": {Name: "com.apple.main-thread"}}, 59 | }, 60 | Measurements: json.RawMessage(`{"first_metric":{"unit":"ms","values":[{"timestamp":1.0,"value":1}]}}`), 61 | }, 62 | }, 63 | want: SampleChunk{ 64 | Profile: SampleData{ 65 | Frames: []frame.Frame{ 66 | {Function: "a"}, 67 | {Function: "b"}, 68 | {Function: "c"}, 69 | {Function: "d"}, 70 | }, 71 | Samples: []Sample{ 72 | {StackID: 0, Timestamp: 1.0}, 73 | {StackID: 1, Timestamp: 2.0}, 74 | {StackID: 2, Timestamp: 3.0}, 75 | {StackID: 3, Timestamp: 4.0}, 76 | }, 77 | Stacks: [][]int{ 78 | {0, 1}, 79 | {0, 1}, 80 | {2, 3}, 81 | {2, 3}, 82 | }, 83 | ThreadMetadata: map[string]sample.ThreadMetadata{"0x0000000102adc700": {Name: "com.apple.main-thread"}, "0x000000016d8fb180": {Name: "com.apple.network.connections"}}, 84 | }, 85 | Measurements: json.RawMessage(`{"first_metric":{"unit":"ms","values":[{"timestamp":1,"value":1},{"timestamp":2,"value":1.2}]}}`), 86 | }, 87 | start: uint64(1e9), 88 | end: uint64(4e9), 89 | }, 90 | } 91 | 92 | for _, test := range tests { 93 | t.Run(test.name, func(t *testing.T) { 94 | have, err := MergeSampleChunks(test.have, test.start, test.end) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | if diff := testutil.Diff(have, test.want); diff != "" { 99 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 100 | } 101 | }) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /internal/clientsdk/clientsdk.go: -------------------------------------------------------------------------------- 1 | package clientsdk 2 | 3 | type ClientSDK struct { 4 | Name string `json:"name"` 5 | Version string `json:"version"` 6 | } 7 | -------------------------------------------------------------------------------- /internal/debugmeta/debugmeta.go: -------------------------------------------------------------------------------- 1 | package debugmeta 2 | 3 | type ( 4 | Features struct { 5 | HasDebugInfo bool `json:"has_debug_info"` 6 | HasSources bool `json:"has_sources"` 7 | HasSymbols bool `json:"has_symbols"` 8 | HasUnwindInfo bool `json:"has_unwind_info"` 9 | } 10 | 11 | Image struct { 12 | Arch string `json:"arch,omitempty"` 13 | CodeFile string `json:"code_file,omitempty"` 14 | DebugID string `json:"debug_id,omitempty"` 15 | DebugStatus string `json:"debug_status,omitempty"` 16 | Features *Features `json:"features,omitempty"` 17 | ImageAddr string `json:"image_addr,omitempty"` 18 | ImageSize uint64 `json:"image_size,omitempty"` 19 | ImageVMAddr string `json:"image_vmaddr,omitempty"` 20 | Type string `json:"type,omitempty"` 21 | UUID string `json:"uuid,omitempty"` 22 | } 23 | 24 | DebugMeta struct { 25 | Images []Image `json:"images,omitempty"` 26 | } 27 | ) 28 | -------------------------------------------------------------------------------- /internal/errorutil/errorutil.go: -------------------------------------------------------------------------------- 1 | package errorutil 2 | 3 | import "errors" 4 | 5 | // ErrDataIntegrity is a base error type to use for failures that are due to 6 | // unrecoverable data integrity issues. 7 | var ErrDataIntegrity = errors.New("data integrity error") 8 | 9 | // ErrNoResults represents situations in which no results were returned by the called API. 10 | var ErrNoResults = errors.New("no results returned") 11 | -------------------------------------------------------------------------------- /internal/flamegraph/span_util.go: -------------------------------------------------------------------------------- 1 | package flamegraph 2 | 3 | import ( 4 | "math" 5 | "sort" 6 | "time" 7 | 8 | "github.com/getsentry/vroom/internal/nodetree" 9 | "github.com/getsentry/vroom/internal/utils" 10 | ) 11 | 12 | func mergeIntervals(intervals *[]utils.Interval) []utils.Interval { 13 | if len(*intervals) == 0 { 14 | return *intervals 15 | } 16 | sort.SliceStable((*intervals), func(i, j int) bool { 17 | if (*intervals)[i].Start == (*intervals)[j].Start { 18 | return (*intervals)[i].End < (*intervals)[j].End 19 | } 20 | return (*intervals)[i].Start < (*intervals)[j].Start 21 | }) 22 | 23 | newIntervals := []utils.Interval{(*intervals)[0]} 24 | for _, interval := range (*intervals)[1:] { 25 | if interval.Start <= newIntervals[len(newIntervals)-1].End { 26 | newIntervals[len(newIntervals)-1].End = max(newIntervals[len(newIntervals)-1].End, interval.End) 27 | } else { 28 | newIntervals = append(newIntervals, interval) 29 | } 30 | } 31 | 32 | return newIntervals 33 | } 34 | 35 | func sliceCallTree(callTree *[]*nodetree.Node, intervals *[]utils.Interval) []*nodetree.Node { 36 | slicedTree := make([]*nodetree.Node, 0) 37 | for _, node := range *callTree { 38 | if duration := getTotalOverlappingDuration(node, intervals); duration > 0 { 39 | sampleCount := int(math.Ceil(float64(duration) / float64(time.Millisecond*10))) 40 | // here we take the minimum between the node sample count and the estimated 41 | // sample count to mitigate the case when we make a wrong estimation due 42 | // to sampling frequency not being respected. (Python native code holding 43 | // the GIL, php, etc.) 44 | node.SampleCount = min(sampleCount, node.SampleCount) 45 | node.DurationNS = duration 46 | if children := sliceCallTree(&node.Children, intervals); len(children) > 0 { 47 | node.Children = children 48 | } else { 49 | node.Children = nil 50 | } 51 | slicedTree = append(slicedTree, node) 52 | } 53 | } // end range callTree 54 | return slicedTree 55 | } 56 | 57 | func getTotalOverlappingDuration(node *nodetree.Node, intervals *[]utils.Interval) uint64 { 58 | var duration uint64 59 | for _, interval := range *intervals { 60 | if node.EndNS <= interval.Start { 61 | // in this case any remaining interval 62 | // starts after the given call frame 63 | // therefeore we can bail out early 64 | break 65 | } 66 | duration += overlappingDuration(node, &interval) 67 | } 68 | return duration 69 | } 70 | 71 | func overlappingDuration(node *nodetree.Node, interval *utils.Interval) uint64 { 72 | end := min(node.EndNS, interval.End) 73 | start := max(node.StartNS, interval.Start) 74 | 75 | if end <= start { 76 | return 0 77 | } 78 | return end - start 79 | } 80 | -------------------------------------------------------------------------------- /internal/frame/python_std_lib.go: -------------------------------------------------------------------------------- 1 | // This file is autogenerated from scripts/make_python_stdlib.py 2 | // To update this file, update the python versions list in 3 | // scripts/make_python_stdlib.py then run `make python-stdlib` 4 | package frame 5 | 6 | var ( 7 | pythonStdlib = map[string]struct{}{ 8 | "__future__": {}, 9 | "__hello__": {}, 10 | "__phello__": {}, 11 | "__phello__.foo": {}, 12 | "_aix_support": {}, 13 | "_ast": {}, 14 | "_bootlocale": {}, 15 | "_bootsubprocess": {}, 16 | "_collections_abc": {}, 17 | "_compat_pickle": {}, 18 | "_compression": {}, 19 | "_dummy_thread": {}, 20 | "_markupbase": {}, 21 | "_osx_support": {}, 22 | "_py_abc": {}, 23 | "_pydecimal": {}, 24 | "_pyio": {}, 25 | "_sitebuiltins": {}, 26 | "_strptime": {}, 27 | "_thread": {}, 28 | "_threading_local": {}, 29 | "_weakrefset": {}, 30 | "abc": {}, 31 | "aifc": {}, 32 | "antigravity": {}, 33 | "argparse": {}, 34 | "array": {}, 35 | "ast": {}, 36 | "asynchat": {}, 37 | "asyncio": {}, 38 | "asyncore": {}, 39 | "atexit": {}, 40 | "audioop": {}, 41 | "base64": {}, 42 | "bdb": {}, 43 | "binascii": {}, 44 | "binhex": {}, 45 | "bisect": {}, 46 | "builtins": {}, 47 | "bz2": {}, 48 | "cProfile": {}, 49 | "calendar": {}, 50 | "cgi": {}, 51 | "cgitb": {}, 52 | "chunk": {}, 53 | "cmath": {}, 54 | "cmd": {}, 55 | "code": {}, 56 | "codecs": {}, 57 | "codeop": {}, 58 | "collections": {}, 59 | "colorsys": {}, 60 | "compileall": {}, 61 | "concurrent": {}, 62 | "configparser": {}, 63 | "contextlib": {}, 64 | "contextvars": {}, 65 | "copy": {}, 66 | "copyreg": {}, 67 | "crypt": {}, 68 | "csv": {}, 69 | "ctypes": {}, 70 | "curses": {}, 71 | "dataclasses": {}, 72 | "datetime": {}, 73 | "dbm": {}, 74 | "decimal": {}, 75 | "difflib": {}, 76 | "dis": {}, 77 | "distutils": {}, 78 | "doctest": {}, 79 | "dummy_threading": {}, 80 | "email": {}, 81 | "encodings": {}, 82 | "ensurepip": {}, 83 | "enum": {}, 84 | "errno": {}, 85 | "faulthandler": {}, 86 | "fcntl": {}, 87 | "filecmp": {}, 88 | "fileinput": {}, 89 | "fnmatch": {}, 90 | "formatter": {}, 91 | "fpectl": {}, 92 | "fractions": {}, 93 | "ftplib": {}, 94 | "functools": {}, 95 | "gc": {}, 96 | "genericpath": {}, 97 | "getopt": {}, 98 | "getpass": {}, 99 | "gettext": {}, 100 | "glob": {}, 101 | "graphlib": {}, 102 | "grp": {}, 103 | "gzip": {}, 104 | "hashlib": {}, 105 | "heapq": {}, 106 | "hmac": {}, 107 | "html": {}, 108 | "http": {}, 109 | "idlelib": {}, 110 | "imaplib": {}, 111 | "imghdr": {}, 112 | "imp": {}, 113 | "importlib": {}, 114 | "inspect": {}, 115 | "io": {}, 116 | "ipaddress": {}, 117 | "itertools": {}, 118 | "json": {}, 119 | "keyword": {}, 120 | "lib2to3": {}, 121 | "linecache": {}, 122 | "locale": {}, 123 | "logging": {}, 124 | "lzma": {}, 125 | "macpath": {}, 126 | "macurl2path": {}, 127 | "mailbox": {}, 128 | "mailcap": {}, 129 | "marshal": {}, 130 | "math": {}, 131 | "mimetypes": {}, 132 | "mmap": {}, 133 | "modulefinder": {}, 134 | "msilib": {}, 135 | "msvcrt": {}, 136 | "multiprocessing": {}, 137 | "netrc": {}, 138 | "nis": {}, 139 | "nntplib": {}, 140 | "ntpath": {}, 141 | "nturl2path": {}, 142 | "numbers": {}, 143 | "opcode": {}, 144 | "operator": {}, 145 | "optparse": {}, 146 | "os": {}, 147 | "os2emxpath": {}, 148 | "ossaudiodev": {}, 149 | "parser": {}, 150 | "pathlib": {}, 151 | "pdb": {}, 152 | "pickle": {}, 153 | "pickletools": {}, 154 | "pipes": {}, 155 | "pkgutil": {}, 156 | "platform": {}, 157 | "plistlib": {}, 158 | "poplib": {}, 159 | "posix": {}, 160 | "posixpath": {}, 161 | "pprint": {}, 162 | "profile": {}, 163 | "pstats": {}, 164 | "pty": {}, 165 | "pwd": {}, 166 | "py_compile": {}, 167 | "pyclbr": {}, 168 | "pydoc": {}, 169 | "pydoc_data": {}, 170 | "queue": {}, 171 | "quopri": {}, 172 | "random": {}, 173 | "re": {}, 174 | "readline": {}, 175 | "reprlib": {}, 176 | "resource": {}, 177 | "rlcompleter": {}, 178 | "runpy": {}, 179 | "sched": {}, 180 | "secrets": {}, 181 | "select": {}, 182 | "selectors": {}, 183 | "shelve": {}, 184 | "shlex": {}, 185 | "shutil": {}, 186 | "signal": {}, 187 | "site": {}, 188 | "smtpd": {}, 189 | "smtplib": {}, 190 | "sndhdr": {}, 191 | "socket": {}, 192 | "socketserver": {}, 193 | "spwd": {}, 194 | "sqlite3": {}, 195 | "sre": {}, 196 | "sre_compile": {}, 197 | "sre_constants": {}, 198 | "sre_parse": {}, 199 | "ssl": {}, 200 | "stat": {}, 201 | "statistics": {}, 202 | "string": {}, 203 | "stringprep": {}, 204 | "struct": {}, 205 | "subprocess": {}, 206 | "sunau": {}, 207 | "symbol": {}, 208 | "symtable": {}, 209 | "sys": {}, 210 | "sysconfig": {}, 211 | "syslog": {}, 212 | "tabnanny": {}, 213 | "tarfile": {}, 214 | "telnetlib": {}, 215 | "tempfile": {}, 216 | "termios": {}, 217 | "test": {}, 218 | "textwrap": {}, 219 | "this": {}, 220 | "threading": {}, 221 | "time": {}, 222 | "timeit": {}, 223 | "tkinter": {}, 224 | "token": {}, 225 | "tokenize": {}, 226 | "tomllib": {}, 227 | "trace": {}, 228 | "traceback": {}, 229 | "tracemalloc": {}, 230 | "tty": {}, 231 | "turtle": {}, 232 | "turtledemo": {}, 233 | "types": {}, 234 | "typing": {}, 235 | "unicodedata": {}, 236 | "unittest": {}, 237 | "urllib": {}, 238 | "uu": {}, 239 | "uuid": {}, 240 | "venv": {}, 241 | "warnings": {}, 242 | "wave": {}, 243 | "weakref": {}, 244 | "webbrowser": {}, 245 | "winreg": {}, 246 | "winsound": {}, 247 | "wsgiref": {}, 248 | "xdrlib": {}, 249 | "xml": {}, 250 | "xmlrpc": {}, 251 | "zipapp": {}, 252 | "zipfile": {}, 253 | "zipimport": {}, 254 | "zlib": {}, 255 | "zoneinfo": {}, 256 | } 257 | ) 258 | -------------------------------------------------------------------------------- /internal/httputil/decompress.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | 7 | "github.com/andybalholm/brotli" 8 | ) 9 | 10 | // DecompressPayload adds a reader of the right type in case you need to decompress the body. 11 | func DecompressPayload(next http.Handler) http.HandlerFunc { 12 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | defer r.Body.Close() 14 | 15 | if r.Header.Get("Content-Encoding") == "br" { 16 | r.Body = io.NopCloser(brotli.NewReader(r.Body)) 17 | } 18 | 19 | next.ServeHTTP(w, r) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /internal/httputil/request.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/getsentry/sentry-go" 9 | "github.com/julienschmidt/httprouter" 10 | ) 11 | 12 | // GetRequiredQueryParameters attempts to read the specified query parameters 13 | // from the request and returns a map of the key value pairs. If any of the required 14 | // query parameters are missing or blank, it'll write a 400 status code as well as 15 | // the reasoning for the error into the ResponseWriter, and also set return false. 16 | func GetRequiredQueryParameters(w http.ResponseWriter, r *http.Request, keys ...string) (map[string]string, bool) { 17 | params := make(map[string]string, len(keys)) 18 | for _, key := range keys { 19 | value := r.URL.Query().Get(key) 20 | if value == "" { 21 | http.Error(w, fmt.Sprintf("expected %s query parameter", key), http.StatusBadRequest) 22 | return nil, false 23 | } 24 | params[key] = value 25 | } 26 | return params, true 27 | } 28 | 29 | func AnonymizeTransactionName(handler http.Handler) http.HandlerFunc { 30 | return func(w http.ResponseWriter, r *http.Request) { 31 | transaction := sentry.TransactionFromContext(r.Context()) 32 | if transaction != nil { 33 | params := httprouter.ParamsFromContext(r.Context()) 34 | path := r.URL.Path 35 | for _, param := range params { 36 | path = strings.Replace(path, param.Value, fmt.Sprintf(":%s", param.Key), 1) 37 | } 38 | 39 | transaction.Name = fmt.Sprintf("%s %s", r.Method, path) 40 | } 41 | 42 | handler.ServeHTTP(w, r) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/httputil/transaction.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/getsentry/sentry-go" 7 | ) 8 | 9 | // HTTPStatusCodeTag is the name of the HTTP status code tag. 10 | const HTTPStatusCodeTag = "http.response.status_code" 11 | 12 | // SetHTTPStatusCodeTag sets the status code tag for the current request to the top-level transaction. 13 | // TODO: Move this to the SDK itself. 14 | func SetHTTPStatusCodeTag(e *sentry.Event, hint *sentry.EventHint) *sentry.Event { 15 | if hint.Response == nil { 16 | return e 17 | } 18 | if e.Tags == nil { 19 | e.Tags = make(map[string]string) 20 | } 21 | if _, exists := e.Tags[HTTPStatusCodeTag]; !exists { 22 | e.Tags[HTTPStatusCodeTag] = strconv.Itoa(hint.Response.StatusCode) 23 | } 24 | return e 25 | } 26 | -------------------------------------------------------------------------------- /internal/logutil/logutil.go: -------------------------------------------------------------------------------- 1 | package logutil 2 | 3 | import ( 4 | "log/slog" 5 | "os" 6 | ) 7 | 8 | func ConfigureLogger() { 9 | slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ 10 | Level: slog.LevelInfo, 11 | ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { 12 | if a.Key == slog.MessageKey { 13 | a.Key = "message" 14 | } 15 | return a 16 | }, 17 | }))) 18 | } 19 | -------------------------------------------------------------------------------- /internal/measurements/measurements.go: -------------------------------------------------------------------------------- 1 | package measurements 2 | 3 | type Measurement struct { 4 | Unit string `json:"unit"` 5 | Values []MeasurementValue `json:"values"` 6 | } 7 | 8 | type MeasurementValue struct { 9 | ElapsedSinceStartNs uint64 `json:"elapsed_since_start_ns"` 10 | Value float64 `json:"value"` 11 | } 12 | 13 | type MeasurementV2 struct { 14 | Unit string `json:"unit"` 15 | Values []MeasurementValueV2 `json:"values"` 16 | } 17 | 18 | // https://github.com/getsentry/relay/blob/master/relay-profiling/src/measurements.rs#L23-L29 19 | type MeasurementValueV2 struct { 20 | // UNIX timestamp in seconds as a float 21 | Timestamp float64 `json:"timestamp"` 22 | Value float64 `json:"value"` 23 | } 24 | -------------------------------------------------------------------------------- /internal/metadata/metadata.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | type Metadata struct { 4 | AndroidAPILevel uint32 `json:"android_api_level"` 5 | Architecture string `json:"architecture"` 6 | DeviceClassification string `json:"device_classification"` 7 | DeviceLocale string `json:"device_locale"` 8 | DeviceManufacturer string `json:"device_manufacturer"` 9 | DeviceModel string `json:"device_model"` 10 | DeviceOSBuildNumber string `json:"device_os_build_number"` 11 | DeviceOSName string `json:"device_os_name"` 12 | DeviceOSVersion string `json:"device_os_version"` 13 | ID string `json:"id"` 14 | ProjectID string `json:"project_id"` 15 | SDKName string `json:"sdk_name"` 16 | SDKVersion string `json:"sdk_version"` 17 | Timestamp int64 `json:"timestamp"` 18 | TraceDurationMs float64 `json:"trace_duration_ms"` 19 | TransactionID string `json:"transaction_id"` 20 | TransactionName string `json:"transaction_name"` 21 | VersionCode string `json:"version_code"` 22 | VersionName string `json:"version_name"` 23 | } 24 | -------------------------------------------------------------------------------- /internal/metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/getsentry/vroom/internal/nodetree" 8 | "github.com/getsentry/vroom/internal/testutil" 9 | "github.com/getsentry/vroom/internal/utils" 10 | ) 11 | 12 | func TestAggregatorAddFunctions(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | calltreeFunctions []nodetree.CallTreeFunction 16 | want Aggregator 17 | }{ 18 | { 19 | name: "addFunctions", 20 | calltreeFunctions: []nodetree.CallTreeFunction{ 21 | { 22 | Function: "a", 23 | Fingerprint: 0, 24 | SelfTimesNS: []uint64{10, 5, 25}, 25 | SumSelfTimeNS: 40, 26 | }, 27 | { 28 | Function: "b", 29 | Fingerprint: 1, 30 | SelfTimesNS: []uint64{45, 60}, 31 | SumSelfTimeNS: 105, 32 | }, 33 | }, 34 | want: Aggregator{ 35 | MaxUniqueFunctions: 100, 36 | MaxNumOfExamples: 5, 37 | CallTreeFunctions: map[uint32]nodetree.CallTreeFunction{ 38 | 0: { 39 | Function: "a", 40 | Fingerprint: 0, 41 | SelfTimesNS: []uint64{10, 5, 25, 10, 5, 25}, 42 | SumSelfTimeNS: 80, 43 | }, 44 | 1: { 45 | Function: "b", 46 | Fingerprint: 1, 47 | SelfTimesNS: []uint64{45, 60, 45, 60}, 48 | SumSelfTimeNS: 210, 49 | }, 50 | }, 51 | FunctionsMetadata: map[uint32]FunctionsMetadata{ 52 | 0: { 53 | MaxVal: 40, 54 | Worst: utils.ExampleMetadata{ProfileID: "1"}, 55 | Examples: []utils.ExampleMetadata{{ProfileID: "1"}, {ProfileID: "2"}}, 56 | }, 57 | 1: { 58 | MaxVal: 105, 59 | Worst: utils.ExampleMetadata{ProfileID: "1"}, 60 | Examples: []utils.ExampleMetadata{{ProfileID: "1"}, {ProfileID: "2"}}, 61 | }, 62 | }, // end want 63 | }, 64 | }, // end first test 65 | } // end tests list 66 | 67 | ma := NewAggregator(100, 5, 0) 68 | for _, test := range tests { 69 | // add the same calltreeFunctions twice: once coming from a profile/chunk with 70 | // ID 1 and the second one with ID 2 71 | ma.AddFunctions(test.calltreeFunctions, utils.ExampleMetadata{ProfileID: "1"}) 72 | ma.AddFunctions(test.calltreeFunctions, utils.ExampleMetadata{ProfileID: "2"}) 73 | if diff := testutil.Diff(ma, test.want); diff != "" { 74 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 75 | } 76 | } 77 | } 78 | 79 | func TestAggregatorToMetrics(t *testing.T) { 80 | tests := []struct { 81 | name string 82 | Aggregator Aggregator 83 | want []utils.FunctionMetrics 84 | }{ 85 | { 86 | name: "toMetrics", 87 | Aggregator: Aggregator{ 88 | MaxUniqueFunctions: 100, 89 | CallTreeFunctions: map[uint32]nodetree.CallTreeFunction{ 90 | 0: { 91 | Function: "a", 92 | Fingerprint: 0, 93 | SelfTimesNS: []uint64{1, 2, 3, 4, 10, 8, 7, 11, 20}, 94 | SumSelfTimeNS: 66, 95 | SampleCount: 2, 96 | }, 97 | 1: { 98 | Function: "b", 99 | Fingerprint: 1, 100 | SelfTimesNS: []uint64{1, 2, 3, 4, 10, 8, 7, 11, 20}, 101 | SumSelfTimeNS: 66, 102 | SampleCount: 2, 103 | }, 104 | }, //end callTreeFunctions 105 | FunctionsMetadata: map[uint32]FunctionsMetadata{ 106 | 0: { 107 | MaxVal: 66, 108 | Worst: utils.ExampleMetadata{ProfileID: "1"}, 109 | Examples: []utils.ExampleMetadata{{ProfileID: "1"}, {ProfileID: "2"}}, 110 | }, 111 | 1: { 112 | MaxVal: 66, 113 | Worst: utils.ExampleMetadata{ProfileID: "3"}, 114 | Examples: []utils.ExampleMetadata{{ProfileID: "1"}, {ProfileID: "3"}}, 115 | }, 116 | }, //end functionsMetadata 117 | }, //end Aggregator 118 | want: []utils.FunctionMetrics{ 119 | { 120 | Name: "a", 121 | Fingerprint: 0, 122 | P75: 10, 123 | P95: 20, 124 | P99: 20, 125 | Count: 2, 126 | Sum: 66, 127 | Avg: float64(66) / float64(9), 128 | Worst: utils.ExampleMetadata{ProfileID: "1"}, 129 | Examples: []utils.ExampleMetadata{{ProfileID: "1"}, {ProfileID: "2"}}, 130 | }, 131 | { 132 | Name: "b", 133 | Fingerprint: 1, 134 | P75: 10, 135 | P95: 20, 136 | P99: 20, 137 | Count: 2, 138 | Sum: 66, 139 | Avg: float64(66) / float64(9), 140 | Worst: utils.ExampleMetadata{ProfileID: "3"}, 141 | Examples: []utils.ExampleMetadata{{ProfileID: "1"}, {ProfileID: "3"}}, 142 | }, 143 | }, //want 144 | }, 145 | } 146 | 147 | for _, test := range tests { 148 | metrics := test.Aggregator.ToMetrics() 149 | sort.Slice(metrics, func(i, j int) bool { 150 | return metrics[i].Name < metrics[j].Name 151 | }) 152 | if diff := testutil.Diff(metrics, test.want); diff != "" { 153 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /internal/occurrence/find.go: -------------------------------------------------------------------------------- 1 | package occurrence 2 | 3 | import ( 4 | "github.com/getsentry/vroom/internal/nodetree" 5 | "github.com/getsentry/vroom/internal/profile" 6 | ) 7 | 8 | func Find(p profile.Profile, callTrees map[uint64][]*nodetree.Node) []*Occurrence { 9 | var occurrences []*Occurrence 10 | if jobs, exists := detectFrameJobs[p.Platform()]; exists { 11 | for _, metadata := range jobs { 12 | detectFrame(p, callTrees, metadata, &occurrences) 13 | } 14 | } 15 | findFrameDropCause(p, callTrees, &occurrences) 16 | return occurrences 17 | } 18 | -------------------------------------------------------------------------------- /internal/occurrence/frame_drop.go: -------------------------------------------------------------------------------- 1 | package occurrence 2 | 3 | import ( 4 | "math" 5 | "time" 6 | 7 | "github.com/getsentry/vroom/internal/frame" 8 | "github.com/getsentry/vroom/internal/nodetree" 9 | "github.com/getsentry/vroom/internal/profile" 10 | ) 11 | 12 | type ( 13 | nodeStack struct { 14 | depth int 15 | n *nodetree.Node 16 | st []*nodetree.Node 17 | } 18 | 19 | frozenFrameStats struct { 20 | durationNS uint64 21 | endNS uint64 22 | minDurationNS uint64 23 | startLimitNS uint64 24 | startNS uint64 25 | } 26 | ) 27 | 28 | func newFrozenFrameStats(endNS uint64, durationNS float64) frozenFrameStats { 29 | margin := uint64(math.Max(durationNS*marginPercent, float64(10*time.Millisecond))) 30 | s := frozenFrameStats{ 31 | endNS: endNS + margin, 32 | durationNS: uint64(durationNS), 33 | minDurationNS: uint64(durationNS * minFrameDurationPercent), 34 | } 35 | if endNS >= (s.durationNS + margin) { 36 | s.startNS = endNS - s.durationNS - margin 37 | } 38 | s.startLimitNS = s.startNS + uint64(durationNS*0.20) 39 | return s 40 | } 41 | 42 | // nodeStackIfValid returns the nodeStack if we consider it valid as 43 | // a frame drop cause. 44 | func (s *frozenFrameStats) IsNodeStackValid(ns *nodeStack) bool { 45 | return ns.n.Frame.Function != "" && 46 | ns.n.IsApplication && 47 | ns.n.StartNS >= s.startNS && 48 | ns.n.EndNS <= s.endNS && 49 | ns.n.DurationNS >= s.minDurationNS && 50 | ns.n.StartNS <= s.startLimitNS 51 | } 52 | 53 | const ( 54 | FrameDrop Category = "frame_drop" 55 | 56 | marginPercent float64 = 0.05 57 | minFrameDurationPercent float64 = 0.5 58 | startLimitPercent float64 = 0.2 59 | unknownFramesInTheStackThreshold float64 = 0.8 60 | ) 61 | 62 | func findFrameDropCause( 63 | p profile.Profile, 64 | callTreesPerThreadID map[uint64][]*nodetree.Node, 65 | occurrences *[]*Occurrence, 66 | ) { 67 | frameDrops, exists := p.Measurements()["frozen_frame_renders"] 68 | if !exists { 69 | return 70 | } 71 | callTrees, exists := callTreesPerThreadID[p.Transaction().ActiveThreadID] 72 | if !exists { 73 | return 74 | } 75 | for _, mv := range frameDrops.Values { 76 | stats := newFrozenFrameStats(mv.ElapsedSinceStartNs, mv.Value) 77 | for _, root := range callTrees { 78 | st := make([]*nodetree.Node, 0, profile.MaxStackDepth) 79 | cause := findFrameDropCauseFrame( 80 | root, 81 | stats, 82 | &st, 83 | 0, 84 | ) 85 | if cause == nil { 86 | continue 87 | } 88 | // We found a potential stacktrace responsible for this frozen frame 89 | stackTrace := make([]frame.Frame, 0, len(cause.st)) 90 | var unknownFramesCount float64 91 | for _, f := range cause.st { 92 | if f.Frame.Function == "" { 93 | unknownFramesCount++ 94 | } 95 | stackTrace = append(stackTrace, f.ToFrame()) 96 | } 97 | // If there are too many unknown frames in the stack, 98 | // we do not create an occurrence. 99 | if unknownFramesCount >= float64(len(stackTrace))*unknownFramesInTheStackThreshold { 100 | continue 101 | } 102 | *occurrences = append( 103 | *occurrences, 104 | NewOccurrence(p, nodeInfo{ 105 | Category: FrameDrop, 106 | Node: *cause.n, 107 | StackTrace: stackTrace, 108 | }), 109 | ) 110 | break 111 | } 112 | } 113 | } 114 | 115 | func findFrameDropCauseFrame( 116 | n *nodetree.Node, 117 | stats frozenFrameStats, 118 | st *[]*nodetree.Node, 119 | depth int, 120 | ) *nodeStack { 121 | *st = append(*st, n) 122 | defer func() { 123 | *st = (*st)[:len(*st)-1] 124 | }() 125 | var longest *nodeStack 126 | 127 | // Explore each branch to find the deepest valid node. 128 | for _, c := range n.Children { 129 | cause := findFrameDropCauseFrame( 130 | c, 131 | stats, 132 | st, 133 | depth+1, 134 | ) 135 | if cause == nil { 136 | continue 137 | } 138 | if longest == nil { 139 | longest = cause 140 | continue 141 | } 142 | 143 | // Only keep the longest node. 144 | if cause.n.DurationNS > longest.n.DurationNS || 145 | cause.n.DurationNS == longest.n.DurationNS && cause.depth > longest.depth { 146 | longest = cause 147 | } 148 | } 149 | 150 | var current *nodeStack 151 | 152 | // Create a nodeStack of the current node 153 | ns := &nodeStack{depth, n, nil} 154 | // Check if current node if valid. 155 | if stats.IsNodeStackValid(ns) { 156 | current = ns 157 | } 158 | 159 | if longest == nil && current == nil { 160 | return nil 161 | } 162 | 163 | // If we didn't find any valid node downstream, we return the current. 164 | if longest == nil { 165 | current.st = make([]*nodetree.Node, len(*st)) 166 | copy(current.st, *st) 167 | return current 168 | } 169 | 170 | // If current is not valid or a node downstream is equal or longer, we return it. 171 | // We gave priority to the child instead of the current node. 172 | if current == nil || longest.n.DurationNS >= current.n.DurationNS { 173 | return longest 174 | } 175 | 176 | current.st = make([]*nodetree.Node, len(*st)) 177 | copy(current.st, *st) 178 | return current 179 | } 180 | -------------------------------------------------------------------------------- /internal/occurrence/kafka.go: -------------------------------------------------------------------------------- 1 | package occurrence 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/segmentio/kafka-go" 7 | ) 8 | 9 | func GenerateKafkaMessageBatch(occurrences []*Occurrence) ([]kafka.Message, error) { 10 | messages := make([]kafka.Message, 0, len(occurrences)) 11 | for _, o := range occurrences { 12 | b, err := json.Marshal(o) 13 | if err != nil { 14 | return nil, err 15 | } 16 | messages = append(messages, kafka.Message{ 17 | Value: b, 18 | }) 19 | } 20 | return messages, nil 21 | } 22 | -------------------------------------------------------------------------------- /internal/occurrence/occurrence_test.go: -------------------------------------------------------------------------------- 1 | package occurrence 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/getsentry/vroom/internal/frame" 7 | "github.com/getsentry/vroom/internal/platform" 8 | "github.com/getsentry/vroom/internal/testutil" 9 | ) 10 | 11 | func TestNormalizeAndroidStackTrace(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | input []frame.Frame 15 | output []frame.Frame 16 | }{ 17 | { 18 | name: "Normalize Android stack trace", 19 | input: []frame.Frame{ 20 | { 21 | Package: "com.google.gson", 22 | Function: "com.google.gson.JSONDecode.decode()", 23 | }, 24 | }, 25 | output: []frame.Frame{ 26 | { 27 | Package: "com.google.gson", 28 | Function: "JSONDecode.decode()", 29 | }, 30 | }, 31 | }, 32 | } 33 | 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | normalizeAndroidStackTrace(tt.input) 37 | if diff := testutil.Diff(tt.input, tt.output); diff != "" { 38 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 39 | } 40 | }) 41 | } 42 | } 43 | 44 | func TestFromRegressedFunction(t *testing.T) { 45 | f := frame.Frame{ 46 | Module: "foo", 47 | Function: "bar", 48 | } 49 | tests := []struct { 50 | name string 51 | frame frame.Frame 52 | function RegressedFunction 53 | expectedType Type 54 | expectedTitle IssueTitle 55 | expectedSubtitle string 56 | }{ 57 | { 58 | name: "released", 59 | frame: f, 60 | function: RegressedFunction{ 61 | OrganizationID: 1, 62 | ProjectID: 1, 63 | ProfileID: "", 64 | Fingerprint: 0, 65 | AggregateRange1: 100_000_000, 66 | AggregateRange2: 200_000_000, 67 | }, 68 | expectedType: 2011, 69 | expectedTitle: "Function Regression", 70 | expectedSubtitle: "Duration increased from 100ms to 200ms (P95).", 71 | }, 72 | } 73 | 74 | for _, tt := range tests { 75 | t.Run(tt.name, func(t *testing.T) { 76 | occ := FromRegressedFunction(platform.Python, tt.function, tt.frame) 77 | if occ.Type != tt.expectedType { 78 | t.Fatalf("Occurrent type mismatch: got %v want %v\n", occ.Type, tt.expectedType) 79 | } 80 | if occ.IssueTitle != tt.expectedTitle { 81 | t.Fatalf("Occurrent title mismatch: got %v want %v\n", occ.IssueTitle, tt.expectedTitle) 82 | } 83 | if occ.Subtitle != tt.expectedSubtitle { 84 | t.Fatalf("Occurrent subtitle mismatch: got %v want %v\n", occ.Subtitle, tt.expectedSubtitle) 85 | } 86 | }) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /internal/occurrence/regressed_frame.go: -------------------------------------------------------------------------------- 1 | package occurrence 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/getsentry/sentry-go" 8 | "github.com/getsentry/vroom/internal/chunk" 9 | "github.com/getsentry/vroom/internal/frame" 10 | "github.com/getsentry/vroom/internal/platform" 11 | "github.com/getsentry/vroom/internal/profile" 12 | "github.com/getsentry/vroom/internal/storageutil" 13 | "github.com/getsentry/vroom/internal/utils" 14 | "gocloud.dev/blob" 15 | ) 16 | 17 | type RegressedFunction struct { 18 | OrganizationID uint64 `json:"organization_id"` 19 | ProjectID uint64 `json:"project_id"` 20 | ProfileID string `json:"profile_id"` 21 | Example utils.ExampleMetadata `json:"example"` 22 | Fingerprint uint32 `json:"fingerprint"` 23 | AbsolutePercentageChange float64 `json:"absolute_percentage_change"` 24 | AggregateRange1 float64 `json:"aggregate_range_1"` 25 | AggregateRange2 float64 `json:"aggregate_range_2"` 26 | Breakpoint uint64 `json:"breakpoint"` 27 | TrendDifference float64 `json:"trend_difference"` 28 | TrendPercentage float64 `json:"trend_percentage"` 29 | UnweightedPValue float64 `json:"unweighted_p_value"` 30 | UnweightedTValue float64 `json:"unweighted_t_value"` 31 | } 32 | 33 | func ProcessRegressedFunction( 34 | ctx context.Context, 35 | profilesBucket *blob.Bucket, 36 | regressedFunction RegressedFunction, 37 | jobs chan storageutil.ReadJob, 38 | ) (*Occurrence, error) { 39 | results := make(chan storageutil.ReadJobResult, 1) 40 | defer close(results) 41 | 42 | if regressedFunction.ProfileID != "" { 43 | // For back compat, we should be use the example moving forwards 44 | jobs <- profile.ReadJob{ 45 | Ctx: ctx, 46 | OrganizationID: regressedFunction.OrganizationID, 47 | ProjectID: regressedFunction.ProjectID, 48 | ProfileID: regressedFunction.ProfileID, 49 | Storage: profilesBucket, 50 | Result: results, 51 | } 52 | } else if regressedFunction.Example.ProfileID != "" { 53 | jobs <- profile.ReadJob{ 54 | Ctx: ctx, 55 | OrganizationID: regressedFunction.OrganizationID, 56 | ProjectID: regressedFunction.ProjectID, 57 | ProfileID: regressedFunction.Example.ProfileID, 58 | Storage: profilesBucket, 59 | Result: results, 60 | } 61 | } else { 62 | jobs <- chunk.ReadJob{ 63 | Ctx: ctx, 64 | OrganizationID: regressedFunction.OrganizationID, 65 | ProjectID: regressedFunction.ProjectID, 66 | ProfilerID: regressedFunction.Example.ProfilerID, 67 | ChunkID: regressedFunction.Example.ChunkID, 68 | Storage: profilesBucket, 69 | Result: results, 70 | } 71 | } 72 | 73 | res := <-results 74 | platform, frame, err := getPlatformAndFrame(ctx, res, regressedFunction.Fingerprint) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return FromRegressedFunction(platform, regressedFunction, frame), nil 79 | } 80 | 81 | func getPlatformAndFrame( 82 | ctx context.Context, 83 | res storageutil.ReadJobResult, 84 | target uint32, 85 | ) (platform.Platform, frame.Frame, error) { 86 | var platform platform.Platform 87 | var frame frame.Frame 88 | 89 | err := res.Error() 90 | if err != nil { 91 | return platform, frame, err 92 | } 93 | 94 | s := sentry.StartSpan(ctx, "processing") 95 | s.Description = "Searching for fingerprint" 96 | defer s.Finish() 97 | 98 | if result, ok := res.(profile.ReadJobResult); ok { 99 | platform = result.Profile.Platform() 100 | frame, err = result.Profile.GetFrameWithFingerprint(target) 101 | if err != nil { 102 | return platform, frame, err 103 | } 104 | } else if result, ok := res.(chunk.ReadJobResult); ok { 105 | platform = result.Chunk.GetPlatform() 106 | frame, err = result.Chunk.GetFrameWithFingerprint(target) 107 | if err != nil { 108 | return platform, frame, err 109 | } 110 | } else { 111 | // This should never happen 112 | return platform, frame, errors.New("unexpected result from storage") 113 | } 114 | 115 | return platform, frame, nil 116 | } 117 | -------------------------------------------------------------------------------- /internal/packageutil/package.go: -------------------------------------------------------------------------------- 1 | package packageutil 2 | 3 | import "strings" 4 | 5 | // IsRustApplicationPackage determines whether the image represents that of the application 6 | // binary (or a binary embedded in the application binary) by checking its package path. 7 | func IsRustApplicationPackage(p string) bool { 8 | return p != "" && 9 | // `/library/std/src/` and `/usr/lib/system/` come from a real profile collected on macos. 10 | // In this case the function belongs to a shared library, not to the profiled application. 11 | !strings.Contains(p, "/library/std/src/") && 12 | !strings.HasPrefix(p, "/usr/lib/system/") && 13 | // the following a prefixes of functions belonging to either core lib 14 | // or third party libs 15 | !strings.HasPrefix(p, "/rustc/") && 16 | !strings.HasPrefix(p, "/usr/local/rustup/") && 17 | !strings.HasPrefix(p, "/usr/local/cargo/") 18 | } 19 | 20 | // IsCocoaApplicationPackage determines whether the image represents that of the application 21 | // binary (or a binary embedded in the application binary) by checking its package path. 22 | func IsCocoaApplicationPackage(p string) bool { 23 | // These are the path patterns that iOS uses for applications, system 24 | // libraries are stored elsewhere. 25 | return strings.HasPrefix(p, "/private/var/containers") || 26 | strings.HasPrefix(p, "/var/containers") || 27 | strings.Contains(p, "/Developer/Xcode/DerivedData") || 28 | strings.Contains(p, "/data/Containers/Bundle/Application") 29 | } 30 | 31 | var ( 32 | androidPackagePrefixes = []string{ 33 | "android.", 34 | "androidx.", 35 | "com.android.", 36 | "com.google.android.", 37 | "com.motorola.", 38 | "java.", 39 | "javax.", 40 | "kotlin.", 41 | "kotlinx.", 42 | "retrofit2.", 43 | "sun.", 44 | } 45 | ) 46 | 47 | // IsAndroidApplicationPackage checks if a symbol belongs to an Android system package. 48 | func IsAndroidApplicationPackage(packageName string) bool { 49 | for _, p := range androidPackagePrefixes { 50 | if strings.HasPrefix(packageName, p) { 51 | return false 52 | } 53 | } 54 | return true 55 | } 56 | -------------------------------------------------------------------------------- /internal/platform/platform.go: -------------------------------------------------------------------------------- 1 | package platform 2 | 3 | type Platform string 4 | 5 | const ( 6 | Android Platform = "android" 7 | Cocoa Platform = "cocoa" 8 | Java Platform = "java" 9 | JavaScript Platform = "javascript" 10 | Node Platform = "node" 11 | PHP Platform = "php" 12 | Python Platform = "python" 13 | Rust Platform = "rust" 14 | ) 15 | -------------------------------------------------------------------------------- /internal/profile/consts.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | const MaxStackDepth = 128 4 | -------------------------------------------------------------------------------- /internal/profile/legacy_test.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/getsentry/vroom/internal/frame" 8 | "github.com/getsentry/vroom/internal/sample" 9 | "github.com/getsentry/vroom/internal/testutil" 10 | ) 11 | 12 | func TestSampleToAndroidFormat(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | input sample.Trace 16 | output Android 17 | }{ 18 | { 19 | name: "Convert Sample Profile to Android profile: stacks {[a,b,c], [a,b,d]}", 20 | input: sample.Trace{ 21 | Frames: []frame.Frame{ 22 | {Function: "a", InApp: &testutil.False}, 23 | {Function: "b", InApp: &testutil.True}, 24 | {Function: "c", InApp: &testutil.False}, 25 | {Function: "d", InApp: &testutil.False}, 26 | }, 27 | Samples: []sample.Sample{ 28 | { 29 | ElapsedSinceStartNS: 0, 30 | StackID: 0, 31 | ThreadID: 1, 32 | }, 33 | { 34 | ElapsedSinceStartNS: 1e7, 35 | StackID: 1, 36 | ThreadID: 1, 37 | }, 38 | { 39 | ElapsedSinceStartNS: 1e7 * 2, 40 | StackID: 0, 41 | ThreadID: 1, 42 | }, 43 | }, 44 | Stacks: []sample.Stack{ 45 | {2, 1, 0}, 46 | {3, 1, 0}, 47 | }, 48 | ThreadMetadata: map[string]sample.ThreadMetadata{ 49 | "1": { 50 | Name: "JavaScriptThread", 51 | }, 52 | }, 53 | }, 54 | output: Android{ 55 | Clock: DualClock, 56 | Events: []AndroidEvent{ 57 | { 58 | Action: "Enter", 59 | ThreadID: 2, 60 | MethodID: 1, 61 | Time: EventTime{}, 62 | }, 63 | { 64 | Action: "Enter", 65 | ThreadID: 2, 66 | MethodID: 2, 67 | Time: EventTime{}, 68 | }, 69 | { 70 | Action: "Enter", 71 | ThreadID: 2, 72 | MethodID: 3, 73 | Time: EventTime{}, 74 | }, 75 | { 76 | Action: "Exit", 77 | ThreadID: 2, 78 | MethodID: 3, 79 | Time: EventTime{ 80 | Monotonic: EventMonotonic{ 81 | Wall: Duration{ 82 | Nanos: 1e7, 83 | Secs: 0, 84 | }, 85 | }, 86 | }, 87 | }, 88 | { 89 | Action: "Enter", 90 | ThreadID: 2, 91 | MethodID: 4, 92 | Time: EventTime{ 93 | Monotonic: EventMonotonic{ 94 | Wall: Duration{ 95 | Nanos: 1e7, 96 | Secs: 0, 97 | }, 98 | }, 99 | }, 100 | }, 101 | { 102 | Action: "Exit", 103 | ThreadID: 2, 104 | MethodID: 4, 105 | Time: EventTime{ 106 | Monotonic: EventMonotonic{ 107 | Wall: Duration{ 108 | Nanos: 1e7 * 2, 109 | Secs: 0, 110 | }, 111 | }, 112 | }, 113 | }, 114 | { 115 | Action: "Exit", 116 | ThreadID: 2, 117 | MethodID: 2, 118 | Time: EventTime{ 119 | Monotonic: EventMonotonic{ 120 | Wall: Duration{ 121 | Nanos: 1e7 * 2, 122 | Secs: 0, 123 | }, 124 | }, 125 | }, 126 | }, 127 | { 128 | Action: "Exit", 129 | ThreadID: 2, 130 | MethodID: 1, 131 | Time: EventTime{ 132 | Monotonic: EventMonotonic{ 133 | Wall: Duration{ 134 | Nanos: 1e7 * 2, 135 | Secs: 0, 136 | }, 137 | }, 138 | }, 139 | }, 140 | }, // end events 141 | Methods: []AndroidMethod{ 142 | { 143 | ID: 1, 144 | Name: "a", 145 | InApp: &testutil.False, 146 | }, 147 | { 148 | ID: 2, 149 | Name: "b", 150 | InApp: &testutil.True, 151 | }, 152 | { 153 | ID: 3, 154 | Name: "c", 155 | InApp: &testutil.False, 156 | }, 157 | { 158 | ID: 4, 159 | Name: "d", 160 | InApp: &testutil.False, 161 | }, 162 | }, 163 | 164 | Threads: []AndroidThread{ 165 | { 166 | ID: 2, 167 | Name: "main", 168 | }, 169 | }, 170 | }, 171 | }, 172 | { 173 | name: "Convert Sample Profile to Android profile: stacks {[a,b,c], [a,b]}", 174 | input: sample.Trace{ 175 | Frames: []frame.Frame{ 176 | {Function: "a", InApp: &testutil.False}, 177 | {Function: "b", InApp: &testutil.True}, 178 | {Function: "c", InApp: &testutil.False}, 179 | }, 180 | Samples: []sample.Sample{ 181 | { 182 | ElapsedSinceStartNS: 0, 183 | StackID: 0, 184 | ThreadID: 1, 185 | }, 186 | { 187 | ElapsedSinceStartNS: 1e7, 188 | StackID: 1, 189 | ThreadID: 1, 190 | }, 191 | { 192 | ElapsedSinceStartNS: 1e7 * 2, 193 | StackID: 0, 194 | ThreadID: 1, 195 | }, 196 | }, 197 | Stacks: []sample.Stack{ 198 | {2, 1, 0}, 199 | {1, 0}, 200 | }, 201 | ThreadMetadata: map[string]sample.ThreadMetadata{ 202 | "1": { 203 | Name: "JavaScriptThread", 204 | }, 205 | }, 206 | }, 207 | output: Android{ 208 | Clock: DualClock, 209 | Events: []AndroidEvent{ 210 | { 211 | Action: "Enter", 212 | ThreadID: 2, 213 | MethodID: 1, 214 | Time: EventTime{}, 215 | }, 216 | { 217 | Action: "Enter", 218 | ThreadID: 2, 219 | MethodID: 2, 220 | Time: EventTime{}, 221 | }, 222 | { 223 | Action: "Enter", 224 | ThreadID: 2, 225 | MethodID: 3, 226 | Time: EventTime{}, 227 | }, 228 | { 229 | Action: "Exit", 230 | ThreadID: 2, 231 | MethodID: 3, 232 | Time: EventTime{ 233 | Monotonic: EventMonotonic{ 234 | Wall: Duration{ 235 | Nanos: 1e7, 236 | Secs: 0, 237 | }, 238 | }, 239 | }, 240 | }, 241 | 242 | { 243 | Action: "Exit", 244 | ThreadID: 2, 245 | MethodID: 2, 246 | Time: EventTime{ 247 | Monotonic: EventMonotonic{ 248 | Wall: Duration{ 249 | Nanos: 1e7 * 2, 250 | Secs: 0, 251 | }, 252 | }, 253 | }, 254 | }, 255 | { 256 | Action: "Exit", 257 | ThreadID: 2, 258 | MethodID: 1, 259 | Time: EventTime{ 260 | Monotonic: EventMonotonic{ 261 | Wall: Duration{ 262 | Nanos: 1e7 * 2, 263 | Secs: 0, 264 | }, 265 | }, 266 | }, 267 | }, 268 | }, // end events 269 | Methods: []AndroidMethod{ 270 | { 271 | ID: 1, 272 | Name: "a", 273 | InApp: &testutil.False, 274 | }, 275 | { 276 | ID: 2, 277 | Name: "b", 278 | InApp: &testutil.True, 279 | }, 280 | { 281 | ID: 3, 282 | Name: "c", 283 | InApp: &testutil.False, 284 | }, 285 | }, 286 | 287 | Threads: []AndroidThread{ 288 | { 289 | ID: 2, 290 | Name: "main", 291 | }, 292 | }, 293 | }, 294 | }, 295 | } 296 | 297 | for _, test := range tests { 298 | t.Run(test.name, func(t *testing.T) { 299 | convertedProfile := sampleToAndroidFormat(test.input, 1, map[uint64]void{1: {}}) 300 | 301 | if diff := testutil.Diff(convertedProfile, test.output); diff != "" { 302 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 303 | } 304 | }) 305 | } 306 | } 307 | 308 | func TestNormalizeLegacyAndroidProfile(t *testing.T) { 309 | tests := []struct { 310 | name string 311 | input LegacyProfile 312 | output LegacyProfile 313 | }{ 314 | { 315 | name: "Classify [Native] frames as system frames", 316 | input: LegacyProfile{ 317 | RawProfile: RawProfile{ 318 | JsProfile: json.RawMessage(`{"profile":{"frames":[{"function":"[Native] functionPrototypeApply"}]}}`), 319 | }, 320 | Trace: &Android{}, 321 | }, 322 | output: LegacyProfile{ 323 | RawProfile: RawProfile{ 324 | JsProfile: json.RawMessage(`{"profile":{"frames":[{"data":{},"function":"[Native] functionPrototypeApply","in_app":false,"platform":"javascript"}],"queue_metadata":null,"samples":null,"stacks":null,"thread_metadata":null}}`), 325 | }, 326 | Trace: &Android{}, 327 | }, 328 | }, 329 | } 330 | 331 | for _, test := range tests { 332 | t.Run(test.name, func(t *testing.T) { 333 | test.input.Normalize() 334 | if diff := testutil.Diff(test.input, test.output); diff != "" { 335 | t.Fatalf("Result mismatch: got - want +\n%s", diff) 336 | } 337 | }) 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /internal/profile/profile.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/getsentry/vroom/internal/debugmeta" 8 | "github.com/getsentry/vroom/internal/frame" 9 | "github.com/getsentry/vroom/internal/measurements" 10 | "github.com/getsentry/vroom/internal/metadata" 11 | "github.com/getsentry/vroom/internal/nodetree" 12 | "github.com/getsentry/vroom/internal/platform" 13 | "github.com/getsentry/vroom/internal/sample" 14 | "github.com/getsentry/vroom/internal/speedscope" 15 | "github.com/getsentry/vroom/internal/transaction" 16 | "github.com/getsentry/vroom/internal/utils" 17 | ) 18 | 19 | type ( 20 | profileInterface interface { 21 | GetDebugMeta() debugmeta.DebugMeta 22 | GetDurationNS() uint64 23 | GetEnvironment() string 24 | GetID() string 25 | GetMeasurements() map[string]measurements.Measurement 26 | GetOrganizationID() uint64 27 | GetPlatform() platform.Platform 28 | GetProjectID() uint64 29 | GetReceived() time.Time 30 | GetRelease() string 31 | GetRetentionDays() int 32 | GetTimestamp() time.Time 33 | GetTransaction() transaction.Transaction 34 | GetTransactionMetadata() transaction.Metadata 35 | GetTransactionTags() map[string]string 36 | 37 | CallTrees() (map[uint64][]*nodetree.Node, error) 38 | IsSampleFormat() bool 39 | Metadata() metadata.Metadata 40 | Normalize() 41 | Speedscope() (speedscope.Output, error) 42 | StoragePath() string 43 | IsSampled() bool 44 | SetProfileID(ID string) 45 | GetOptions() utils.Options 46 | GetFrameWithFingerprint(uint32) (frame.Frame, error) 47 | } 48 | 49 | Profile struct { 50 | profile profileInterface 51 | } 52 | 53 | version struct { 54 | Version string `json:"version"` 55 | } 56 | ) 57 | 58 | func New(p profileInterface) Profile { 59 | return Profile{ 60 | profile: p, 61 | } 62 | } 63 | 64 | func (p *Profile) UnmarshalJSON(b []byte) error { 65 | var v version 66 | err := json.Unmarshal(b, &v) 67 | if err != nil { 68 | return err 69 | } 70 | switch v.Version { 71 | case "": 72 | p.profile = new(LegacyProfile) 73 | default: 74 | p.profile = new(sample.Profile) 75 | } 76 | return json.Unmarshal(b, &p.profile) 77 | } 78 | 79 | func (p Profile) MarshalJSON() ([]byte, error) { 80 | return json.Marshal(p.profile) 81 | } 82 | 83 | func (p *Profile) CallTrees() (map[uint64][]*nodetree.Node, error) { 84 | return p.profile.CallTrees() 85 | } 86 | 87 | func (p *Profile) DebugMeta() debugmeta.DebugMeta { 88 | return p.profile.GetDebugMeta() 89 | } 90 | 91 | func (p *Profile) ID() string { 92 | return p.profile.GetID() 93 | } 94 | 95 | func (p *Profile) OrganizationID() uint64 { 96 | return p.profile.GetOrganizationID() 97 | } 98 | 99 | func (p *Profile) ProjectID() uint64 { 100 | return p.profile.GetProjectID() 101 | } 102 | 103 | func (p *Profile) StoragePath() string { 104 | return p.profile.StoragePath() 105 | } 106 | 107 | func (p *Profile) IsSampleFormat() bool { 108 | return p.profile.IsSampleFormat() 109 | } 110 | 111 | func (p *Profile) Speedscope() (speedscope.Output, error) { 112 | return p.profile.Speedscope() 113 | } 114 | 115 | func (p *Profile) Metadata() metadata.Metadata { 116 | return p.profile.Metadata() 117 | } 118 | 119 | func (p *Profile) Platform() platform.Platform { 120 | return p.profile.GetPlatform() 121 | } 122 | 123 | func (p *Profile) Normalize() { 124 | p.profile.Normalize() 125 | } 126 | 127 | func (p *Profile) Transaction() transaction.Transaction { 128 | return p.profile.GetTransaction() 129 | } 130 | 131 | func (p *Profile) Environment() string { 132 | return p.profile.GetEnvironment() 133 | } 134 | 135 | func (p *Profile) Timestamp() time.Time { 136 | return p.profile.GetTimestamp() 137 | } 138 | 139 | func (p *Profile) StartAndEndEpoch() (uint64, uint64) { 140 | startEpoch := uint64(p.Timestamp().UnixNano()) 141 | duration := p.DurationNS() 142 | return startEpoch, startEpoch + duration 143 | } 144 | 145 | func (p *Profile) Received() time.Time { 146 | return p.profile.GetReceived() 147 | } 148 | 149 | func (p *Profile) Release() string { 150 | return p.profile.GetRelease() 151 | } 152 | 153 | func (p *Profile) RetentionDays() int { 154 | return p.profile.GetRetentionDays() 155 | } 156 | 157 | func (p *Profile) DurationNS() uint64 { 158 | return p.profile.GetDurationNS() 159 | } 160 | 161 | func (p *Profile) TransactionMetadata() transaction.Metadata { 162 | return p.profile.GetTransactionMetadata() 163 | } 164 | 165 | func (p *Profile) TransactionTags() map[string]string { 166 | return p.profile.GetTransactionTags() 167 | } 168 | 169 | func (p *Profile) IsSampled() bool { 170 | return p.profile.IsSampled() 171 | } 172 | 173 | func (p *Profile) SetProfileID(ID string) { 174 | p.profile.SetProfileID(ID) 175 | } 176 | 177 | func (p *Profile) Measurements() map[string]measurements.Measurement { 178 | return p.profile.GetMeasurements() 179 | } 180 | 181 | func (p *Profile) GetOptions() utils.Options { 182 | return p.profile.GetOptions() 183 | } 184 | 185 | func (p *Profile) GetFrameWithFingerprint(target uint32) (frame.Frame, error) { 186 | return p.profile.GetFrameWithFingerprint(target) 187 | } 188 | -------------------------------------------------------------------------------- /internal/profile/readjob.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/getsentry/vroom/internal/nodetree" 7 | "github.com/getsentry/vroom/internal/storageutil" 8 | "gocloud.dev/blob" 9 | ) 10 | 11 | type ( 12 | ReadJob struct { 13 | Ctx context.Context 14 | Storage *blob.Bucket 15 | OrganizationID uint64 16 | ProjectID uint64 17 | ProfileID string 18 | Result chan<- storageutil.ReadJobResult 19 | } 20 | 21 | ReadJobResult struct { 22 | Err error 23 | Profile *Profile 24 | } 25 | ) 26 | 27 | func (job ReadJob) Read() { 28 | var profile Profile 29 | 30 | err := storageutil.UnmarshalCompressed( 31 | job.Ctx, 32 | job.Storage, 33 | StoragePath(job.OrganizationID, job.ProjectID, job.ProfileID), 34 | &profile, 35 | ) 36 | 37 | job.Result <- ReadJobResult{Profile: &profile, Err: err} 38 | } 39 | 40 | func (result ReadJobResult) Error() error { 41 | return result.Err 42 | } 43 | 44 | type ( 45 | CallTreesReadJob ReadJob 46 | 47 | CallTreesReadJobResult struct { 48 | Err error 49 | CallTrees map[uint64][]*nodetree.Node 50 | Profile *Profile 51 | } 52 | ) 53 | 54 | func (job CallTreesReadJob) Read() { 55 | var profile Profile 56 | 57 | err := storageutil.UnmarshalCompressed( 58 | job.Ctx, 59 | job.Storage, 60 | StoragePath(job.OrganizationID, job.ProjectID, job.ProfileID), 61 | &profile, 62 | ) 63 | 64 | if err != nil { 65 | job.Result <- CallTreesReadJobResult{Err: err} 66 | return 67 | } 68 | 69 | callTrees, err := profile.CallTrees() 70 | 71 | job.Result <- CallTreesReadJobResult{ 72 | CallTrees: callTrees, 73 | Profile: &profile, 74 | Err: err, 75 | } 76 | } 77 | 78 | func (result CallTreesReadJobResult) Error() error { 79 | return result.Err 80 | } 81 | -------------------------------------------------------------------------------- /internal/profile/trace.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import ( 4 | "github.com/getsentry/vroom/internal/frame" 5 | "github.com/getsentry/vroom/internal/nodetree" 6 | "github.com/getsentry/vroom/internal/speedscope" 7 | ) 8 | 9 | type ( 10 | Trace interface { 11 | ActiveThreadID() uint64 12 | CallTrees() map[uint64][]*nodetree.Node 13 | Speedscope() (speedscope.Output, error) 14 | GetFrameWithFingerprint(uint32) (frame.Frame, error) 15 | } 16 | ) 17 | -------------------------------------------------------------------------------- /internal/profile/version.go: -------------------------------------------------------------------------------- 1 | package profile 2 | 3 | import "fmt" 4 | 5 | func FormatVersion(name, code interface{}) string { 6 | if c, ok := code.(string); !ok || c == "" { 7 | return fmt.Sprintf("%v", name) 8 | } 9 | return fmt.Sprintf("%v (%v)", name, code) 10 | } 11 | -------------------------------------------------------------------------------- /internal/speedscope/speedscope.go: -------------------------------------------------------------------------------- 1 | package speedscope 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/getsentry/vroom/internal/clientsdk" 8 | "github.com/getsentry/vroom/internal/debugmeta" 9 | "github.com/getsentry/vroom/internal/measurements" 10 | "github.com/getsentry/vroom/internal/platform" 11 | "github.com/getsentry/vroom/internal/timeutil" 12 | "github.com/getsentry/vroom/internal/transaction" 13 | "github.com/getsentry/vroom/internal/utils" 14 | ) 15 | 16 | const ( 17 | ValueUnitNanoseconds ValueUnit = "nanoseconds" 18 | ValueUnitCount ValueUnit = "count" 19 | 20 | EventTypeOpenFrame EventType = "O" 21 | EventTypeCloseFrame EventType = "C" 22 | 23 | ProfileTypeEvented ProfileType = "evented" 24 | ProfileTypeSampled ProfileType = "sampled" 25 | ) 26 | 27 | type ( 28 | Frame struct { 29 | Col uint32 `json:"col,omitempty"` 30 | File string `json:"file,omitempty"` 31 | Image string `json:"image,omitempty"` 32 | Inline bool `json:"inline,omitempty"` 33 | IsApplication bool `json:"is_application"` 34 | Line uint32 `json:"line,omitempty"` 35 | Name string `json:"name"` 36 | Path string `json:"path,omitempty"` 37 | } 38 | 39 | Event struct { 40 | Type EventType `json:"type"` 41 | Frame int `json:"frame"` 42 | At uint64 `json:"at"` 43 | } 44 | 45 | Queue struct { 46 | Label string `json:"name"` 47 | StartNS uint64 `json:"start_ns"` 48 | EndNS uint64 `json:"end_ns"` 49 | } 50 | 51 | EventedProfile struct { 52 | EndValue uint64 `json:"endValue"` 53 | Events []Event `json:"events"` 54 | Name string `json:"name"` 55 | StartValue uint64 `json:"startValue"` 56 | ThreadID uint64 `json:"threadID"` 57 | Type ProfileType `json:"type"` 58 | Unit ValueUnit `json:"unit"` 59 | } 60 | 61 | SampledProfile struct { 62 | EndValue uint64 `json:"endValue"` 63 | IsMainThread bool `json:"isMainThread"` 64 | Name string `json:"name"` 65 | Priority int `json:"priority,omitempty"` 66 | Queues map[string]Queue `json:"queues,omitempty"` 67 | Samples [][]int `json:"samples"` 68 | SamplesProfiles [][]int `json:"samples_profiles,omitempty"` 69 | SamplesExamples [][]int `json:"samples_examples,omitempty"` 70 | StartValue uint64 `json:"startValue"` 71 | State string `json:"state,omitempty"` 72 | ThreadID uint64 `json:"threadID"` 73 | Type ProfileType `json:"type"` 74 | Unit ValueUnit `json:"unit"` 75 | Weights []uint64 `json:"weights"` 76 | SampleDurationsNs []uint64 `json:"sample_durations_ns"` 77 | SampleCounts []uint64 `json:"sample_counts,omitempty"` 78 | } 79 | 80 | SharedData struct { 81 | Frames []Frame `json:"frames"` 82 | ProfileIDs []string `json:"profile_ids,omitempty"` 83 | Profiles []utils.ExampleMetadata `json:"profiles,omitempty"` 84 | } 85 | 86 | EventType string 87 | ProfileType string 88 | ValueUnit string 89 | 90 | Output struct { 91 | ActiveProfileIndex int `json:"activeProfileIndex"` 92 | AndroidClock string `json:"androidClock,omitempty"` 93 | DurationNS uint64 `json:"durationNS,omitempty"` 94 | Images []debugmeta.Image `json:"images,omitempty"` 95 | Measurements interface{} `json:"measurements,omitempty"` 96 | Metadata ProfileMetadata `json:"metadata"` 97 | Platform platform.Platform `json:"platform"` 98 | ProfileID string `json:"profileID,omitempty"` 99 | ChunkID string `json:"chunkID,omitempty"` 100 | Profiles []interface{} `json:"profiles"` 101 | ProjectID uint64 `json:"projectID"` 102 | Shared SharedData `json:"shared"` 103 | TransactionName string `json:"transactionName"` 104 | Version string `json:"version,omitempty"` 105 | Metrics *[]utils.FunctionMetrics `json:"metrics"` 106 | } 107 | 108 | ProfileMetadata struct { 109 | ProfileView 110 | 111 | Version string `json:"version"` 112 | } 113 | 114 | ProfileView struct { 115 | AndroidAPILevel uint32 `json:"androidAPILevel,omitempty"` //nolint:unused 116 | Architecture string `json:"architecture,omitempty"` //nolint:unused 117 | BuildID string `json:"-"` //nolint:unused 118 | ClientSDK clientsdk.ClientSDK `json:"-"` 119 | DebugMeta debugmeta.DebugMeta `json:"-"` //nolint:unused 120 | DeviceClassification string `json:"deviceClassification"` //nolint:unused 121 | DeviceLocale string `json:"deviceLocale"` //nolint:unused 122 | DeviceManufacturer string `json:"deviceManufacturer"` //nolint:unused 123 | DeviceModel string `json:"deviceModel"` //nolint:unused 124 | DeviceOSBuildNumber string `json:"deviceOSBuildNumber,omitempty"` //nolint:unused 125 | DeviceOSName string `json:"deviceOSName"` //nolint:unused 126 | DeviceOSVersion string `json:"deviceOSVersion"` //nolint:unused 127 | DurationNS uint64 `json:"durationNS"` //nolint:unused 128 | Environment string `json:"environment,omitempty"` //nolint:unused 129 | JsProfile json.RawMessage `json:"-"` //nolint:unused 130 | Measurements map[string]measurements.Measurement `json:"-"` //nolint:unused 131 | Options utils.Options `json:"-"` //nolint:unused 132 | OrganizationID uint64 `json:"organizationID"` 133 | Platform platform.Platform `json:"platform"` //nolint:unused 134 | Profile json.RawMessage `json:"-"` //nolint:unused 135 | ProfileID string `json:"profileID"` //nolint:unused 136 | ProjectID uint64 `json:"projectID"` //nolint:unused 137 | Received timeutil.Time `json:"received"` //nolint:unused 138 | RetentionDays int `json:"-"` //nolint:unused 139 | Sampled bool `json:"sampled"` //nolint:unused 140 | Timestamp time.Time `json:"timestamp,omitempty"` //nolint:unused 141 | TraceID string `json:"traceID"` //nolint:unused 142 | TransactionID string `json:"transactionID"` //nolint:unused 143 | TransactionMetadata transaction.Metadata `json:"-"` //nolint:unused 144 | TransactionName string `json:"transactionName"` //nolint:unused 145 | TransactionTags map[string]string `json:"-"` //nolint:unused 146 | VersionCode string `json:"-"` //nolint:unused 147 | VersionName string `json:"-"` //nolint:unused 148 | } 149 | ) 150 | -------------------------------------------------------------------------------- /internal/storageutil/storageutil.go: -------------------------------------------------------------------------------- 1 | package storageutil 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "time" 9 | 10 | "cloud.google.com/go/storage" 11 | "github.com/pierrec/lz4/v4" 12 | "gocloud.dev/blob" 13 | "gocloud.dev/gcerrors" 14 | ) 15 | 16 | // ErrObjectNotFound indicates an object was not found. 17 | var ErrObjectNotFound = errors.New("object not found") 18 | 19 | // CompressedWrite compresses and writes data to Google Cloud Storage. 20 | func CompressedWrite(ctx context.Context, b *blob.Bucket, objectName string, d interface{}) error { 21 | ctx, cancel := context.WithTimeout(ctx, 5*time.Second) 22 | defer cancel() 23 | writerOptions := &blob.WriterOptions{ 24 | BeforeWrite: func(asFunc func(interface{}) bool) error { 25 | var objp **storage.ObjectHandle 26 | // If it's not a GCS resource, we just move on. 27 | if !asFunc(&objp) { 28 | return nil 29 | } 30 | // Replace the ObjectHandle with a new one that adds Conditions. 31 | *objp = (*objp).If(storage.Conditions{DoesNotExist: true}) 32 | return nil 33 | }, 34 | } 35 | ow, err := b.NewWriter(ctx, objectName, writerOptions) 36 | if err != nil { 37 | return err 38 | } 39 | zw := lz4.NewWriter(ow) 40 | _ = zw.Apply(lz4.CompressionLevelOption(lz4.Level9)) 41 | jw := json.NewEncoder(zw) 42 | err = jw.Encode(d) 43 | if err != nil { 44 | cancel() 45 | ow.Close() 46 | return err 47 | } 48 | err = zw.Close() 49 | if err != nil { 50 | cancel() 51 | ow.Close() 52 | return err 53 | } 54 | return ow.Close() 55 | } 56 | 57 | // UnmarshalCompressed reads compressed JSON data from GCS and unmarshals it. 58 | func UnmarshalCompressed( 59 | ctx context.Context, 60 | b *blob.Bucket, 61 | objectName string, 62 | d interface{}, 63 | ) error { 64 | ctx, cancel := context.WithTimeout(ctx, 5*time.Second) 65 | defer cancel() 66 | 67 | or, err := b.NewReader(ctx, objectName, nil) 68 | if err != nil { 69 | if gcerrors.Code(err) == gcerrors.NotFound { 70 | return fmt.Errorf("%w: %s", ErrObjectNotFound, objectName) 71 | } 72 | 73 | return err 74 | } 75 | defer or.Close() 76 | zr := lz4.NewReader(or) 77 | err = json.NewDecoder(zr).Decode(d) 78 | if err != nil { 79 | return err 80 | } 81 | return nil 82 | } 83 | 84 | type ( 85 | ReadJob interface { 86 | Read() 87 | } 88 | 89 | ReadJobResult interface { 90 | Error() error 91 | } 92 | ) 93 | 94 | func ReadWorker(jobs <-chan ReadJob) { 95 | for job := range jobs { 96 | job.Read() 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /internal/storageutil/storageutil_test.go: -------------------------------------------------------------------------------- 1 | package storageutil 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "log" 11 | "os" 12 | "testing" 13 | 14 | "github.com/fsouza/fake-gcs-server/fakestorage" 15 | "github.com/getsentry/vroom/internal/sample" 16 | "github.com/google/uuid" 17 | "github.com/phayes/freeport" 18 | "github.com/pierrec/lz4/v4" 19 | "gocloud.dev/blob" 20 | _ "gocloud.dev/blob/fileblob" 21 | _ "gocloud.dev/blob/gcsblob" 22 | 23 | gojson "github.com/goccy/go-json" 24 | jsoniter "github.com/json-iterator/go" 25 | ) 26 | 27 | const bucketName = "profiles" 28 | 29 | var gcsServer *fakestorage.Server 30 | var gcsBlobBucket *blob.Bucket 31 | var fileBlobBucket *blob.Bucket 32 | 33 | type Profile struct { 34 | Samples []int `json:"samples"` 35 | Frames []int `json:"frames"` 36 | } 37 | 38 | func TestMain(m *testing.M) { 39 | port, err := freeport.GetFreePort() 40 | if err != nil { 41 | log.Fatalf("no free port found: %v", err) 42 | } 43 | publicHost := fmt.Sprintf("127.0.0.1:%d", port) 44 | gcsServer, err = fakestorage.NewServerWithOptions(fakestorage.Options{ 45 | PublicHost: publicHost, 46 | Host: "127.0.0.1", 47 | Port: uint16(port), 48 | Scheme: "http", 49 | }) 50 | if err != nil { 51 | log.Fatalf("couldn't set up gcs server: %v", err) 52 | } 53 | os.Setenv("STORAGE_EMULATOR_HOST", publicHost) 54 | gcsServer.CreateBucketWithOpts(fakestorage.CreateBucketOpts{Name: bucketName}) 55 | 56 | temporaryDirectory, err := os.MkdirTemp(os.TempDir(), "sentry-profiles-*") 57 | if err != nil { 58 | log.Fatalf("couldn't create a temporary directory: %s", err.Error()) 59 | } 60 | 61 | gcsBlobBucket, err = blob.OpenBucket(context.Background(), "gs://"+bucketName) 62 | if err != nil { 63 | log.Fatalf("couldn't open a local gcs bucket: %s", err.Error()) 64 | } 65 | fileBlobBucket, err = blob.OpenBucket(context.Background(), "file://localhost/"+temporaryDirectory) 66 | if err != nil { 67 | log.Fatalf("couldn't open a local filesystem bucket: %s", err.Error()) 68 | } 69 | 70 | code := m.Run() 71 | 72 | if err := gcsBlobBucket.Close(); err != nil { 73 | log.Printf("couldn't close the local gcs bucket: %s", err.Error()) 74 | } 75 | 76 | if err := fileBlobBucket.Close(); err != nil { 77 | log.Printf("couldn't close the local filesystem bucket: %s", err.Error()) 78 | } 79 | 80 | err = os.RemoveAll(temporaryDirectory) 81 | if err != nil { 82 | log.Printf("couldn't remove the temporary directory: %s", err.Error()) 83 | } 84 | 85 | gcsServer.Stop() 86 | 87 | os.Exit(code) 88 | } 89 | 90 | func TestUploadProfile(t *testing.T) { 91 | ctx := context.Background() 92 | objectName := uuid.New().String() 93 | originalData := struct { 94 | Samples []uint64 `json:"samples"` 95 | Frames []uint64 `json:"frames"` 96 | }{ 97 | Samples: []uint64{1, 2, 3, 4}, 98 | Frames: []uint64{1, 2, 3, 4}, 99 | } 100 | 101 | tests := []struct { 102 | name string 103 | blobBucket *blob.Bucket 104 | }{ 105 | { 106 | name: "GCS", 107 | blobBucket: gcsBlobBucket, 108 | }, 109 | { 110 | name: "Filesystem", 111 | blobBucket: fileBlobBucket, 112 | }, 113 | } 114 | 115 | for _, test := range tests { 116 | t.Run(test.name, func(t *testing.T) { 117 | err := CompressedWrite(ctx, test.blobBucket, objectName, originalData) 118 | if err != nil { 119 | t.Fatalf("we should be able to write: %s", err.Error()) 120 | } 121 | 122 | objectReader, err := test.blobBucket.NewReader(ctx, objectName, nil) 123 | if err != nil { 124 | t.Fatalf("we should be able to read the object: %s", err.Error()) 125 | } 126 | defer func() { 127 | err := objectReader.Close() 128 | if err != nil { 129 | t.Logf("closing the filereader: %s", err.Error()) 130 | } 131 | }() 132 | 133 | r := lz4.NewReader(objectReader) 134 | uncompressedData, err := io.ReadAll(r) 135 | if err != nil { 136 | t.Fatalf("we should be able to uncompress the data: %v", err) 137 | } 138 | b, err := json.Marshal(originalData) 139 | if err != nil { 140 | t.Fatalf("we should be able to marshal this: %v", err) 141 | } 142 | if !bytes.Equal(b, bytes.TrimSpace(uncompressedData)) { 143 | t.Fatal("data should be identical") 144 | } 145 | }) 146 | } 147 | } 148 | 149 | func TestDownloadProfile(t *testing.T) { 150 | ctx := context.Background() 151 | objectName := uuid.New().String() 152 | originalData := []byte(`{"samples":[1,2,3,4],"frames":[1,2,3,4]}`) 153 | 154 | var compressedData bytes.Buffer 155 | w := lz4.NewWriter(&compressedData) 156 | _, _ = w.Write(originalData) 157 | err := w.Close() 158 | if err != nil { 159 | t.Fatalf("we should be able to close the writer: %v", err) 160 | } 161 | 162 | tests := []struct { 163 | name string 164 | blobBucket *blob.Bucket 165 | }{ 166 | { 167 | name: "GCS", 168 | blobBucket: gcsBlobBucket, 169 | }, 170 | { 171 | name: "Filesystem", 172 | blobBucket: fileBlobBucket, 173 | }, 174 | } 175 | 176 | for _, test := range tests { 177 | t.Run(test.name, func(t *testing.T) { 178 | wr, err := fileBlobBucket.NewWriter(ctx, objectName, nil) 179 | if err != nil { 180 | t.Fatalf("we should write an object: %s", err.Error()) 181 | } 182 | 183 | _, err = wr.Write(compressedData.Bytes()) 184 | if err != nil { 185 | t.Fatalf("we should write an object: %s", err.Error()) 186 | } 187 | 188 | err = wr.Close() 189 | if err != nil { 190 | t.Fatalf("closing the filewriter: %s", err.Error()) 191 | } 192 | 193 | var profile Profile 194 | err = UnmarshalCompressed(ctx, fileBlobBucket, objectName, &profile) 195 | if err != nil { 196 | t.Fatalf("we should be able to read the object: %v", err) 197 | } 198 | 199 | uncompressedData, err := json.Marshal(profile) 200 | if err != nil { 201 | t.Fatalf("we should be able to marshal back to JSON: %v", err) 202 | } 203 | if !bytes.Equal(originalData, uncompressedData) { 204 | t.Fatalf("data should be identical: %v %v", string(originalData), string(uncompressedData)) 205 | } 206 | }) 207 | } 208 | } 209 | 210 | func TestDownloadProfileNotFound(t *testing.T) { 211 | ctx := context.Background() 212 | objectName := uuid.NewString() 213 | 214 | tests := []struct { 215 | name string 216 | blobBucket *blob.Bucket 217 | }{ 218 | { 219 | name: "GCS", 220 | blobBucket: gcsBlobBucket, 221 | }, 222 | { 223 | name: "Filesystem", 224 | blobBucket: fileBlobBucket, 225 | }, 226 | } 227 | 228 | for _, test := range tests { 229 | t.Run(test.name, func(t *testing.T) { 230 | var profile Profile 231 | err := UnmarshalCompressed(ctx, test.blobBucket, objectName, &profile) 232 | if err == nil { 233 | t.Error("expecting an error, got nil") 234 | } 235 | 236 | if !errors.Is(err, ErrObjectNotFound) { 237 | t.Errorf("expecting an error of ErrObjectNotFound, instead got %s", err.Error()) 238 | } 239 | }) 240 | } 241 | } 242 | 243 | func BenchmarkGoJSON(b *testing.B) { 244 | b.ReportAllocs() 245 | testProfile, err := os.ReadFile("../../test/data/node.json") 246 | if err != nil { 247 | b.Fatal(err) 248 | } 249 | for i := 0; i < b.N; i++ { 250 | var result sample.Profile 251 | if err := gojson.Unmarshal(testProfile, &result); err != nil { 252 | b.Fatal(err) 253 | } 254 | } 255 | } 256 | 257 | func BenchmarkJsonIterator(b *testing.B) { 258 | b.ReportAllocs() 259 | testProfile, err := os.ReadFile("../../test/data/node.json") 260 | if err != nil { 261 | b.Fatal(err) 262 | } 263 | for n := 0; n < b.N; n++ { 264 | var result sample.Profile 265 | if err := jsoniter.Unmarshal(testProfile, &result); err != nil { 266 | b.Fatal(err) 267 | } 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /internal/testutil/testutil.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/getsentry/vroom/internal/timeutil" 7 | "github.com/google/go-cmp/cmp" 8 | ) 9 | 10 | var ( 11 | alwaysEqual = cmp.Comparer(func(_, _ interface{}) bool { return true }) 12 | defaultCmpOptions = []cmp.Option{ 13 | // NaNs compare equal 14 | cmp.FilterValues(func(x, y float64) bool { 15 | return math.IsNaN(x) && math.IsNaN(y) 16 | }, alwaysEqual), 17 | cmp.FilterValues(func(x, y float32) bool { 18 | return math.IsNaN(float64(x)) && math.IsNaN(float64(y)) 19 | }, alwaysEqual), 20 | cmp.AllowUnexported(timeutil.Time{}), 21 | } 22 | 23 | False = false 24 | True = true 25 | ) 26 | 27 | func Diff(a, b interface{}, opts ...cmp.Option) string { 28 | opts = append(opts, defaultCmpOptions...) 29 | return cmp.Diff(a, b, opts...) 30 | } 31 | -------------------------------------------------------------------------------- /internal/timeutil/time.go: -------------------------------------------------------------------------------- 1 | package timeutil 2 | 3 | import ( 4 | "encoding/json" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | type Time time.Time 10 | 11 | func (t *Time) UnmarshalJSON(b []byte) error { 12 | s := string(b) 13 | if s == "null" || s == "{}" { 14 | return nil 15 | } 16 | if s[0] == '"' { 17 | tt, err := time.Parse(`"`+time.RFC3339+`"`, s) 18 | if err != nil { 19 | return err 20 | } 21 | *t = Time(tt) 22 | } else { 23 | i, err := strconv.ParseInt(s, 10, 64) 24 | if err != nil { 25 | return err 26 | } 27 | *t = Time(time.Unix(i, 0)) 28 | } 29 | return nil 30 | } 31 | 32 | func (t Time) MarshalJSON() ([]byte, error) { 33 | return json.Marshal(time.Time(t)) 34 | } 35 | 36 | func (t Time) Time() time.Time { 37 | return time.Time(t) 38 | } 39 | -------------------------------------------------------------------------------- /internal/timeutil/time_test.go: -------------------------------------------------------------------------------- 1 | package timeutil 2 | 3 | import ( 4 | "encoding/json" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestParseInt64Timeutil(t *testing.T) { 10 | var tt Time 11 | b := []byte(`1675277158`) 12 | err := json.Unmarshal(b, &tt) 13 | if err != nil { 14 | t.Fatalf("error while parsing: %+v\n", err) 15 | } 16 | if string(b) != strconv.FormatInt(tt.Time().Unix(), 10) { 17 | t.Fatalf("wanted: %+v, got: %+v\n", string(b), tt.Time().Unix()) 18 | } 19 | } 20 | func TestParseStringTimeutil(t *testing.T) { 21 | var tt Time 22 | b := []byte(`"2023-01-01T12:00:00+00:00"`) 23 | err := json.Unmarshal(b, &tt) 24 | if err != nil { 25 | t.Fatalf("error while parsing: %+v\n", err) 26 | } 27 | ttf := tt.Time().Format(`"2006-01-02T15:04:05-07:00"`) 28 | if string(b) != ttf { 29 | t.Fatalf("wanted: %+v, got: %+v\n", string(b), ttf) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/transaction/metadata.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | import "time" 4 | 5 | type ( 6 | Metadata struct { 7 | AppIdentifier string `json:"app.identifier,omitempty"` 8 | Dist string `json:"dist,omitempty"` 9 | Environment string `json:"environment,omitempty"` 10 | HTTPMethod string `json:"http.method,omitempty"` 11 | Release string `json:"release,omitempty"` 12 | Transaction string `json:"transaction,omitempty"` 13 | TransactionEnd time.Time `json:"transaction.end"` 14 | TransactionOp string `json:"transaction.op,omitempty"` 15 | TransactionStart time.Time `json:"transaction.start"` 16 | TransactionStatus string `json:"transaction.status,omitempty"` 17 | SegmentID string `json:"segment_id,omitempty"` 18 | } 19 | ) 20 | -------------------------------------------------------------------------------- /internal/transaction/transaction.go: -------------------------------------------------------------------------------- 1 | package transaction 2 | 3 | type ( 4 | Transaction struct { 5 | ActiveThreadID uint64 `json:"active_thread_id"` 6 | DurationNS uint64 `json:"duration_ns,omitempty"` 7 | ID string `json:"id"` 8 | Name string `json:"name"` 9 | TraceID string `json:"trace_id"` 10 | SegmentID string `json:"segment_id"` 11 | } 12 | ) 13 | -------------------------------------------------------------------------------- /internal/utils/options.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "encoding/json" 4 | 5 | type Options struct { 6 | ProjectDSN string `json:"dsn"` 7 | } 8 | 9 | func (o Options) MarshalJSON() ([]byte, error) { 10 | return json.Marshal(nil) 11 | } 12 | -------------------------------------------------------------------------------- /internal/utils/structs.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type ( 4 | Interval struct { 5 | Start uint64 `json:"start,string"` 6 | End uint64 `json:"end,string"` 7 | ActiveThreadID string `json:"active_thread_id,omitempty"` 8 | } 9 | 10 | TransactionProfileCandidate struct { 11 | ProjectID uint64 `json:"project_id"` 12 | ProfileID string `json:"profile_id"` 13 | } 14 | 15 | ContinuousProfileCandidate struct { 16 | ProjectID uint64 `json:"project_id"` 17 | ProfilerID string `json:"profiler_id"` 18 | ChunkID string `json:"chunk_id"` 19 | TransactionID string `json:"transaction_id"` 20 | ThreadID *string `json:"thread_id"` 21 | Start uint64 `json:"start,string"` 22 | End uint64 `json:"end,string"` 23 | } 24 | 25 | // ExampleMetadata and FunctionMetrics have been moved here, although they'd 26 | // belong more to the metrics package, in order to avoid the circular dependency 27 | // hell that'd be introduced following the optimization to support metrics 28 | // generation within the flamegraph logic. 29 | ExampleMetadata struct { 30 | ProjectID uint64 `json:"project_id,omitempty"` 31 | ProfileID string `json:"profile_id,omitempty"` 32 | ProfilerID string `json:"profiler_id,omitempty"` 33 | ChunkID string `json:"chunk_id,omitempty"` 34 | TransactionID string `json:"transaction_id,omitempty"` 35 | ThreadID *string `json:"thread_id,omitempty"` 36 | Start float64 `json:"start,omitempty"` 37 | End float64 `json:"end,omitempty"` 38 | } 39 | 40 | FunctionMetrics struct { 41 | Name string `json:"name"` 42 | Package string `json:"package"` 43 | Fingerprint uint64 `json:"fingerprint"` 44 | InApp bool `json:"in_app"` 45 | P75 uint64 `json:"p75"` 46 | P95 uint64 `json:"p95"` 47 | P99 uint64 `json:"p99"` 48 | Avg float64 `json:"avg"` 49 | Sum uint64 `json:"sum"` 50 | Count uint64 `json:"count"` 51 | Worst ExampleMetadata `json:"worst"` 52 | Examples []ExampleMetadata `json:"examples"` 53 | } 54 | ) 55 | 56 | func NewExampleFromProfileID( 57 | projectID uint64, 58 | profileID string, 59 | start uint64, 60 | end uint64, 61 | ) ExampleMetadata { 62 | return ExampleMetadata{ 63 | ProjectID: projectID, 64 | ProfileID: profileID, 65 | Start: float64(start) / 1e9, 66 | End: float64(end) / 1e9, 67 | } 68 | } 69 | 70 | func NewExampleFromProfilerChunk( 71 | projectID uint64, 72 | profilerID string, 73 | chunkID string, 74 | transactionID string, 75 | threadID *string, 76 | start uint64, 77 | end uint64, 78 | ) ExampleMetadata { 79 | return ExampleMetadata{ 80 | ProjectID: projectID, 81 | ProfilerID: profilerID, 82 | ChunkID: chunkID, 83 | TransactionID: transactionID, 84 | ThreadID: threadID, 85 | Start: float64(start) / 1e9, 86 | End: float64(end) / 1e9, 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | go build -o . -ldflags="-s -w" ./cmd/vroom 6 | -------------------------------------------------------------------------------- /scripts/make_python_stdlib.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import logging 4 | import os 5 | import os.path 6 | import tempfile 7 | 8 | from git import Repo 9 | from sphinx.ext.intersphinx import fetch_inventory 10 | 11 | logging.getLogger().setLevel(logging.INFO) 12 | 13 | 14 | SPHINX_OBJECTS_URL = "https://docs.python.org/{}/objects.inv" 15 | CPYTHON_GIT_URL = "https://github.com/python/cpython.git" 16 | VERSIONS = [ 17 | "3.3", 18 | "3.4", 19 | "3.5", 20 | "3.6", 21 | "3.7", 22 | "3.8", 23 | "3.9", 24 | "3.10", 25 | "3.11", 26 | ] 27 | TARGET_GO_MODULE = "internal/frame/python_std_lib.go" 28 | 29 | 30 | class FakeConfig: 31 | intersphinx_timeout = None 32 | tls_verify = True 33 | user_agent = "" 34 | 35 | 36 | class FakeApp: 37 | srcdir = "" 38 | config = FakeConfig() 39 | 40 | 41 | SKIPPED_MODULES = {"__main__"} 42 | 43 | 44 | def fetch_documented_modules(versions): 45 | """ 46 | Fetches the list of publically documented modules found on https//docs.python.org. 47 | This list is incomplete as the private internal modules used by Cpython will not be found here. 48 | 49 | Adapted from https://github.com/PyCQA/isort/blob/e321a670d0fefdea0e04ed9d8d696434cf49bdec/scripts/mkstdlibs.py 50 | """ 51 | 52 | logging.info("Fetching documented modules") 53 | 54 | for version in versions: 55 | logging.info("Fetching documented modules for %s", version) 56 | 57 | url = SPHINX_OBJECTS_URL.format(version) 58 | invdata = fetch_inventory(FakeApp(), "", url) 59 | 60 | for module in invdata["py:module"]: 61 | root = module.split(".", 1)[0] 62 | if root not in SKIPPED_MODULES: 63 | yield root 64 | 65 | 66 | def fetch_git_modules(versions): 67 | """ 68 | Fetches a list of python modules (not native modules) by traversing CPython source. 69 | This list captures the internal modules used by CPython that is undocumented. 70 | 71 | This does NOT find the native modules at the moment. Native modules also do not 72 | appear in python profiles at the moment so this is a non issue for now. 73 | """ 74 | 75 | logging.info("Fetching git modules") 76 | 77 | with tempfile.TemporaryDirectory() as dir_name: 78 | logging.info("Cloning CPython") 79 | 80 | repo = Repo.clone_from(CPYTHON_GIT_URL, dir_name) 81 | 82 | # most of the python libraries are defined in this folder 83 | lib_dir = os.path.join(dir_name, "Lib") 84 | 85 | for version in versions: 86 | logging.info("Fetching git modules for %s", version) 87 | 88 | repo.git.checkout(version) 89 | 90 | for name in os.listdir(lib_dir): 91 | path = os.path.join(lib_dir, name) 92 | 93 | if os.path.isdir(path): 94 | # if it's a directory, it should contain an `__init__.py` 95 | # if it's also a module 96 | init_path = os.path.join(path, "__init__.py") 97 | if not os.path.exists(init_path): 98 | continue 99 | module = os.path.basename(path) 100 | 101 | elif os.path.isfile(path): 102 | # if it's a file, it should end with `.py` 103 | # if it's also a module 104 | module, ext = os.path.splitext(path) 105 | if ext != ".py": 106 | continue 107 | module = os.path.basename(module) 108 | 109 | yield module 110 | 111 | 112 | PYTHON_STD_LIB_GO_TEMPLATE = \ 113 | """// This file is autogenerated from scripts/make_python_stdlib.py 114 | // To update this file, update the python versions list in 115 | // scripts/make_python_stdlib.py then run `make python-stdlib` 116 | package frame 117 | 118 | var ( 119 | \tpythonStdlib = map[string]struct{{}}{{ 120 | {modules} 121 | \t}} 122 | ) 123 | """ 124 | 125 | 126 | def generate_python_std_lib_go(modules): 127 | logging.info("Generating Python stdlib module") 128 | max_len = max(len(module) for module in modules) 129 | module_keys = [ 130 | f'"{module}":'.ljust(max_len + 3, " ") 131 | for module in sorted(modules) 132 | ] 133 | formatted_modules = "\n".join( 134 | f'\t\t{module} {{}},' for module in module_keys 135 | ) 136 | return PYTHON_STD_LIB_GO_TEMPLATE.format(modules=formatted_modules) 137 | 138 | 139 | def main(): 140 | 141 | # Any modules we want to enforce across Python versions stdlib can be 142 | # included in set init 143 | modules = { 144 | "_ast", 145 | "posixpath", 146 | "ntpath", 147 | "sre_constants", 148 | "sre_parse", 149 | "sre_compile", 150 | "sre", 151 | } 152 | 153 | for module in fetch_documented_modules(VERSIONS): 154 | modules.add(module) 155 | 156 | for module in fetch_git_modules(VERSIONS): 157 | modules.add(module) 158 | 159 | with open(TARGET_GO_MODULE, "w") as f: 160 | f.write(generate_python_std_lib_go(modules)) 161 | 162 | 163 | if __name__ == "__main__": 164 | main() 165 | -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | export KAFKA_AUTO_CREATE_TOPICS_ENABLE="true" 6 | export PORT="8085" 7 | export STORAGE_EMULATOR_HOST="http://0.0.0.0:8888/" 8 | 9 | ./vroom 10 | -------------------------------------------------------------------------------- /test/gcs/sentry-profiles/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | */ 3 | !.gitignore 4 | --------------------------------------------------------------------------------