├── .github ├── CLA.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md ├── ISSUE_TEMPLATE │ ├── 1-bug-report.md │ ├── 2-feature-request.md │ └── 3-help.md ├── PULL_REQUEST_TEMPLATE.md ├── SECURITY.md ├── assets │ ├── qpoint-open-light.svg │ ├── qpoint-open.svg │ ├── qtap-header-dark.svg │ ├── qtap-header-light.svg │ ├── qtap-overview-dark.svg │ ├── qtap-overview-light.svg │ └── qtap_demo.gif └── workflows │ ├── ci.yaml │ ├── stars.yaml │ └── test.yaml ├── .gitignore ├── .golangci.yaml ├── .vscode └── c_cpp_properties.json ├── LICENSE ├── Makefile ├── README.md ├── bpf ├── .clang-format ├── headers │ ├── bpf_core_read.h │ ├── bpf_endian.h │ ├── bpf_helper_defs.h │ ├── bpf_helpers.h │ ├── bpf_tracing.h │ ├── vmlinux.h │ ├── vmlinux_amd64.h │ └── vmlinux_arm64.h └── tap │ ├── bpf2go.c │ ├── common.bpf.h │ ├── net.bpf.h │ ├── openssl.bpf.c │ ├── openssl.bpf.h │ ├── process.bpf.c │ ├── process.bpf.h │ ├── protocol.bpf.c │ ├── protocol.bpf.h │ ├── settings.bpf.c │ ├── settings.bpf.h │ ├── sock_pid_fd.bpf.c │ ├── sock_pid_fd.bpf.h │ ├── sock_utils.bpf.h │ ├── socket.bpf.c │ ├── socket.bpf.h │ ├── tap.bpf.h │ └── trace.bpf.h ├── cmd └── qtap │ └── main.go ├── e2e ├── http_test.go ├── main_linux_test.go ├── main_other_test.go └── main_test.go ├── examples ├── fluentbit │ ├── basic-axiom.conf │ ├── basic-stdout.conf │ ├── docker-compose.yaml │ ├── example.env │ ├── extract-payloads-s3.conf │ ├── extract-payloads-stdout.conf │ ├── lua │ │ ├── extract-payloads.lua │ │ └── s3-payloads.lua │ └── parsers.conf ├── http-access-logs │ ├── docker-compose.yaml │ ├── httpbin.http │ └── qtap.yaml ├── sample-axiom.yaml ├── sample-capture-minio.yaml ├── sample-capture.yaml ├── sample-config.yaml ├── sample-otel-grpc-eventstore.yaml ├── sample-otel-http-eventstore.yaml └── sample-otel-stdout-eventstore.yaml ├── go.mod ├── go.sum ├── internal └── tap │ ├── gen.go │ ├── tap_arm64_bpfel.go │ ├── tap_arm64_bpfel.o │ ├── tap_x86_bpfel.go │ └── tap_x86_bpfel.o ├── pkg ├── binutils │ ├── elf.go │ ├── strings.go │ └── strings_test.go ├── buildinfo │ └── buildinfo.go ├── cap │ ├── cap.go │ └── cap_test.go ├── cmd │ ├── reload.go │ ├── root.go │ ├── tap_linux.go │ └── tap_other.go ├── config │ ├── config.go │ ├── config_test.go │ ├── default.yaml │ ├── default_provider.go │ ├── eventstore.go │ ├── eventstore_test.go │ ├── filters.go │ ├── filters_test.go │ ├── local_provider.go │ ├── objectstore.go │ ├── objectstore_test.go │ ├── provider.go │ ├── qscan.go │ ├── tap.go │ ├── testdata │ │ ├── cert_valid.yaml │ │ ├── eventstore_console.yaml │ │ ├── eventstore_disabled.yaml │ │ ├── eventstore_otel.yaml │ │ ├── eventstore_pulse.yaml │ │ ├── objectstore_console.yaml │ │ ├── objectstore_disabled.yaml │ │ ├── objectstore_qpoint.yaml │ │ ├── objectstore_s3.yaml │ │ ├── value_source_env.yaml │ │ └── value_source_text.yaml │ ├── types.go │ └── types_test.go ├── connection │ ├── auditlog.go │ ├── auditlog_test.go │ ├── connection.go │ ├── connection_test.go │ ├── events.go │ ├── manager.go │ ├── metrics.go │ ├── reader.go │ ├── report.go │ ├── types.go │ └── writer.go ├── container │ ├── container.go │ ├── containerd.go │ ├── docker.go │ ├── kubernetes.go │ └── types.go ├── dns │ ├── dns.go │ └── manager.go ├── e2e │ ├── config.go │ ├── e2e.go │ └── eventstore.go ├── ebpf │ ├── common │ │ ├── ftrace.go │ │ ├── kprobe.go │ │ ├── probe.go │ │ ├── tracepoint.go │ │ └── uprobe.go │ ├── process │ │ ├── event.go │ │ ├── manager.go │ │ ├── reader.go │ │ └── reader_test.go │ ├── socket │ │ ├── manager.go │ │ ├── reader.go │ │ ├── reader_test.go │ │ ├── settings.go │ │ └── socket.go │ ├── tls │ │ ├── manager.go │ │ ├── manager_test.go │ │ └── openssl │ │ │ ├── container.go │ │ │ ├── manager.go │ │ │ ├── manager_test.go │ │ │ ├── symaddr.go │ │ │ └── target.go │ └── trace │ │ ├── entry.go │ │ ├── manager.go │ │ ├── reader.go │ │ ├── toggle.go │ │ └── trace.go ├── plugins │ ├── accesslogs │ │ ├── access_logs_http.go │ │ ├── access_logs_plugin.go │ │ ├── console.go │ │ ├── json.go │ │ └── printer.go │ ├── connection.go │ ├── context.go │ ├── deployment.go │ ├── headers.go │ ├── httpcapture │ │ ├── capture_level_test.go │ │ ├── config_test.go │ │ ├── formatter_test.go │ │ ├── httpcapture_http.go │ │ ├── httpcapture_plugin.go │ │ ├── httpcapture_plugin_test.go │ │ ├── httpcapture_test.go │ │ ├── httptransaction.go │ │ └── httptransaction_test.go │ ├── logger │ │ └── logger.go │ ├── manager.go │ ├── metadata │ │ └── metadata.go │ ├── plugins.go │ ├── plugintest │ │ └── helpers.go │ ├── registry.go │ ├── report │ │ ├── README.md │ │ ├── report_http.go │ │ ├── report_plugin.go │ │ ├── token.go │ │ └── token_test.go │ ├── stack.go │ ├── tools │ │ ├── mime.go │ │ ├── mime_test.go │ │ ├── path.go │ │ ├── path_test.go │ │ ├── txs.go │ │ └── txs_test.go │ └── wrapper │ │ └── wrapper.go ├── process │ ├── container.go │ ├── filter.go │ ├── filter_groups.go │ ├── manager.go │ ├── metrics.go │ ├── mocks │ │ ├── eventer.go │ │ └── receiver.go │ ├── observer.go │ ├── process.go │ ├── procfs.go │ ├── qpoint.go │ └── qpoint_test.go ├── qnet │ ├── addr.go │ └── addr_test.go ├── rulekitext │ ├── ext.go │ └── ext_test.go ├── services │ ├── adapters.go │ ├── client │ │ └── client.go │ ├── eventstore │ │ ├── axiom │ │ │ ├── README.md │ │ │ ├── axiom.go │ │ │ ├── axiom_test.go │ │ │ ├── factory.go │ │ │ ├── factory_test.go │ │ │ ├── integration_test.go │ │ │ └── metrics.go │ │ ├── console │ │ │ └── console.go │ │ ├── eventstore.go │ │ ├── eventstore_mock.go │ │ ├── noop │ │ │ └── noop.go │ │ └── otel │ │ │ ├── README.md │ │ │ ├── factory.go │ │ │ ├── factory_test.go │ │ │ ├── integration_test.go │ │ │ ├── metrics.go │ │ │ ├── otel.go │ │ │ └── otel_test.go │ ├── manager.go │ ├── mocks │ │ ├── registry_accessor.go │ │ └── service_factory.go │ ├── objectstore │ │ ├── console │ │ │ └── console.go │ │ ├── noop │ │ │ └── noop.go │ │ ├── objectstore.go │ │ └── s3 │ │ │ ├── factory.go │ │ │ ├── minio │ │ │ └── minio.go │ │ │ ├── s3.go │ │ │ └── s3_test.go │ ├── registry.go │ ├── rulekitsvc │ │ └── macros.go │ └── services.go ├── status │ └── server.go ├── stream │ ├── factory.go │ └── protocols │ │ ├── dns │ │ └── stream.go │ │ ├── http1 │ │ ├── diagnostics │ │ │ └── diagnostics.go │ │ ├── parser.go │ │ ├── parser_test.go │ │ ├── reader.go │ │ ├── reader_test.go │ │ ├── session.go │ │ └── stream.go │ │ └── http2 │ │ ├── headers.go │ │ ├── session.go │ │ └── stream.go ├── synq │ ├── linked_buffer.go │ ├── linked_buffer_test.go │ ├── map.go │ ├── map_test.go │ ├── queue.go │ ├── queue_test.go │ ├── ttlcache.go │ └── ttlcache_test.go ├── tags │ ├── tags.go │ └── tags_test.go ├── telemetry │ ├── collector.go │ ├── collector_test.go │ ├── counter.go │ ├── factory.go │ ├── factory_test.go │ ├── gauge.go │ ├── instance.go │ ├── instance_darwin.go │ ├── instance_linux.go │ ├── observable_gauge.go │ ├── options_test.go │ ├── opts.go │ ├── opts_test.go │ ├── telemetry.go │ ├── trace.go │ └── trace_noop_exporter.go └── tlsutils │ ├── tlsutils.go │ └── tlsutils_test.go └── scripts └── update-libbpf-headers.sh /.github/CLA.md: -------------------------------------------------------------------------------- 1 | # Contributor License Agreement 2 | 3 | This Contributor License Agreement ("CLA") documents the rights granted by Contributors to Qtap. 4 | 5 | ## 1. Definitions 6 | 7 | "You" or "Your" means the individual Copyright owner who Submits a Contribution to the Project. 8 | 9 | "Contribution" means any work of authorship that is Submitted by You to the Project. 10 | 11 | "Submit" means any form of electronic, verbal, or written communication sent to the Project, including code, documentation, or issue tracking. 12 | 13 | "Project" means Qtap. 14 | 15 | ## 2. Grant of Rights 16 | 17 | ### 2.1 Copyright License 18 | 19 | You grant the Project maintainers a perpetual, worldwide, non-exclusive, transferable, royalty-free, irrevocable license to reproduce, modify, display, perform, sublicense, and distribute Your Contributions and derivative works as part of the Project under: 20 | (a) the AGPLv3.0 license; and 21 | (b) any other license applied by the Project maintainers to the Project, including commercial licenses. 22 | 23 | ### 2.2 Patent License 24 | 25 | You grant the Project maintainers a perpetual, worldwide, non-exclusive, transferable, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Project, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Project to which such Contribution(s) was submitted. 26 | 27 | ## 3. Representations 28 | 29 | You represent that: 30 | (a) You are legally entitled to grant the above licenses. 31 | (b) Each of Your Contributions is Your original creation. 32 | (c) Your Contribution submissions include complete details of any third-party license or other restriction of which You are aware and which are associated with any part of Your Contributions. 33 | 34 | ## 4. Disclaimer 35 | 36 | Your Contributions are provided "as is", without warranty of any kind, express or implied. -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jonfriesen @tylerflint @kamaln7 2 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, 4 | email, or any other method with a member of the Qpoint technical team before making a change. 5 | 6 | Please note we have a code of conduct, please follow it in all your interactions with the project. 7 | 8 | By submitting a pull request or contribution to this project, you are agreeing to the terms outlined in the [Contributor License Agreement](./CLA.md) file. 9 | 10 | ## Pull Request Process 11 | 12 | 1. Ensure any install or build dependencies are removed before the end of the layer when doing a 13 | build. 14 | 2. Update the README.md with details of changes to the interface, this includes new environment 15 | variables, exposed ports, useful file locations and container parameters. 16 | 3. Increase the version numbers in any examples files and the README.md to the new version that this 17 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 18 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 19 | do not have permission to do that, you may request the second reviewer to merge it for you. 20 | 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 18 | 19 | * **Version**: 20 | * **Platform**: 22 | * **Subsystem**: 23 | 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1-bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug report" 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | 8 | 9 | ## Description 10 | 11 | 12 | ## Expected Behavior 13 | 14 | 15 | ## Actual Behavior 16 | 17 | 18 | ## Possible Fix 19 | 20 | 21 | ## Steps to Reproduce 22 | 23 | 24 | 1. 25 | 2. 26 | 3. 27 | 4. 28 | 29 | ## Context 30 | 31 | 32 | ## Your Environment 33 | 34 | * Version used: 35 | * Environment name and version (e.g. Docker? Containerd? RaspberryPi?): 36 | * Server type and version: 37 | * Operating System and version: 38 | * Link to your project: 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | 8 | 9 | ## Detailed Description 10 | 11 | 12 | ## Context 13 | 14 | 15 | 16 | ## Possible Implementation 17 | 18 | 19 | ## Your Environment 20 | 21 | * Version used: 22 | * Environment name and version (e.g. Docker? Containerd? RaspberryPi?): 23 | * Server type and version: 24 | * Operating System and version: 25 | * Link to your project: 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/3-help.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "⁉️ Need help with Qtap?" 3 | about: Please file an issue in our help repo. 4 | 5 | --- 6 | 7 | If you have a question about Node.js that is not a bug report or feature 8 | request, please post it in https://github.com/qpoint-io/qtap/discussions! 9 | 10 | Questions posted to this repository will be closed. -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 19 | -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | If you discover a security vulnerability in Qtap, please report it privately. Do **not** create a public issue. 6 | 7 | - Email: [security@qpoint.io](mailto:security@qpoint.io) 8 | - Please include as much detail as possible to help us reproduce and address the issue quickly. 9 | - We aim to respond within 10 business days. 10 | - Please do not send us binaries. 11 | 12 | After triage, we will work with you on a coordinated disclosure timeline if needed. Once resolved, we will publish an advisory and credit the reporter if desired. 13 | 14 | ## Further Reading 15 | - [GitHub Security Advisories](https://docs.github.com/en/code-security/security-advisories) 16 | - [Coordinated Disclosure](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability) 17 | -------------------------------------------------------------------------------- /.github/assets/qtap_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qpoint-io/qtap/20f2c095a247e2846497e87a3ad673e99a5a511f/.github/assets/qtap_demo.gif -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - v* 10 | pull_request: 11 | branches: [ '*' ] 12 | 13 | jobs: 14 | ci: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | with: 19 | token: ${{ secrets.GITHUB_TOKEN }} 20 | 21 | - name: Install system dependencies 22 | run: | 23 | sudo apt-get update 24 | sudo apt-get install -y --no-install-recommends \ 25 | ca-certificates \ 26 | build-essential \ 27 | pkg-config \ 28 | clang \ 29 | clang-format \ 30 | llvm \ 31 | libelf-dev \ 32 | linux-headers-generic \ 33 | linux-libc-dev 34 | 35 | - name: Set up Clang 36 | uses: egor-tensin/setup-clang@v1 37 | with: 38 | version: 14 39 | platform: any 40 | 41 | - name: Set up Go 42 | uses: actions/setup-go@v5 43 | with: 44 | go-version: '1.24.3' 45 | cache: true 46 | 47 | - name: Run CI 48 | run: make ci -------------------------------------------------------------------------------- /.github/workflows/stars.yaml: -------------------------------------------------------------------------------- 1 | name: Star Notification 2 | 3 | on: 4 | workflow_dispatch: 5 | watch: 6 | types: [started] 7 | 8 | jobs: 9 | notify: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Send to Slack 13 | uses: slackapi/slack-github-action@v1.23.0 14 | with: 15 | # For posting a rich message using Block Kit 16 | payload: | 17 | { 18 | "blocks": [ 19 | { 20 | "type": "header", 21 | "text": { 22 | "type": "plain_text", 23 | "text": "🌟 New GitHub Star! 🌟", 24 | "emoji": true 25 | } 26 | }, 27 | { 28 | "type": "section", 29 | "text": { 30 | "type": "mrkdwn", 31 | "text": "*${{ github.repository }}* was just starred by _${{ github.actor }}_!\n\n*Total Stars:* ${{ github.event.repository.stargazers_count }}" 32 | }, 33 | "accessory": { 34 | "type": "image", 35 | "image_url": "${{ github.event.sender.avatar_url }}", 36 | "alt_text": "${{ github.actor }}'s avatar" 37 | } 38 | }, 39 | { 40 | "type": "context", 41 | "elements": [ 42 | { 43 | "type": "image", 44 | "image_url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png", 45 | "alt_text": "GitHub" 46 | }, 47 | { 48 | "type": "mrkdwn", 49 | "text": "GitHub Star Notification • ${{ github.repository }}" 50 | } 51 | ] 52 | } 53 | ] 54 | } 55 | env: 56 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BOT_SOCIAL }} 57 | SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK 58 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - v* 10 | pull_request: 11 | branches: [ '*' ] 12 | 13 | jobs: 14 | e2e: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | with: 19 | token: ${{ secrets.GITHUB_TOKEN }} 20 | 21 | - name: Install system dependencies 22 | run: | 23 | sudo apt-get update 24 | sudo apt-get install -y --no-install-recommends \ 25 | ca-certificates \ 26 | build-essential \ 27 | pkg-config \ 28 | clang \ 29 | clang-format \ 30 | llvm \ 31 | libelf-dev \ 32 | linux-headers-generic \ 33 | linux-libc-dev 34 | 35 | - name: Set up Clang 36 | uses: egor-tensin/setup-clang@v1 37 | with: 38 | version: 14 39 | platform: any 40 | 41 | - name: Set up Go 42 | uses: actions/setup-go@v5 43 | with: 44 | go-version: '1.24.3' 45 | cache: true 46 | 47 | - name: Run E2E tests 48 | run: sudo make e2e -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | 23 | # final dist 24 | dist/ 25 | bin/ 26 | 27 | # tmp 28 | tmp/ 29 | 30 | # cruft 31 | .DS_Store 32 | 33 | # Clang 34 | *.d 35 | *.o.tmp 36 | 37 | # env 38 | .env 39 | 40 | # certs 41 | *.crt 42 | *.csr 43 | *.key 44 | *.srl 45 | *.pem 46 | -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "libbpf", 5 | "includePath": [ 6 | "${workspaceFolder}/**", 7 | "${workspaceFolder}/bpf/tap/**", 8 | ], 9 | "defines": [ 10 | "__TARGET_ARCH_arm64" // this enables lsp for headers on arm64 when using macos 11 | ], 12 | "compilerPath": "${default}", 13 | "cStandard": "${default}", 14 | "cppStandard": "${default}", 15 | "intelliSenseMode": "${default}" 16 | } 17 | ], 18 | "version": 4 19 | } -------------------------------------------------------------------------------- /bpf/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: LLVM 3 | AlignAfterOpenBracket: DontAlign 4 | AlignConsecutiveAssignments: true 5 | AlignEscapedNewlines: DontAlign 6 | # mkdocs annotations in source code are written as trailing comments 7 | # and alignment pushes these really far away from the content. 8 | AlignTrailingComments: false 9 | AlwaysBreakBeforeMultilineStrings: true 10 | AlwaysBreakTemplateDeclarations: false 11 | AllowAllParametersOfDeclarationOnNextLine: false 12 | AllowShortFunctionsOnASingleLine: false 13 | BreakBeforeBraces: Attach 14 | IndentWidth: 4 15 | KeepEmptyLinesAtTheStartOfBlocks: false 16 | TabWidth: 4 17 | UseTab: ForContinuationAndIndentation 18 | ColumnLimit: 150 19 | # Go compiler comments need to stay unindented. 20 | CommentPragmas: '^go:.*' 21 | # linux/bpf.h needs to be included before bpf/bpf_helpers.h for types like __u64 22 | # and sorting makes this impossible. 23 | SortIncludes: false 24 | BraceWrapping: 25 | AfterControlStatement: true 26 | 27 | # Macro formatting 28 | AlignConsecutiveMacros: true 29 | AlwaysBreakAfterReturnType: None 30 | BreakBeforeBinaryOperators: None 31 | BreakBeforeTernaryOperators: true 32 | BreakStringLiterals: false 33 | 34 | ... 35 | 36 | -------------------------------------------------------------------------------- /bpf/headers/vmlinux.h: -------------------------------------------------------------------------------- 1 | #ifndef __VMLINUX_H_PARENT_ 2 | #define __VMLINUX_H_PARENT_ 3 | 4 | #if defined(__TARGET_ARCH_x86) 5 | 6 | #include "vmlinux_amd64.h" 7 | 8 | #elif defined(__TARGET_ARCH_arm64) 9 | 10 | #include "vmlinux_arm64.h" 11 | 12 | #endif 13 | 14 | #endif /*__VMLINUX_H_PARENT_*/ 15 | -------------------------------------------------------------------------------- /bpf/tap/bpf2go.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 - The Qpoint Authors 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation; either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License along 15 | * with this program; if not, write to the Free Software Foundation, Inc., 16 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17 | * 18 | * SPDX-License-Identifier: GPL-2.0 19 | */ 20 | 21 | #include "openssl.bpf.c" 22 | #include "process.bpf.c" 23 | #include "protocol.bpf.c" 24 | #include "socket.bpf.c" 25 | #include "sock_pid_fd.bpf.c" 26 | #include "settings.bpf.c" 27 | 28 | char _license[] SEC("license") = "GPL"; 29 | -------------------------------------------------------------------------------- /bpf/tap/common.bpf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This code runs using libbpf in the Linux kernel. 3 | * Copyright 2025 - The Qpoint Authors 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation; either version 2 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along 16 | * with this program; if not, write to the Free Software Foundation, Inc., 17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | * 19 | * SPDX-License-Identifier: GPL-2.0 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "vmlinux.h" 25 | #include "bpf_helpers.h" 26 | #include "bpf_tracing.h" 27 | #include "net.bpf.h" 28 | 29 | // This keeps instruction count below BPF's limit of 4096 per probe 30 | #define LOOP_LIMIT 100 31 | #define LOOP_LIMIT_SM 25 32 | 33 | // Check if BPF_UPROBE is not defined 34 | #ifndef BPF_UPROBE 35 | // If BPF_UPROBE is not defined, define it as BPF_KPROBE 36 | // This equates user-space probes to kernel-space probes if they are not separately defined 37 | #define BPF_UPROBE BPF_KPROBE 38 | #endif 39 | 40 | // Check if BPF_URETPROBE is not defined 41 | #ifndef BPF_URETPROBE 42 | // If BPF_URETPROBE is not defined, define it as BPF_KRETPROBE 43 | // This equates user-space return probes to kernel-space return probes if they are not separately defined 44 | #define BPF_URETPROBE BPF_KRETPROBE 45 | #endif 46 | 47 | // Invalid file descriptor 48 | const __s32 INVALID_FD = -1; 49 | 50 | // Qpoint PID (this is set by the user-space program) 51 | const volatile u32 qpid = 0; 52 | 53 | static __inline int _strncmp(const char *s1, const char *s2, const uint32_t n) { 54 | for (uint32_t i = 0; i < n; ++i) { 55 | if (s1[i] != s2[i] || s1[i] == '\0') 56 | return s1[i] - s2[i]; 57 | } 58 | return 0; 59 | } 60 | 61 | static __inline char *_strstr(const char *haystack, const char *needle) { 62 | if (!*needle) 63 | return (char *)haystack; 64 | 65 | for (; *haystack; ++haystack) { 66 | if (*haystack == *needle) { 67 | const char *h = haystack, *n = needle; 68 | while (*h && *n && *h == *n) { 69 | ++h; 70 | ++n; 71 | } 72 | if (!*n) 73 | return (char *)haystack; 74 | } 75 | } 76 | return NULL; 77 | } 78 | 79 | static inline __u64 _strlen(const char *s, __u64 max) { 80 | __u64 len = 0; 81 | while (len < max && s[len]) 82 | len++; 83 | return len; 84 | } 85 | -------------------------------------------------------------------------------- /bpf/tap/openssl.bpf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This code runs using libbpf in the Linux kernel. 3 | * Copyright 2025 - The Qpoint Authors 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation; either version 2 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along 16 | * with this program; if not, write to the Free Software Foundation, Inc., 17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | * 19 | * SPDX-License-Identifier: GPL-2.0 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "vmlinux.h" 25 | 26 | /* 27 | * This file provides a generic interface for TLS implementations to register 28 | * themselves with OpenSSL. For example, instead of OpenSSL directly calling 29 | * NodeTLS functions, NodeTLS can register its presence with OpenSSL through this 30 | * interface. 31 | */ 32 | 33 | // Helper functions that OpenSSL will call 34 | static inline int ssl_register_handle(uintptr_t ssl) { 35 | #ifdef ENABLE_NODETLS 36 | // Call into the NodeTLS module using the extern declaration 37 | extern int update_node_ssl_tls_wrap_map(uintptr_t ssl); 38 | return update_node_ssl_tls_wrap_map(ssl); 39 | #else 40 | return 0; 41 | #endif 42 | } 43 | 44 | static inline int32_t ssl_get_fd(uint64_t pid_tgid, uintptr_t ssl) { 45 | #ifdef ENABLE_NODETLS 46 | // Call into the NodeTLS module using the extern declaration 47 | extern int32_t get_fd_from_node(uint64_t pid_tgid, uintptr_t ssl); 48 | return get_fd_from_node(pid_tgid, ssl); 49 | #else 50 | return 0; 51 | #endif 52 | } 53 | 54 | static inline int ssl_remove_handle(uintptr_t ssl) { 55 | #ifdef ENABLE_NODETLS 56 | // Call into the NodeTLS module using the extern declaration 57 | extern int remove_node_ssl_tls_wrap_map(uintptr_t ssl); 58 | return remove_node_ssl_tls_wrap_map(ssl); 59 | #else 60 | return 0; 61 | #endif 62 | } 63 | -------------------------------------------------------------------------------- /bpf/tap/process.bpf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This code runs using libbpf in the Linux kernel. 3 | * Copyright 2025 - The Qpoint Authors 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation; either version 2 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along 16 | * with this program; if not, write to the Free Software Foundation, Inc., 17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | * 19 | * SPDX-License-Identifier: GPL-2.0 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "vmlinux.h" 25 | #include "bpf_tracing.h" 26 | 27 | // flag definitions (matching the Go constants) 28 | #define SKIP_DATA_FLAG (1 << 0) 29 | #define SKIP_DNS_FLAG (1 << 1) 30 | #define SKIP_TLS_FLAG (1 << 2) 31 | #define SKIP_HTTP_FLAG (1 << 3) 32 | 33 | // all flags 34 | #define SKIP_ALL_FLAGS (SKIP_DATA_FLAG | SKIP_DNS_FLAG | SKIP_TLS_FLAG | SKIP_HTTP_FLAG) 35 | 36 | enum QPOINT_STRATEGY { 37 | QP_OBSERVE, 38 | QP_IGNORE, 39 | QP_AUDIT, 40 | QP_FORWARD, 41 | QP_PROXY, 42 | }; 43 | 44 | // any process meta available to Qtap that can be helpful within eBPF 45 | struct process_meta { 46 | __u64 root_id; // 8 bytes 47 | __u32 qpoint_strategy; // 4 bytes 48 | __u8 filter; // 1 byte 49 | bool tls_ok; // 1 byte 50 | char container_id[13]; // 13 bytes 51 | char _pad[3]; // 3 bytes padding to maintain 8-byte alignment 52 | }; 53 | 54 | static __always_inline struct process_meta *get_process_meta(__u32 pid); 55 | 56 | // macros for checking individual flags 57 | #define SKIP_DATA(pid) \ 58 | ({ \ 59 | struct process_meta *meta = get_process_meta(pid); \ 60 | (bool)((pid == qpid) || (meta && (meta->filter & SKIP_DATA_FLAG))); \ 61 | }) 62 | #define SKIP_DNS(pid) \ 63 | ({ \ 64 | struct process_meta *meta = get_process_meta(pid); \ 65 | (bool)(meta && (meta->filter & SKIP_DNS_FLAG)); \ 66 | }) 67 | #define SKIP_TLS(pid) \ 68 | ({ \ 69 | struct process_meta *meta = get_process_meta(pid); \ 70 | (bool)(meta && (meta->filter & SKIP_TLS_FLAG)); \ 71 | }) 72 | #define SKIP_HTTP(pid) \ 73 | ({ \ 74 | struct process_meta *meta = get_process_meta(pid); \ 75 | (bool)(meta && (meta->filter & SKIP_HTTP_FLAG)); \ 76 | }) 77 | 78 | // macro for checking all flags 79 | #define SKIP_ALL(pid) \ 80 | ({ \ 81 | struct process_meta *meta = get_process_meta(pid); \ 82 | (bool)(meta && ((meta->filter & (SKIP_ALL_FLAGS & ~SKIP_DATA_FLAG)) == (SKIP_ALL_FLAGS & ~SKIP_DATA_FLAG)) && SKIP_DATA(pid)); \ 83 | }) -------------------------------------------------------------------------------- /bpf/tap/protocol.bpf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This code runs using libbpf in the Linux kernel. 3 | * Copyright 2025 - The Qpoint Authors 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation; either version 2 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along 16 | * with this program; if not, write to the Free Software Foundation, Inc., 17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | * 19 | * SPDX-License-Identifier: GPL-2.0 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "vmlinux.h" 25 | #include "tap.bpf.h" 26 | 27 | // buffer info struct for protocol detection 28 | struct buf_info { 29 | const void *buf; 30 | size_t iovcnt; 31 | }; 32 | 33 | // given a buffer and connection information, dynamically detect the protocol 34 | static bool detect_protocol(struct conn_info *conn_info, struct buf_info *buf_info, size_t count); 35 | 36 | // given a buffer and connection information, detect if the connection is TLS 37 | // sets conn_info->is_ssl to true if TLS is detected 38 | static bool detect_tls(struct conn_info *conn_info, struct buf_info *buf_info, size_t count); 39 | 40 | // given a buffer and connection information, detect if the connection is DNS 41 | static bool is_dns(const struct conn_info *conn_info); 42 | 43 | // given a buffer and connection information, extract the tls handshake 44 | static bool capture_tls_client_hello(struct socket_tls_client_hello_event *handshake, struct buf_info *buf_info, size_t count); 45 | -------------------------------------------------------------------------------- /bpf/tap/settings.bpf.c: -------------------------------------------------------------------------------- 1 | #include "settings.bpf.h" 2 | 3 | // extract the capture direction from settings 4 | static __always_inline bool get_ignore_loopback_setting() { 5 | // define the settings key 6 | enum SOCKET_SETTINGS key = SOCK_SETTING_IGNORE_LOOPBACK; 7 | 8 | // init setting value 9 | __u32 *setting_value; 10 | 11 | // try to fetch the entry 12 | setting_value = bpf_map_lookup_elem(&socket_settings_map, &key); 13 | 14 | // if it's empty, return the default 15 | if (setting_value == NULL) { 16 | // bpf_printk("socket: get_ignore_loopback_setting = NULL"); 17 | return false; 18 | } 19 | 20 | // debug 21 | // bpf_printk("socket: get_ignore_loopback_setting = %d", *setting_value); 22 | 23 | // return the value 24 | return (bool)*setting_value != 0; 25 | } 26 | 27 | // extract the capture direction from settings 28 | static __always_inline enum DIRECTION get_direction_setting() { 29 | // define the settings key 30 | enum SOCKET_SETTINGS key = SOCK_SETTING_DIRECTION; 31 | 32 | // init setting value 33 | __u32 *setting_value; 34 | 35 | // try to fetch the entry 36 | setting_value = bpf_map_lookup_elem(&socket_settings_map, &key); 37 | 38 | // if it's empty, return the default 39 | if (setting_value == NULL) { 40 | // bpf_printk("socket: get_direction_setting = NULL"); 41 | return D_ALL; 42 | } 43 | 44 | // debug 45 | // bpf_printk("socket: get_direction_setting = %d", *setting_value); 46 | 47 | // return the value 48 | return (enum DIRECTION) * setting_value; 49 | } 50 | 51 | // extract the stream http setting 52 | static __always_inline bool get_stream_http_setting() { 53 | // define the settings key 54 | enum SOCKET_SETTINGS key = SOCK_SETTING_STREAM_HTTP; 55 | 56 | // init setting value 57 | __u32 *stream_http; 58 | 59 | // try to fetch the entry 60 | stream_http = bpf_map_lookup_elem(&socket_settings_map, &key); 61 | 62 | // if it's empty, return the default 63 | if (stream_http == NULL) { 64 | // bpf_printk("socket: get_ignore_loopback_setting = NULL"); 65 | return false; 66 | } 67 | 68 | // debug 69 | // bpf_printk("socket: get_ignore_loopback_setting = %d", *stream_http); 70 | 71 | // return the value 72 | return (bool)*stream_http != 0; 73 | } 74 | -------------------------------------------------------------------------------- /bpf/tap/settings.bpf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This code runs using libbpf in the Linux kernel. 3 | * Copyright 2025 - The Qpoint Authors 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation; either version 2 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License along 16 | * with this program; if not, write to the Free Software Foundation, Inc., 17 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | * 19 | * SPDX-License-Identifier: GPL-2.0 20 | */ 21 | 22 | #pragma once 23 | 24 | #include "vmlinux.h" 25 | #include "bpf_helpers.h" 26 | #include "tap.bpf.h" 27 | 28 | // Settings keys 29 | enum SOCKET_SETTINGS { 30 | SOCK_SETTING_IGNORE_LOOPBACK, 31 | SOCK_SETTING_DIRECTION, 32 | SOCK_SETTING_STREAM_HTTP, 33 | }; 34 | 35 | // Settings value types 36 | union socket_setting_value { 37 | // ignore loopback 38 | bool ignore_loopback; 39 | 40 | // direction 41 | enum DIRECTION direction; 42 | 43 | // stream http 44 | bool stream_http; 45 | }; 46 | 47 | // Socket settings (from loader app) 48 | struct { 49 | __uint(type, BPF_MAP_TYPE_HASH); 50 | __uint(max_entries, 12); 51 | __type(key, enum SOCKET_SETTINGS); 52 | __type(value, union socket_setting_value); 53 | } socket_settings_map SEC(".maps"); 54 | 55 | static __always_inline bool get_ignore_loopback_setting(); 56 | static __always_inline enum DIRECTION get_direction_setting(); 57 | static __always_inline bool get_stream_http_setting(); 58 | -------------------------------------------------------------------------------- /bpf/tap/sock_pid_fd.bpf.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "vmlinux.h" 4 | #include "bpf_helpers.h" 5 | #include "tap.bpf.h" 6 | #include "socket.bpf.h" 7 | 8 | // this map allows a process to find the underlying socket from fd 9 | struct { 10 | __uint(type, BPF_MAP_TYPE_HASH); 11 | __uint(max_entries, 16384); 12 | __type(key, struct pid_fd_key); // the file pointer 13 | __type(value, uintptr_t); // the socket pointer 14 | } pid_fd_to_sock_map SEC(".maps"); 15 | 16 | // this map allows a socket program to find the pid from the address and port 17 | struct { 18 | __uint(type, BPF_MAP_TYPE_HASH); 19 | __uint(max_entries, 16384); 20 | __type(key, struct addr_port_key); // the address and port 21 | __type(value, uint32_t); // the pid 22 | } addr_port_to_pid_map SEC(".maps"); 23 | -------------------------------------------------------------------------------- /cmd/qtap/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/qpoint-io/qtap/pkg/cmd" 8 | ) 9 | 10 | func main() { 11 | if err := cmd.Execute(); err != nil { 12 | fmt.Fprintf(os.Stderr, "Error: %v\n", err) 13 | os.Exit(1) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /e2e/main_linux_test.go: -------------------------------------------------------------------------------- 1 | //go:build e2e && linux 2 | 3 | package e2e 4 | 5 | import ( 6 | "github.com/qpoint-io/qtap/internal/tap" 7 | "github.com/qpoint-io/qtap/pkg/cmd" 8 | "github.com/qpoint-io/qtap/pkg/connection" 9 | ebpfProcess "github.com/qpoint-io/qtap/pkg/ebpf/process" 10 | "github.com/qpoint-io/qtap/pkg/ebpf/socket" 11 | "github.com/qpoint-io/qtap/pkg/ebpf/tls" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | func NewEbpfProcManager(logger *zap.Logger, objs *tap.TapObjects) (*ebpfProcess.Manager, error) { 16 | return cmd.NewEbpfProcManager(logger, objs) 17 | } 18 | 19 | func NewEbpfSockManager(logger *zap.Logger, connMan *connection.Manager, objs *tap.TapObjects) (*socket.SocketEventManager, error) { 20 | return cmd.NewEbpfSockManager(logger, connMan, objs) 21 | } 22 | 23 | func InitTLSProbes(logger *zap.Logger, tlsProbesStr string, objs *tap.TapObjects) (*tls.TlsManager, error) { 24 | return cmd.InitTLSProbes(logger, tlsProbesStr, objs) 25 | } 26 | -------------------------------------------------------------------------------- /e2e/main_other_test.go: -------------------------------------------------------------------------------- 1 | //go:build e2e && !linux 2 | 3 | package e2e 4 | 5 | import ( 6 | "fmt" 7 | 8 | "github.com/qpoint-io/qtap/internal/tap" 9 | "github.com/qpoint-io/qtap/pkg/connection" 10 | ebpfProcess "github.com/qpoint-io/qtap/pkg/ebpf/process" 11 | "github.com/qpoint-io/qtap/pkg/ebpf/socket" 12 | "github.com/qpoint-io/qtap/pkg/ebpf/tls" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | func NewEbpfProcManager(logger *zap.Logger, objs *tap.TapObjects) (*ebpfProcess.Manager, error) { 17 | return nil, fmt.Errorf("not supported on this platform") 18 | } 19 | 20 | func NewEbpfSockManager(logger *zap.Logger, connMan *connection.Manager, objs *tap.TapObjects) (*socket.SocketEventManager, error) { 21 | return nil, fmt.Errorf("not supported on this platform") 22 | } 23 | 24 | func InitTLSProbes(logger *zap.Logger, tlsProbesStr string, objs *tap.TapObjects) (*tls.TlsManager, error) { 25 | return nil, fmt.Errorf("not supported on this platform") 26 | } 27 | -------------------------------------------------------------------------------- /examples/fluentbit/basic-axiom.conf: -------------------------------------------------------------------------------- 1 | # Fluent Bit Configuration for Qtap HTTP Transaction Logging to Axiom 2 | # ===================================================================== 3 | # 4 | # This configuration file sets up Fluent Bit to: 5 | # 6 | # 1. Receive logs from Docker containers using the Fluentd forward protocol 7 | # 2. Filter logs specifically from the Qtap container (tagged as docker.qtap) 8 | # 3. Parse the logs as JSON using the generic_json_parser defined in parsers.conf 9 | # 4. Identify HTTP transaction logs by matching the message "HTTP transaction" 10 | # 5. Rewrite the tag for these HTTP transaction logs to "qtap.http" 11 | # 6. Upload the HTTP transaction logs to axiom 12 | # 13 | # This setup allows for easy viewing and processing of Qtap's HTTP transaction 14 | # monitoring data while filtering out other container logs. 15 | 16 | # Ensure the parsers are loaded 17 | [SERVICE] 18 | Parsers_File parsers.conf 19 | 20 | # Listen for logs forwarded via fluentd log driver 21 | [INPUT] 22 | Name forward 23 | Listen 0.0.0.0 24 | Port 24224 25 | 26 | # Filter the logs to only tagged with docker.qtap, and parse them as JSON 27 | [FILTER] 28 | Name parser 29 | Match docker.qtap 30 | Key_Name log 31 | Parser generic_json_parser 32 | 33 | # Rewrite tag for logs with the exact message "HTTP transaction" 34 | [FILTER] 35 | Name rewrite_tag 36 | Match docker.qtap 37 | Rule $msg "^HTTP transaction$" qtap.http false 38 | 39 | # Rewrite tag for logs headed to axiom so we can filter them out later 40 | [FILTER] 41 | Name rewrite_tag 42 | Match qtap.http 43 | Rule $url "^https://api.axiom.co" axiom.http false 44 | 45 | # Output the HTTP transaction logs to axiom 46 | [OUTPUT] 47 | Name http 48 | Match qtap.http 49 | Host api.axiom.co 50 | Port 443 51 | URI /v1/datasets/${AXIOM_DATASET}/ingest 52 | Header Authorization Bearer ${AXIOM_API_KEY} 53 | compress gzip 54 | format json 55 | json_date_key date 56 | json_date_format double 57 | tls On 58 | 59 | # Output all other docker.qtap logs to stdout 60 | # [OUTPUT] 61 | # Name stdout 62 | # Match docker.qtap 63 | # Format json_lines 64 | -------------------------------------------------------------------------------- /examples/fluentbit/basic-stdout.conf: -------------------------------------------------------------------------------- 1 | # Fluent Bit Configuration for Qtap HTTP Transaction Logging 2 | # ========================================================= 3 | # 4 | # This configuration file sets up Fluent Bit to: 5 | # 6 | # 1. Receive logs from Docker containers using the Fluentd forward protocol 7 | # 2. Filter logs specifically from the Qtap container (tagged as docker.qtap) 8 | # 3. Parse the logs as JSON using the generic_json_parser defined in parsers.conf 9 | # 4. Identify HTTP transaction logs by matching the message "HTTP transaction" 10 | # 5. Rewrite the tag for these HTTP transaction logs to "qtap.http" 11 | # 6. Output the HTTP transaction logs to stdout in JSON lines format 12 | # 13 | # This setup allows for easy viewing and processing of Qtap's HTTP transaction 14 | # monitoring data while filtering out other container logs. 15 | 16 | # Ensure the parsers are loaded 17 | [SERVICE] 18 | Parsers_File parsers.conf 19 | 20 | # Listen for logs forwarded via fluentd log driver 21 | [INPUT] 22 | Name forward 23 | Listen 0.0.0.0 24 | Port 24224 25 | 26 | # Filter the logs to only tagged with docker.qtap, and parse them as JSON 27 | [FILTER] 28 | Name parser 29 | Match docker.qtap 30 | Key_Name log 31 | Parser generic_json_parser 32 | 33 | # Rewrite tag for logs with the exact message "HTTP transaction" 34 | [FILTER] 35 | Name rewrite_tag 36 | Match docker.qtap 37 | Rule $msg "^HTTP transaction$" qtap.http false 38 | 39 | # Output the HTTP transaction logs to stdout, as individual lines of JSON 40 | [OUTPUT] 41 | Name stdout 42 | Match qtap.http 43 | Format json_lines 44 | 45 | # Output all other docker.qtap logs to stdout 46 | # [OUTPUT] 47 | # Name stdout 48 | # Match docker.qtap 49 | # Format json_lines 50 | -------------------------------------------------------------------------------- /examples/fluentbit/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Qtap with Fluent Bit Docker Compose Configuration 2 | # ================================================= 3 | # 4 | # This docker-compose file runs two services: 5 | # 6 | # 1. fluent-bit: Collects and processes logs from the Qtap container 7 | # - Configured via mounted configuration files 8 | # - Exposes port 24224 for log collection 9 | # - Can forward logs to various destinations (Axiom, S3, etc.) 10 | # - Uses environment variables for credentials 11 | # 12 | # 2. qtap: Monitors HTTP transactions using eBPF 13 | # - Runs with host networking and privileged access for eBPF functionality 14 | # - Forwards logs to fluent-bit for processing 15 | # - Requires elevated privileges for eBPF functionality 16 | # 17 | # To run this setup from the project root directory, use: 18 | # 19 | # docker compose -f examples/fluentbit/docker-compose.yaml --env-file examples/fluentbit/.env up 20 | # 21 | version: '3' 22 | 23 | services: 24 | fluent-bit: 25 | image: fluent/fluent-bit:latest 26 | volumes: 27 | - ./:/fluent-bit/etc 28 | - /var/run/docker.sock:/var/run/docker.sock 29 | command: [ "fluent-bit", "-c", "/fluent-bit/etc/${FLUENTBIT_CONFIG:-basic-stdout.conf}" ] 30 | container_name: fluent-bit 31 | ports: 32 | - "24224:24224" 33 | environment: 34 | - AXIOM_DATASET=${AXIOM_DATASET} 35 | - AXIOM_API_KEY=${AXIOM_API_KEY} 36 | - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} 37 | - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} 38 | - AWS_REGION=${AWS_REGION} 39 | - S3_BUCKET=${S3_BUCKET} 40 | - S3_ENDPOINT=${S3_ENDPOINT} 41 | 42 | qtap: 43 | image: us-docker.pkg.dev/qpoint-edge/public/qtap:v0.9.7 44 | pid: host 45 | network_mode: host 46 | logging: 47 | driver: "fluentd" 48 | options: 49 | tag: docker.qtap 50 | fluentd-address: 127.0.0.1:24224 51 | depends_on: 52 | - fluent-bit 53 | user: "0:0" 54 | privileged: true 55 | cap_add: 56 | - CAP_BPF 57 | - CAP_SYS_ADMIN 58 | volumes: 59 | - /sys:/sys 60 | - ../:/qtap 61 | environment: 62 | - TINI_SUBREAPER=1 63 | ulimits: 64 | memlock: 65 | soft: -1 66 | hard: -1 67 | command: > 68 | --log-level=info 69 | --config=/qtap/sample-config.yaml 70 | -------------------------------------------------------------------------------- /examples/fluentbit/example.env: -------------------------------------------------------------------------------- 1 | FLUENTBIT_CONFIG=basic-stdout.conf 2 | AXIOM_DATASET= 3 | AXIOM_API_KEY= 4 | AWS_ACCESS_KEY_ID= 5 | AWS_SECRET_ACCESS_KEY= 6 | S3_BUCKET= 7 | S3_REGION= 8 | S3_ENDPOINT= 9 | -------------------------------------------------------------------------------- /examples/fluentbit/extract-payloads-s3.conf: -------------------------------------------------------------------------------- 1 | # Fluent Bit Configuration for Qtap HTTP Transaction Logging 2 | # splitting the payloads out into separate records and storing them in S3 3 | # ===================================================== 4 | # 5 | # This configuration file sets up Fluent Bit to: 6 | # 7 | # 1. Receive logs from Docker containers using the Fluentd forward protocol 8 | # 2. Filter logs specifically from the Qtap container (tagged as docker.qtap) 9 | # 3. Parse the logs as JSON using the generic_json_parser defined in parsers.conf 10 | # 4. Identify HTTP transaction logs by matching the message "HTTP transaction" 11 | # 5. Rewrite the tag for these HTTP transaction logs to "qtap.http" 12 | # 6. Process payloads using a Lua script (s3-payloads.lua) to generate S3 URLs 13 | # 7. Rewrite the tag for payload records to "qtap.http.payload" 14 | # 8. Output HTTP transaction logs to stdout in JSON lines format 15 | # 9. Store payload records in S3 using the configured endpoint, bucket, and region 16 | 17 | # Ensure the parsers are loaded 18 | [SERVICE] 19 | Parsers_File parsers.conf 20 | 21 | # Listen for logs forwarded via fluentd log driver 22 | [INPUT] 23 | Name forward 24 | Listen 0.0.0.0 25 | Port 24224 26 | 27 | # Filter the logs to only tagged with docker.qtap, and parse them as JSON 28 | [FILTER] 29 | Name parser 30 | Match docker.qtap 31 | Key_Name log 32 | Parser generic_json_parser 33 | 34 | # Rewrite tag for logs with the exact message "HTTP transaction" 35 | [FILTER] 36 | Name rewrite_tag 37 | Match docker.qtap 38 | Rule $msg "^HTTP transaction$" qtap.http false 39 | 40 | # Lua filter to process payloads and generate S3 URLs 41 | [FILTER] 42 | Name lua 43 | Match qtap.http 44 | Script lua/s3-payloads.lua 45 | Call process_payloads 46 | Protected_Mode true 47 | 48 | # Rewrite tag for payload records 49 | [FILTER] 50 | Name rewrite_tag 51 | Match qtap.http 52 | Rule $msg "^HTTP payload$" qtap.http.payload false 53 | 54 | # Output the HTTP transaction logs to stdout, as individual lines of JSON 55 | [OUTPUT] 56 | Name stdout 57 | Match qtap.http 58 | Format json_lines 59 | 60 | # Output the payload records to S3 61 | [OUTPUT] 62 | Name s3 63 | Match qtap.http.payload 64 | endpoint ${S3_ENDPOINT} 65 | bucket ${S3_BUCKET} 66 | region ${S3_REGION} 67 | s3_key_format /$s3_key 68 | log_key data 69 | 70 | # Output all other docker.qtap logs to stdout 71 | # [OUTPUT] 72 | # Name stdout 73 | # Match docker.qtap 74 | # Format json_lines 75 | -------------------------------------------------------------------------------- /examples/fluentbit/extract-payloads-stdout.conf: -------------------------------------------------------------------------------- 1 | # Fluent Bit Configuration for Qtap HTTP Transaction Logging 2 | # splitting the payloads out into separate records 3 | # ===================================================== 4 | # 5 | # This configuration file sets up Fluent Bit to: 6 | # 7 | # 1. Receive logs from Docker containers using the Fluentd forward protocol 8 | # 2. Filter logs specifically from the Qtap container (tagged as docker.qtap) 9 | # 3. Parse the logs as JSON using the generic_json_parser defined in parsers.conf 10 | # 4. Identify HTTP transaction logs by matching the message "HTTP transaction" 11 | # 5. Rewrite the tag for these HTTP transaction logs to "qtap.http" 12 | # 6. Use a Lua script to extract and split HTTP payloads into separate records 13 | # 7. Rewrite the tag for payload records to "qtap.http.payload" 14 | # 8. Output both HTTP transaction logs and payload records to stdout in JSON lines format 15 | # 16 | # This setup allows for easy viewing and processing of Qtap's HTTP transaction 17 | # monitoring data and associated payloads while filtering out other container logs. 18 | 19 | # Ensure the parsers are loaded 20 | [SERVICE] 21 | Parsers_File parsers.conf 22 | 23 | # Listen for logs forwarded via fluentd log driver 24 | [INPUT] 25 | Name forward 26 | Listen 0.0.0.0 27 | Port 24224 28 | 29 | # Filter the logs to only tagged with docker.qtap, and parse them as JSON 30 | [FILTER] 31 | Name parser 32 | Match docker.qtap 33 | Key_Name log 34 | Parser generic_json_parser 35 | 36 | # Rewrite tag for logs with the exact message "HTTP transaction" 37 | [FILTER] 38 | Name rewrite_tag 39 | Match docker.qtap 40 | Rule $msg "^HTTP transaction$" qtap.http false 41 | 42 | # Lua filter to split payloads into separate records 43 | [FILTER] 44 | Name lua 45 | Match qtap.http 46 | Script lua/extract-payloads.lua 47 | Call extract_payloads 48 | Protected_Mode true 49 | 50 | # Rewrite tag for payload records 51 | [FILTER] 52 | Name rewrite_tag 53 | Match qtap.http 54 | Rule $msg "^HTTP payload$" qtap.http.payload false 55 | 56 | # Output the HTTP transaction logs to stdout, as individual lines of JSON 57 | [OUTPUT] 58 | Name stdout 59 | Match qtap.http 60 | Format json_lines 61 | 62 | # Output the payload records to stdout 63 | [OUTPUT] 64 | Name stdout 65 | Match qtap.http.payload 66 | Format json_lines 67 | 68 | # Output all other docker.qtap logs to stdout 69 | # [OUTPUT] 70 | # Name stdout 71 | # Match docker.qtap 72 | # Format json_lines 73 | -------------------------------------------------------------------------------- /examples/fluentbit/parsers.conf: -------------------------------------------------------------------------------- 1 | # A generic JSON parser 2 | [PARSER] 3 | Name generic_json_parser 4 | Format json 5 | -------------------------------------------------------------------------------- /examples/http-access-logs/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # Qtap Docker Compose Configuration 2 | # ================================================= 3 | # 4 | # This docker-compose file runs two services: 5 | # 6 | # 1. httpbin: A simple HTTP request & response service for testing 7 | # - Exposes port 8080 for HTTP requests 8 | # - Provides various endpoints for testing HTTP methods and responses 9 | # - Useful for generating HTTP traffic that Qtap can monitor 10 | # 11 | # 2. qtap: Monitors HTTP transactions using eBPF 12 | # - Runs with host networking and privileged access for eBPF functionality 13 | # - Forwards logs to fluent-bit for processing 14 | # - Requires elevated privileges for eBPF functionality 15 | # 16 | # To run this setup from the project root directory, use: 17 | # 18 | # docker compose -f examples/http-access-logs/docker-compose.yaml up 19 | # 20 | version: '3' 21 | 22 | services: 23 | httpbin: 24 | image: ghcr.io/mccutchen/go-httpbin 25 | ports: 26 | - "8080:8080" 27 | restart: unless-stopped 28 | 29 | qtap: 30 | image: us-docker.pkg.dev/qpoint-edge/public/qtap:v0.9.7 31 | pid: host 32 | network_mode: host 33 | user: "0:0" 34 | privileged: true 35 | cap_add: 36 | - CAP_BPF 37 | - CAP_SYS_ADMIN 38 | volumes: 39 | - /sys:/sys 40 | - ./:/qtap 41 | environment: 42 | - TINI_SUBREAPER=1 43 | ulimits: 44 | memlock: 45 | soft: -1 46 | hard: -1 47 | command: > 48 | --log-level=info 49 | --config=/qtap/qtap.yaml 50 | -------------------------------------------------------------------------------- /examples/http-access-logs/qtap.yaml: -------------------------------------------------------------------------------- 1 | # plugin stacks 2 | stacks: 3 | default: 4 | plugins: 5 | - type: access_logs 6 | config: 7 | # default mode (none, summary, details, full) 8 | mode: summary 9 | 10 | # display format (console, json) 11 | format: console 12 | 13 | # custom rules 14 | rules: 15 | # # ignore sshd docker forwarding 16 | # - name: ignore sshd docker forwarding 17 | # expr: exe matches |sshd$| 18 | # mode: none 19 | 20 | # # show all details when not GET 21 | # - name: details when not GET 22 | # expr: request.method != "GET" 23 | # mode: details 24 | 25 | # # capture details and payloads on errors 26 | # - name: full log on errors 27 | # expr: response.status >= 400 28 | # mode: full 29 | 30 | # # show full details and payloads for example.com 31 | # - name: details for example.com 32 | # expr: request.host == "example.com" 33 | # mode: full 34 | 35 | # # show full details and payloads when the container name is qpoint_qtap_demo 36 | # - name: details for qpoint_qtap_demo 37 | # expr: container_name == "qpoint_qtap_demo" 38 | # mode: full 39 | 40 | # # show full details and payloads when the container image is alpine 41 | # - name: details for alpine 42 | # expr: container_image == "alpine" 43 | # mode: full 44 | 45 | # # show full details and payloads when the pod name is qpoint_qtap_demo 46 | # - name: details for qpoint_qtap_demo 47 | # expr: pod_name == "qpoint_qtap_demo" 48 | # mode: full 49 | 50 | # # show full details and payloads when the pod namespace is qpoint_qtap_demo_ns 51 | # - name: details for qpoint_qtap_demo_ns 52 | # expr: pod_namespace == "qpoint_qtap_demo_ns" 53 | # mode: full 54 | 55 | # tap config 56 | tap: 57 | direction: all # (all|egress|egress-external|egress-internal) 58 | ignore_loopback: false 59 | audit_include_dns: false 60 | http: 61 | stack: default 62 | -------------------------------------------------------------------------------- /examples/sample-axiom.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | services: 3 | event_stores: 4 | - type: axiom 5 | dataset: 6 | type: text 7 | value: qpoint-events # The name of the Axiom dataset to send events to 8 | token: 9 | type: env # or "text" for direct value 10 | value: AXIOM_TOKEN # Environment variable name or direct token value 11 | object_stores: 12 | - type: stdout 13 | stacks: 14 | report_to_axiom: 15 | plugins: 16 | - type: access_logs 17 | config: 18 | mode: summary 19 | format: console 20 | - type: report_usage 21 | tap: 22 | direction: egress-external 23 | ignore_loopback: true 24 | audit_include_dns: false 25 | http: 26 | stack: report_to_axiom -------------------------------------------------------------------------------- /examples/sample-capture-minio.yaml: -------------------------------------------------------------------------------- 1 | # This is a simple example of a qtap config that captures HTTP transactions and 2 | # stores writes the transaction to an S3 bucket and prints a record to stdout 3 | # which contains a URL to the transaction bucket item. 4 | # 5 | # A Minio instance be be started easily with docker: 6 | # ``` 7 | # docker run \ 8 | # -p 9000:9000 \ 9 | # -p 9001:9001 \ 10 | # --name minio \ 11 | # -v ~/minio/data:/data \ 12 | # -e "MINIO_ROOT_USER=root" \ 13 | # -e "MINIO_ROOT_PASSWORD=password123" \ 14 | # quay.io/minio/minio:RELEASE.2025-04-08T15-41-24Z-cpuv1 server /data --console-address ":9001" 15 | # ``` 16 | # Ensure the correct IP that is is set in the qpoint config, so that other containers can reach it. 17 | # This will likely be different from the IP used to access minio from your local machine. 18 | # 19 | # Note: We are using an older version of MinIO because they removed all the admin functionality 20 | # from the webui in latest version which is a huge PITA. 21 | 22 | version: 2 23 | services: 24 | event_stores: 25 | - type: stdout 26 | object_stores: 27 | - type: s3 28 | endpoint: 172.17.0.3:9000 # REPLACE: with your minio endpoint 29 | bucket: qpoint-warehouse # REPLACE: with your minio bucket name 30 | insecure: true 31 | access_key: 32 | type: text 33 | value: # REPLACE: with your minio access key 34 | secret_key: 35 | type: text 36 | value: # REPLACE: with your minio secret key 37 | stacks: 38 | basic_reporting: 39 | plugins: 40 | - type: http_capture 41 | config: 42 | level: summary # (none|summary|details|full) 43 | format: text # (json|text) 44 | rules: 45 | - name: full log httpbin.org 46 | expr: request.host == "httpbin.org" 47 | level: full 48 | - name: details log on 500 49 | expr: response.status >= 500 50 | level: details 51 | - name: full log on 400 52 | expr: response.status >= 400 && response.status < 500 53 | level: full 54 | 55 | - type: report_usage 56 | tap: 57 | direction: egress # (egress|egress-external|egress-internal) 58 | ignore_loopback: true 59 | audit_include_dns: false 60 | http: 61 | stack: basic_reporting 62 | -------------------------------------------------------------------------------- /examples/sample-capture.yaml: -------------------------------------------------------------------------------- 1 | # This is a simple example of a qtap config that captures HTTP transactions and 2 | # stores writes them to stdout. This includes headers and bodies if the level is 3 | # set to details or full. 4 | # 5 | version: 2 6 | services: 7 | event_stores: 8 | - type: stdout 9 | object_stores: 10 | - type: stdout 11 | stacks: 12 | basic_reporting: 13 | plugins: 14 | - type: report_usage 15 | - type: http_capture 16 | config: 17 | level: summary # (none|summary|headers|full) 18 | format: text # (json|text) 19 | rules: 20 | - name: full log httpbin.org 21 | expr: request.host == "httpbin.org" 22 | level: full 23 | - name: headers log on 500 24 | expr: response.status >= 500 25 | level: headers 26 | - name: full log on 400 27 | expr: is_4xx() # see rulekit config below 28 | level: full 29 | 30 | # rulekit config 31 | rulekit: 32 | macros: 33 | - name: is_4xx 34 | expr: response.status >= 400 && response.status < 500 35 | 36 | # tap config 37 | tap: 38 | direction: egress # (egress|egress-external|egress-internal) 39 | ignore_loopback: true 40 | audit_include_dns: false 41 | http: 42 | stack: basic_reporting 43 | -------------------------------------------------------------------------------- /examples/sample-config.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | services: 3 | event_stores: 4 | - type: stdout 5 | object_stores: 6 | - type: stdout 7 | stacks: 8 | basic_reporting: 9 | plugins: 10 | - type: access_logs 11 | config: 12 | mode: summary # (summary|details|full|none) 13 | format: console # (json|console) 14 | rules: 15 | - name: full log httpbin.org 16 | expr: request.host == "httpbin.org" 17 | mode: full 18 | - name: details log on 500 19 | expr: response.status >= 500 20 | mode: details 21 | - name: details log on example.com and any subdomains 22 | expr: in_zone(request.host, "example.com") 23 | mode: details 24 | - name: full log on 400 25 | expr: response.status >= 400 && response.status < 500 26 | mode: full 27 | - type: report_usage 28 | tap: 29 | direction: egress # (all|egress|egress-external|egress-internal|ingress) 30 | ignore_loopback: true 31 | audit_include_dns: false 32 | http: 33 | stack: basic_reporting 34 | -------------------------------------------------------------------------------- /examples/sample-otel-grpc-eventstore.yaml: -------------------------------------------------------------------------------- 1 | # This is a simple example of a qtap config that captures HTTP transactions and 2 | # stores writes them to stdout. This includes headers and bodies if the level is 3 | # set to details or full. 4 | # 5 | version: 2 6 | services: 7 | event_stores: 8 | - type: otel 9 | endpoint: "host.docker.internal:4317" # gRPC endpoint 10 | protocol: grpc # "grpc", "http", or "stdout" 11 | service_name: "qtap" 12 | environment: "production" 13 | # headers: 14 | # api-key: 15 | # type: env 16 | # value: OTEL_API_KEY 17 | tls: 18 | enabled: false 19 | object_stores: 20 | - type: stdout 21 | stacks: 22 | basic_reporting: 23 | plugins: 24 | - type: http_capture 25 | config: 26 | level: summary # (none|summary|details|full) 27 | format: json # (json|text) 28 | rules: 29 | - name: full log httpbin.org 30 | expr: request.host == "httpbin.org" 31 | level: full 32 | - name: details log on 500 33 | expr: response.status >= 500 34 | level: details 35 | - name: full log on 400 36 | expr: response.status >= 400 && response.status < 500 37 | level: full 38 | 39 | - type: report_usage 40 | tap: 41 | direction: egress # (egress|egress-external|egress-internal) 42 | ignore_loopback: true 43 | audit_include_dns: false 44 | http: 45 | stack: basic_reporting 46 | -------------------------------------------------------------------------------- /examples/sample-otel-http-eventstore.yaml: -------------------------------------------------------------------------------- 1 | # This is a simple example of a qtap config that captures HTTP transactions and 2 | # stores writes them to stdout. This includes headers and bodies if the level is 3 | # set to details or full. 4 | # 5 | version: 2 6 | services: 7 | event_stores: 8 | - type: otel 9 | endpoint: "host.docker.internal:4318" # http endpoint 10 | protocol: http # "grpc", "http", or "stdout" 11 | service_name: "qtap" 12 | environment: "production" 13 | # headers: 14 | # api-key: 15 | # type: env 16 | # value: OTEL_API_KEY 17 | tls: 18 | enabled: false 19 | object_stores: 20 | - type: stdout 21 | stacks: 22 | basic_reporting: 23 | plugins: 24 | - type: http_capture 25 | config: 26 | level: summary # (none|summary|details|full) 27 | format: json # (json|text) 28 | rules: 29 | - name: full log httpbin.org 30 | expr: request.host == "httpbin.org" 31 | level: full 32 | - name: details log on 500 33 | expr: response.status >= 500 34 | level: details 35 | - name: full log on 400 36 | expr: response.status >= 400 && response.status < 500 37 | level: full 38 | 39 | - type: report_usage 40 | tap: 41 | direction: egress # (egress|egress-external|egress-internal) 42 | ignore_loopback: true 43 | audit_include_dns: false 44 | http: 45 | stack: basic_reporting 46 | -------------------------------------------------------------------------------- /examples/sample-otel-stdout-eventstore.yaml: -------------------------------------------------------------------------------- 1 | # This is a simple example of a qtap config that captures HTTP transactions and 2 | # stores writes them to stdout. This includes headers and bodies if the level is 3 | # set to details or full. 4 | # 5 | version: 2 6 | services: 7 | event_stores: 8 | - type: otel 9 | protocol: stdout 10 | service_name: "qtap-sample" 11 | environment: "development" 12 | object_stores: 13 | - type: stdout 14 | stacks: 15 | basic_reporting: 16 | plugins: 17 | - type: http_capture 18 | config: 19 | level: summary # (none|summary|details|full) 20 | format: text # (json|text) 21 | rules: 22 | - name: full log httpbin.org 23 | expr: request.host == "httpbin.org" 24 | level: full 25 | - name: details log on 500 26 | expr: response.status >= 500 27 | level: details 28 | - name: full log on 400 29 | expr: response.status >= 400 && response.status < 500 30 | level: full 31 | 32 | - type: report_usage 33 | tap: 34 | direction: egress # (egress|egress-external|egress-internal) 35 | ignore_loopback: true 36 | audit_include_dns: false 37 | http: 38 | stack: basic_reporting 39 | -------------------------------------------------------------------------------- /internal/tap/gen.go: -------------------------------------------------------------------------------- 1 | package tap 2 | 3 | //go:generate go tool github.com/cilium/ebpf/cmd/bpf2go -target arm64,amd64 Tap ../../bpf/tap/bpf2go.c -- -O2 -target bpf -g -I../../bpf/headers -I../../bpf/tap -DBPF_NO_PRESERVE_ACCESS_INDEX 4 | -------------------------------------------------------------------------------- /internal/tap/tap_arm64_bpfel.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qpoint-io/qtap/20f2c095a247e2846497e87a3ad673e99a5a511f/internal/tap/tap_arm64_bpfel.o -------------------------------------------------------------------------------- /internal/tap/tap_x86_bpfel.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qpoint-io/qtap/20f2c095a247e2846497e87a3ad673e99a5a511f/internal/tap/tap_x86_bpfel.o -------------------------------------------------------------------------------- /pkg/binutils/strings.go: -------------------------------------------------------------------------------- 1 | package binutils 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "unicode" 9 | ) 10 | 11 | const minStrLength = 4 12 | 13 | func (e *Elf) SearchString(searchStr string, strategy MatchStrategy) (string, error) { 14 | if e.isClosed { 15 | return "", errors.New("ELF file is closed") 16 | } 17 | 18 | var buffer [bufferSize]byte 19 | var window [bufferSize * 2]byte 20 | windowLen := 0 21 | offset := int64(0) 22 | 23 | for { 24 | n, err := e.file.ReadAt(buffer[:], offset) 25 | if err != nil && err != io.EOF { 26 | return "", fmt.Errorf("error reading file: %w", err) 27 | } 28 | 29 | // Copy buffer to window 30 | copy(window[windowLen:], buffer[:n]) 31 | windowLen += n 32 | 33 | for windowLen >= minStrLength { 34 | idx := bytes.IndexFunc(window[:windowLen], func(r rune) bool { 35 | return !unicode.IsPrint(r) 36 | }) 37 | 38 | if idx == -1 { 39 | break 40 | } 41 | 42 | if idx >= minStrLength { 43 | str := string(window[:idx]) 44 | if match(str, searchStr, strategy) { 45 | return str, nil 46 | } 47 | } 48 | 49 | // Shift window contents 50 | copy(window[:], window[idx+1:windowLen]) 51 | windowLen -= idx + 1 52 | } 53 | 54 | if windowLen > bufferSize { 55 | copy(window[:], window[windowLen-bufferSize:windowLen]) 56 | windowLen = bufferSize 57 | } 58 | 59 | offset += int64(n) 60 | 61 | if err == io.EOF { 62 | break 63 | } 64 | } 65 | 66 | // Check the last string in the window 67 | if windowLen >= minStrLength { 68 | str := string(window[:windowLen]) 69 | if match(str, searchStr, strategy) { 70 | return str, nil 71 | } 72 | } 73 | 74 | return "", fmt.Errorf("no match found for '%s' using strategy %v", searchStr, strategy) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/buildinfo/buildinfo.go: -------------------------------------------------------------------------------- 1 | package buildinfo 2 | 3 | // this is set by the build process 4 | var ( 5 | version string 6 | commit string 7 | branch string 8 | buildTime string 9 | ) 10 | 11 | func Version() string { 12 | if version == "" { 13 | return "dev" 14 | } 15 | return version 16 | } 17 | 18 | func Commit() string { 19 | if commit == "" { 20 | return "unknown" 21 | } 22 | return commit 23 | } 24 | 25 | func Branch() string { 26 | if branch == "" { 27 | return "unknown" 28 | } 29 | return branch 30 | } 31 | 32 | func BuildTime() string { 33 | if buildTime == "" { 34 | return "unknown" 35 | } 36 | return buildTime 37 | } 38 | -------------------------------------------------------------------------------- /pkg/cmd/reload.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "strconv" 7 | "strings" 8 | "syscall" 9 | 10 | "github.com/spf13/cobra" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | const binaryName = "qtap" 15 | 16 | var reloadConfigCmd = &cobra.Command{ 17 | Use: "reload-config", 18 | Short: "Live reload the current config", 19 | Long: `Reload the current configuration without restarting the application. 20 | Example usage: 21 | qtap reload-config`, 22 | Run: func(cmd *cobra.Command, args []string) { 23 | logger := initLogger() 24 | defer syncLogger(logger) 25 | 26 | runReloadCmd(logger) 27 | }, 28 | } 29 | 30 | func runReloadCmd(logger *zap.Logger) { 31 | pid, err := findPIDByBinaryName(binaryName) 32 | if err != nil { 33 | if errors.Is(err, os.ErrNotExist) { 34 | logger.Fatal("could not find a running qtap process") 35 | } 36 | logger.Fatal("error finding running qtap process", zap.Error(err)) 37 | } 38 | 39 | logger.Info("sending SIGHUP signal to process", zap.Int("pid", pid)) 40 | err = syscall.Kill(pid, syscall.SIGHUP) 41 | if err != nil { 42 | logger.Fatal("error sending signal", zap.Error(err)) 43 | } 44 | } 45 | 46 | func findPIDByBinaryName(name string) (int, error) { 47 | currentPID := os.Getpid() // Get the current process PID 48 | procs, err := os.ReadDir("/proc") 49 | if err != nil { 50 | return 0, err 51 | } 52 | for _, proc := range procs { 53 | if pid, err := strconv.Atoi(proc.Name()); err == nil { 54 | if pid == currentPID { 55 | continue // Skip the current process 56 | } 57 | cmdline, err := os.ReadFile("/proc/" + proc.Name() + "/cmdline") 58 | if err == nil { 59 | cmds := strings.Split(string(cmdline), "\x00") 60 | if len(cmds) > 0 && strings.Contains(cmds[0], name) { 61 | return pid, nil 62 | } 63 | } 64 | } 65 | } 66 | return 0, os.ErrNotExist 67 | } 68 | -------------------------------------------------------------------------------- /pkg/cmd/tap_other.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | 3 | package cmd 4 | 5 | import ( 6 | "go.uber.org/zap" 7 | ) 8 | 9 | func runTapCmd(logger *zap.Logger) { 10 | logger.Warn("'tap' leverages eBPF probes which can only run on Linux.") 11 | } 12 | -------------------------------------------------------------------------------- /pkg/config/default.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | services: 3 | event_stores: 4 | - type: stdout 5 | object_stores: 6 | - type: stdout 7 | stacks: 8 | basic_reporting: 9 | plugins: 10 | - type: access_logs 11 | config: 12 | mode: full 13 | format: console 14 | - type: report_usage 15 | tap: 16 | direction: egress 17 | ignore_loopback: true 18 | audit_include_dns: false 19 | http: 20 | stack: basic_reporting 21 | -------------------------------------------------------------------------------- /pkg/config/default_provider.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | _ "embed" 5 | "fmt" 6 | 7 | "go.uber.org/zap" 8 | ) 9 | 10 | //go:embed default.yaml 11 | var defaultConfigBytes []byte 12 | 13 | // DefaultConfigProvider loads configuration from a local file and reloads on SIGHUP 14 | type DefaultConfigProvider struct { 15 | logger *zap.Logger 16 | cfg *Config 17 | callback func(*Config) (func(), error) 18 | } 19 | 20 | // NewDefaultConfigProvider creates a new provider for default config 21 | func NewDefaultConfigProvider(logger *zap.Logger) *DefaultConfigProvider { 22 | return &DefaultConfigProvider{ 23 | logger: logger, 24 | } 25 | } 26 | 27 | // Start watching for config changes via SIGHUP 28 | func (p *DefaultConfigProvider) Start() error { 29 | cfg, err := UnmarshalConfig(defaultConfigBytes) 30 | if err != nil { 31 | return fmt.Errorf("failed to unmarshal default config: %w", err) 32 | } 33 | 34 | p.cfg = cfg 35 | 36 | if p.callback != nil { 37 | _, err := p.callback(p.cfg) 38 | return err 39 | } 40 | 41 | return nil 42 | } 43 | 44 | // Stop watching for config changes 45 | func (p *DefaultConfigProvider) Stop() {} 46 | 47 | // OnConfigChange registers a callback for config changes 48 | func (p *DefaultConfigProvider) OnConfigChange(callback func(*Config) (func(), error)) { 49 | p.callback = callback 50 | } 51 | 52 | // Reload forces a configuration reload 53 | func (p *DefaultConfigProvider) Reload() error { 54 | _, err := p.callback(p.cfg) 55 | return err 56 | } 57 | -------------------------------------------------------------------------------- /pkg/config/eventstore.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type EventStoreType string 4 | 5 | const ( 6 | EventStoreType_DISABLED EventStoreType = "disabled" 7 | EventStoreType_CONSOLE EventStoreType = "stdout" 8 | EventStoreType_PULSE EventStoreType = "pulse" 9 | EventStoreType_PULSE_STREAMING EventStoreType = "pulse-streaming" 10 | EventStoreType_PULSE_LEGACY EventStoreType = "pulse-legacy" 11 | EventStoreType_AXIOM EventStoreType = "axiom" 12 | EventStoreType_OTEL EventStoreType = "otel" 13 | ) 14 | 15 | type ServiceEventStore struct { 16 | Type EventStoreType `yaml:"type" validate:"required"` 17 | ID string `yaml:"id"` 18 | EventStoreConfig `yaml:",inline,omitempty"` 19 | } 20 | 21 | func (s ServiceEventStore) ServiceType() string { 22 | switch s.Type { 23 | case EventStoreType_PULSE: 24 | return "eventstore.eventstorev1_nonstreaming" 25 | case EventStoreType_PULSE_STREAMING: 26 | return "eventstore.eventstorev1" 27 | case EventStoreType_PULSE_LEGACY: 28 | return "eventstore.pulse" 29 | case EventStoreType_CONSOLE: 30 | return "eventstore.console" 31 | case EventStoreType_DISABLED: 32 | return "eventstore.noop" 33 | case EventStoreType_AXIOM: 34 | return "eventstore.axiom" 35 | case EventStoreType_OTEL: 36 | return "eventstore.otel" 37 | case "e2e": // TODO(e2e) 38 | return "eventstore.e2e" 39 | default: 40 | return "eventstore.console" 41 | } 42 | } 43 | 44 | type EventStoreConfig struct { 45 | Token ValueSource `yaml:"token"` 46 | EventStorePulseConfig `yaml:",inline,omitempty"` 47 | EventStoreAxiomConfig `yaml:",inline,omitempty"` 48 | EventStoreOTelConfig `yaml:",inline,omitempty"` 49 | } 50 | 51 | type EventStorePulseConfig struct { 52 | URL string `yaml:"url"` 53 | AllowInsecure bool `yaml:"allow_insecure"` 54 | } 55 | 56 | type EventStoreAxiomConfig struct { 57 | Dataset ValueSource `yaml:"dataset"` 58 | } 59 | 60 | type EventStoreOTelConfig struct { 61 | Endpoint string `yaml:"endpoint"` 62 | Protocol string `yaml:"protocol"` // "grpc", "http", "stdout" 63 | Headers map[string]ValueSource `yaml:"headers"` // Optional headers for auth 64 | ServiceName string `yaml:"service_name"` 65 | Environment string `yaml:"environment"` 66 | TLS EventStoreOTelTLS `yaml:"tls"` 67 | } 68 | 69 | type EventStoreOTelTLS struct { 70 | Enabled bool `yaml:"enabled"` 71 | InsecureSkipVerify bool `yaml:"insecure_skip_verify"` 72 | } 73 | -------------------------------------------------------------------------------- /pkg/config/eventstore_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "os" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | func TestEventStoreUnmarshal(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | filename string 16 | want ServiceEventStore 17 | wantErr bool 18 | }{ 19 | { 20 | name: "console eventstore", 21 | filename: "testdata/eventstore_console.yaml", 22 | want: ServiceEventStore{ 23 | Type: EventStoreType_CONSOLE, 24 | ID: "console-store", 25 | }, 26 | wantErr: false, 27 | }, 28 | { 29 | name: "pulse eventstore", 30 | filename: "testdata/eventstore_pulse.yaml", 31 | want: ServiceEventStore{ 32 | Type: EventStoreType_PULSE, 33 | ID: "pulse-store", 34 | EventStoreConfig: EventStoreConfig{ 35 | Token: ValueSource{ 36 | Value: "secret-token", 37 | }, 38 | EventStorePulseConfig: EventStorePulseConfig{ 39 | URL: "https://pulse.example.com", 40 | }, 41 | }, 42 | }, 43 | wantErr: false, 44 | }, 45 | { 46 | name: "disabled eventstore", 47 | filename: "testdata/eventstore_disabled.yaml", 48 | want: ServiceEventStore{ 49 | Type: EventStoreType_DISABLED, 50 | ID: "disabled-store", 51 | }, 52 | wantErr: false, 53 | }, 54 | { 55 | name: "otel eventstore", 56 | filename: "testdata/eventstore_otel.yaml", 57 | want: ServiceEventStore{ 58 | Type: EventStoreType_OTEL, 59 | ID: "otel-store", 60 | EventStoreConfig: EventStoreConfig{ 61 | EventStoreOTelConfig: EventStoreOTelConfig{ 62 | Endpoint: "localhost:4317", 63 | Protocol: "grpc", 64 | ServiceName: "qtap-test", 65 | Environment: "test", 66 | Headers: map[string]ValueSource{ 67 | "api-key": { 68 | Type: "text", 69 | Value: "test-key", 70 | }, 71 | }, 72 | TLS: EventStoreOTelTLS{ 73 | Enabled: false, 74 | InsecureSkipVerify: false, 75 | }, 76 | }, 77 | }, 78 | }, 79 | wantErr: false, 80 | }, 81 | } 82 | 83 | for _, tt := range tests { 84 | t.Run(tt.name, func(t *testing.T) { 85 | data, err := os.ReadFile(tt.filename) 86 | if err != nil { 87 | t.Fatalf("failed to read test file: %v", err) 88 | } 89 | 90 | var got ServiceEventStore 91 | err = yaml.Unmarshal(data, &got) 92 | if (err != nil) != tt.wantErr { 93 | t.Errorf("yaml.Unmarshal() error = %v, wantErr %v", err, tt.wantErr) 94 | return 95 | } 96 | 97 | assert.Equal(t, tt.want, got) 98 | }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /pkg/config/objectstore.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type ObjectStoreType string 4 | 5 | const ( 6 | ObjectStoreType_DISABLED ObjectStoreType = "disabled" 7 | ObjectStoreType_CONSOLE ObjectStoreType = "stdout" 8 | ObjectStoreType_QPOINT ObjectStoreType = "qpoint" 9 | ObjectStoreType_S3 ObjectStoreType = "s3" 10 | ) 11 | 12 | type ServiceObjectStore struct { 13 | Type ObjectStoreType `yaml:"type" validate:"required"` 14 | ID string `yaml:"id"` 15 | ObjectStoreConfig `yaml:",inline,omitempty"` 16 | } 17 | 18 | func (s ServiceObjectStore) ServiceType() string { 19 | switch s.Type { 20 | case ObjectStoreType_QPOINT: 21 | return "objectstore.warehouse" 22 | case ObjectStoreType_S3: 23 | return "objectstore.s3" 24 | case ObjectStoreType_CONSOLE: 25 | return "objectstore.console" 26 | case ObjectStoreType_DISABLED: 27 | return "objectstore.noop" 28 | default: 29 | return "objectstore.console" 30 | } 31 | } 32 | 33 | type ObjectStoreConfig struct { 34 | ObjectStoreQPointWarehouseConfig `yaml:",inline,omitempty"` 35 | ObjectStoreS3Config `yaml:",inline,omitempty"` 36 | } 37 | 38 | type ObjectStoreQPointWarehouseConfig struct { 39 | URL string `yaml:"url"` 40 | Token ValueSource `yaml:"token"` 41 | } 42 | 43 | type ObjectStoreS3Config struct { 44 | Endpoint string `yaml:"endpoint"` 45 | Bucket string `yaml:"bucket"` 46 | Region string `yaml:"region"` 47 | AccessKey ValueSource `yaml:"access_key"` 48 | SecretKey ValueSource `yaml:"secret_key"` 49 | AccessURL string `yaml:"access_url"` 50 | Insecure bool `yaml:"insecure"` 51 | } 52 | -------------------------------------------------------------------------------- /pkg/config/objectstore_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "os" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | func TestObjectStoreUnmarshal(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | filename string 16 | want ServiceObjectStore 17 | wantErr bool 18 | }{ 19 | { 20 | name: "console objectstore", 21 | filename: "testdata/objectstore_console.yaml", 22 | want: ServiceObjectStore{ 23 | Type: ObjectStoreType_CONSOLE, 24 | ID: "console-store", 25 | }, 26 | wantErr: false, 27 | }, 28 | { 29 | name: "qpoint objectstore", 30 | filename: "testdata/objectstore_qpoint.yaml", 31 | want: ServiceObjectStore{ 32 | Type: ObjectStoreType_QPOINT, 33 | ID: "qpoint-store", 34 | ObjectStoreConfig: ObjectStoreConfig{ 35 | ObjectStoreQPointWarehouseConfig: ObjectStoreQPointWarehouseConfig{ 36 | URL: "https://warehouse.example.com", 37 | Token: ValueSource{ 38 | Value: "qpoint-token", 39 | }, 40 | }, 41 | }, 42 | }, 43 | wantErr: false, 44 | }, 45 | { 46 | name: "s3 objectstore", 47 | filename: "testdata/objectstore_s3.yaml", 48 | want: ServiceObjectStore{ 49 | Type: ObjectStoreType_S3, 50 | ID: "s3-store", 51 | ObjectStoreConfig: ObjectStoreConfig{ 52 | ObjectStoreS3Config: ObjectStoreS3Config{ 53 | Endpoint: "custom.s3.endpoint.com", 54 | Bucket: "my-test-bucket", 55 | Region: "us-west-2", 56 | AccessURL: "https://custom.cdn.example.com", 57 | Insecure: false, 58 | AccessKey: ValueSource{ 59 | Value: "access-key-123", 60 | }, 61 | SecretKey: ValueSource{ 62 | Value: "secret-key-456", 63 | }, 64 | }, 65 | }, 66 | }, 67 | wantErr: false, 68 | }, 69 | { 70 | name: "disabled objectstore", 71 | filename: "testdata/objectstore_disabled.yaml", 72 | want: ServiceObjectStore{ 73 | Type: ObjectStoreType_DISABLED, 74 | ID: "disabled-store", 75 | }, 76 | wantErr: false, 77 | }, 78 | } 79 | 80 | for _, tt := range tests { 81 | t.Run(tt.name, func(t *testing.T) { 82 | data, err := os.ReadFile(tt.filename) 83 | if err != nil { 84 | t.Fatalf("failed to read test file: %v", err) 85 | } 86 | 87 | var got ServiceObjectStore 88 | err = yaml.Unmarshal(data, &got) 89 | if (err != nil) != tt.wantErr { 90 | t.Errorf("yaml.Unmarshal() error = %v, wantErr %v", err, tt.wantErr) 91 | return 92 | } 93 | 94 | assert.Equal(t, tt.want, got) 95 | }) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /pkg/config/qscan.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type QscanType string 4 | 5 | const ( 6 | QscanType_DISABLED QscanType = "disabled" 7 | QscanType_CONSOLE QscanType = "stdout" 8 | QscanType_Client QscanType = "client" 9 | ) 10 | 11 | type ServiceQscan struct { 12 | Type QscanType `yaml:"type" validate:"required"` 13 | URL string `yaml:"url"` 14 | Token ValueSource `yaml:"token"` 15 | } 16 | 17 | func (s ServiceQscan) ServiceType() string { 18 | switch s.Type { 19 | case QscanType_CONSOLE: 20 | return "qscan.console" 21 | case QscanType_Client: 22 | return "qscan.client" 23 | case QscanType_DISABLED: 24 | return "qscan.noop" 25 | default: 26 | return "qscan.console" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/config/tap.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type TrafficDirection string 4 | 5 | var ( 6 | TrafficDirection_ALL TrafficDirection = "all" 7 | TrafficDirection_INGRESS TrafficDirection = "ingress" 8 | TrafficDirection_EGRESS TrafficDirection = "egress" 9 | TrafficDirection_EGRESS_INTERNAL TrafficDirection = "egress-internal" 10 | TrafficDirection_EGRESS_EXTERNAL TrafficDirection = "egress-external" 11 | ) 12 | 13 | type TapHttpConfig struct { 14 | Stack string `yaml:"stack"` 15 | } 16 | 17 | func (c *TapHttpConfig) HasStack() bool { 18 | return c.Stack != "" && c.Stack != "none" 19 | } 20 | 21 | type TapEndpointConfig struct { 22 | Domain string `yaml:"domain" validate:"required,hostname"` 23 | Http TapHttpConfig `yaml:"http"` 24 | } 25 | 26 | type EnvTag struct { 27 | Env string `yaml:"env"` 28 | Key string `yaml:"key"` 29 | } 30 | 31 | type TapConfig struct { 32 | Direction TrafficDirection `yaml:"direction"` 33 | IgnoreLoopback bool `yaml:"ignore_loopback"` 34 | AuditIncludeDNS bool `yaml:"audit_include_dns"` 35 | Http TapHttpConfig `yaml:"http"` 36 | Filters TapFilters `yaml:"filters,omitempty"` 37 | Endpoints []TapEndpointConfig `yaml:"endpoints" validate:"dive"` 38 | EnvTags []EnvTag `yaml:"env_tags,omitempty"` 39 | } 40 | 41 | func (c *TapConfig) HasAnyStack() bool { 42 | if c.Http.HasStack() { 43 | return true 44 | } 45 | 46 | for _, e := range c.Endpoints { 47 | if e.Http.HasStack() { 48 | return true 49 | } 50 | } 51 | 52 | return false 53 | } 54 | -------------------------------------------------------------------------------- /pkg/config/testdata/cert_valid.yaml: -------------------------------------------------------------------------------- 1 | ca: ca-cert-data 2 | crt: certificate-data 3 | key: private-key-data -------------------------------------------------------------------------------- /pkg/config/testdata/eventstore_console.yaml: -------------------------------------------------------------------------------- 1 | type: stdout 2 | id: console-store -------------------------------------------------------------------------------- /pkg/config/testdata/eventstore_disabled.yaml: -------------------------------------------------------------------------------- 1 | type: disabled 2 | id: disabled-store -------------------------------------------------------------------------------- /pkg/config/testdata/eventstore_otel.yaml: -------------------------------------------------------------------------------- 1 | type: otel 2 | id: otel-store 3 | endpoint: "localhost:4317" 4 | protocol: "grpc" 5 | service_name: "qtap-test" 6 | environment: "test" 7 | headers: 8 | api-key: 9 | type: text 10 | value: test-key 11 | tls: 12 | enabled: false 13 | insecure_skip_verify: false 14 | -------------------------------------------------------------------------------- /pkg/config/testdata/eventstore_pulse.yaml: -------------------------------------------------------------------------------- 1 | type: pulse 2 | id: pulse-store 3 | url: https://pulse.example.com 4 | token: 5 | value: secret-token -------------------------------------------------------------------------------- /pkg/config/testdata/objectstore_console.yaml: -------------------------------------------------------------------------------- 1 | type: stdout 2 | id: console-store -------------------------------------------------------------------------------- /pkg/config/testdata/objectstore_disabled.yaml: -------------------------------------------------------------------------------- 1 | type: disabled 2 | id: disabled-store -------------------------------------------------------------------------------- /pkg/config/testdata/objectstore_qpoint.yaml: -------------------------------------------------------------------------------- 1 | type: qpoint 2 | id: qpoint-store 3 | url: https://warehouse.example.com 4 | token: 5 | value: qpoint-token -------------------------------------------------------------------------------- /pkg/config/testdata/objectstore_s3.yaml: -------------------------------------------------------------------------------- 1 | type: s3 2 | id: s3-store 3 | endpoint: custom.s3.endpoint.com 4 | bucket: my-test-bucket 5 | region: us-west-2 6 | access_url: https://custom.cdn.example.com 7 | insecure: false 8 | access_key: 9 | value: access-key-123 10 | secret_key: 11 | value: secret-key-456 -------------------------------------------------------------------------------- /pkg/config/testdata/value_source_env.yaml: -------------------------------------------------------------------------------- 1 | type: env 2 | value: TEST_ENV_VAR -------------------------------------------------------------------------------- /pkg/config/testdata/value_source_text.yaml: -------------------------------------------------------------------------------- 1 | type: text 2 | value: some-text-value -------------------------------------------------------------------------------- /pkg/config/types.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "os" 4 | 5 | type ValueSourceType string 6 | 7 | const ( 8 | ValueSourceType_ENV ValueSourceType = "env" 9 | ValueSourceType_TEXT ValueSourceType = "text" 10 | ) 11 | 12 | type ValueSource struct { 13 | Type ValueSourceType `yaml:"type"` 14 | Value string `yaml:"value"` 15 | } 16 | 17 | func (vs ValueSource) String() string { 18 | switch vs.Type { 19 | case ValueSourceType_ENV: 20 | return os.Getenv(vs.Value) 21 | case ValueSourceType_TEXT: 22 | return vs.Value 23 | default: 24 | return "" 25 | } 26 | } 27 | 28 | type AccessControlAction string 29 | 30 | var ( 31 | AccessControlAction_UNKNOWN AccessControlAction = "" 32 | AccessControlAction_ALLOW AccessControlAction = "allow" 33 | AccessControlAction_DENY AccessControlAction = "deny" 34 | AccessControlAction_LOG AccessControlAction = "log" 35 | ) 36 | 37 | func (a AccessControlAction) String() string { 38 | return string(a) 39 | } 40 | 41 | type Cert struct { 42 | Ca string `yaml:"ca" validate:"required,stringnotempty"` 43 | Crt string `yaml:"crt" validate:"required,stringnotempty"` 44 | Key string `yaml:"key" validate:"required,stringnotempty"` 45 | } 46 | -------------------------------------------------------------------------------- /pkg/connection/metrics.go: -------------------------------------------------------------------------------- 1 | package connection 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/telemetry" 5 | "github.com/qpoint-io/qtap/pkg/tlsutils" 6 | ) 7 | 8 | func newManagerMetrics(m *Manager) *managerMetrics { 9 | return &managerMetrics{m: m} 10 | } 11 | 12 | type managerMetrics struct { 13 | m *Manager 14 | 15 | activeConnections telemetry.GaugeFn 16 | activeConnectionsTLS telemetry.GaugeFn 17 | } 18 | 19 | func (m *managerMetrics) Register(factory telemetry.Factory) { 20 | m.activeConnections = factory.Gauge( 21 | "tap_active_connections", 22 | telemetry.WithDescription("The number of active connections"), 23 | telemetry.WithLabels( 24 | // connection.SocketType 25 | "socket_type", 26 | ), 27 | ) 28 | 29 | m.activeConnectionsTLS = factory.Gauge( 30 | "tap_active_connections_tls", 31 | telemetry.WithDescription("The number of active TLS connections"), 32 | telemetry.WithLabels( 33 | // tlsutils.TLSVersion 34 | "version", 35 | ), 36 | ) 37 | } 38 | 39 | func (m *managerMetrics) Collect() { 40 | var ( 41 | // activeConnections 42 | connectionsByType = map[SocketType]int{} 43 | // activeConnectionsTLS 44 | connectionsByTLSVersion = map[tlsutils.TLSVersion]int{} 45 | ) 46 | 47 | m.m.connections.Iter(func(key Cookie, conn *Connection) bool { 48 | if conn.OpenEvent != nil { 49 | connectionsByType[conn.OpenEvent.SocketType]++ 50 | } 51 | if conn.TLSClientHello != nil { 52 | connectionsByTLSVersion[conn.TLSClientHello.Version]++ 53 | } 54 | return true 55 | }) 56 | 57 | for socketType, count := range connectionsByType { 58 | m.activeConnections(float64(count), string(socketType)) 59 | } 60 | 61 | for tlsVersion, count := range connectionsByTLSVersion { 62 | m.activeConnectionsTLS(float64(count), tlsVersion.String()) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /pkg/connection/report.go: -------------------------------------------------------------------------------- 1 | package connection 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "strings" 7 | "time" 8 | 9 | "go.opentelemetry.io/otel/attribute" 10 | "go.opentelemetry.io/otel/trace" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | func (c *Connection) logConnectionReport() { 15 | span := trace.SpanFromContext(c.ctx) 16 | var errorMsgs []string 17 | var logFn func(msg string, fields ...zap.Field) = c.logger.Debug 18 | 19 | fields := []zap.Field{ 20 | zap.Any("cookie", c.cookie), 21 | zap.String("destinationProtocol", c.OpenEvent.SocketType.String()), 22 | zap.Dict("report", c.report.reportFields()...), 23 | } 24 | 25 | // add handler type 26 | fields = append(fields, zap.String("handler", c.HandlerType.String())) 27 | span.SetAttributes(attribute.String("connection.handler", c.HandlerType.String())) 28 | 29 | // add strategy 30 | if proc := c.Process(); proc != nil { 31 | span.SetAttributes(attribute.String("connection.strategy", proc.Strategy.String())) 32 | fields = append(fields, zap.String("strategy", proc.Strategy.String())) 33 | } 34 | 35 | if len(errorMsgs) > 0 { 36 | fields = append(fields, zap.Strings("errors", errorMsgs)) 37 | } 38 | 39 | // send it 40 | logFn("connection report", fields...) 41 | } 42 | 43 | type report struct { 44 | ctx context.Context 45 | openTime time.Time 46 | closeTime time.Time 47 | dataEventCount uint64 48 | gotOrigDestEvent bool 49 | gotTLSClientHelloEvent bool 50 | gotProtocolEvent bool 51 | gotHandlerTypeEvent bool 52 | } 53 | 54 | // reportEvent is called when the event is first received 55 | func (r *report) reportEvent(event any) { 56 | span := trace.SpanFromContext(r.ctx) 57 | eventName := strings.TrimPrefix(reflect.TypeOf(event).String(), "connection.") 58 | var eventAttrs []attribute.KeyValue 59 | 60 | switch v := event.(type) { 61 | case OpenEvent: 62 | r.openTime = time.Now() 63 | case CloseEvent: 64 | r.closeTime = time.Now() 65 | case ProtocolEvent: 66 | eventAttrs = append(eventAttrs, attribute.String("protocol", v.Protocol.String())) 67 | r.gotProtocolEvent = true 68 | case DataEvent: 69 | eventAttrs = append(eventAttrs, attribute.Int("data_event.size", v.Size)) 70 | r.dataEventCount++ 71 | case TLSClientHelloEvent: 72 | r.gotTLSClientHelloEvent = true 73 | case OriginalDestinationEvent: 74 | r.gotOrigDestEvent = true 75 | case HandlerTypeEvent: 76 | r.gotHandlerTypeEvent = true 77 | } 78 | 79 | span.AddEvent(eventName, trace.WithAttributes(eventAttrs...)) 80 | } 81 | 82 | func (r *report) reportFields() []zap.Field { 83 | return []zap.Field{ 84 | zap.Duration("duration", r.closeTime.Sub(r.openTime)), 85 | zap.Bool("gotTLSClientHelloEvent", r.gotTLSClientHelloEvent), 86 | zap.Bool("gotProtocolEvent", r.gotProtocolEvent), 87 | zap.Bool("gotHandlerTypeEvent", r.gotHandlerTypeEvent), 88 | zap.Uint64("dataEventCount", r.dataEventCount), 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /pkg/connection/types.go: -------------------------------------------------------------------------------- 1 | package connection 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type Cookie uint64 8 | 9 | func (c Cookie) Key() Cookie { 10 | return c 11 | } 12 | 13 | type pidfd struct { 14 | PID uint32 15 | FD int32 16 | } 17 | 18 | // a unique connection composite key 19 | type ConnPIDKey struct { 20 | PID uint32 // Process PID 21 | TGID uint32 // Process TGID 22 | FD int32 // The file descriptor to the opened network connection 23 | FUNCTION Source // The function of the connection 24 | TSID uint64 // Timestamp at the initialization of the struct 25 | } 26 | 27 | // returns a string representation of connID 28 | func (c ConnPIDKey) String() string { 29 | return fmt.Sprintf("PID:%d TGID:%d FD:%d FUNCTION:%s TSID:%d", 30 | c.PID, 31 | c.TGID, 32 | c.FD, 33 | c.FUNCTION.String(), 34 | c.TSID) 35 | } 36 | 37 | func (c ConnPIDKey) PIDFD() pidfd { 38 | return pidfd{PID: c.PID, FD: c.FD} 39 | } 40 | 41 | type SocketType string 42 | 43 | const ( 44 | SocketType_UNKNOWN SocketType = "" 45 | SocketType_TCP SocketType = "tcp" 46 | SocketType_UDP SocketType = "udp" 47 | SocketType_RAW SocketType = "raw" 48 | SocketType_ICMP SocketType = "icmp" 49 | ) 50 | 51 | func (t SocketType) String() string { 52 | return string(t) 53 | } 54 | 55 | type Source uint32 56 | 57 | const ( 58 | Client Source = iota + 1 // iota is 1 for the first constant 59 | Server 60 | ) 61 | 62 | func (s Source) String() string { 63 | switch s { 64 | case Client: 65 | return "client" 66 | case Server: 67 | return "server" 68 | default: 69 | return "unknown" 70 | } 71 | } 72 | 73 | type HandlerType string 74 | 75 | const ( 76 | HandlerType_RAW HandlerType = "raw" 77 | HandlerType_REDIRECTED HandlerType = "redirected" 78 | HandlerType_FORWARDING HandlerType = "forwarding" 79 | ) 80 | 81 | func (t HandlerType) String() string { 82 | return string(t) 83 | } 84 | 85 | type Direction string 86 | 87 | func (d Direction) String() string { 88 | return string(d) 89 | } 90 | 91 | // directions 92 | const ( 93 | Ingress Direction = "ingress" 94 | Egress Direction = "egress" 95 | ) 96 | 97 | // L7 protocol 98 | type Protocol string 99 | 100 | const ( 101 | Protocol_UNKNOWN Protocol = "unknown" 102 | Protocol_HTTP1 Protocol = "http1" 103 | Protocol_HTTP2 Protocol = "http2" 104 | Protocol_DNS Protocol = "dns" 105 | Protocol_GRPC Protocol = "grpc" 106 | ) 107 | 108 | func (c Protocol) String() string { 109 | return string(c) 110 | } 111 | -------------------------------------------------------------------------------- /pkg/container/types.go: -------------------------------------------------------------------------------- 1 | package container 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type Container struct { 8 | ID string `json:"id"` 9 | Name string `json:"name"` 10 | Labels map[string]string `json:"labels"` 11 | 12 | RootPID int `json:"rootPid"` 13 | Image string `json:"image"` 14 | ImageDigest string `json:"imageDigest"` 15 | RootFS string `json:"-"` 16 | 17 | p *Pod 18 | } 19 | 20 | func (c Container) TidyName() string { 21 | return strings.TrimLeft(c.Name, "/") 22 | } 23 | 24 | // when a pod is created, the container runtime first creates a "sandbox" container 25 | // that sets up the shared Linux namespaces (network, IPC, etc.) for the pod. Other 26 | // containers in the pod then join these namespaces. This function helps identify 27 | // these special sandbox containers by their labels. 28 | func (c Container) IsSandbox() bool { 29 | if len(c.Labels) == 0 { 30 | return false 31 | } 32 | 33 | return c.Labels["io.cri-containerd.kind"] == "sandbox" || 34 | c.Labels["io.kubernetes.docker.type"] == "sandbox" || 35 | c.Labels["io.kubernetes.docker.type"] == "podsandbox" 36 | } 37 | 38 | func (c *Container) Pod() *Pod { 39 | if c.p != nil { 40 | return c.p 41 | } 42 | 43 | var p Pod 44 | p.LoadFromContainer(c) 45 | c.p = &p 46 | 47 | return &p 48 | } 49 | 50 | func (c *Container) SetPod(p *Pod) { 51 | c.p = p 52 | } 53 | 54 | const ( 55 | ContainerLabelKeyPodName = "io.kubernetes.pod.name" 56 | ContainerLabelKeyPodNamespace = "io.kubernetes.pod.namespace" 57 | ContainerLabelKeyPodUID = "io.kubernetes.pod.uid" 58 | ) 59 | 60 | type Pod struct { 61 | Name string 62 | Namespace string 63 | UID string 64 | Labels map[string]string 65 | Annotations map[string]string 66 | } 67 | 68 | func (p *Pod) LoadFromContainer(c *Container) { 69 | labels := c.Labels 70 | if len(labels) == 0 { 71 | return 72 | } 73 | p.Name = labels[ContainerLabelKeyPodName] 74 | p.Namespace = labels[ContainerLabelKeyPodNamespace] 75 | p.UID = labels[ContainerLabelKeyPodUID] 76 | } 77 | -------------------------------------------------------------------------------- /pkg/dns/dns.go: -------------------------------------------------------------------------------- 1 | package dns 2 | 3 | import ( 4 | "net" 5 | "syscall" 6 | ) 7 | 8 | // DNS record entry 9 | type Record struct { 10 | SaFamily uint16 // Address family 11 | Addr [16]byte // ipv4 or ipv6 address raw bytes 12 | Domain string // the domain name 13 | } 14 | 15 | func NewRecord(saFamily uint16, addr [16]byte, domain string) *Record { 16 | return &Record{ 17 | SaFamily: saFamily, 18 | Addr: addr, 19 | Domain: domain, 20 | } 21 | } 22 | 23 | func (r Record) AddrSize() uint16 { 24 | switch r.SaFamily { 25 | case syscall.AF_INET: 26 | return 4 27 | case syscall.AF_INET6: 28 | return 16 29 | default: 30 | return 0 31 | } 32 | } 33 | 34 | func (r Record) IP() net.IP { 35 | return net.IP(r.Addr[:r.AddrSize()]) 36 | } 37 | 38 | func (r Record) IpString() string { 39 | return r.IP().String() 40 | } 41 | -------------------------------------------------------------------------------- /pkg/e2e/config.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/config" 5 | ) 6 | 7 | // ConfigProvider loads configuration from an in-memory object 8 | type ConfigProvider struct { 9 | cfg *config.Config 10 | callback func(*config.Config) (func(), error) 11 | } 12 | 13 | // NewDefaultConfigProvider creates a new provider for default config 14 | func NewConfigProvider(cfg *config.Config) *ConfigProvider { 15 | return &ConfigProvider{ 16 | cfg: cfg, 17 | callback: func(cfg *config.Config) (func(), error) { return nil, nil }, 18 | } 19 | } 20 | 21 | // Start watching for config changes via SIGHUP 22 | func (p *ConfigProvider) Start() error { 23 | _, err := p.callback(p.cfg) 24 | return err 25 | } 26 | 27 | // Stop watching for config changes 28 | func (p *ConfigProvider) Stop() {} 29 | 30 | // SetConfig sets the config 31 | func (p *ConfigProvider) SetConfig(cfg *config.Config) (func(), error) { 32 | p.cfg = cfg 33 | return p.callback(p.cfg) 34 | } 35 | 36 | // OnConfigChange registers a callback for config changes 37 | func (p *ConfigProvider) OnConfigChange(callback func(*config.Config) (func(), error)) { 38 | p.callback = callback 39 | } 40 | 41 | // Reload forces a configuration reload 42 | func (p *ConfigProvider) Reload() error { 43 | _, err := p.callback(p.cfg) 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /pkg/ebpf/common/ftrace.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/cilium/ebpf" 7 | "github.com/cilium/ebpf/link" 8 | ) 9 | 10 | type Ftrace struct { 11 | // meta 12 | Function string 13 | Prog *ebpf.Program 14 | IsExit bool 15 | 16 | // state 17 | conn link.Link 18 | } 19 | 20 | func NewFentry(function string, prog *ebpf.Program) *Ftrace { 21 | return &Ftrace{ 22 | Function: function, 23 | Prog: prog, 24 | IsExit: false, 25 | } 26 | } 27 | 28 | func NewFexit(function string, prog *ebpf.Program) *Ftrace { 29 | return &Ftrace{ 30 | Function: function, 31 | Prog: prog, 32 | IsExit: true, 33 | } 34 | } 35 | 36 | func (ft *Ftrace) Attach() error { 37 | // Establish the link 38 | conn, err := link.AttachTracing(link.TracingOptions{ 39 | Program: ft.Prog, 40 | }) 41 | 42 | // Set the state 43 | ft.conn = conn 44 | 45 | // Return the error 46 | return err 47 | } 48 | 49 | func (ft *Ftrace) Detach() error { 50 | if ft.conn == nil { 51 | return nil 52 | } 53 | return ft.conn.Close() 54 | } 55 | 56 | func (ft *Ftrace) ID() string { 57 | prefix := "fentry" 58 | if ft.IsExit { 59 | prefix = "fexit" 60 | } 61 | return fmt.Sprintf("%s/%s", prefix, ft.Function) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/ebpf/common/kprobe.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/cilium/ebpf" 8 | "github.com/cilium/ebpf/link" 9 | ) 10 | 11 | type Kprobe struct { 12 | // meta 13 | Functions []string 14 | Prog *ebpf.Program 15 | IsRet bool 16 | 17 | // state 18 | conn link.Link 19 | } 20 | 21 | func NewKprobe(prog *ebpf.Program, functions ...string) *Kprobe { 22 | return &Kprobe{ 23 | Functions: functions, 24 | Prog: prog, 25 | } 26 | } 27 | 28 | func NewKretprobe(prog *ebpf.Program, functions ...string) *Kprobe { 29 | return &Kprobe{ 30 | Functions: functions, 31 | Prog: prog, 32 | IsRet: true, 33 | } 34 | } 35 | 36 | func (k *Kprobe) Attach() error { 37 | var conn link.Link 38 | var err error 39 | 40 | // establish the link 41 | if k.IsRet { 42 | conn, err = tryAttach(link.Kretprobe, k.Prog, k.Functions) 43 | } else { 44 | conn, err = tryAttach(link.Kprobe, k.Prog, k.Functions) 45 | } 46 | 47 | // set the state 48 | k.conn = conn 49 | 50 | // return the error 51 | return err 52 | } 53 | 54 | func (k *Kprobe) Detach() error { 55 | if k.conn == nil { 56 | return nil 57 | } 58 | 59 | return k.conn.Close() 60 | } 61 | 62 | func (k *Kprobe) ID() string { 63 | return "kprobe/" + strings.Join(k.Functions, ", ") 64 | } 65 | 66 | type kprobeLinkerFn func(symbol string, prog *ebpf.Program, opts *link.KprobeOptions) (link.Link, error) 67 | 68 | func tryAttach(fn kprobeLinkerFn, prog *ebpf.Program, functions []string) (link.Link, error) { 69 | var conn link.Link 70 | var err error 71 | 72 | for _, function := range functions { 73 | conn, err = fn(function, prog, nil) 74 | 75 | if err == nil { 76 | return conn, nil 77 | } 78 | } 79 | 80 | return nil, fmt.Errorf("failed to attach to any of the functions: %s, error: %w", strings.Join(functions, ", "), err) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/ebpf/common/probe.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type Probe interface { 4 | Attach() error 5 | Detach() error 6 | ID() string 7 | } 8 | -------------------------------------------------------------------------------- /pkg/ebpf/common/tracepoint.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/cilium/ebpf" 7 | "github.com/cilium/ebpf/link" 8 | ) 9 | 10 | type Tracepoint struct { 11 | // meta 12 | Group string 13 | Name string 14 | Prog *ebpf.Program 15 | 16 | // state 17 | conn link.Link 18 | } 19 | 20 | func NewTracepoint(group, name string, prog *ebpf.Program) *Tracepoint { 21 | return &Tracepoint{ 22 | Group: group, 23 | Name: name, 24 | Prog: prog, 25 | } 26 | } 27 | 28 | func (t *Tracepoint) Attach() error { 29 | // establish the link 30 | conn, err := link.Tracepoint(t.Group, t.Name, t.Prog, nil) 31 | 32 | // set the state 33 | t.conn = conn 34 | 35 | // return the error 36 | return err 37 | } 38 | 39 | func (t *Tracepoint) Detach() error { 40 | if t.conn == nil { 41 | return nil 42 | } 43 | return t.conn.Close() 44 | } 45 | 46 | func (t *Tracepoint) ID() string { 47 | return fmt.Sprintf("tracepoint/%s/%s", t.Group, t.Name) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/ebpf/common/uprobe.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/cilium/ebpf" 7 | "github.com/cilium/ebpf/link" 8 | ) 9 | 10 | type Uprobe struct { 11 | // meta 12 | Function string 13 | Prog *ebpf.Program 14 | IsRet bool 15 | 16 | // state 17 | conn link.Link 18 | } 19 | 20 | func NewUprobe(function string, prog *ebpf.Program) *Uprobe { 21 | return &Uprobe{ 22 | Function: function, 23 | Prog: prog, 24 | } 25 | } 26 | 27 | func NewUretprobe(function string, prog *ebpf.Program) *Uprobe { 28 | return &Uprobe{ 29 | Function: function, 30 | Prog: prog, 31 | IsRet: true, 32 | } 33 | } 34 | 35 | func (k *Uprobe) Attach(exe *link.Executable, addr uint64) error { 36 | if exe == nil { 37 | return errors.New("executable is nil") 38 | } 39 | 40 | var conn link.Link 41 | var err error 42 | 43 | // establish the link 44 | if k.IsRet { 45 | conn, err = exe.Uretprobe(k.Function, k.Prog, &link.UprobeOptions{Address: addr}) 46 | } else { 47 | conn, err = exe.Uprobe(k.Function, k.Prog, &link.UprobeOptions{Address: addr}) 48 | } 49 | 50 | // set the state 51 | k.conn = conn 52 | 53 | // return the error 54 | return err 55 | } 56 | 57 | func (k *Uprobe) Detach() error { 58 | if k.conn == nil { 59 | return nil 60 | } 61 | 62 | return k.conn.Close() 63 | } 64 | 65 | func (k *Uprobe) ID() string { 66 | return "uprobe/" + k.Function 67 | } 68 | -------------------------------------------------------------------------------- /pkg/ebpf/process/event.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | // events represents process event types 4 | type events uint64 5 | 6 | // this must align with the events enum in bpf/capture/process.bpf.c 7 | const ( 8 | EVENT_EXEC_START events = iota + 1 9 | EVENT_EXEC_ARGV 10 | EVENT_EXEC_END 11 | EVENT_EXIT 12 | EVENT_MMAP 13 | EVENT_RENAME 14 | ) 15 | 16 | // event is the base struct for all events 17 | type event struct { 18 | Type events 19 | } 20 | 21 | // execStartEvent corresponds to exec_start_event 22 | type execStartEvent struct { 23 | Pid int32 24 | ExeSize uint32 25 | } 26 | 27 | // execArgvEvent corresponds to exec_argv_event 28 | type execArgvEvent struct { 29 | Pid int32 30 | ArgvSize uint32 31 | } 32 | 33 | // execEndEvent corresponds to exec_end_event 34 | type execEndEvent struct { 35 | Pid int32 36 | } 37 | 38 | // exitEvent corresponds to exit_info_event 39 | type exitEvent struct { 40 | Pid int32 41 | } 42 | -------------------------------------------------------------------------------- /pkg/ebpf/socket/reader_test.go: -------------------------------------------------------------------------------- 1 | package socket 2 | 3 | import ( 4 | "encoding/binary" 5 | "testing" 6 | ) 7 | 8 | func Test_fixPortEndianness(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | inputPort uint16 12 | systemEndian binary.ByteOrder 13 | expectedPort uint16 14 | }{ 15 | { 16 | name: "big endian system - port 80", 17 | inputPort: 0x0050, // 80 in big endian 18 | systemEndian: binary.BigEndian, 19 | expectedPort: 80, 20 | }, 21 | { 22 | name: "big endian system - port 443", 23 | inputPort: 0x01BB, // 443 in big endian 24 | systemEndian: binary.BigEndian, 25 | expectedPort: 443, 26 | }, 27 | { 28 | name: "little endian system - port 80", 29 | inputPort: 0x5000, // 80 in big endian read as little endian 30 | systemEndian: binary.LittleEndian, 31 | expectedPort: 80, 32 | }, 33 | { 34 | name: "little endian system - port 443", 35 | inputPort: 0xBB01, // 443 in big endian read as little endian 36 | systemEndian: binary.LittleEndian, 37 | expectedPort: 443, 38 | }, 39 | } 40 | 41 | for _, tt := range tests { 42 | t.Run(tt.name, func(t *testing.T) { 43 | got := fixPortEndianness(tt.systemEndian, tt.inputPort) 44 | if got != tt.expectedPort { 45 | t.Errorf("fixPortEndianness() = %d, want %d", got, tt.expectedPort) 46 | } 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/ebpf/tls/manager_test.go: -------------------------------------------------------------------------------- 1 | package tls 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | func TestTlsManager(t *testing.T) { 11 | // test for panics while creating the metrics 12 | m := NewTlsManager(zap.NewNop()) 13 | require.NoError(t, m.Start()) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/ebpf/tls/openssl/manager_test.go: -------------------------------------------------------------------------------- 1 | package openssl 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/qpoint-io/qtap/pkg/ebpf/common" 7 | "github.com/stretchr/testify/require" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | func TestOpenSSLManager(t *testing.T) { 12 | // test for panics while setting up metrics 13 | m := NewOpenSSLManager(zap.NewNop(), func() []*common.Uprobe { 14 | return []*common.Uprobe{ 15 | common.NewUprobe("SSL_read", nil), 16 | common.NewUprobe("SSL_write", nil), 17 | } 18 | }) 19 | require.NoError(t, m.Start()) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/ebpf/trace/entry.go: -------------------------------------------------------------------------------- 1 | package trace 2 | 3 | import "go.uber.org/zap" 4 | 5 | type TraceEntry struct { 6 | msg string 7 | fields []zap.Field 8 | } 9 | 10 | func NewTraceEntry(msg string) *TraceEntry { 11 | return &TraceEntry{msg: msg} 12 | } 13 | 14 | func (e *TraceEntry) AddField(field zap.Field) { 15 | e.fields = append(e.fields, field) 16 | } 17 | 18 | func (e *TraceEntry) Print(logger *zap.Logger) { 19 | // construct the fields 20 | fields := append([]zap.Field{zap.String("msg", e.msg)}, e.fields...) 21 | 22 | // generate the log entry 23 | logger.Info("eBPF trace", fields...) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/ebpf/trace/toggle.go: -------------------------------------------------------------------------------- 1 | package trace 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type Toggle struct { 9 | Key string 10 | Value string 11 | } 12 | 13 | type Matcher struct { 14 | toggles []*Toggle 15 | } 16 | 17 | func NewMatcher(toggleQuery string) (*Matcher, error) { 18 | matcher := &Matcher{} 19 | if toggleQuery == "" { 20 | return matcher, nil 21 | } 22 | 23 | toggles := strings.Split(toggleQuery, ",") 24 | for _, toggle := range toggles { 25 | parts := strings.Split(toggle, ":") 26 | if len(parts) != 2 { 27 | return nil, fmt.Errorf("invalid toggle format: %s", toggle) 28 | } 29 | 30 | matcher.toggles = append(matcher.toggles, &Toggle{ 31 | Key: parts[0], 32 | Value: parts[1], 33 | }) 34 | } 35 | 36 | return matcher, nil 37 | } 38 | 39 | func (m *Matcher) MatchExe(exe string) bool { 40 | for _, t := range m.toggles { 41 | switch t.Key { 42 | case "exe": 43 | if exe == t.Value || t.Value == "*" { 44 | return true 45 | } 46 | case "exe.contains": 47 | if strings.Contains(exe, t.Value) { 48 | return true 49 | } 50 | case "exe.startsWith": 51 | if strings.HasPrefix(exe, t.Value) { 52 | return true 53 | } 54 | case "exe.endsWith": 55 | if strings.HasSuffix(exe, t.Value) { 56 | return true 57 | } 58 | } 59 | } 60 | return false 61 | } 62 | 63 | func (m *Matcher) HasProcToggles() bool { 64 | for _, t := range m.toggles { 65 | if strings.HasPrefix(t.Key, "exe") { 66 | return true 67 | } 68 | } 69 | return false 70 | } 71 | 72 | func (m *Matcher) GetModuleToggles() []string { 73 | var modules []string 74 | for _, t := range m.toggles { 75 | if t.Key == "mod" { 76 | modules = append(modules, t.Value) 77 | } 78 | } 79 | return modules 80 | } 81 | -------------------------------------------------------------------------------- /pkg/ebpf/trace/trace.go: -------------------------------------------------------------------------------- 1 | package trace 2 | 3 | // represents the different bpf components that can be traced. 4 | type QtapComponent uint32 5 | 6 | const ( 7 | QtapCa QtapComponent = iota 8 | QtapDebug 9 | QtapGotls 10 | QtapJavassl 11 | QtapNodetls 12 | QtapOpenssl 13 | QtapProcess 14 | QtapProtocol 15 | QtapRedirector 16 | QtapSocket 17 | ) 18 | 19 | func QtapComponentFromString(s string) (QtapComponent, bool) { 20 | switch s { 21 | case "ca": 22 | return QtapCa, true 23 | case "debug": 24 | return QtapDebug, true 25 | case "gotls": 26 | return QtapGotls, true 27 | case "javassl": 28 | return QtapJavassl, true 29 | case "nodetls": 30 | return QtapNodetls, true 31 | case "openssl": 32 | return QtapOpenssl, true 33 | case "process": 34 | return QtapProcess, true 35 | case "protocol": 36 | return QtapProtocol, true 37 | case "redirector": 38 | return QtapRedirector, true 39 | case "socket": 40 | return QtapSocket, true 41 | default: 42 | return 0, false 43 | } 44 | } 45 | 46 | // TraceEvent represents the different types of trace events 47 | type TraceEvent uint64 48 | 49 | const ( 50 | TraceMsg TraceEvent = 1 + iota 51 | TraceAttr 52 | TraceEnd 53 | ) 54 | 55 | // TraceAttrType represents the different types of trace attributes 56 | type TraceAttrType uint64 57 | 58 | const ( 59 | TraceString TraceAttrType = 1 + iota 60 | TraceInt 61 | TraceUint 62 | TracePointer 63 | TraceBool 64 | TraceIP4 65 | TraceIP6 66 | ) 67 | 68 | type TraceEventMeta struct { 69 | Type TraceEvent 70 | Tsid uint64 71 | } 72 | 73 | type TraceMsgEvent struct { 74 | MsgSize uint32 75 | } 76 | 77 | type TraceAttrEvent struct { 78 | AttrType TraceAttrType 79 | TitleSize uint32 80 | Title [256]int8 81 | _ [4]byte // padding 82 | } 83 | -------------------------------------------------------------------------------- /pkg/plugins/accesslogs/printer.go: -------------------------------------------------------------------------------- 1 | package accesslogs 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/plugins" 5 | "go.uber.org/zap" 6 | ) 7 | 8 | // Printer defines the interface for different access log formatting implementations 9 | type Printer interface { 10 | // PrintSummary prints a brief summary of the HTTP transaction 11 | PrintSummary() 12 | 13 | // PrintDetails prints detailed information about the HTTP transaction 14 | PrintDetails() error 15 | 16 | // PrintFull prints detailed information including request and response bodies 17 | PrintFull() error 18 | } 19 | 20 | // LoggerFactory creates a new Logger based on the specified format 21 | func NewPrinter( 22 | format outputFormat, 23 | ctx plugins.PluginContext, 24 | reqheaders plugins.Headers, 25 | resheaders plugins.Headers, 26 | logger *zap.Logger, 27 | writer *zap.Logger, 28 | ) Printer { 29 | switch format { 30 | case outputFormatJSON: 31 | return NewJSONPrinter(ctx, reqheaders, resheaders, logger, writer) 32 | case outputFormatConsole: 33 | return NewConsolePrinter(ctx, reqheaders, resheaders, logger, writer) 34 | default: 35 | return NewJSONPrinter(ctx, reqheaders, resheaders, logger, writer) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pkg/plugins/context.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/qpoint-io/qtap/pkg/plugins/metadata" 7 | "github.com/qpoint-io/qtap/pkg/synq" 8 | "github.com/qpoint-io/qtap/pkg/tags" 9 | ) 10 | 11 | type ConnectionContext struct { 12 | connection *Connection 13 | } 14 | 15 | // HttpPluginInstance interface implementation 16 | // this is the client side of the connection that filters 17 | // can use to interact with the connection 18 | func (c *ConnectionContext) GetRequestBodyBuffer() BodyBuffer { 19 | if c.connection.reqBody == nil { 20 | c.connection.reqBody = synq.NewLinkedBuffer(c.connection.bufferSize) 21 | } 22 | 23 | return c.connection.reqBody 24 | } 25 | 26 | func (c *ConnectionContext) GetResponseBodyBuffer() BodyBuffer { 27 | if c.connection.resBody == nil { 28 | c.connection.resBody = synq.NewLinkedBuffer(c.connection.bufferSize) 29 | } 30 | 31 | return c.connection.resBody 32 | } 33 | 34 | // Metadata returns connection specific metadata in a map[string]any. 35 | func (c *ConnectionContext) Metadata() map[string]MetadataValue { 36 | return c.connection.metadata 37 | } 38 | 39 | // GetMetadata returns a key value of type any, if the key exists. 40 | func (c *ConnectionContext) GetMetadata(key string) MetadataValue { 41 | if c.connection.metadata == nil { 42 | return &metadata.MetadataValue{} 43 | } 44 | 45 | if value, ok := c.connection.metadata[key]; ok { 46 | return value 47 | } 48 | 49 | // if the key doesn't exist, return an empty value 50 | // this is to avoid nil pointers 51 | // an OK() method is provided to check if the value is set 52 | return &metadata.MetadataValue{} 53 | } 54 | 55 | func (c *ConnectionContext) Tags() tags.List { 56 | return c.connection.tags 57 | } 58 | 59 | func (c *ConnectionContext) Context() context.Context { 60 | return c.connection.ctx 61 | } 62 | -------------------------------------------------------------------------------- /pkg/plugins/deployment.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/config" 5 | "github.com/qpoint-io/qtap/pkg/services" 6 | "go.uber.org/zap" 7 | ) 8 | 9 | // A collection of plugin instances for a single connection 10 | type StackInstance []HttpPluginInstance 11 | 12 | // StackDeployment manages the lifecycle of a collection of plugins 13 | // and plugin instances. This is a one off deployment and does not 14 | // support configuration changes. 15 | type StackDeployment struct { 16 | // logger 17 | logger *zap.Logger 18 | 19 | // name 20 | name string 21 | 22 | // plugins 23 | plugins []HttpPlugin 24 | 25 | // required services 26 | requiredServices []services.ServiceType 27 | 28 | // plugin accessor 29 | pluginAccessor PluginAccessor 30 | } 31 | 32 | func NewStackDeployment(logger *zap.Logger, name string, pluginAccessor PluginAccessor) *StackDeployment { 33 | return &StackDeployment{ 34 | name: name, 35 | logger: logger, 36 | pluginAccessor: pluginAccessor, 37 | } 38 | } 39 | 40 | func (d *StackDeployment) Setup(conf *config.Stack) error { 41 | // initilize the plugins 42 | for _, cp := range conf.Plugins { 43 | // create an plugin 44 | plugin := d.pluginAccessor.Get(PluginType(cp.Type)) 45 | if plugin == nil { 46 | d.logger.Warn("plugin not found", zap.String("type", cp.Type)) 47 | continue 48 | } 49 | plugin.Init(d.logger.With(zap.String("plugin", cp.Type)), cp.Config) 50 | 51 | // add the required services 52 | rm := map[services.ServiceType]struct{}{} 53 | for _, rs := range plugin.RequiredServices() { 54 | if _, ok := rm[rs]; !ok { 55 | rm[rs] = struct{}{} 56 | d.requiredServices = append(d.requiredServices, rs) 57 | } 58 | } 59 | 60 | // add to the list of plugins 61 | d.plugins = append(d.plugins, plugin) 62 | } 63 | 64 | return nil 65 | } 66 | 67 | func (d *StackDeployment) NewInstance(connection *Connection) StackInstance { 68 | instances := make(StackInstance, 0, len(d.plugins)) 69 | for _, p := range d.plugins { 70 | instances = append(instances, p.NewInstance(connection.Context(), connection.services...)) 71 | } 72 | return instances 73 | } 74 | 75 | func (d *StackDeployment) Teardown() { 76 | for _, p := range d.plugins { 77 | p.Destroy() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /pkg/plugins/headers.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "bytes" 5 | "net/http" 6 | "sync" 7 | ) 8 | 9 | var _ HeaderValue = (*Header)(nil) 10 | 11 | type Header []byte 12 | 13 | func NewHeaderValue(str string) Header { 14 | return Header([]byte(str)) 15 | } 16 | 17 | // Bytes implements HeaderValue. 18 | func (h Header) Bytes() []byte { 19 | return h 20 | } 21 | 22 | // Equal implements HeaderValue. 23 | func (h Header) Equal(str string) bool { 24 | return bytes.Equal(h, []byte(str)) 25 | } 26 | 27 | // String implements HeaderValue. 28 | func (h Header) String() string { 29 | return string(h) 30 | } 31 | 32 | type HttpHeaderMap struct { 33 | mu sync.RWMutex 34 | header http.Header 35 | } 36 | 37 | func NewHeaders(header http.Header) *HttpHeaderMap { 38 | return &HttpHeaderMap{ 39 | header: header, 40 | } 41 | } 42 | 43 | func (h *HttpHeaderMap) Get(key string) (HeaderValue, bool) { 44 | h.mu.RLock() 45 | defer h.mu.RUnlock() 46 | 47 | if h.header != nil { 48 | val := h.header.Get(key) 49 | return NewHeaderValue(val), val != "" 50 | } 51 | return nil, false 52 | } 53 | 54 | func (h *HttpHeaderMap) Values(key string, iter func(value HeaderValue)) { 55 | h.mu.RLock() 56 | defer h.mu.RUnlock() 57 | 58 | if h.header != nil { 59 | for _, value := range h.header.Values(key) { 60 | iter(NewHeaderValue(value)) 61 | } 62 | } 63 | } 64 | 65 | func (h *HttpHeaderMap) Set(key, value string) { 66 | h.mu.Lock() 67 | defer h.mu.Unlock() 68 | 69 | if h.header != nil { 70 | h.header.Set(key, value) 71 | } 72 | } 73 | 74 | func (h *HttpHeaderMap) Remove(key string) { 75 | h.mu.Lock() 76 | defer h.mu.Unlock() 77 | 78 | if h.header != nil { 79 | h.header.Del(key) 80 | } 81 | } 82 | 83 | func (h *HttpHeaderMap) All() map[string]string { 84 | h.mu.RLock() 85 | defer h.mu.RUnlock() 86 | 87 | // This follows Go's http.Header.Clone() implementation 88 | all := make(map[string]string) 89 | for k, v := range h.header { 90 | var val string 91 | if len(v) > 0 { 92 | val = v[0] 93 | } 94 | all[k] = val 95 | } 96 | 97 | return all 98 | } 99 | 100 | func (h *HttpHeaderMap) StdlibHeader() http.Header { 101 | h.mu.RLock() 102 | defer h.mu.RUnlock() 103 | 104 | return h.header.Clone() 105 | } 106 | -------------------------------------------------------------------------------- /pkg/plugins/httpcapture/httpcapture_test.go: -------------------------------------------------------------------------------- 1 | package httpcapture 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestHttpTransactionSerialization(t *testing.T) { 13 | // Create a simple transaction for testing 14 | tx := &HttpTransaction{ 15 | TransactionTime: time.Now(), 16 | DurationMs: 100, 17 | Direction: "egress-external", 18 | Metadata: Metadata{ 19 | ProcessID: "12345", 20 | ProcessExe: "/usr/bin/app", 21 | ContainerName: "test-container", 22 | EndpointID: "test-endpoint", 23 | ConnectionID: "test-connection", 24 | QpointRequestID: "test-request-id", 25 | BytesSent: 100, 26 | BytesReceived: 200, 27 | }, 28 | Request: Request{ 29 | Method: "GET", 30 | URL: "https://example.com/api/users", 31 | Path: "/api/users", 32 | Authority: "example.com", 33 | Scheme: "https", 34 | UserAgent: "test-agent", 35 | ContentType: "application/json", 36 | Headers: map[string]string{ 37 | ":method": "GET", 38 | ":path": "/api/users", 39 | ":authority": "example.com", 40 | ":scheme": "https", 41 | "User-Agent": "test-agent", 42 | "Content-Type": "application/json", 43 | "qpoint-request-id": "test-request-id", 44 | }, 45 | Body: []byte(`{"test":"request"}`), 46 | }, 47 | Response: Response{ 48 | Status: 200, 49 | ContentType: "application/json", 50 | Headers: map[string]string{ 51 | ":status": "200", 52 | "Content-Type": "application/json", 53 | }, 54 | Body: []byte(`{"result":"success"}`), 55 | }, 56 | } 57 | 58 | // Test JSON serialization 59 | jsonData, err := tx.ToJSON() 60 | require.NoError(t, err) 61 | 62 | // Verify JSON can be parsed back 63 | var parsedTx HttpTransaction 64 | err = json.Unmarshal(jsonData, &parsedTx) 65 | require.NoError(t, err) 66 | 67 | // Verify key fields 68 | assert.Equal(t, tx.Request.Method, parsedTx.Request.Method) 69 | assert.Equal(t, tx.Response.Status, parsedTx.Response.Status) 70 | assert.Equal(t, tx.Metadata.ProcessID, parsedTx.Metadata.ProcessID) 71 | assert.Equal(t, tx.Direction, parsedTx.Direction) 72 | 73 | // Test text format 74 | textOutput := tx.ToString() 75 | 76 | // Verify text contains key information 77 | assert.Contains(t, textOutput, "HTTP Transaction") 78 | assert.Contains(t, textOutput, "Method: GET") 79 | assert.Contains(t, textOutput, "URL: https://example.com/api/users") 80 | assert.Contains(t, textOutput, "Status: 200") 81 | assert.Contains(t, textOutput, `{"test":"request"}`) 82 | assert.Contains(t, textOutput, `{"result":"success"}`) 83 | } 84 | -------------------------------------------------------------------------------- /pkg/plugins/metadata/metadata.go: -------------------------------------------------------------------------------- 1 | package metadata 2 | 3 | import "fmt" 4 | 5 | // MetadataValue holds a value of type any. 6 | type MetadataValue struct { 7 | Value any 8 | } 9 | 10 | func (m *MetadataValue) OK() bool { 11 | return m.Value != nil 12 | } 13 | 14 | func (m *MetadataValue) Raw() any { 15 | return m.Value 16 | } 17 | 18 | func (m *MetadataValue) String() string { 19 | return fmt.Sprintf("%v", m.Value) 20 | } 21 | 22 | func (m *MetadataValue) Int64() int64 { 23 | if m.Value == nil { 24 | return 0 25 | } 26 | 27 | switch v := m.Value.(type) { 28 | case int64: 29 | return v 30 | case int: 31 | return int64(v) 32 | case float64: 33 | return int64(v) 34 | default: 35 | return 0 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pkg/plugins/plugins.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/qpoint-io/qtap/pkg/services" 8 | "github.com/qpoint-io/qtap/pkg/tags" 9 | "go.uber.org/zap" 10 | "gopkg.in/yaml.v3" 11 | ) 12 | 13 | type PluginType string 14 | 15 | func (p PluginType) String() string { 16 | return string(p) 17 | } 18 | 19 | type HeadersStatus int 20 | 21 | const ( 22 | HeadersStatusContinue HeadersStatus = 0 23 | HeadersStatusStopIteration HeadersStatus = 1 24 | ) 25 | 26 | type BodyStatus int 27 | 28 | const ( 29 | BodyStatusContinue BodyStatus = 0 30 | BodyStatusStopIterationAndBuffer BodyStatus = 1 31 | ) 32 | 33 | var NewHttpPlugin func(config map[string]any) HttpPlugin 34 | 35 | type HttpPlugin interface { 36 | Init(logger *zap.Logger, config yaml.Node) 37 | NewInstance(PluginContext, ...services.Service) HttpPluginInstance 38 | RequiredServices() []services.ServiceType 39 | Destroy() 40 | PluginType() PluginType 41 | } 42 | 43 | type HttpPluginInstance interface { 44 | RequestHeaders(requestHeaders Headers, endOfStream bool) HeadersStatus 45 | RequestBody(frame BodyBuffer, endOfStream bool) BodyStatus 46 | ResponseHeaders(responseHeaders Headers, endOfStream bool) HeadersStatus 47 | ResponseBody(frame BodyBuffer, endOfStream bool) BodyStatus 48 | Destroy() 49 | } 50 | 51 | type PluginContext interface { 52 | GetRequestBodyBuffer() BodyBuffer 53 | GetResponseBodyBuffer() BodyBuffer 54 | 55 | // TODO(Jon): these should be "services" 56 | Metadata() map[string]MetadataValue 57 | GetMetadata(key string) MetadataValue 58 | Tags() tags.List 59 | Context() context.Context 60 | } 61 | 62 | type Headers interface { 63 | Get(key string) (HeaderValue, bool) 64 | Values(key string, iter func(value HeaderValue)) 65 | Set(key, value string) 66 | Remove(key string) 67 | All() map[string]string 68 | } 69 | 70 | type BodyBuffer interface { 71 | io.ReaderAt 72 | Length() int 73 | Slices(iter func(view []byte)) 74 | Copy() []byte 75 | NewReader() io.Reader 76 | } 77 | 78 | type HeaderValue interface { 79 | String() string 80 | Bytes() []byte 81 | Equal(str string) bool 82 | } 83 | 84 | type MetadataValue interface { 85 | OK() bool 86 | Raw() any 87 | String() string 88 | Int64() int64 89 | } 90 | 91 | // PluginAccessor is a type that can access the plugin registry 92 | type PluginAccessor interface { 93 | // Get retrieves a plugin by type 94 | Get(pluginType PluginType) HttpPlugin 95 | } 96 | -------------------------------------------------------------------------------- /pkg/plugins/registry.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sync" 7 | ) 8 | 9 | // PluginRegistry holds references to active plugin instances 10 | type PluginRegistry struct { 11 | plugins map[PluginType]HttpPlugin 12 | mu sync.RWMutex 13 | } 14 | 15 | // NewRegistry creates a new service registry 16 | func NewRegistry(plugins ...HttpPlugin) *PluginRegistry { 17 | registry := &PluginRegistry{ 18 | plugins: make(map[PluginType]HttpPlugin), 19 | } 20 | 21 | for _, p := range plugins { 22 | registry.Register(p) 23 | } 24 | 25 | return registry 26 | } 27 | 28 | // Register adds or replaces a service in the registry 29 | func (sr *PluginRegistry) Register(svc HttpPlugin) { 30 | sr.mu.Lock() 31 | defer sr.mu.Unlock() 32 | 33 | // Close old plugin if it implements Closer 34 | if old, exists := sr.plugins[svc.PluginType()]; exists { 35 | if closer, ok := old.(io.Closer); ok { 36 | closer.Close() 37 | } 38 | } 39 | 40 | sr.plugins[svc.PluginType()] = svc 41 | } 42 | 43 | // Get retrieves a service by type 44 | func (sr *PluginRegistry) Get(pluginType PluginType) HttpPlugin { 45 | sr.mu.RLock() 46 | defer sr.mu.RUnlock() 47 | return sr.plugins[pluginType] 48 | } 49 | 50 | // Close closes all registered services that implement CloseableService 51 | func (sr *PluginRegistry) Close() error { 52 | sr.mu.Lock() 53 | defer sr.mu.Unlock() 54 | 55 | var errs []error 56 | for _, svc := range sr.plugins { 57 | if closer, ok := svc.(io.Closer); ok { 58 | if err := closer.Close(); err != nil { 59 | errs = append(errs, err) 60 | } 61 | } 62 | } 63 | 64 | if len(errs) > 0 { 65 | return fmt.Errorf("errors closing services: %v", errs) 66 | } 67 | 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /pkg/plugins/report/README.md: -------------------------------------------------------------------------------- 1 | # QPoint Report Plugin 2 | 3 | This plugin captures the request and response payload and passes these requests to Qtap ingestion (out of band). This plugin runs on 10080 by default. This plugin submits batches of request reports on a 5 second tick, out of band of the user. 4 | 5 | #### Configuration options: 6 | 7 | - `pulse_endpoint`: an endpoint for submitting pulse report requests. 8 | - `pulse_token`: the bearer token to access a pulse endpoint within the Authorization header (eg. `Authorization: Bearer `). 9 | - `batch_period_ms`: delay between batch submissions in milliseconds (defaults to 5 seconds). 10 | - `tags`: a string array of static tags that will be appended to every report request. 11 | -------------------------------------------------------------------------------- /pkg/plugins/report/report_plugin.go: -------------------------------------------------------------------------------- 1 | package report 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/plugins" 5 | "github.com/qpoint-io/qtap/pkg/services" 6 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 7 | "gopkg.in/yaml.v3" 8 | 9 | "go.uber.org/zap" 10 | ) 11 | 12 | const ( 13 | PluginTypeReport plugins.PluginType = "report_usage" 14 | ) 15 | 16 | type Config struct { 17 | Tags []string `json:"tags"` 18 | } 19 | 20 | type Factory struct { 21 | logger *zap.Logger 22 | } 23 | 24 | func (f *Factory) Init(logger *zap.Logger, config yaml.Node) { 25 | f.logger = logger 26 | } 27 | 28 | func (f *Factory) NewInstance(ctx plugins.PluginContext, svcs ...services.Service) plugins.HttpPluginInstance { 29 | f.logger.Debug("new plugin instance created") 30 | fi := &filterInstance{ 31 | logger: f.logger, 32 | ctx: ctx, 33 | } 34 | 35 | for _, s := range svcs { 36 | if i, ok := s.(eventstore.EventStore); ok { 37 | fi.eventstore = i 38 | } 39 | } 40 | 41 | return fi 42 | } 43 | 44 | func (f *Factory) RequiredServices() []services.ServiceType { 45 | return []services.ServiceType{eventstore.TypeEventStore} 46 | } 47 | 48 | func (f *Factory) Destroy() { 49 | f.logger.Debug("filter destroyed") 50 | } 51 | 52 | func (f *Factory) PluginType() plugins.PluginType { 53 | return PluginTypeReport 54 | } 55 | -------------------------------------------------------------------------------- /pkg/plugins/stack.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/qpoint-io/qtap/pkg/config" 8 | "go.uber.org/zap" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | // Stack manages the lifecycle of a StackDeployment 13 | // which contains a list of plugins. 14 | type Stack struct { 15 | // name 16 | name string 17 | 18 | // logger 19 | logger *zap.Logger 20 | 21 | // plugin accessor 22 | pluginAccessor PluginAccessor 23 | 24 | // activeDeployment 25 | activeDeployment *StackDeployment 26 | inactiveDeployment *StackDeployment 27 | 28 | // stack config snapshot (JSON) 29 | configSnapshot string 30 | 31 | // mutex 32 | mu sync.Mutex 33 | } 34 | 35 | func NewStack(name string, logger *zap.Logger, pluginAccessor PluginAccessor) *Stack { 36 | s := &Stack{ 37 | name: name, 38 | logger: logger, 39 | pluginAccessor: pluginAccessor, 40 | } 41 | 42 | return s 43 | } 44 | 45 | func (s *Stack) SetConfig(conf *config.Stack) error { 46 | // generate a snapshot of the incoming config 47 | snapshot, err := yaml.Marshal(conf) 48 | if err != nil { 49 | return fmt.Errorf("marshalling stack config: %w", err) 50 | } 51 | 52 | // if the snapshot is the same, don't do anything 53 | if s.configSnapshot == string(snapshot) { 54 | return nil 55 | } 56 | 57 | // iniialize deployment 58 | deployment := NewStackDeployment(s.logger, s.name, s.pluginAccessor) 59 | err = deployment.Setup(conf) 60 | if err != nil { 61 | return fmt.Errorf("setting up deployment: %w", err) 62 | } 63 | 64 | // lock the stack 65 | s.mu.Lock() 66 | defer s.mu.Unlock() 67 | 68 | // set the deployments 69 | if s.inactiveDeployment != nil { 70 | s.inactiveDeployment.Teardown() 71 | } 72 | s.inactiveDeployment = s.activeDeployment 73 | s.activeDeployment = deployment 74 | 75 | return nil 76 | } 77 | 78 | func (s *Stack) GetActiveDeployment() *StackDeployment { 79 | return s.activeDeployment 80 | } 81 | 82 | func (s *Stack) Teardown() { 83 | s.mu.Lock() 84 | defer s.mu.Unlock() 85 | 86 | if s.activeDeployment != nil { 87 | s.activeDeployment.Teardown() 88 | } 89 | if s.inactiveDeployment != nil { 90 | s.inactiveDeployment.Teardown() 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /pkg/plugins/tools/mime.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func MimeCategory(contentType string) string { 8 | // Default value if content type is empty or undefined 9 | if contentType == "" { 10 | return "other" 11 | } 12 | 13 | // Truncate to base content type without any parameters 14 | if idx := strings.Index(contentType, ";"); idx != -1 { 15 | contentType = contentType[:idx] 16 | } 17 | 18 | // Check for various content types 19 | switch { 20 | case isApp(contentType): 21 | return "app" 22 | case isCss(contentType): 23 | return "css" 24 | case isJs(contentType): 25 | return "js" 26 | case isFont(contentType): 27 | return "font" 28 | case isImage(contentType): 29 | return "image" 30 | case isMedia(contentType): 31 | return "media" 32 | default: 33 | return "other" 34 | } 35 | } 36 | 37 | func isApp(contentType string) bool { 38 | appTypes := []string{"text/html", "application/json", "application/grpc", "text/xml", "application/xml", "text/plain"} 39 | for _, t := range appTypes { 40 | if contentType == t { 41 | return true 42 | } 43 | } 44 | return false 45 | } 46 | 47 | func isCss(contentType string) bool { 48 | return contentType == "text/css" 49 | } 50 | 51 | func isJs(contentType string) bool { 52 | return contentType == "text/javascript" 53 | } 54 | 55 | func isFont(contentType string) bool { 56 | return strings.HasPrefix(contentType, "font") 57 | } 58 | 59 | func isImage(contentType string) bool { 60 | return strings.HasPrefix(contentType, "image") 61 | } 62 | 63 | func isMedia(contentType string) bool { 64 | return strings.HasPrefix(contentType, "audio") || strings.HasPrefix(contentType, "video") 65 | } 66 | -------------------------------------------------------------------------------- /pkg/plugins/tools/path.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | func NormalizeURLPath(path string) string { 9 | tokens := tokenizePath(path) 10 | normalizedTokens := normalizeSegments(tokens) 11 | 12 | newPath := assemblePath(normalizedTokens) 13 | if newPath[0] != '/' { 14 | newPath = "/" + newPath 15 | } 16 | 17 | return newPath 18 | } 19 | 20 | func tokenizePath(path string) []string { 21 | tokens := strings.Split(path, "/") 22 | filteredTokens := make([]string, 0) 23 | 24 | for _, token := range tokens { 25 | if len(token) > 0 { 26 | filteredTokens = append(filteredTokens, token) 27 | } 28 | } 29 | 30 | return filteredTokens 31 | } 32 | 33 | func assemblePath(segments []string) string { 34 | return "/" + strings.Join(segments, "/") 35 | } 36 | 37 | func isLikelyWord(input string) bool { 38 | matched, _ := regexp.MatchString("^[A-Za-z]+$", input) 39 | isReasonableLength := len(input) > 3 && len(input) <= 20 40 | 41 | return matched && isReasonableLength 42 | } 43 | 44 | func normalizeSegments(segments []string) []string { 45 | if len(segments) <= 1 { 46 | return segments 47 | } 48 | 49 | n := []string{} 50 | 51 | for i := range segments { 52 | if segments[i] == "" { 53 | continue 54 | } 55 | 56 | if !isLikelyWord(segments[i]) { 57 | resourceName := "id" 58 | if i > 0 { 59 | if isLikelyWord(segments[i-1]) { 60 | resourceName = segments[i-1] 61 | } else if strings.HasPrefix(segments[i-1], "{") { 62 | re := regexp.MustCompile(`\{(.+?)Id\}`) 63 | match := re.FindStringSubmatch(segments[i-1]) 64 | if match != nil { 65 | resourceName = match[1] 66 | } 67 | } 68 | } 69 | segments[i] = "{" + resourceName + "Id}" 70 | } 71 | 72 | n = append(n, segments[i]) 73 | } 74 | return n 75 | } 76 | -------------------------------------------------------------------------------- /pkg/plugins/tools/path_test.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestTokenizePath(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | path string 13 | expected []string 14 | }{ 15 | {"Normal path", "/path/to/resource", []string{"path", "to", "resource"}}, 16 | {"Trailing slash", "/path/to/resource/", []string{"path", "to", "resource"}}, 17 | {"Leading and trailing slash", "/path/to/resource/", []string{"path", "to", "resource"}}, 18 | {"Multiple slashes", "//path//to//resource//", []string{"path", "to", "resource"}}, 19 | {"Empty path", "", []string{}}, 20 | } 21 | 22 | for _, tc := range tests { 23 | t.Run(tc.name, func(t *testing.T) { 24 | result := tokenizePath(tc.path) 25 | assert.Equal(t, tc.expected, result) 26 | }) 27 | } 28 | } 29 | 30 | func TestIsLikelyWord(t *testing.T) { 31 | tests := []struct { 32 | name string 33 | input string 34 | expected bool 35 | }{ 36 | {"Valid short word", "word", true}, 37 | {"Too short", "w", false}, 38 | {"Too long", "thisiswaytoolongforaword", false}, 39 | {"Contains numbers", "word123", false}, 40 | {"Contains special characters", "word-word", false}, 41 | {"Empty string", "", false}, 42 | } 43 | 44 | for _, tc := range tests { 45 | t.Run(tc.name, func(t *testing.T) { 46 | result := isLikelyWord(tc.input) 47 | assert.Equal(t, tc.expected, result) 48 | }) 49 | } 50 | } 51 | 52 | func TestNormalizeSegments(t *testing.T) { 53 | tests := []struct { 54 | name string 55 | segments []string 56 | expected []string 57 | }{ 58 | {"Single word", []string{"article"}, []string{"article"}}, 59 | {"Valid words", []string{"user", "name"}, []string{"user", "name"}}, 60 | {"Contains id", []string{"user", "123"}, []string{"user", "{userId}"}}, 61 | {"Multiple ids", []string{"user", "123", "page", "456"}, []string{"user", "{userId}", "page", "{pageId}"}}, 62 | {"Mixed valid and invalid", []string{"user", "", "page", "123"}, []string{"user", "page", "{pageId}"}}, 63 | {"Placeholder transformation", []string{"user", "{userId}", "page", "123"}, []string{"user", "{userId}", "page", "{pageId}"}}, 64 | } 65 | 66 | for _, tc := range tests { 67 | t.Run(tc.name, func(t *testing.T) { 68 | result := normalizeSegments(tc.segments) 69 | assert.Equal(t, tc.expected, result) 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pkg/process/container.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import "go.uber.org/zap" 4 | 5 | type Container struct { 6 | ID string `json:"container_id,omitempty"` 7 | Name string `json:"container_name,omitempty"` 8 | Labels map[string]string `json:"container_labels,omitempty"` 9 | Image string `json:"container_image,omitempty"` 10 | ImageDigest string `json:"container_imageDigest,omitempty"` 11 | RootFS string `json:"-"` 12 | } 13 | 14 | func (c Container) Fields() []zap.Field { 15 | f := []zap.Field{ 16 | // zap.String("containerId", c.ID), 17 | zap.String("containerName", c.Name), 18 | // zap.Any("containerLabels", c.Labels), 19 | zap.String("containerImage", c.Image), 20 | // zap.String("containerImageDigest", c.ImageDigest), 21 | } 22 | 23 | return f 24 | } 25 | 26 | func (c Container) ControlValues() map[string]any { 27 | id := c.ID 28 | if len(id) > 12 { 29 | id = id[:12] 30 | } 31 | 32 | v := map[string]any{ 33 | "id": id, 34 | "name": c.Name, 35 | "image": c.Image, 36 | } 37 | 38 | if len(c.Labels) > 0 { 39 | l := make(map[string]any, len(c.Labels)) 40 | for k, v := range c.Labels { 41 | l[k] = v 42 | } 43 | v["labels"] = l 44 | } 45 | 46 | return v 47 | } 48 | 49 | type Pod struct { 50 | Name string `json:"pod_name,omitempty"` 51 | Namespace string `json:"pod_namespace,omitempty"` 52 | Labels map[string]string `json:"pod_labels,omitempty"` 53 | Annotations map[string]string `json:"pod_annotations,omitempty"` 54 | } 55 | 56 | func (p Pod) Fields() []zap.Field { 57 | return []zap.Field{ 58 | zap.String("podName", p.Name), 59 | zap.String("podNamespace", p.Namespace), 60 | // zap.String("podUID", p.UID), 61 | // zap.Any("podLabels", p.Labels), 62 | // zap.Any("podAnnotations", p.Annotations), 63 | } 64 | } 65 | 66 | func (p Pod) ControlValues() map[string]any { 67 | v := map[string]any{ 68 | "name": p.Name, 69 | "namespace": p.Namespace, 70 | } 71 | 72 | if len(p.Labels) > 0 { 73 | l := make(map[string]any, len(p.Labels)) 74 | for k, v := range p.Labels { 75 | l[k] = v 76 | } 77 | v["labels"] = l 78 | } 79 | 80 | if len(p.Annotations) > 0 { 81 | a := make(map[string]any, len(p.Annotations)) 82 | for k, v := range p.Annotations { 83 | a[k] = v 84 | } 85 | v["annotations"] = a 86 | } 87 | 88 | return v 89 | } 90 | -------------------------------------------------------------------------------- /pkg/process/metrics.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import "github.com/qpoint-io/qtap/pkg/telemetry" 4 | 5 | var ( 6 | // processAddTotal tracks the number of processes added 7 | processAddTotal = telemetry.Counter("processes_added", 8 | telemetry.WithDescription("Total number of processes added")) 9 | 10 | // processRemoveTotal tracks the number of processes removed 11 | processRemoveTotal = telemetry.Counter("processes_removed", 12 | telemetry.WithDescription("Total number of processes removed")) 13 | 14 | // processRenamedTotal tracks the number of processes renamed 15 | processRenamedTotal = telemetry.Counter("processes_renamed", 16 | telemetry.WithDescription("Total number of processes renamed")) 17 | ) 18 | 19 | // trackActiveProcessCount tracks the number of active processes as an observable gauge 20 | func trackActiveProcessCount(fn func() int) { 21 | telemetry.ObservableGauge("processes_active", 22 | func() float64 { 23 | return float64(fn()) 24 | }, 25 | telemetry.WithDescription("Total number of active monitored processes"), 26 | ) 27 | } 28 | 29 | // IncrementProcessAdd increments the process add counter 30 | func incrementProcessAdd() { 31 | processAddTotal(1) 32 | } 33 | 34 | // IncrementProcessRemove increments the process remove counter 35 | func incrementProcessRemove() { 36 | processRemoveTotal(1) 37 | } 38 | 39 | // IncrementProcessRenamed increments the process renamed counter 40 | func incrementProcessRenamed() { 41 | processRenamedTotal(1) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/process/mocks/receiver.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/qpoint-io/qtap/pkg/process (interfaces: Receiver) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination ./mocks/receiver.go -package mocks . Receiver 7 | // 8 | 9 | // Package mocks is a generated GoMock package. 10 | package mocks 11 | 12 | import ( 13 | reflect "reflect" 14 | 15 | process "github.com/qpoint-io/qtap/pkg/process" 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockReceiver is a mock of Receiver interface. 20 | type MockReceiver struct { 21 | ctrl *gomock.Controller 22 | recorder *MockReceiverMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockReceiverMockRecorder is the mock recorder for MockReceiver. 27 | type MockReceiverMockRecorder struct { 28 | mock *MockReceiver 29 | } 30 | 31 | // NewMockReceiver creates a new mock instance. 32 | func NewMockReceiver(ctrl *gomock.Controller) *MockReceiver { 33 | mock := &MockReceiver{ctrl: ctrl} 34 | mock.recorder = &MockReceiverMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockReceiver) EXPECT() *MockReceiverMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // RegisterProcess mocks base method. 44 | func (m *MockReceiver) RegisterProcess(p *process.Process) error { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "RegisterProcess", p) 47 | ret0, _ := ret[0].(error) 48 | return ret0 49 | } 50 | 51 | // RegisterProcess indicates an expected call of RegisterProcess. 52 | func (mr *MockReceiverMockRecorder) RegisterProcess(p any) *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterProcess", reflect.TypeOf((*MockReceiver)(nil).RegisterProcess), p) 55 | } 56 | 57 | // UnregisterProcess mocks base method. 58 | func (m *MockReceiver) UnregisterProcess(pid int) error { 59 | m.ctrl.T.Helper() 60 | ret := m.ctrl.Call(m, "UnregisterProcess", pid) 61 | ret0, _ := ret[0].(error) 62 | return ret0 63 | } 64 | 65 | // UnregisterProcess indicates an expected call of UnregisterProcess. 66 | func (mr *MockReceiverMockRecorder) UnregisterProcess(pid any) *gomock.Call { 67 | mr.mock.ctrl.T.Helper() 68 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnregisterProcess", reflect.TypeOf((*MockReceiver)(nil).UnregisterProcess), pid) 69 | } 70 | -------------------------------------------------------------------------------- /pkg/process/observer.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | type Observer interface { 4 | ProcessStarted(*Process) error 5 | ProcessReplaced(*Process) error 6 | ProcessStopped(*Process) error 7 | } 8 | 9 | type DefaultObserver struct{} 10 | 11 | func (d *DefaultObserver) ProcessStarted(proc *Process) error { 12 | return nil 13 | } 14 | 15 | func (d *DefaultObserver) ProcessReplaced(proc *Process) error { 16 | return nil 17 | } 18 | 19 | func (d *DefaultObserver) ProcessStopped(proc *Process) error { 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /pkg/process/qpoint.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | 7 | "github.com/qpoint-io/qtap/pkg/config" 8 | ) 9 | 10 | const ( 11 | QpointStrategyEnvVar = "QPOINT_STRATEGY" 12 | QpointTagsEnvVar = "QPOINT_TAGS" 13 | ) 14 | 15 | // QpointStrategy represents the different qpoint strategies that can be used. 16 | type QpointStrategy uint32 17 | 18 | const ( 19 | StrategyObserve QpointStrategy = iota 20 | StrategyIgnore 21 | StrategyAudit 22 | StrategyForward 23 | StrategyProxy 24 | ) 25 | 26 | // createTapFilter parses filter strings and creates a TapFilter 27 | func createTapFilter(filterStr string) (*config.TapFilter, error) { 28 | filterStr, _ = strings.CutPrefix(filterStr, "exe.") 29 | 30 | strategy, matchValue, found := strings.Cut(filterStr, ":") 31 | if !found { 32 | return nil, errors.New("invalid filter format") 33 | } 34 | 35 | var matchStrategy config.MatchStrategy 36 | if !matchStrategy.Parse(strategy) { 37 | return nil, errors.New("invalid match strategy") 38 | } 39 | 40 | return &config.TapFilter{ 41 | Exe: matchValue, 42 | Strategy: matchStrategy, 43 | }, nil 44 | } 45 | 46 | func QpointStrategyFromString(s string, p *Process) (QpointStrategy, error) { 47 | strat, filterStr, found := strings.Cut(s, ",") 48 | if found { 49 | var match bool 50 | filterStrs := strings.Split(filterStr, ",") 51 | for _, filterStr := range filterStrs { 52 | filterConfig, err := createTapFilter(filterStr) 53 | if err != nil { 54 | return StrategyObserve, err 55 | } 56 | 57 | filter, err := FromConfigFilter(filterConfig) 58 | if err != nil { 59 | return StrategyObserve, err 60 | } 61 | 62 | // evaluate the filter 63 | match, err = filter.Evaluate(p) 64 | if err != nil { 65 | return StrategyObserve, err 66 | } 67 | 68 | if !match { 69 | continue 70 | } 71 | 72 | // we found a match 73 | match = true 74 | break 75 | } 76 | 77 | if !match { 78 | return StrategyObserve, nil 79 | } 80 | } 81 | 82 | switch strat { 83 | case "observe": 84 | return StrategyObserve, nil 85 | case "ignore": 86 | return StrategyIgnore, nil 87 | case "audit": 88 | return StrategyAudit, nil 89 | case "forward": 90 | return StrategyForward, nil 91 | case "proxy": 92 | return StrategyProxy, nil 93 | default: 94 | return StrategyObserve, nil 95 | } 96 | } 97 | 98 | func (s QpointStrategy) String() string { 99 | switch s { 100 | case StrategyObserve: 101 | return "observe" 102 | case StrategyIgnore: 103 | return "ignore" 104 | case StrategyAudit: 105 | return "audit" 106 | case StrategyForward: 107 | return "forward" 108 | case StrategyProxy: 109 | return "proxy" 110 | default: 111 | return "observe" 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pkg/qnet/addr.go: -------------------------------------------------------------------------------- 1 | package qnet 2 | 3 | import ( 4 | "net" 5 | "strconv" 6 | ) 7 | 8 | type NetFamily string 9 | 10 | const ( 11 | NetFamily_Unknown NetFamily = "unknown" 12 | NetFamily_IPv4 NetFamily = "ipv4" 13 | NetFamily_IPv6 NetFamily = "ipv6" 14 | ) 15 | 16 | func (f NetFamily) String() string { 17 | return string(f) 18 | } 19 | 20 | // ensure NetAddr fulfills net.Addr interface 21 | var _ net.Addr = NetAddr{} 22 | 23 | type NetAddr struct { 24 | Family NetFamily `json:"family,omitempty"` 25 | IP net.IP `json:"ip,omitempty"` 26 | Port uint16 `json:"port,omitempty"` 27 | } 28 | 29 | // Network returns the network type (e.g., "ipv4" or "ipv6") 30 | func (na NetAddr) Network() string { 31 | return string(na.Family) 32 | } 33 | 34 | // String returns a string representation of the address 35 | func (na NetAddr) String() string { 36 | switch na.Family { 37 | case NetFamily_IPv4, NetFamily_IPv6: 38 | return net.JoinHostPort(na.IP.String(), strconv.Itoa(int(na.Port))) 39 | default: 40 | return "unknown" 41 | } 42 | } 43 | 44 | func NetAddrFromTCPAddr(addr *net.TCPAddr) NetAddr { 45 | a := NetAddr{ 46 | Port: uint16(addr.Port), 47 | } 48 | 49 | if ip4 := addr.IP.To4(); ip4 != nil { 50 | a.Family = NetFamily_IPv4 51 | } else { 52 | a.Family = NetFamily_IPv6 53 | } 54 | 55 | a.IP = addr.IP 56 | 57 | return a 58 | } 59 | 60 | func (na NetAddr) ToBytes() [16]byte { 61 | var result [16]byte 62 | switch na.Family { 63 | case NetFamily_IPv4: 64 | copy(result[:], na.IP.To4()) 65 | case NetFamily_IPv6: 66 | copy(result[:], na.IP.To16()) 67 | } 68 | return result 69 | } 70 | 71 | func (na NetAddr) Equal(other NetAddr) bool { 72 | return na.Family == other.Family && na.IP.Equal(other.IP) && na.Port == other.Port 73 | } 74 | 75 | func (na NetAddr) Empty() bool { 76 | return na.Equal(NetAddr{}) 77 | } 78 | 79 | func (na NetAddr) ControlValues() map[string]any { 80 | return map[string]any{ 81 | "ip": na.IP, 82 | "port": int(na.Port), 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pkg/rulekitext/ext.go: -------------------------------------------------------------------------------- 1 | package rulekitext 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/qpoint-io/rulekit" 7 | "golang.org/x/net/publicsuffix" 8 | ) 9 | 10 | // register all functions 11 | var Functions = map[string]*rulekit.Function{ 12 | // zone 13 | // 14 | // zone(dst.domain) 15 | // -> {"dst.domain": "example.com"} -> "example.com" 16 | // -> {"dst.domain": "api.example.com"} -> "example.com" 17 | "zone": { 18 | Args: []rulekit.FunctionArg{ 19 | {Name: "domain"}, 20 | }, 21 | Eval: func(args rulekit.KV) rulekit.Result { 22 | domain, err := rulekit.IndexFuncArg[string](args, "domain") 23 | if err != nil { 24 | return rulekit.Result{Error: err} 25 | } 26 | 27 | return rulekit.Result{Value: Zone(domain)} 28 | }, 29 | }, 30 | 31 | // in_zone 32 | // 33 | // in_zone(dst.domain, "example.com") 34 | // -> {"dst.domain": "example.com"} -> true 35 | // -> {"dst.domain": "api.example.com"} -> true 36 | "in_zone": { 37 | Args: []rulekit.FunctionArg{ 38 | {Name: "domain"}, 39 | {Name: "zone"}, 40 | }, 41 | Eval: func(args rulekit.KV) rulekit.Result { 42 | domain, err := rulekit.IndexFuncArg[string](args, "domain") 43 | if err != nil { 44 | return rulekit.Result{Error: err} 45 | } 46 | 47 | zone, err := rulekit.IndexFuncArg[string](args, "zone") 48 | if err != nil { 49 | return rulekit.Result{Error: err} 50 | } 51 | 52 | return rulekit.Result{Value: InZone(domain, zone)} 53 | }, 54 | }, 55 | } 56 | 57 | // Zone returns the effective TLD+1 of a domain. 58 | // 59 | // Zone("example.com") -> "example.com" 60 | // Zone("sub.example.com") -> "example.com" 61 | var Zone = func(domain string) string { 62 | domain = strings.TrimSuffix(domain, ".") 63 | if zone, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil { 64 | return zone 65 | } 66 | 67 | return domain 68 | } 69 | 70 | // InZone returns true if the domain is the same as the zone or is a subdomain of the zone. 71 | // 72 | // InZone( "example.com", "example.com") -> true 73 | // InZone("sub.example.com", "example.com") -> true 74 | // InZone( "test.com", "example.com") -> false 75 | var InZone = func(domain, zone string) bool { 76 | return domain == zone || strings.HasSuffix(domain, "."+zone) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/rulekitext/ext_test.go: -------------------------------------------------------------------------------- 1 | package rulekitext 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/qpoint-io/rulekit" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestZone(t *testing.T) { 12 | tcs := map[string]string{ 13 | "example.com": "example.com", 14 | "sub.example.com": "example.com", 15 | "test.some-unknown-tld": "test.some-unknown-tld", 16 | "localhost": "localhost", 17 | "some.sub.test.s3-website.us-west-1.amazonaws.com": "test.s3-website.us-west-1.amazonaws.com", 18 | } 19 | 20 | for domain, want := range tcs { 21 | t.Run(domain, func(t *testing.T) { 22 | r, err := rulekit.Parse("zone(domain)") 23 | require.NoError(t, err) 24 | 25 | res := r.Eval(&rulekit.Ctx{ 26 | Functions: Functions, 27 | KV: rulekit.KV{ 28 | "domain": domain, 29 | }, 30 | }) 31 | require.True(t, res.Ok()) 32 | require.Equal(t, want, res.Value) 33 | 34 | require.Equal(t, want, Zone(domain)) 35 | }) 36 | } 37 | } 38 | 39 | func TestInZone(t *testing.T) { 40 | tcs := []struct { 41 | domain string 42 | zone string 43 | want bool 44 | }{ 45 | { 46 | domain: "example.com", 47 | zone: "example.com", 48 | want: true, 49 | }, 50 | { 51 | domain: "sub.example.com", 52 | zone: "example.com", 53 | want: true, 54 | }, 55 | { 56 | domain: "api.example.com", 57 | zone: "internal.example.com", 58 | want: false, 59 | }, 60 | { 61 | domain: "test.com", 62 | zone: "example.com", 63 | want: false, 64 | }, 65 | } 66 | 67 | for _, tc := range tcs { 68 | t.Run(fmt.Sprintf("%s,%s", tc.domain, tc.zone), func(t *testing.T) { 69 | require.Equal(t, tc.want, InZone(tc.domain, tc.zone)) 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /pkg/services/adapters.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | ) 6 | 7 | type LoggerAdapter interface { 8 | SetLogger(*zap.Logger) 9 | } 10 | 11 | type LogHelper struct { 12 | logger *zap.Logger 13 | } 14 | 15 | func (l *LogHelper) SetLogger(logger *zap.Logger) { 16 | l.logger = logger 17 | } 18 | 19 | func (l *LogHelper) Log() *zap.Logger { 20 | if l.logger == nil { 21 | l.logger = zap.NewNop() 22 | } 23 | 24 | return l.logger 25 | } 26 | -------------------------------------------------------------------------------- /pkg/services/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/qpoint-io/qtap/pkg/telemetry" 8 | ) 9 | 10 | // NewHttpClient creates a custom HTTP client with optimized settings for service-to-service communication. 11 | // It configures: 12 | // - Connection pooling with controlled limits to prevent resource exhaustion 13 | // - Timeouts at various levels (idle, response, TLS, overall) to ensure responsive behavior 14 | // - Keep-alive settings for connection reuse to reduce latency 15 | // 16 | // The client is specifically tuned for internal service calls with: 17 | // - Limited concurrent connections (10 per host) to prevent overwhelming downstream services 18 | // - Short idle timeouts (5s) to free up resources quickly 19 | // - Conservative overall timeout (30s) to prevent hung operations 20 | func NewHttpClient() *http.Client { 21 | // Clone the default transport 22 | t := http.DefaultTransport.(*http.Transport).Clone() 23 | 24 | // Connection pool settings 25 | t.IdleConnTimeout = 5 * time.Second 26 | t.MaxIdleConnsPerHost = 100 27 | t.MaxConnsPerHost = 0 // Limit total connections per host 28 | 29 | // Timeout settings 30 | t.ResponseHeaderTimeout = 10 * time.Second 31 | t.ExpectContinueTimeout = 1 * time.Second 32 | 33 | // Keep-alive settings 34 | t.DisableKeepAlives = false // Enable keep-alives for connection reuse 35 | t.MaxIdleConns = 0 // Overall connection pool limit 36 | 37 | // TLS settings 38 | t.TLSHandshakeTimeout = 5 * time.Second 39 | 40 | cl := &http.Client{ 41 | Transport: t, 42 | Timeout: 30 * time.Second, // Overall request timeout 43 | } 44 | telemetry.InstrumentHTTPClient(cl) 45 | return cl 46 | } 47 | -------------------------------------------------------------------------------- /pkg/services/eventstore/axiom/README.md: -------------------------------------------------------------------------------- 1 | # Axiom EventStore Service 2 | 3 | The Axiom EventStore service submits events to [Axiom](https://axiom.co) using their official Go SDK. 4 | 5 | ## Configuration 6 | 7 | To use the Axiom EventStore service, configure it in your YAML configuration file: 8 | 9 | ```yaml 10 | eventstore: 11 | type: axiom 12 | dataset: # The name of the Axiom dataset to send events to 13 | type: env # or "text" for direct value 14 | value: AXIOM_DATASET # Environment variable name or direct dataset value 15 | token: 16 | type: env # or "text" for direct value 17 | value: AXIOM_TOKEN # Environment variable name or direct token value 18 | ``` 19 | 20 | ### Configuration Fields 21 | 22 | - `type`: Must be `"axiom"` to use this service 23 | - `dataset`: The name of the Axiom dataset where events will be sent. Defaults to `"qtap-events"` if not specified 24 | - `type`: Either `"env"` (to read from environment variable) or `"text"` (direct value) 25 | - `value`: Environment variable name (if type is `"env"`) or the actual token (if type is `"text"") 26 | - `token`: The Axiom API token configuration 27 | - `type`: Either `"env"` (to read from environment variable) or `"text"` (direct value) 28 | - `value`: Environment variable name (if type is `"env"`) or the actual token (if type is `"text"") 29 | 30 | ## Environment Setup 31 | 32 | If using environment variable for the token: 33 | 34 | ```bash 35 | export AXIOM_TOKEN="your-axiom-api-token" 36 | ``` 37 | 38 | ## Supported Event Types 39 | 40 | The service supports the following event types from the `eventstore` package: 41 | 42 | - `*eventstore.Connection` - Connection Audit Log 43 | - `*eventstore.Request` - HTTP request events 44 | - `*eventstore.Issue` - Error/issue events 45 | - `*eventstore.PIIEntity` - PII detection events 46 | - `*eventstore.ArtifactRecord` - Artifact storage records 47 | 48 | ## Metrics 49 | 50 | The service exposes the following metrics: 51 | 52 | - `tap_axiom_records_submitted`: Total number of records submitted to Axiom 53 | - `tap_axiom_records_ingested`: Total number of records successfully ingested by Axiom 54 | - `tap_axiom_records_failed`: Total number of records that failed to be ingested by Axiom 55 | 56 | ## Error Handling 57 | 58 | The service includes robust error handling: 59 | 60 | - Validation of required configuration (API token) 61 | - Graceful handling of unsupported event types 62 | - Detailed error logging for failed submissions 63 | - Metrics tracking for monitoring ingestion success/failure rates 64 | 65 | ## Example Usage 66 | 67 | Once configured, the service will automatically receive and submit events as they are generated by the application. No additional code changes are required - the service integrates with the existing eventstore infrastructure. 68 | -------------------------------------------------------------------------------- /pkg/services/eventstore/axiom/factory.go: -------------------------------------------------------------------------------- 1 | package axiom 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/axiomhq/axiom-go/axiom" 9 | "github.com/qpoint-io/qtap/pkg/config" 10 | "github.com/qpoint-io/qtap/pkg/services" 11 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 12 | "github.com/qpoint-io/qtap/pkg/services/objectstore" 13 | "github.com/qpoint-io/qtap/pkg/telemetry" 14 | "go.uber.org/zap" 15 | ) 16 | 17 | var tracer = telemetry.Tracer() 18 | 19 | const ( 20 | Type services.ServiceType = "axiom" 21 | ) 22 | 23 | type Factory struct { 24 | eventstore.BaseEventStore 25 | 26 | logger *zap.Logger 27 | axiomClient *axiom.Client 28 | dataset string 29 | } 30 | 31 | // Init initializes the Axiom service factory 32 | func (f *Factory) Init(ctx context.Context, cfg any) error { 33 | c, ok := cfg.(config.ServiceEventStore) 34 | if !ok { 35 | return fmt.Errorf("invalid config type: %T wanted config.ServiceEventStore", cfg) 36 | } 37 | 38 | f.logger = zap.L().With(zap.String("service_factory", f.FactoryType().String())) 39 | 40 | // Extract Axiom configuration from the service config 41 | token := c.Token.String() 42 | if token == "" { 43 | return errors.New("Axiom API token is required") 44 | } 45 | 46 | // Use URL as dataset name if provided, otherwise use a default 47 | f.dataset = "qtap-events" 48 | if c.Dataset.String() != "" { 49 | f.dataset = c.Dataset.String() // URL field can be repurposed as dataset name 50 | } 51 | 52 | // Create Axiom client 53 | client, err := axiom.NewClient( 54 | axiom.SetToken(token), 55 | ) 56 | if err != nil { 57 | return fmt.Errorf("failed to create Axiom client: %w", err) 58 | } 59 | 60 | f.axiomClient = client 61 | 62 | f.logger.Info("Axiom service factory initialized", 63 | zap.String("dataset", f.dataset)) 64 | 65 | return nil 66 | } 67 | 68 | // Create creates a new Axiom EventStore service instance 69 | func (f *Factory) Create(ctx context.Context) (services.Service, error) { 70 | of := f.Registry.Get(objectstore.TypeObjectStore) 71 | s, err := of.Create(ctx) 72 | if err != nil { 73 | return nil, fmt.Errorf("creating object store: %w", err) 74 | } 75 | o, ok := s.(objectstore.ObjectStore) 76 | if !ok { 77 | return nil, errors.New("object store service is not an objectstore.ObjectStore") 78 | } 79 | 80 | return &EventStore{ 81 | axiomClient: f.axiomClient, 82 | dataset: f.dataset, 83 | objectStore: o, 84 | }, nil 85 | } 86 | 87 | // FactoryType returns the service factory type 88 | func (f *Factory) FactoryType() services.ServiceType { 89 | return services.ServiceType(fmt.Sprintf("%s.%s", eventstore.TypeEventStore, Type)) 90 | } 91 | 92 | // Close cleans up resources (Axiom client handles its own cleanup) 93 | func (f *Factory) Close() error { 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/services/eventstore/axiom/metrics.go: -------------------------------------------------------------------------------- 1 | package axiom 2 | 3 | import "github.com/qpoint-io/qtap/pkg/telemetry" 4 | 5 | var ( 6 | submittedRecords = telemetry.Counter( 7 | "tap_axiom_records_submitted", 8 | telemetry.WithDescription("The number of records submitted to Axiom")) 9 | 10 | ingestedRecords = telemetry.Counter( 11 | "tap_axiom_records_ingested", 12 | telemetry.WithDescription("The number of records successfully ingested by Axiom")) 13 | 14 | failedRecords = telemetry.Counter( 15 | "tap_axiom_records_failed", 16 | telemetry.WithDescription("The number of records that failed to be ingested by Axiom")) 17 | ) 18 | 19 | func incrementSubmittedRecords() { 20 | submittedRecords(1) 21 | } 22 | 23 | func incrementIngestedRecords(count float64) { 24 | ingestedRecords(count) 25 | } 26 | 27 | func incrementFailedRecords(count float64) { 28 | failedRecords(count) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/services/eventstore/console/console.go: -------------------------------------------------------------------------------- 1 | package console 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/qpoint-io/qtap/pkg/connection" 9 | "github.com/qpoint-io/qtap/pkg/services" 10 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 11 | "github.com/qpoint-io/qtap/pkg/services/objectstore" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | const ( 16 | TypeConsoleEventStore services.ServiceType = "console" 17 | ) 18 | 19 | type connidable interface { 20 | SetConnectionID(id string) 21 | } 22 | 23 | type taggable interface { 24 | AddTags(tag ...string) 25 | } 26 | 27 | type Factory struct { 28 | eventstore.BaseEventStore 29 | } 30 | 31 | func (f *Factory) Init(ctx context.Context, cfg any) error { 32 | return nil 33 | } 34 | 35 | func (f *Factory) Create(ctx context.Context) (services.Service, error) { 36 | of := f.Registry.Get(objectstore.TypeObjectStore) 37 | s, err := of.Create(ctx) 38 | if err != nil { 39 | return nil, fmt.Errorf("creating object store: %w", err) 40 | } 41 | 42 | o, ok := s.(objectstore.ObjectStore) 43 | if !ok { 44 | return nil, errors.New("object store service is not an objectstore.ObjectStore") 45 | } 46 | 47 | return &EventStore{ 48 | objectStore: o, 49 | }, nil 50 | } 51 | 52 | // ServiceType returns the service type 53 | func (f *Factory) FactoryType() services.ServiceType { 54 | return services.ServiceType(fmt.Sprintf("%s.%s", eventstore.TypeEventStore, TypeConsoleEventStore)) 55 | } 56 | 57 | // EventStore implements the EventStore interface with Postgres 58 | type EventStore struct { 59 | services.LogHelper 60 | eventstore.BaseEventStore 61 | 62 | conn *connection.Connection 63 | objectStore objectstore.ObjectStore 64 | } 65 | 66 | func (s *EventStore) SetConnection(conn *connection.Connection) { 67 | s.conn = conn 68 | } 69 | 70 | // Save stores an event 71 | func (s *EventStore) Save(ctx context.Context, item any) { 72 | if l, ok := s.objectStore.(services.LoggerAdapter); ok { 73 | l.SetLogger(s.Log()) 74 | } 75 | 76 | if s.conn != nil { 77 | if c, ok := item.(connidable); ok { 78 | c.SetConnectionID(s.conn.ID()) 79 | } 80 | 81 | if t, ok := item.(taggable); ok { 82 | t.AddTags(s.conn.Tags().List()...) 83 | } 84 | } 85 | 86 | switch i := item.(type) { 87 | case *eventstore.Artifact: 88 | go func() { 89 | ar, err := s.objectStore.Put(*i) 90 | if err != nil { 91 | s.Log().Error("failed to put artifact", zap.Error(err)) 92 | return 93 | } 94 | s.Save(ctx, ar) 95 | }() 96 | default: 97 | s.Log().Info("event store submission", 98 | zap.String("type", fmt.Sprintf("%T", item)), 99 | zap.Any("item", item)) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pkg/services/eventstore/eventstore_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/qpoint-io/qtap/pkg/services/eventstore (interfaces: EventStore) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination ./eventstore_mock.go -package eventstore . EventStore 7 | // 8 | 9 | // Package eventstore is a generated GoMock package. 10 | package eventstore 11 | 12 | import ( 13 | context "context" 14 | reflect "reflect" 15 | 16 | services "github.com/qpoint-io/qtap/pkg/services" 17 | gomock "go.uber.org/mock/gomock" 18 | ) 19 | 20 | // MockEventStore is a mock of EventStore interface. 21 | type MockEventStore struct { 22 | ctrl *gomock.Controller 23 | recorder *MockEventStoreMockRecorder 24 | isgomock struct{} 25 | } 26 | 27 | // MockEventStoreMockRecorder is the mock recorder for MockEventStore. 28 | type MockEventStoreMockRecorder struct { 29 | mock *MockEventStore 30 | } 31 | 32 | // NewMockEventStore creates a new mock instance. 33 | func NewMockEventStore(ctrl *gomock.Controller) *MockEventStore { 34 | mock := &MockEventStore{ctrl: ctrl} 35 | mock.recorder = &MockEventStoreMockRecorder{mock} 36 | return mock 37 | } 38 | 39 | // EXPECT returns an object that allows the caller to indicate expected use. 40 | func (m *MockEventStore) EXPECT() *MockEventStoreMockRecorder { 41 | return m.recorder 42 | } 43 | 44 | // Save mocks base method. 45 | func (m *MockEventStore) Save(ctx context.Context, item any) { 46 | m.ctrl.T.Helper() 47 | m.ctrl.Call(m, "Save", ctx, item) 48 | } 49 | 50 | // Save indicates an expected call of Save. 51 | func (mr *MockEventStoreMockRecorder) Save(ctx, item any) *gomock.Call { 52 | mr.mock.ctrl.T.Helper() 53 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockEventStore)(nil).Save), ctx, item) 54 | } 55 | 56 | // ServiceType mocks base method. 57 | func (m *MockEventStore) ServiceType() services.ServiceType { 58 | m.ctrl.T.Helper() 59 | ret := m.ctrl.Call(m, "ServiceType") 60 | ret0, _ := ret[0].(services.ServiceType) 61 | return ret0 62 | } 63 | 64 | // ServiceType indicates an expected call of ServiceType. 65 | func (mr *MockEventStoreMockRecorder) ServiceType() *gomock.Call { 66 | mr.mock.ctrl.T.Helper() 67 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServiceType", reflect.TypeOf((*MockEventStore)(nil).ServiceType)) 68 | } 69 | -------------------------------------------------------------------------------- /pkg/services/eventstore/noop/noop.go: -------------------------------------------------------------------------------- 1 | package noop 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qpoint-io/qtap/pkg/services" 8 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 9 | ) 10 | 11 | const ( 12 | TypeNoopEventStore services.ServiceType = "noop" 13 | ) 14 | 15 | type Factory struct { 16 | eventstore.BaseEventStore 17 | } 18 | 19 | func (f *Factory) Init(ctx context.Context, cfg any) error { 20 | return nil 21 | } 22 | 23 | func (f *Factory) Create(ctx context.Context) (services.Service, error) { 24 | return &EventStore{}, nil 25 | } 26 | 27 | // ServiceType returns the service type 28 | func (f *Factory) FactoryType() services.ServiceType { 29 | return services.ServiceType(fmt.Sprintf("%s.%s", eventstore.TypeEventStore, TypeNoopEventStore)) 30 | } 31 | 32 | // EventStore implements the EventStore interface with Postgres 33 | type EventStore struct { 34 | eventstore.BaseEventStore 35 | } 36 | 37 | // Save stores an event 38 | func (s *EventStore) Save(ctx context.Context, item any) {} 39 | -------------------------------------------------------------------------------- /pkg/services/eventstore/otel/metrics.go: -------------------------------------------------------------------------------- 1 | // pkg/services/eventstore/otel/metrics.go 2 | package otel 3 | 4 | import "github.com/qpoint-io/qtap/pkg/telemetry" 5 | 6 | var submittedRecords = telemetry.Counter( 7 | "tap_otel_records_submitted", 8 | telemetry.WithDescription("The number of records submitted to OpenTelemetry")) 9 | 10 | func incrementSubmittedRecords() { 11 | submittedRecords(1) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/services/manager.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/qpoint-io/qtap/pkg/config" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | // ServiceManager handles service creation and lifecycle 12 | type ServiceManager struct { 13 | ctx context.Context 14 | logger *zap.Logger 15 | registry *ServiceRegistry 16 | factories map[ServiceType]FactoryFactory 17 | } 18 | 19 | // NewServiceManager creates a new service manager 20 | func NewServiceManager(ctx context.Context, logger *zap.Logger, registry *ServiceRegistry) *ServiceManager { 21 | return &ServiceManager{ 22 | ctx: ctx, 23 | logger: logger, 24 | registry: registry, 25 | factories: make(map[ServiceType]FactoryFactory), 26 | } 27 | } 28 | 29 | // RegisterFactory registers a service factory 30 | func (sm *ServiceManager) RegisterFactory(fns ...FactoryFactory) { 31 | for _, fn := range fns { 32 | factory := fn() 33 | if _, exists := sm.factories[factory.FactoryType()]; !exists { 34 | sm.logger.Debug("registering factory", zap.String("factory_type", factory.FactoryType().String())) 35 | sm.factories[factory.FactoryType()] = fn 36 | } 37 | } 38 | } 39 | 40 | // SetConfig processes a config update and creates/updates services 41 | func (sm *ServiceManager) SetConfig(config *config.Config) { 42 | if config == nil { 43 | return 44 | } 45 | 46 | svcs := config.Services.ToMap() 47 | svcs["rulekit"] = config.Rulekit 48 | for key, svcConfig := range svcs { 49 | fn, exists := sm.factories[ServiceType(key)] 50 | if !exists { 51 | sm.logger.Debug("no factory registered for service type", zap.String("service_type", key)) 52 | continue 53 | } 54 | 55 | factory := fn() 56 | 57 | // Close old service if it exists and implements Closer 58 | if old := sm.registry.Get(factory.ServiceType()); old != nil { 59 | // closes factories that are closeable 60 | if closer, ok := old.(io.Closer); ok { 61 | defer func() { 62 | closer.Close() 63 | }() 64 | } 65 | 66 | // sends replacement factory to services that support it 67 | if next, ok := old.(NextFactory); ok { 68 | defer func() { 69 | next.Next(factory) 70 | }() 71 | } 72 | } 73 | 74 | // Set the registry for the factory if it implements the SetRegistry interface 75 | if sr, ok := factory.(SetRegistry); ok { 76 | sr.SetRegistry(sm.registry) 77 | } 78 | 79 | sm.logger.Info("initializing service factory", zap.String("factory_type", key)) 80 | if err := factory.Init(sm.ctx, svcConfig); err != nil { 81 | sm.logger.Error("failed to initialize service factory", zap.String("factory_type", key), zap.Error(err)) 82 | continue 83 | } 84 | 85 | sm.registry.Register(factory) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /pkg/services/mocks/registry_accessor.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/qpoint-io/qtap/pkg/services (interfaces: RegistryAccessor) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination ./mocks/registry_accessor.go -package mocks . RegistryAccessor 7 | // 8 | 9 | // Package mocks is a generated GoMock package. 10 | package mocks 11 | 12 | import ( 13 | reflect "reflect" 14 | 15 | services "github.com/qpoint-io/qtap/pkg/services" 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockRegistryAccessor is a mock of RegistryAccessor interface. 20 | type MockRegistryAccessor struct { 21 | ctrl *gomock.Controller 22 | recorder *MockRegistryAccessorMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockRegistryAccessorMockRecorder is the mock recorder for MockRegistryAccessor. 27 | type MockRegistryAccessorMockRecorder struct { 28 | mock *MockRegistryAccessor 29 | } 30 | 31 | // NewMockRegistryAccessor creates a new mock instance. 32 | func NewMockRegistryAccessor(ctrl *gomock.Controller) *MockRegistryAccessor { 33 | mock := &MockRegistryAccessor{ctrl: ctrl} 34 | mock.recorder = &MockRegistryAccessorMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockRegistryAccessor) EXPECT() *MockRegistryAccessorMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // Get mocks base method. 44 | func (m *MockRegistryAccessor) Get(serviceType services.ServiceType) services.ServiceFactory { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "Get", serviceType) 47 | ret0, _ := ret[0].(services.ServiceFactory) 48 | return ret0 49 | } 50 | 51 | // Get indicates an expected call of Get. 52 | func (mr *MockRegistryAccessorMockRecorder) Get(serviceType any) *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRegistryAccessor)(nil).Get), serviceType) 55 | } 56 | -------------------------------------------------------------------------------- /pkg/services/objectstore/console/console.go: -------------------------------------------------------------------------------- 1 | package console 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qpoint-io/qtap/pkg/services" 8 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 9 | "github.com/qpoint-io/qtap/pkg/services/objectstore" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | const ( 14 | TypeConsoleEventStore services.ServiceType = "console" 15 | ) 16 | 17 | type Factory struct { 18 | objectstore.BaseObjectStore 19 | } 20 | 21 | func (f *Factory) Init(ctx context.Context, cfg any) error { 22 | return nil 23 | } 24 | 25 | func (f *Factory) Create(ctx context.Context) (services.Service, error) { 26 | return &ObjectStore{}, nil 27 | } 28 | 29 | func (f *Factory) FactoryType() services.ServiceType { 30 | return services.ServiceType(fmt.Sprintf("%s.%s", objectstore.TypeObjectStore, TypeConsoleEventStore)) 31 | } 32 | 33 | type ObjectStore struct { 34 | services.LogHelper 35 | objectstore.BaseObjectStore 36 | } 37 | 38 | func (s *ObjectStore) Put(artifact eventstore.Artifact) (*eventstore.ArtifactRecord, error) { 39 | s.Log().Info("object store submission", 40 | zap.Dict("artifact", artifact.Fields()...)) 41 | 42 | fmt.Println(string(artifact.Data)) 43 | 44 | return artifact.Record("stdout://" + artifact.Digest()), nil 45 | } 46 | -------------------------------------------------------------------------------- /pkg/services/objectstore/noop/noop.go: -------------------------------------------------------------------------------- 1 | package noop 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qpoint-io/qtap/pkg/services" 8 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 9 | "github.com/qpoint-io/qtap/pkg/services/objectstore" 10 | ) 11 | 12 | const ( 13 | TypeNoopObjectStore services.ServiceType = "noop" 14 | ) 15 | 16 | type Factory struct { 17 | objectstore.BaseObjectStore 18 | } 19 | 20 | func (f *Factory) Init(ctx context.Context, cfg any) error { 21 | return nil 22 | } 23 | 24 | func (f *Factory) Create(ctx context.Context) (services.Service, error) { 25 | return &ObjectStore{}, nil 26 | } 27 | 28 | func (f *Factory) FactoryType() services.ServiceType { 29 | return services.ServiceType(fmt.Sprintf("%s.%s", objectstore.TypeObjectStore, TypeNoopObjectStore)) 30 | } 31 | 32 | type ObjectStore struct { 33 | objectstore.BaseObjectStore 34 | } 35 | 36 | func (s *ObjectStore) Put(artifact eventstore.Artifact) (*eventstore.ArtifactRecord, error) { 37 | return nil, nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/services/objectstore/objectstore.go: -------------------------------------------------------------------------------- 1 | package objectstore 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/services" 5 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 6 | ) 7 | 8 | const ( 9 | TypeObjectStore services.ServiceType = "objectstore" 10 | ) 11 | 12 | // ObjectStore defines the interface for object storage services 13 | type ObjectStore interface { 14 | services.Service 15 | Put(artifact eventstore.Artifact) (*eventstore.ArtifactRecord, error) 16 | } 17 | 18 | // BaseObjectStore provides common functionality for ObjectStore implementations 19 | type BaseObjectStore struct{} 20 | 21 | // ServiceType returns the service type 22 | func (b *BaseObjectStore) ServiceType() services.ServiceType { 23 | return TypeObjectStore 24 | } 25 | -------------------------------------------------------------------------------- /pkg/services/objectstore/s3/minio/minio.go: -------------------------------------------------------------------------------- 1 | package minio 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | 8 | "github.com/minio/minio-go/v7" 9 | "github.com/minio/minio-go/v7/pkg/credentials" 10 | "github.com/qpoint-io/qtap/pkg/services/client" 11 | "github.com/qpoint-io/qtap/pkg/telemetry" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | var tracer = telemetry.Tracer() 16 | 17 | type S3ObjectStore struct { 18 | logger *zap.Logger 19 | insecure bool 20 | client *minio.Client 21 | bucket string 22 | } 23 | 24 | func NewS3ObjectStore(logger *zap.Logger, endpoint, bucket, region, accessKey, secretKey string, insecure bool) (*S3ObjectStore, error) { 25 | client, err := minio.New(endpoint, &minio.Options{ 26 | Creds: credentials.NewStaticV4(accessKey, secretKey, ""), 27 | Secure: !insecure, 28 | Region: region, 29 | Transport: client.NewHttpClient().Transport, 30 | }) 31 | if err != nil { 32 | return nil, fmt.Errorf("failed to create S3 client: %w", err) 33 | } 34 | 35 | return &S3ObjectStore{ 36 | logger: logger, 37 | insecure: insecure, 38 | client: client, 39 | bucket: bucket, 40 | }, nil 41 | } 42 | 43 | func (s *S3ObjectStore) Put(ctx context.Context, digest string, contentType string, data []byte) (map[string]string, error) { 44 | ctx, span := tracer.Start(ctx, "S3.Put") 45 | defer span.End() 46 | 47 | reader := bytes.NewReader(data) 48 | 49 | _, err := s.client.PutObject(ctx, s.bucket, digest, reader, int64(len(data)), minio.PutObjectOptions{ 50 | ContentType: contentType, 51 | }) 52 | if err != nil { 53 | return nil, fmt.Errorf("failed to upload object: %w", err) 54 | } 55 | 56 | // construct the permanent URL 57 | scheme := "http" 58 | if !s.insecure { 59 | scheme = "https" 60 | } 61 | 62 | s.logger.Debug("s3 object uploaded successfully", 63 | zap.String("bucket", s.bucket), 64 | zap.String("key", digest), 65 | zap.Int("size", len(data))) 66 | 67 | return map[string]string{ 68 | "SCHEME": scheme, 69 | "ENDPOINT": s.client.EndpointURL().Host, 70 | "BUCKET": s.bucket, 71 | "DIGEST": digest, 72 | }, nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/services/objectstore/s3/s3.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/qpoint-io/qtap/pkg/services" 9 | "github.com/qpoint-io/qtap/pkg/services/eventstore" 10 | "github.com/qpoint-io/qtap/pkg/services/objectstore" 11 | ) 12 | 13 | // templateURL takes a URL template string containing {{KEY}} placeholders 14 | // and replaces them with corresponding values from the provided map. 15 | // If a key in the template is not found in the values map, it remains unchanged. 16 | func templateURL(template string, values map[string]string) string { 17 | result := template 18 | for key, value := range values { 19 | placeholder := fmt.Sprintf("{{%s}}", key) 20 | result = strings.ReplaceAll(result, placeholder, value) 21 | } 22 | return result 23 | } 24 | 25 | type ObjectStore struct { 26 | services.LogHelper 27 | objectstore.BaseObjectStore 28 | putFn func(ctx context.Context, digest string, contentType string, data []byte) (map[string]string, error) 29 | accessURL string 30 | } 31 | 32 | func (s *ObjectStore) Put(artifact eventstore.Artifact) (*eventstore.ArtifactRecord, error) { 33 | ctx := context.Background() 34 | m, err := s.putFn(ctx, artifact.Digest(), artifact.ContentType, artifact.Data) 35 | if err != nil { 36 | return nil, fmt.Errorf("failed to put artifact: %w", err) 37 | } 38 | 39 | // build the URL 40 | url := fmt.Sprintf("%s://%s/%s/%s", m["SCHEME"], m["ENDPOINT"], m["BUCKET"], m["DIGEST"]) 41 | if s.accessURL != "" { 42 | url = templateURL(s.accessURL, m) 43 | } 44 | 45 | return artifact.Record(url), nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/services/objectstore/s3/s3_test.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import "testing" 4 | 5 | func Test_templateURL(t *testing.T) { 6 | tests := []struct { 7 | name string 8 | template string 9 | values map[string]string 10 | want string 11 | }{ 12 | { 13 | name: "simple replacement", 14 | template: "https://storage.googleapis.com/{{BUCKET_NAME}}/file.txt", 15 | values: map[string]string{"BUCKET_NAME": "my_gcs_bucket"}, 16 | want: "https://storage.googleapis.com/my_gcs_bucket/file.txt", 17 | }, 18 | { 19 | name: "multiple replacements", 20 | template: "{{SCHEME}}://{{HOST}}/{{PATH}}/{{FILE}}", 21 | values: map[string]string{ 22 | "SCHEME": "https", 23 | "HOST": "example.com", 24 | "PATH": "data", 25 | "FILE": "report.pdf", 26 | }, 27 | want: "https://example.com/data/report.pdf", 28 | }, 29 | { 30 | name: "missing value", 31 | template: "https://{{HOST}}/{{PATH}}", 32 | values: map[string]string{ 33 | "HOST": "example.com", 34 | }, 35 | want: "https://example.com/{{PATH}}", 36 | }, 37 | { 38 | name: "empty template", 39 | template: "", 40 | values: map[string]string{"KEY": "value"}, 41 | want: "", 42 | }, 43 | { 44 | name: "no placeholders", 45 | template: "https://example.com/static/file.txt", 46 | values: map[string]string{"KEY": "value"}, 47 | want: "https://example.com/static/file.txt", 48 | }, 49 | } 50 | 51 | for _, tt := range tests { 52 | t.Run(tt.name, func(t *testing.T) { 53 | got := templateURL(tt.template, tt.values) 54 | if got != tt.want { 55 | t.Errorf("TemplateURL() = %v, want %v", got, tt.want) 56 | } 57 | }) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/services/registry.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sync" 7 | ) 8 | 9 | // ServiceRegistry holds references to active service instances 10 | type ServiceRegistry struct { 11 | services map[ServiceType]ServiceFactory 12 | mu sync.RWMutex 13 | } 14 | 15 | // NewServiceRegistry creates a new service registry 16 | func NewServiceRegistry() *ServiceRegistry { 17 | return &ServiceRegistry{ 18 | services: make(map[ServiceType]ServiceFactory), 19 | } 20 | } 21 | 22 | // Register adds or replaces a service in the registry 23 | func (sr *ServiceRegistry) Register(svc ServiceFactory) { 24 | sr.mu.Lock() 25 | defer sr.mu.Unlock() 26 | 27 | sr.services[svc.ServiceType()] = svc 28 | } 29 | 30 | // Get retrieves a service by type 31 | func (sr *ServiceRegistry) Get(serviceType ServiceType) ServiceFactory { 32 | sr.mu.RLock() 33 | defer sr.mu.RUnlock() 34 | return sr.services[serviceType] 35 | } 36 | 37 | // Close closes all registered services that implement CloseableService 38 | func (sr *ServiceRegistry) Close() error { 39 | sr.mu.Lock() 40 | defer sr.mu.Unlock() 41 | 42 | var errs []error 43 | for _, svc := range sr.services { 44 | if closer, ok := svc.(io.Closer); ok { 45 | if err := closer.Close(); err != nil { 46 | errs = append(errs, err) 47 | } 48 | } 49 | } 50 | 51 | if len(errs) > 0 { 52 | return fmt.Errorf("errors closing services: %v", errs) 53 | } 54 | 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /pkg/services/rulekitsvc/macros.go: -------------------------------------------------------------------------------- 1 | package rulekitsvc 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/qpoint-io/qtap/pkg/config" 8 | "github.com/qpoint-io/qtap/pkg/rulekitext" 9 | "github.com/qpoint-io/qtap/pkg/services" 10 | "github.com/qpoint-io/rulekit" 11 | ) 12 | 13 | const ( 14 | TypeRulekit services.ServiceType = "rulekit" 15 | ) 16 | 17 | type Service interface { 18 | services.Service 19 | Macros() map[string]rulekit.Rule 20 | Functions() map[string]*rulekit.Function 21 | } 22 | 23 | type Factory struct { 24 | Macros map[string]rulekit.Rule 25 | } 26 | 27 | func (f *Factory) FactoryType() services.ServiceType { 28 | return TypeRulekit 29 | } 30 | 31 | func (f *Factory) ServiceType() services.ServiceType { 32 | return TypeRulekit 33 | } 34 | 35 | func (f *Factory) Init(ctx context.Context, cfg any) error { 36 | c, ok := cfg.(*config.Rulekit) 37 | if !ok { 38 | return fmt.Errorf("invalid config type: %T wanted *config.Rulekit", cfg) 39 | } 40 | 41 | if c != nil { 42 | macros, err := c.ParseMacros() 43 | if err != nil { 44 | return fmt.Errorf("parsing rulekit macros: %w", err) 45 | } 46 | if err := config.ValidateRulekitMacros(macros); err != nil { 47 | return fmt.Errorf("validating rulekit macros: %w", err) 48 | } 49 | 50 | f.Macros = macros 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func (f *Factory) Create(ctx context.Context) (services.Service, error) { 57 | return &service{f: f}, nil 58 | } 59 | 60 | type service struct { 61 | f *Factory 62 | } 63 | 64 | func (s *service) ServiceType() services.ServiceType { 65 | return TypeRulekit 66 | } 67 | 68 | func (s *service) Macros() map[string]rulekit.Rule { 69 | return s.f.Macros 70 | } 71 | 72 | func (s *service) Functions() map[string]*rulekit.Function { 73 | return rulekitext.Functions 74 | } 75 | -------------------------------------------------------------------------------- /pkg/services/services.go: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // ServiceType represents a type identifier for services 8 | type ServiceType string 9 | 10 | func (s ServiceType) String() string { 11 | return string(s) 12 | } 13 | 14 | // Service is the base interface that all services must implement 15 | type Service interface { 16 | // ServiceType returns the type of service 17 | ServiceType() ServiceType 18 | } 19 | 20 | // FactoryFactory creates a service factory 🎶 21 | type FactoryFactory func() ServiceFactory 22 | 23 | // ServiceFactory creates service instances 24 | // 25 | //go:generate go tool go.uber.org/mock/mockgen -destination ./mocks/service_factory.go -package mocks . ServiceFactory 26 | type ServiceFactory interface { 27 | // Init initializes the service factory 28 | Init(ctx context.Context, config any) error 29 | // Create creates a new service instance 30 | Create(ctx context.Context) (Service, error) 31 | // FactoryType returns the type of factory 32 | FactoryType() ServiceType 33 | // ServiceType returns the type of service this factory creates 34 | ServiceType() ServiceType 35 | } 36 | 37 | // SetRegistry sets the registry for the service 38 | type SetRegistry interface { 39 | SetRegistry(registry RegistryAccessor) 40 | } 41 | 42 | // RegistryAccessor is a type that can access the service registry 43 | // 44 | //go:generate go tool go.uber.org/mock/mockgen -destination ./mocks/registry_accessor.go -package mocks . RegistryAccessor 45 | type RegistryAccessor interface { 46 | // Get retrieves a service factory by type 47 | Get(serviceType ServiceType) ServiceFactory 48 | } 49 | 50 | // NextFactory indicates that a factory will handle graceful replacements 51 | // of itself to the replacement factory 52 | type NextFactory interface { 53 | Next(ServiceFactory) 54 | } 55 | -------------------------------------------------------------------------------- /pkg/status/server.go: -------------------------------------------------------------------------------- 1 | package status 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | _ "net/http/pprof" 7 | "time" 8 | 9 | "go.uber.org/zap" 10 | ) 11 | 12 | type StatusServer interface { 13 | Start() error 14 | Stop() error 15 | IsReady() bool 16 | } 17 | 18 | type BaseStatusServer struct { 19 | listen string 20 | logger *zap.Logger 21 | metricsHandler http.Handler 22 | server *http.Server 23 | readyCheck func() bool 24 | } 25 | 26 | func NewBaseStatusServer(listen string, logger *zap.Logger, metricsHandler http.Handler, readyCheck func() bool) *BaseStatusServer { 27 | return &BaseStatusServer{ 28 | listen: listen, 29 | logger: logger, 30 | metricsHandler: metricsHandler, 31 | readyCheck: readyCheck, 32 | } 33 | } 34 | 35 | func (s *BaseStatusServer) Start() error { 36 | mux := http.DefaultServeMux 37 | s.setupRoutes(mux) 38 | 39 | s.server = &http.Server{ 40 | Addr: s.listen, 41 | Handler: mux, 42 | } 43 | 44 | go func() { 45 | if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 46 | s.logger.Error("unable to start status server", zap.Error(err)) 47 | } 48 | }() 49 | 50 | s.logger.Info("status server listening", zap.String("url", "http://"+s.listen)) 51 | return nil 52 | } 53 | 54 | func (s *BaseStatusServer) Stop() error { 55 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 56 | defer cancel() 57 | return s.server.Shutdown(ctx) 58 | } 59 | 60 | func (s *BaseStatusServer) IsReady() bool { 61 | return s.readyCheck() 62 | } 63 | 64 | func (s *BaseStatusServer) setupRoutes(mux *http.ServeMux) { 65 | mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { 66 | if s.IsReady() { 67 | w.WriteHeader(http.StatusOK) 68 | if _, err := w.Write([]byte("ready")); err != nil { 69 | s.logger.Error("failed to write response", zap.Error(err)) 70 | } 71 | } else { 72 | w.WriteHeader(http.StatusServiceUnavailable) 73 | if _, err := w.Write([]byte("not ready")); err != nil { 74 | s.logger.Error("failed to write response", zap.Error(err)) 75 | } 76 | } 77 | }) 78 | 79 | mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { 80 | if _, err := w.Write([]byte("healthy")); err != nil { 81 | s.logger.Error("failed to write response", zap.Error(err)) 82 | } 83 | }) 84 | 85 | if s.metricsHandler != nil { 86 | mux.Handle("GET /metrics", s.metricsHandler) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /pkg/stream/factory.go: -------------------------------------------------------------------------------- 1 | package stream 2 | 3 | import ( 4 | "github.com/qpoint-io/qtap/pkg/connection" 5 | "github.com/qpoint-io/qtap/pkg/dns" 6 | "github.com/qpoint-io/qtap/pkg/plugins" 7 | dnsStream "github.com/qpoint-io/qtap/pkg/stream/protocols/dns" 8 | "github.com/qpoint-io/qtap/pkg/stream/protocols/http1" 9 | "github.com/qpoint-io/qtap/pkg/stream/protocols/http2" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | type StreamFactory struct { 14 | // logger 15 | logger *zap.Logger 16 | 17 | // dns manager 18 | dnsManager *dns.DNSManager 19 | 20 | // plugin manager 21 | pluginManager *plugins.Manager 22 | } 23 | 24 | type StreamFactoryOpt func(*StreamFactory) 25 | 26 | func SetDnsManager(manager *dns.DNSManager) StreamFactoryOpt { 27 | return func(m *StreamFactory) { 28 | m.dnsManager = manager 29 | } 30 | } 31 | 32 | func SetPluginManager(manager *plugins.Manager) StreamFactoryOpt { 33 | return func(m *StreamFactory) { 34 | m.pluginManager = manager 35 | } 36 | } 37 | 38 | func NewStreamFactory(logger *zap.Logger, opts ...StreamFactoryOpt) *StreamFactory { 39 | m := &StreamFactory{ 40 | logger: logger, 41 | } 42 | 43 | for _, opt := range opts { 44 | opt(m) 45 | } 46 | 47 | return m 48 | } 49 | 50 | func (m *StreamFactory) OnConnection(conn *connection.Connection) connection.StreamProcessor { 51 | logger := conn.Logger() 52 | 53 | // parse dns streams 54 | if conn.Protocol == connection.Protocol_DNS && conn.OpenEvent.Source == connection.Client && m.dnsManager != nil { 55 | return dnsStream.NewDNSStream(conn.Context(), logger, conn, m.dnsManager) 56 | } 57 | 58 | // parse http streams 59 | if conn.Protocol == connection.Protocol_HTTP1 || conn.Protocol == connection.Protocol_HTTP2 { 60 | // extract the domain 61 | domain := conn.Domain() 62 | 63 | // if the domain does not have a stack and no default stack is set, skip it 64 | if _, exists := m.pluginManager.GetDomainStack(domain, "http"); !exists { 65 | return nil 66 | } 67 | 68 | // parse http/1 streams 69 | if conn.Protocol == connection.Protocol_HTTP1 { 70 | return http1.NewHTTPStream(conn.Context(), domain, logger, conn, 71 | http1.SetPluginManager(m.pluginManager), 72 | ) 73 | } 74 | 75 | // parse http/2 streams 76 | if conn.Protocol == connection.Protocol_HTTP2 { 77 | return http2.NewHTTPStream(conn.Context(), domain, logger, conn, 78 | http2.SetPluginManager(m.pluginManager), 79 | ) 80 | } 81 | } 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/stream/protocols/http1/reader.go: -------------------------------------------------------------------------------- 1 | package http1 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "sync" 9 | ) 10 | 11 | // BufferedReader provides blocking reads over a bytes buffer with context cancellation support. 12 | type BufferedReader struct { 13 | ctx context.Context 14 | buf *bytes.Buffer 15 | mu sync.RWMutex 16 | notify chan struct{} 17 | } 18 | 19 | // NewBufferedReader creates a new BufferedReader instance. 20 | func NewBufferedReader(ctx context.Context) *BufferedReader { 21 | return &BufferedReader{ 22 | ctx: ctx, 23 | buf: bytes.NewBuffer(nil), 24 | notify: make(chan struct{}, 1), 25 | } 26 | } 27 | 28 | // Read implements io.Reader. It blocks until data is available or the context is cancelled. 29 | func (r *BufferedReader) Read(p []byte) (n int, err error) { 30 | r.mu.RLock() 31 | if r.buf == nil { 32 | r.mu.RUnlock() 33 | return 0, fmt.Errorf("buffer is nil on read: %w", io.ErrUnexpectedEOF) 34 | } 35 | 36 | for r.buf.Len() == 0 { 37 | // we need to reset the buffer here because our readWaiter prevents 38 | // the bytes.Buffer from hitting a zero length read until the stream 39 | // is closed. For long running chunked streams, this will cause the 40 | // buffer to constantly grow, while all the previous data has already 41 | // been read. 42 | r.buf.Reset() 43 | r.mu.RUnlock() 44 | 45 | // wait for the buffer to be written to 46 | select { 47 | case <-r.notify: 48 | case <-r.ctx.Done(): 49 | return 0, r.ctx.Err() 50 | } 51 | 52 | r.mu.RLock() 53 | if r.buf == nil { 54 | r.mu.RUnlock() 55 | return 0, fmt.Errorf("notify: buffer is nil: %w", io.ErrUnexpectedEOF) 56 | } 57 | } 58 | 59 | n, err = r.buf.Read(p) 60 | r.mu.RUnlock() 61 | return n, err 62 | } 63 | 64 | // Write adds data to the internal buffer. 65 | func (r *BufferedReader) Write(p []byte) (n int, err error) { 66 | r.mu.Lock() 67 | defer r.mu.Unlock() 68 | 69 | if r.buf == nil { 70 | return 0, io.ErrClosedPipe 71 | } 72 | 73 | n, err = r.buf.Write(p) 74 | select { 75 | case r.notify <- struct{}{}: 76 | default: 77 | } 78 | return n, err 79 | } 80 | 81 | // Close implements io.Closer. 82 | func (r *BufferedReader) Close() error { 83 | r.mu.Lock() 84 | defer r.mu.Unlock() 85 | 86 | r.buf = nil 87 | select { 88 | case r.notify <- struct{}{}: 89 | default: 90 | } 91 | 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /pkg/synq/ttlcache.go: -------------------------------------------------------------------------------- 1 | package synq 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var now = time.Now 8 | 9 | type TTLCache[K comparable, V any] struct { 10 | // container entries 11 | entries *Map[K, V] 12 | 13 | // expiration entries 14 | expirations *Map[K, time.Time] 15 | 16 | // expiration watcher 17 | ticker *time.Ticker 18 | 19 | // stop channel 20 | stop chan bool 21 | 22 | // expiration duration 23 | expireDuration time.Duration 24 | } 25 | 26 | func NewTTLCache[K comparable, V any](expireDuration, cleanupDuration time.Duration) *TTLCache[K, V] { 27 | // init the container 28 | container := &TTLCache[K, V]{ 29 | entries: NewMap[K, V](), 30 | expirations: NewMap[K, time.Time](), 31 | ticker: time.NewTicker(cleanupDuration), 32 | stop: make(chan bool), 33 | expireDuration: expireDuration, 34 | } 35 | 36 | // start the container 37 | container.Start() 38 | 39 | // return the container 40 | return container 41 | } 42 | 43 | func (c *TTLCache[K, V]) Start() { 44 | go func() { 45 | for { 46 | select { 47 | case <-c.ticker.C: 48 | c.ExpireRecords() 49 | case <-c.stop: 50 | return 51 | } 52 | } 53 | }() 54 | } 55 | 56 | func (c *TTLCache[K, V]) Stop() { 57 | // stop the ticker 58 | c.ticker.Stop() 59 | 60 | // stop the goroutine 61 | c.stop <- true 62 | } 63 | 64 | func (c *TTLCache[K, V]) ExpireRecords() { 65 | // check for any records that have expired and delete 66 | for key, exp := range c.expirations.Copy() { 67 | if now().After(exp) { 68 | c.entries.Delete(key) 69 | c.expirations.Delete(key) 70 | } 71 | } 72 | } 73 | 74 | func (c *TTLCache[K, V]) Load(key K) (value V, ok bool) { 75 | return c.entries.Load(key) 76 | } 77 | 78 | func (c *TTLCache[K, V]) Store(key K, value V) { 79 | // add entry to the store 80 | c.entries.Store(key, value) 81 | 82 | // add an entry to the expirations entry 83 | c.expirations.Store(key, now().Add(c.expireDuration)) 84 | } 85 | 86 | func (c *TTLCache[K, V]) Delete(key K) { 87 | // remove the entry from the store 88 | c.entries.Delete(key) 89 | 90 | // remove the entry from the expirations 91 | c.expirations.Delete(key) 92 | } 93 | 94 | func (c *TTLCache[K, V]) Renew(key K) { 95 | c.expirations.Store(key, now().Add(c.expireDuration)) 96 | } 97 | 98 | func (c *TTLCache[K, V]) Len() int { 99 | return c.entries.Len() 100 | } 101 | 102 | func (c *TTLCache[K, V]) Copy() map[K]V { 103 | return c.entries.Copy() 104 | } 105 | -------------------------------------------------------------------------------- /pkg/synq/ttlcache_test.go: -------------------------------------------------------------------------------- 1 | package synq 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestTTLCache(t *testing.T) { 9 | // Create a mock time that we can control 10 | mockTime := time.Now() 11 | originalNow := now 12 | now = func() time.Time { return mockTime } 13 | defer func() { now = originalNow }() 14 | 15 | // Create container with 100ms expiration and 50ms cleanup 16 | container := NewTTLCache[string, int](100*time.Millisecond, 50*time.Millisecond) 17 | defer container.Stop() 18 | 19 | // Test Store and Load 20 | t.Run("Store and Load", func(t *testing.T) { 21 | container.Store("key1", 123) 22 | 23 | if val, ok := container.Load("key1"); !ok || val != 123 { 24 | t.Errorf("Expected to load 123, got %v, exists: %v", val, ok) 25 | } 26 | }) 27 | 28 | // Test Delete 29 | t.Run("Delete", func(t *testing.T) { 30 | container.Store("key2", 456) 31 | container.Delete("key2") 32 | 33 | if _, ok := container.Load("key2"); ok { 34 | t.Error("Expected key2 to be deleted") 35 | } 36 | }) 37 | 38 | // Test Expiration 39 | t.Run("Expiration", func(t *testing.T) { 40 | container.Store("key3", 789) 41 | 42 | // Advance time beyond expiration 43 | mockTime = mockTime.Add(150 * time.Millisecond) 44 | container.ExpireRecords() 45 | 46 | if _, ok := container.Load("key3"); ok { 47 | t.Error("Expected key3 to be expired") 48 | } 49 | }) 50 | 51 | // Test Renew 52 | t.Run("Renew", func(t *testing.T) { 53 | container.Store("key4", 999) 54 | 55 | // Advance half the expiration time 56 | mockTime = mockTime.Add(50 * time.Millisecond) 57 | 58 | // Renew the key 59 | container.Renew("key4") 60 | 61 | // Advance another half of expiration time 62 | mockTime = mockTime.Add(50 * time.Millisecond) 63 | 64 | if val, ok := container.Load("key4"); !ok || val != 999 { 65 | t.Errorf("Expected renewed key to still exist with value 999, got %v, exists: %v", val, ok) 66 | } 67 | }) 68 | 69 | // Test Length 70 | t.Run("Length", func(t *testing.T) { 71 | // Advance beyond expiration time to clear previous entries 72 | mockTime = mockTime.Add(150 * time.Millisecond) 73 | container.ExpireRecords() 74 | container.Store("key5", 555) 75 | container.Store("key6", 666) 76 | 77 | if length := container.Len(); length != 2 { 78 | t.Errorf("Expected length 2, got %d", length) 79 | } 80 | }) 81 | 82 | // Test Copy 83 | t.Run("Copy", func(t *testing.T) { 84 | container.Store("key7", 777) 85 | copied := container.Copy() 86 | 87 | if len(copied) != container.Len() { 88 | t.Errorf("Expected copied map to have same length as container") 89 | } 90 | 91 | if val, ok := copied["key7"]; !ok || val != 777 { 92 | t.Errorf("Expected copied map to contain key7 with value 777") 93 | } 94 | }) 95 | } 96 | -------------------------------------------------------------------------------- /pkg/telemetry/collector.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | // Collector provides a way to batch collect metrics. 8 | // 9 | // Register() is called upon creation to register the metrics. All metrics must be registered 10 | // on the provided Factory. The global functions such as telemetry.Counter() may not be used. 11 | // 12 | // On collection, it will signal Collect(). The collector must compute the latest values 13 | // synchronously and set them on its metrics. 14 | type Collector interface { 15 | Register(Factory) 16 | Collect() 17 | } 18 | 19 | // RegisterCollector registers a collector. 20 | func RegisterCollector(c Collector) { 21 | registerCollector(c, defaultRegisterer) 22 | } 23 | 24 | func registerCollector(c Collector, registerer prometheus.Registerer) { 25 | // create a registry and have the collector register its metrics on it 26 | registry := prometheus.NewRegistry() 27 | factory := &factory{ 28 | registerer: registry, 29 | } 30 | collector := &collector{ 31 | collector: c, 32 | registry: registry, 33 | factory: factory, 34 | } 35 | 36 | // register the metrics 37 | c.Register(factory) 38 | // register the collector with the actual prometheus registry 39 | registerer.MustRegister(collector) 40 | } 41 | 42 | // collector bridges between Collector and prometheus.Collector. 43 | type collector struct { 44 | collector Collector 45 | registry *prometheus.Registry 46 | factory *factory 47 | } 48 | 49 | // Describe implements prometheus.Collector. 50 | func (c *collector) Describe(ch chan<- *prometheus.Desc) { 51 | c.registry.Describe(ch) 52 | } 53 | 54 | // Collect implements prometheus.Collector. 55 | func (c *collector) Collect(ch chan<- prometheus.Metric) { 56 | // reset all gauges to avoid orphaned label values 57 | for _, gauge := range c.factory.gauges { 58 | gauge.Reset() 59 | } 60 | 61 | // signal that we are collecting metrics so that the collector can compute the latest values 62 | c.collector.Collect() 63 | 64 | // collect the metrics from the registry 65 | c.registry.Collect(ch) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/telemetry/collector_test.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestCollector(t *testing.T) { 13 | registry := prometheus.NewRegistry() 14 | c := &testCollector{} 15 | registerCollector(c, registry) 16 | 17 | // first scrape 18 | wantOutput := ` 19 | # HELP test_counter_total test counter 20 | # TYPE test_counter_total counter 21 | test_counter_total 100 22 | # HELP test_gauge test gauge 23 | # TYPE test_gauge gauge 24 | test_gauge{label1="one",label2="two"} 200 25 | test_gauge{label1="uno",label2="dos"} 300 26 | # HELP test_observable_gauge test observable gauge 27 | # TYPE test_observable_gauge gauge 28 | test_observable_gauge 42 29 | ` 30 | err := testutil.CollectAndCompare( 31 | registry, strings.NewReader(wantOutput), 32 | "test_counter_total", 33 | "test_gauge", 34 | "test_observable_gauge", 35 | ) 36 | require.NoError(t, err) 37 | 38 | // second scrape 39 | // the counter should increase 40 | // disable second set of labels (uno, dos) on test_gauge. it should not be exported anymore. 41 | c.disableSecondSetOfLabels = true 42 | 43 | wantOutput = ` 44 | # HELP test_counter_total test counter 45 | # TYPE test_counter_total counter 46 | test_counter_total 200 47 | # HELP test_gauge test gauge 48 | # TYPE test_gauge gauge 49 | test_gauge{label1="one",label2="two"} 200 50 | # HELP test_observable_gauge test observable gauge 51 | # TYPE test_observable_gauge gauge 52 | test_observable_gauge 42 53 | ` 54 | err = testutil.CollectAndCompare( 55 | registry, strings.NewReader(wantOutput), 56 | "test_counter_total", 57 | "test_gauge", 58 | "test_observable_gauge", 59 | ) 60 | require.NoError(t, err) 61 | } 62 | 63 | type testCollector struct { 64 | testCounter CounterFn 65 | testGauge GaugeFn 66 | 67 | disableSecondSetOfLabels bool 68 | } 69 | 70 | func (c *testCollector) Register(f Factory) { 71 | c.testCounter = f.Counter( 72 | "test_counter", 73 | WithDescription("test counter"), 74 | ) 75 | c.testGauge = f.Gauge( 76 | "test_gauge", 77 | WithDescription("test gauge"), 78 | WithLabels("label1", "label2"), 79 | ) 80 | 81 | f.ObservableGauge( 82 | "test_observable_gauge", 83 | func() float64 { 84 | return 42 85 | }, 86 | WithDescription("test observable gauge"), 87 | ) 88 | } 89 | 90 | func (c *testCollector) Collect() { 91 | c.testCounter(100) 92 | c.testGauge(200, "one", "two") 93 | if !c.disableSecondSetOfLabels { 94 | c.testGauge(300, "uno", "dos") 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pkg/telemetry/counter.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | // Counter registers a counter and returns an increment function 9 | func (f *factory) Counter(name string, opts ...Option) CounterFn { 10 | options := &CommonOptions{} 11 | for _, opt := range opts { 12 | opt(options) 13 | } 14 | 15 | metricOpts := prometheus.CounterOpts{ 16 | Name: name + "_total", 17 | Help: "Counter for " + name, 18 | } 19 | if options.description != "" { 20 | metricOpts.Help = options.description 21 | } 22 | 23 | counter := promauto.With(f.registerer).NewCounterVec(metricOpts, options.labels) 24 | return func(value float64, labels ...string) { 25 | counter.WithLabelValues(labels...).Add(value) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /pkg/telemetry/factory.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | var defaultRegisterer = prometheus.DefaultRegisterer 10 | 11 | var ( 12 | onceMu sync.Mutex 13 | observableOnce = make(map[string]*sync.Once) 14 | ) 15 | 16 | // Factory produces metrics. 17 | type Factory interface { 18 | Counter(name string, opts ...Option) CounterFn 19 | Gauge(name string, opts ...Option) GaugeFn 20 | ObservableGauge(name string, fn func() float64, opts ...Option) 21 | } 22 | 23 | type factory struct { 24 | registerer prometheus.Registerer 25 | gauges []*prometheus.GaugeVec 26 | } 27 | 28 | func Counter(name string, opts ...Option) CounterFn { 29 | return (&factory{ 30 | registerer: defaultRegisterer, 31 | }).Counter(name, opts...) 32 | } 33 | 34 | func Gauge(name string, opts ...Option) GaugeFn { 35 | return (&factory{ 36 | registerer: defaultRegisterer, 37 | }).Gauge(name, opts...) 38 | } 39 | 40 | func ObservableGauge(name string, fn func() float64, opts ...Option) { 41 | onceMu.Lock() 42 | o, ok := observableOnce[name] 43 | if !ok { 44 | o = &sync.Once{} 45 | observableOnce[name] = o 46 | } 47 | onceMu.Unlock() 48 | 49 | o.Do(func() { 50 | (&factory{ 51 | registerer: defaultRegisterer, 52 | }).ObservableGauge(name, fn, opts...) 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /pkg/telemetry/gauge.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | // Gauge registers a gauge and returns a set function 9 | func (f *factory) Gauge(name string, opts ...Option) GaugeFn { 10 | options := &CommonOptions{} 11 | for _, opt := range opts { 12 | opt(options) 13 | } 14 | 15 | metricOpts := prometheus.GaugeOpts{ 16 | Name: name, 17 | Help: "Gauge for " + name, 18 | } 19 | if options.description != "" { 20 | metricOpts.Help = options.description 21 | } 22 | 23 | gauge := promauto.With(f.registerer).NewGaugeVec(metricOpts, options.labels) 24 | if f.registerer != defaultRegisterer { 25 | // this gauge is for a custom collector 26 | 27 | // store references to all creates gauges so we can reset them when the collector is scraped 28 | // otherwise, orphaned label values will continue to be exported 29 | f.gauges = append(f.gauges, gauge) 30 | } 31 | return func(value float64, labels ...string) { 32 | gauge.WithLabelValues(labels...).Set(value) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pkg/telemetry/instance.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/rs/xid" 7 | ) 8 | 9 | // set on init 10 | var instanceID string = xid.New().String() 11 | var hostname string 12 | var configVersion string 13 | 14 | func init() { 15 | var err error 16 | hostname, err = os.Hostname() 17 | if err != nil { 18 | hostname = "unknown" 19 | } 20 | } 21 | 22 | func InstanceID() string { 23 | return instanceID 24 | } 25 | 26 | func Hostname() string { 27 | return hostname 28 | } 29 | 30 | func ConfigVersion() string { 31 | return configVersion 32 | } 33 | -------------------------------------------------------------------------------- /pkg/telemetry/instance_darwin.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "errors" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | func GetSysInfo() (map[string]map[string]string, error) { 10 | return nil, errors.New("not implemented") 11 | } 12 | 13 | func GetSysInfoAsFields() zap.Field { 14 | return zap.Field{} 15 | } 16 | -------------------------------------------------------------------------------- /pkg/telemetry/instance_linux.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "fmt" 5 | "syscall" 6 | 7 | "go.uber.org/zap" 8 | ) 9 | 10 | var kernelFields map[string]string 11 | var systemFields map[string]string 12 | 13 | // GetSysInfo initializes the global kernel and system field maps 14 | func GetSysInfo() (map[string]map[string]string, error) { 15 | if kernelFields == nil || systemFields == nil { 16 | var uname syscall.Utsname 17 | if err := syscall.Uname(&uname); err != nil { 18 | return nil, fmt.Errorf("failed to get system information: %w", err) 19 | } 20 | 21 | kernelFields = map[string]string{ 22 | "name": int8ToStr(uname.Sysname[:]), 23 | "release": int8ToStr(uname.Release[:]), 24 | "version": int8ToStr(uname.Version[:]), 25 | } 26 | 27 | systemFields = map[string]string{ 28 | "hostname": int8ToStr(uname.Nodename[:]), 29 | "architecture": int8ToStr(uname.Machine[:]), 30 | } 31 | } 32 | 33 | return map[string]map[string]string{ 34 | "kernel": kernelFields, 35 | "system": systemFields, 36 | }, nil 37 | } 38 | 39 | // GetSysInfoAsFields returns system information as zap.Fields 40 | func GetSysInfoAsFields() zap.Field { 41 | sysInfo, err := GetSysInfo() 42 | if err != nil { 43 | return zap.Error(err) 44 | } 45 | 46 | return zap.Any("sysinfo", sysInfo) 47 | } 48 | 49 | // int8ToStr converts []int8 to string, stopping at null terminator 50 | func int8ToStr(arr []int8) string { 51 | b := make([]byte, 0, len(arr)) 52 | for _, v := range arr { 53 | if v == 0x00 { 54 | break 55 | } 56 | b = append(b, byte(v)) 57 | } 58 | return string(b) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/telemetry/observable_gauge.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | // ObservableGauge registers a gauge that uses a callback function to compute its value 9 | func (f *factory) ObservableGauge(name string, fn func() float64, opts ...Option) { 10 | options := CommonOptions{} 11 | for _, opt := range opts { 12 | opt(&options) 13 | } 14 | if len(options.labels) > 0 { 15 | panic("ObservableGauge does not support labels") 16 | } 17 | 18 | metricOpts := prometheus.GaugeOpts{ 19 | Name: name, 20 | Help: "Observable gauge for " + name, 21 | } 22 | if options.description != "" { 23 | metricOpts.Help = options.description 24 | } 25 | 26 | promauto.With(f.registerer).NewGaugeFunc(metricOpts, fn) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/telemetry/options_test.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWithDescription(t *testing.T) { 10 | options := &CommonOptions{} 11 | opt := WithDescription("Test description") 12 | 13 | // Apply the option 14 | opt(options) 15 | 16 | assert.Equal(t, "Test description", options.description, "Description should be set correctly") 17 | } 18 | 19 | func TestWithLabels(t *testing.T) { 20 | tests := []struct { 21 | name string 22 | inputLabels []string 23 | expectedLabels []string 24 | }{ 25 | { 26 | name: "Empty labels", 27 | inputLabels: []string{}, 28 | expectedLabels: []string{}, 29 | }, 30 | { 31 | name: "Single label", 32 | inputLabels: []string{"label1"}, 33 | expectedLabels: []string{"label1"}, 34 | }, 35 | { 36 | name: "Multiple labels", 37 | inputLabels: []string{"label1", "label2", "label3"}, 38 | expectedLabels: []string{"label1", "label2", "label3"}, 39 | }, 40 | } 41 | 42 | for _, tt := range tests { 43 | t.Run(tt.name, func(t *testing.T) { 44 | options := &CommonOptions{} 45 | opt := WithLabels(tt.inputLabels...) 46 | 47 | // Apply the option 48 | opt(options) 49 | 50 | assert.Equal(t, tt.expectedLabels, options.labels, "Labels should be set correctly") 51 | }) 52 | } 53 | } 54 | 55 | func TestSnakeCase_Additional(t *testing.T) { 56 | tests := []struct { 57 | name string 58 | segments []string 59 | expected string 60 | }{ 61 | { 62 | name: "All empty segments", 63 | segments: []string{"", "", ""}, 64 | expected: "", 65 | }, 66 | { 67 | name: "Mix of empty and non-empty segments", 68 | segments: []string{"", "foo", "", "bar", ""}, 69 | expected: "foo_bar", 70 | }, 71 | { 72 | name: "No segments", 73 | segments: []string{}, 74 | expected: "", 75 | }, 76 | { 77 | name: "Non-empty segments", 78 | segments: []string{"test", "snake", "case"}, 79 | expected: "test_snake_case", 80 | }, 81 | } 82 | 83 | for _, tt := range tests { 84 | t.Run(tt.name, func(t *testing.T) { 85 | result := SnakeCase(tt.segments...) 86 | assert.Equal(t, tt.expected, result, "SnakeCase should correctly join segments with underscores, excluding empty segments") 87 | }) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/telemetry/opts.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "slices" 5 | "strings" 6 | ) 7 | 8 | // CounterFn is a function type that increments a counter, accepting optional labels 9 | type CounterFn func(float64, ...string) 10 | 11 | // GaugeFn is a function type that sets a value, accepting optional labels 12 | type GaugeFn func(float64, ...string) 13 | 14 | // CommonOptions holds the common options for both counters and gauges 15 | type CommonOptions struct { 16 | description string 17 | labels []string 18 | } 19 | 20 | // Option is a function type that modifies CommonOptions 21 | type Option func(*CommonOptions) 22 | 23 | // WithDescription sets the description for the metric 24 | func WithDescription(description string) Option { 25 | return func(o *CommonOptions) { 26 | o.description = description 27 | } 28 | } 29 | 30 | // WithLabels sets the labels for the metric 31 | func WithLabels(labels ...string) Option { 32 | return func(o *CommonOptions) { 33 | o.labels = labels 34 | } 35 | } 36 | 37 | // SnakeCase joins non empty string segments with underscores 38 | func SnakeCase(segments ...string) string { 39 | segments = slices.DeleteFunc(segments, func(s string) bool { 40 | return s == "" 41 | }) 42 | return strings.Join(segments, "_") 43 | } 44 | -------------------------------------------------------------------------------- /pkg/telemetry/opts_test.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSnakeCase(t *testing.T) { 10 | assert.Equal(t, "foo_bar_baz", SnakeCase("foo", "bar", "baz")) 11 | assert.Equal(t, "foo_bar", SnakeCase("foo", "", "bar", "")) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/telemetry/telemetry.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/prometheus/client_golang/prometheus/promhttp" 7 | ) 8 | 9 | func Handler() http.Handler { 10 | return promhttp.Handler() 11 | } 12 | -------------------------------------------------------------------------------- /pkg/telemetry/trace.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "runtime" 7 | "strings" 8 | 9 | "github.com/qpoint-io/qtap/pkg/buildinfo" 10 | "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" 11 | "go.opentelemetry.io/otel" 12 | "go.opentelemetry.io/otel/baggage" 13 | "go.opentelemetry.io/otel/sdk/resource" 14 | semconv "go.opentelemetry.io/otel/semconv/v1.27.0" 15 | "go.opentelemetry.io/otel/trace" 16 | ) 17 | 18 | func Tracer() trace.Tracer { 19 | pkg, _ := callerInfo(1) 20 | return otel.Tracer(pkg) 21 | } 22 | 23 | func callerInfo(skip int) (pkg, fn string) { 24 | pc, _, _, _ := runtime.Caller(1 + skip) 25 | funcName := runtime.FuncForPC(pc).Name() 26 | lastSlash := strings.LastIndexByte(funcName, '/') 27 | if lastSlash < 0 { 28 | lastSlash = 0 29 | } 30 | lastDot := strings.LastIndexByte(funcName[lastSlash:], '.') + lastSlash 31 | 32 | pkg = funcName[:lastDot] 33 | fn = funcName[lastDot+1:] 34 | 35 | return 36 | } 37 | 38 | func OtelResource(ctx context.Context, name string) (*resource.Resource, error) { 39 | return resource.New(ctx, 40 | resource.WithAttributes(semconv.ServiceNamespaceKey.String("qpoint")), 41 | resource.WithAttributes(semconv.ServiceNameKey.String(name)), 42 | resource.WithAttributes(semconv.ServiceVersionKey.String(buildinfo.Version())), 43 | resource.WithAttributes(semconv.ServiceInstanceIDKey.String(Hostname())), 44 | ) 45 | } 46 | 47 | func InstrumentHTTPClient(client *http.Client) { 48 | client.Transport = otelhttp.NewTransport(client.Transport) 49 | } 50 | 51 | func WithBaggage(ctx context.Context, values map[string]string) context.Context { 52 | members := make([]baggage.Member, 0, len(values)) 53 | for k, v := range values { 54 | m, _ := baggage.NewMember(k, v) 55 | members = append(members, m) 56 | } 57 | bag, _ := baggage.New(members...) 58 | return baggage.ContextWithBaggage(ctx, bag) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/telemetry/trace_noop_exporter.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "context" 5 | 6 | "go.opentelemetry.io/otel/sdk/trace" 7 | ) 8 | 9 | // NoopSpanExporter is an implementation of trace.SpanExporter that performs no operations. 10 | type NoopSpanExporter struct{} 11 | 12 | var _ trace.SpanExporter = NoopSpanExporter{} 13 | 14 | // ExportSpans is part of trace.SpanExporter interface. 15 | func (e NoopSpanExporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error { 16 | return nil 17 | } 18 | 19 | // Shutdown is part of trace.SpanExporter interface. 20 | func (e NoopSpanExporter) Shutdown(ctx context.Context) error { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /pkg/tlsutils/tlsutils_test.go: -------------------------------------------------------------------------------- 1 | package tlsutils 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestTLSVersionString(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | version TLSVersion 12 | expected string 13 | }{ 14 | {"TLS 1.0", VersionTLS10, "TLS 1.0"}, 15 | {"TLS 1.1", VersionTLS11, "TLS 1.1"}, 16 | {"TLS 1.2", VersionTLS12, "TLS 1.2"}, 17 | {"TLS 1.3", VersionTLS13, "TLS 1.3"}, 18 | {"Unknown", TLSVersion(0x0305), "0x0305"}, 19 | } 20 | 21 | for _, tt := range tests { 22 | t.Run(tt.name, func(t *testing.T) { 23 | if got := tt.version.String(); got != tt.expected { 24 | t.Errorf("TLSVersion.String() = %v, want %v", got, tt.expected) 25 | } 26 | }) 27 | } 28 | } 29 | 30 | func TestTLSVersionFloat(t *testing.T) { 31 | tests := []struct { 32 | name string 33 | version TLSVersion 34 | expected float64 35 | }{ 36 | {"TLS 1.0", VersionTLS10, 1.0}, 37 | {"TLS 1.1", VersionTLS11, 1.1}, 38 | {"TLS 1.2", VersionTLS12, 1.2}, 39 | {"TLS 1.3", VersionTLS13, 1.3}, 40 | {"Unknown", TLSVersion(0x0305), 0.0}, 41 | } 42 | 43 | for _, tt := range tests { 44 | t.Run(tt.name, func(t *testing.T) { 45 | if got := tt.version.Float(); got != tt.expected { 46 | t.Errorf("TLSVersion.Float() = %v, want %v", got, tt.expected) 47 | } 48 | }) 49 | } 50 | } 51 | 52 | func TestClientHelloControlValues(t *testing.T) { 53 | tests := []struct { 54 | name string 55 | hello *ClientHello 56 | expected map[string]any 57 | }{ 58 | { 59 | "Complete ClientHello", 60 | &ClientHello{ 61 | SNI: "example.com", 62 | Version: VersionTLS13, 63 | ALPNs: []string{"h2", "http/1.1"}, 64 | }, 65 | map[string]any{ 66 | "enabled": true, 67 | "version": 1.3, 68 | "sni": "example.com", 69 | "alpn": []string{"h2", "http/1.1"}, 70 | }, 71 | }, 72 | { 73 | "Empty ClientHello", 74 | &ClientHello{}, 75 | map[string]any{ 76 | "enabled": true, 77 | "version": 0.0, 78 | "sni": "", 79 | "alpn": []string(nil), 80 | }, 81 | }, 82 | } 83 | 84 | for _, tt := range tests { 85 | t.Run(tt.name, func(t *testing.T) { 86 | if got := tt.hello.ControlValues(); !reflect.DeepEqual(got, tt.expected) { 87 | t.Errorf("ClientHello.ControlValues() = %v, want %v", got, tt.expected) 88 | } 89 | }) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /scripts/update-libbpf-headers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # You don't need to run this script unless you're updating the libbpf headers. 4 | # This script is used to fetch the libbpf headers from the libbpf GitHub repository. 5 | 6 | # Version of libbpf to fetch headers from 7 | LIBBPF_VERSION=1.4.7 8 | 9 | # The headers we want 10 | prefix=libbpf-"$LIBBPF_VERSION" 11 | headers=( 12 | "$prefix"/src/bpf_helper_defs.h 13 | "$prefix"/src/bpf_helpers.h 14 | "$prefix"/src/bpf_tracing.h 15 | "$prefix"/src/bpf_core_read.h 16 | "$prefix"/src/bpf_endian.h 17 | ) 18 | 19 | # Define output directory 20 | OUTPUT_DIR="internal/tap/bpf/headers" 21 | 22 | # Create output directory if it doesn't exist 23 | mkdir -p "$OUTPUT_DIR" 24 | 25 | # Create a temporary directory for extraction 26 | TEMP_DIR=$(mktemp -d) 27 | trap 'rm -rf "$TEMP_DIR"' EXIT 28 | 29 | # Fetch libbpf release and extract to temp directory 30 | curl -sL "https://github.com/libbpf/libbpf/archive/refs/tags/v${LIBBPF_VERSION}.tar.gz" | \ 31 | tar -xz -C "$TEMP_DIR" 32 | 33 | # Copy headers to final destination, stripping directory structure 34 | for header in "${headers[@]}"; do 35 | find "$TEMP_DIR" -name "$(basename "$header")" -exec cp {} "$OUTPUT_DIR/" \; 36 | done 37 | --------------------------------------------------------------------------------