├── ADOPTERS.md ├── .gitignore ├── requirements.txt ├── assets ├── architecture.png ├── complex-trace.png ├── tracepusher-gitlab.jpg └── subspan.schematic.excalidraw.png ├── docs ├── reference │ ├── debug-mode.md │ ├── dry-run-mode.md │ ├── time-shifting.md │ ├── index.md │ ├── span-kind.md │ ├── start-time.md │ ├── duration-type.md │ ├── span-attribute-types.md │ ├── span-status.md │ ├── insecure-flag.md │ ├── span-events.md │ ├── otel-col.md │ └── multi-span-traces.md ├── assets │ ├── architecture.png │ ├── complex-trace.png │ └── subspan.schematic.excalidraw.png ├── usage │ ├── assets │ │ ├── har-to-otel.jpg │ │ └── tracepusher-k8s-jobs.png │ ├── helm.md │ ├── ci.md │ ├── index.md │ ├── docker.md │ ├── standalone.md │ ├── python.md │ ├── har-to-otel.md │ └── k8sjobs.md ├── adopters.md ├── breaking-changes.md ├── try.md ├── faq.md └── index.md ├── operator ├── requirements-operator.txt ├── README.md ├── Dockerfile ├── crds.yml └── operator-logic.py ├── requirements-dev.txt ├── docker ├── ci │ └── Dockerfile └── standard │ └── Dockerfile ├── har-to-otel ├── README.md ├── Dockerfile └── har-to-otel.py ├── samples ├── jobtraceroperator │ ├── jobtracer.yml │ ├── job.yml │ └── multicontainerjob.yml ├── gitlab │ ├── README.md │ └── .gitlab-ci.yml ├── otel │ └── collector │ │ └── config.conf ├── script.sh └── github_codespaces │ └── creation_log_parser.py ├── .github └── workflows │ ├── deploy_docs.yml │ ├── build_standalone_tracepusher_binaries.yml │ └── build_standalone_har_to_otel_binaries.yml ├── mkdocs.yml ├── README.md ├── LICENSE ├── tracepusher_test.py └── tracepusher.py /ADOPTERS.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__ -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.4 2 | pyinstaller==6.9.0 3 | -------------------------------------------------------------------------------- /assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/assets/architecture.png -------------------------------------------------------------------------------- /assets/complex-trace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/assets/complex-trace.png -------------------------------------------------------------------------------- /docs/reference/debug-mode.md: -------------------------------------------------------------------------------- 1 | ## Debug Mode 2 | 3 | Add `-x true` or `--debug true` for extra output 4 | -------------------------------------------------------------------------------- /assets/tracepusher-gitlab.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/assets/tracepusher-gitlab.jpg -------------------------------------------------------------------------------- /docs/assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/docs/assets/architecture.png -------------------------------------------------------------------------------- /docs/assets/complex-trace.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/docs/assets/complex-trace.png -------------------------------------------------------------------------------- /operator/requirements-operator.txt: -------------------------------------------------------------------------------- 1 | kopf==1.36.2 2 | kubernetes==27.2.0 3 | requests==2.32.4 4 | charset_normalizer<3.0 -------------------------------------------------------------------------------- /docs/usage/assets/har-to-otel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/docs/usage/assets/har-to-otel.jpg -------------------------------------------------------------------------------- /assets/subspan.schematic.excalidraw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/assets/subspan.schematic.excalidraw.png -------------------------------------------------------------------------------- /docs/usage/assets/tracepusher-k8s-jobs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/docs/usage/assets/tracepusher-k8s-jobs.png -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.4 2 | pyinstaller==6.3.0 3 | pytest==7.4.0 4 | charset_normalizer<3.0 5 | pytest-opentelemetry==1.0.0 6 | -------------------------------------------------------------------------------- /docs/assets/subspan.schematic.excalidraw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agardnerIT/tracepusher/HEAD/docs/assets/subspan.schematic.excalidraw.png -------------------------------------------------------------------------------- /docs/reference/dry-run-mode.md: -------------------------------------------------------------------------------- 1 | ## Dry Run Mode 2 | 3 | Add `--dr true`, `--dry-run true` or `--dry true` to run without actually pushing any data. 4 | -------------------------------------------------------------------------------- /docker/ci/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:23.10 2 | WORKDIR /app 3 | COPY tracepusher.py /app 4 | RUN apt update 5 | RUN apt install -y bsdmainutils wget curl python3-requests -------------------------------------------------------------------------------- /docs/adopters.md: -------------------------------------------------------------------------------- 1 | Do you use tracepusher? Thanks! 2 | 3 | We'd love to know. 4 | 5 | Add your details to [ADOPTERS.md](https://github.com/agardnerIT/tracepusher/blob/main/ADOPTERS.md) -------------------------------------------------------------------------------- /har-to-otel/README.md: -------------------------------------------------------------------------------- 1 | # Chrome DevTools HAR file to OpenTelemetry converter 2 | 3 | The documentation has moved. See [HAR File to OpenTelemetry Converter](https://agardnerit.github.io/tracepusher/usage/har-to-otel/) -------------------------------------------------------------------------------- /samples/jobtraceroperator/jobtracer.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: tracers.tracepusher.github.io/v1 3 | kind: JobTracer 4 | metadata: 5 | name: tracer 6 | namespace: default 7 | spec: 8 | collectorEndpoint: "http://your-collector.namespace.svc.cluster.local:4318" -------------------------------------------------------------------------------- /operator/README.md: -------------------------------------------------------------------------------- 1 | # tracepusher for Kubernetes: Job tracing operator 2 | 3 | This is a Kubernetes operator which will automatically generate OpenTelemetry traces for any k8s `Job` / `CronJobs`. 4 | 5 | Documentation is here: [tracepusher Kubernetes Operator](https://agardnerit.github.io/tracepusher/usage/k8sjobs/) -------------------------------------------------------------------------------- /docs/breaking-changes.md: -------------------------------------------------------------------------------- 1 | ## v0.3.0 to v0.4.0 2 | Argument handling was entirely re-written for `v0.4.0` and `tracepusher` expects different arguments for [v0.3.0](https://github.com/agardnerIT/tracepusher/releases/tag/0.3.0) and [v0.4.0](https://github.com/agardnerIT/tracepusher/releases/tag/0.4.0). 3 | 4 | [Here is the readme for v0.3.0](https://github.com/agardnerIT/tracepusher/tree/88e6479213a952eed7985d28b1ef49a4396fe992). -------------------------------------------------------------------------------- /docs/reference/time-shifting.md: -------------------------------------------------------------------------------- 1 | ## Time Shifting 2 | 3 | In "default mode" tracepusher starts a trace `now` and finishes it `SPAN_TIME_IN_SECONDS` in the future. 4 | 5 | You may want to push timings for traces that have already occurred (eg. shell scripts). See https://github.com/agardnerIT/tracepusher/issues/4. 6 | 7 | `--time-shift true` means `start` and `end` times will be shifted back by whatever is specified as the `--duration`. 8 | -------------------------------------------------------------------------------- /samples/jobtraceroperator/job.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: pi 6 | namespace: default 7 | #annotations: 8 | # tracepusher/ignore: "true" 9 | # tracepusher/collector: "http://example.com:4318" 10 | spec: 11 | template: 12 | spec: 13 | containers: 14 | - name: pi 15 | image: perl:5.34.0 16 | command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] 17 | restartPolicy: Never 18 | backoffLimit: 0 -------------------------------------------------------------------------------- /samples/gitlab/README.md: -------------------------------------------------------------------------------- 1 | # Tracing Gitlab Pipelines with OpenTelemetry 2 | 3 | ![tracepusher generating an opentelemetry trace from a gitlab pipeline](../../assets/tracepusher-gitlab.jpg) 4 | 5 | ## Video Overview 6 | 7 | Click the image to view on YouTube. 8 | 9 | [![Tracing Gitlab Pipelines with OpenTelemetry](https://img.youtube.com/vi/zZDFQNHepyI/0.jpg)](https://www.youtube.com/watch?v=zZDFQNHepyI) 10 | 11 | ## Code 12 | 13 | For example code, see [.gitlab-ci.yml](.gitlab-ci.yml) 14 | -------------------------------------------------------------------------------- /.github/workflows/deploy_docs.yml: -------------------------------------------------------------------------------- 1 | name: Publish docs via GitHub Pages 2 | on: 3 | push: 4 | branches: 5 | - main 6 | paths: 7 | - docs/** 8 | 9 | jobs: 10 | build: 11 | name: Deploy docs 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout master 15 | uses: actions/checkout@v1 16 | 17 | - name: Deploy docs 18 | uses: mhausenblas/mkdocs-deploy-gh-pages@master 19 | env: 20 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 21 | -------------------------------------------------------------------------------- /docs/try.md: -------------------------------------------------------------------------------- 1 | # Try Tracepusher without Installation 2 | 3 | Try tracepusher from your browser, with nothing to install. 4 | 5 | - [tracepusher with open source software (Jaeger)](https://killercoda.com/agardnerit/scenario/tracepusherOSS) 6 | - [tracepusher with Dynatrace](https://killercoda.com/agardnerit/scenario/tracepusherDT) 7 | 8 | 9 | ## Use tracepusher 10 | 11 | tracepusher can [downloaded as a standalone binary](usage/standalone.md), as a [docker image](usage/docker.md), [CI-ready docker image](usage/ci.md) or [Python script](usage/python.md). -------------------------------------------------------------------------------- /docs/usage/helm.md: -------------------------------------------------------------------------------- 1 | # Trace Helm with Tracepusher 2 | 3 | It is possible to trace `helm` commands using tracepusher. 4 | 5 | Just add the word `trace` to any regular `helm` command. 6 | 7 | For example, `helm version` becomes `helm trace version` 8 | 9 | ## Instructions 10 | 11 | 1. Download and add a tracepusher binary to your `PATH` 12 | 1. Install `helm trace` plugin: 13 | 14 | ``` 15 | helm plugin install https://github.com/agardnerit/helm-trace 16 | ``` 17 | 18 | Full documentation is available on the [helm trace GitHub repo](https://github.com/agardnerIT/helm-trace). -------------------------------------------------------------------------------- /operator/Dockerfile: -------------------------------------------------------------------------------- 1 | # Run build command from root directory 2 | # docker buildx build --platform linux/arm64,linux/amd64 -f Dockerfile --push -t gardnera/tracepusher/operator-v0.1.0 -f operator/Dockerfile . 3 | 4 | FROM python:3.11-alpine 5 | COPY operator/requirements-operator.txt / 6 | COPY operator/operator-logic.py / 7 | COPY tracepusher.py / 8 | RUN apk --update add gcc build-base # required to build some of the following pip packages 9 | RUN pip install -r /requirements-operator.txt # install our dependencies 10 | CMD kopf run -A operator-logic.py # start our operator on container creation -------------------------------------------------------------------------------- /docs/reference/index.md: -------------------------------------------------------------------------------- 1 | ## Technical Reference Pages 2 | 3 | ### Advances Usecases 4 | - [Complex (Multi Span) Traces](multi-span-traces.md) 5 | - [OpenTelemetry Collector Configuration](otel-col.md) 6 | 7 | ### Flags Reference 8 | - [Debug Mode Flag](debug-mode.md) 9 | - [Dry Run Flag](dry-run-mode.md) 10 | - [Span Attribute Types](span-attribute-types.md) 11 | - [Time Shifting](time-shifting.md) 12 | - [Span Events](span-events.md) 13 | - [Span Kind](span-kind.md) 14 | - [Span Durations and Duration Types](duration-type.md) 15 | - [Span Status](span-status.md) 16 | - [Insecure flag](insecure-flag.md) 17 | - [Start Time flag](start-time.md) -------------------------------------------------------------------------------- /samples/jobtraceroperator/multicontainerjob.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: pi-multicontainer 6 | namespace: default 7 | #annotations: 8 | # tracepusher/ignore: "true" 9 | # tracepusher/collector: "http://example.com:4318" 10 | spec: 11 | template: 12 | spec: 13 | containers: 14 | - name: first 15 | image: perl:5.34.0 16 | command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] 17 | - name: second 18 | image: perl:5.34.0 19 | command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(1000)"] 20 | restartPolicy: Never 21 | backoffLimit: 0 -------------------------------------------------------------------------------- /docs/reference/span-kind.md: -------------------------------------------------------------------------------- 1 | ## Span Kind 2 | 3 | The optional flag `-sk` or `--span-kind` allows users to specify the span kind. 4 | 5 | If not specified, tracepusher generates `INTERNAL` type spans. But using the above parameter, a user can override this. 6 | 7 | ### Valid Span Types 8 | 9 | - `UNSPECIFIED` (tracepusher automatically transforms this to `INTERNAL` as per the spec) 10 | - `INTERNAL` (default) 11 | - `CLIENT` 12 | - `SERVER` 13 | - `CONSUMER` 14 | - `PRODUCER` 15 | 16 | ### Example 17 | 18 | ```shell 19 | ./tracepusher \ 20 | -ep http://localhost:4318 \ 21 | -sen serviceA \ 22 | -spn span1 \ 23 | -dur 2 \ 24 | --span-kind CONSUMER 25 | ``` -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: OpenTelemetry tracepusher 2 | nav: 3 | - Home: index.md 4 | - Try tracepusher: try.md 5 | - Usage: usage/index.md 6 | - Usage > Standalone Binary: usage/standalone.md 7 | - Usage > Python: usage/python.md 8 | - Usage > Docker: usage/docker.md 9 | - Usage > CI: usage/ci.md 10 | - Usage > Kubernetes Jobs and CronJobs: usage/k8sjobs.md 11 | - Usage > DevTools HAR File to OpenTelemetry Converter: usage/har-to-otel.md 12 | - Multi Span Traces: reference/multi-span-traces.md 13 | - Reference Pages: reference/index.md 14 | - FAQ: faq.md 15 | - Adopters: adopters.md 16 | - tracepusher on GitHub: https://github.com/agardnerit/tracepusher 17 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | ## Why Does This Exist? 2 | Why, when [tracegen](https://www.jaegertracing.io/docs/1.42/tools/) and the replacement [telemetrygen](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/9597) exist, does this exist? 3 | 4 | This tool does not replace or supercede those tools in any way. For lots of usecases and people, those tools will be better. 5 | 6 | However, they hide the inner-workings (the *how*). For someone getting started or wanting to truly understand what is happening, there is "too much magic". Stuff "just works" whereas tracepusher is more explicit - and thus (I believe) easier to see how the pieces fit together. 7 | 8 | The trace data that tracepusher generates is also customisable whereas "you get what you get" with `tracegen / telemetrygen`. -------------------------------------------------------------------------------- /docker/standard/Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Copy all files into the build-env container 2 | FROM python:3-slim AS build-env 3 | WORKDIR /app 4 | COPY tracepusher.py /app 5 | COPY requirements.txt /app 6 | RUN pip install --no-cache-dir -r /app/requirements.txt 7 | 8 | 9 | # Stage 2: Use Google distroless and only copy in the app files 10 | FROM gcr.io/distroless/python3 11 | 12 | COPY --from=build-env /app /app 13 | # Copy site-packages (which contains the 'requests' module from 'build-env' into the new image) 14 | COPY --from=build-env /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages 15 | WORKDIR /app 16 | # Make sure Python knows where to find the 'requests' module 17 | ENV PYTHONPATH=/usr/local/lib/python3.11/site-packages 18 | 19 | ENTRYPOINT ["python", "./tracepusher.py"] 20 | -------------------------------------------------------------------------------- /samples/otel/collector/config.conf: -------------------------------------------------------------------------------- 1 | receivers: 2 | otlp: 3 | protocols: 4 | grpc: 5 | http: 6 | 7 | processors: 8 | batch: 9 | send_batch_max_size: 1 10 | timeout: 1s 11 | send_batch_size : 1 12 | 13 | memory_limiter: 14 | check_interval: 1s 15 | limit_percentage: 70 16 | spike_limit_percentage: 30 17 | 18 | exporters: 19 | logging: 20 | verbosity: detailed 21 | 22 | otlphttp: 23 | endpoint: https://abc12345.live.dynatrace.com/api/v2/otlp 24 | headers: 25 | Authorization: "Api-Token dt0c01.sample.secret" 26 | 27 | service: 28 | extensions: [] 29 | pipelines: 30 | traces: 31 | receivers: [otlp] 32 | processors: [batch] 33 | exporters: [otlphttp,logging] 34 | metrics: 35 | receivers: [otlp] 36 | processors: [memory_limiter,batch] 37 | exporters: [otlphttp] 38 | -------------------------------------------------------------------------------- /docs/usage/ci.md: -------------------------------------------------------------------------------- 1 | ## tracepusher CI-ready image 2 | 3 | > ℹ️ v0.8.0 and above have standalone, platform-specific binaries which are probably easier to use and better suited to this usecase. 4 | > 5 | > We suggest trying the standalone binary (attached to every GitHub release) first - before using this docker image. 6 | > 7 | > We would love your feedback as we consider retiring this `-ci` image in future. 8 | 9 | The `gardnera/tracepusher:v0.8.0-ci` image is CI ready. 10 | 11 | This containers drops you into a normal shell where you have access to various tools like openssl (for generating UUIDs). 12 | 13 | Tracepusher can be executed using Python from within this container. See [Python usage instructions](python.md) for more info. 14 | 15 | ### Example: Tracing GitLab pipelines 16 | 17 | See [Tracing GitLab pipelines with tracepusher on YouTube](https://youtu.be/zZDFQNHepyI) for a walkthrough and get started with [the sample script](../../samples/gitlab/README.md). 18 | -------------------------------------------------------------------------------- /docs/reference/start-time.md: -------------------------------------------------------------------------------- 1 | ## Start Time 2 | 3 | > Introduced in v0.10.0 4 | 5 | The optional flag `-st` or `--start-time` allows users to specify the span start time. 6 | 7 | If not specified, tracepusher assumes a start time of `now`. 8 | 9 | The two valid formats are: 10 | 11 | 1) A 19 digit string representing milliseconds since the epoch: eg. 1700967916494000000 12 | 2) "%Y-%m-%dT%H:%M:%S.%fZ" eg. "2023-11-26T03:05:16.844Z" 13 | 14 | ## Example 1: Unix timestamp 15 | 16 | ``` 17 | ./tracepusher \ 18 | --endpoint http://localhost:4318 \ 19 | --span-name spanOne \ 20 | --service-name serviceOne \ 21 | --duration 2 \ 22 | --duration-type s \ 23 | --start-time 1700967916494000000 24 | ``` 25 | 26 | ## Example 2: DateTime Format 27 | 28 | ``` 29 | ./tracepusher \ 30 | --endpoint http://localhost:4318 \ 31 | --span-name spanOne \ 32 | --service-name serviceOne \ 33 | --duration 2 \ 34 | --duration-type s \ 35 | --start-time 2023-11-26T03:05:16.844Z 36 | ``` -------------------------------------------------------------------------------- /docs/reference/duration-type.md: -------------------------------------------------------------------------------- 1 | ## Span Duration and Duration Type 2 | 3 | The optional flag `-dt` or `--duration-type` allows users to specify the span duration type. 4 | 5 | If not specified, tracepusher generates spans of a duration type in `seconds`. Using the above parameter, a user can override this. 6 | 7 | ### Valid Span Duration Types 8 | 9 | - `s`: seconds (default) 10 | - `ms` 11 | 12 | ### Examples 13 | 14 | Generate a 2 second long span: 15 | 16 | ```shell 17 | ./tracepusher \ 18 | -ep http://localhost:4318 \ 19 | -sen serviceA \ 20 | -spn span1 \ 21 | -dur 2 22 | ``` 23 | 24 | equivalent to: 25 | 26 | ```shell 27 | ./tracepusher \ 28 | -ep http://localhost:4318 \ 29 | -sen serviceA \ 30 | -spn span1 \ 31 | -dur 2 \ 32 | --duration-type s 33 | ``` 34 | 35 | Generate a span of `1234` milliseconds: 36 | 37 | ```shell 38 | ./tracepusher \ 39 | -ep http://localhost:4318 \ 40 | -sen serviceA \ 41 | -spn span1 \ 42 | -dur 1234 \ 43 | --duration-type ms 44 | ``` -------------------------------------------------------------------------------- /har-to-otel/Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Copy all files into the build-env container 2 | FROM python:3-slim AS build-env 3 | WORKDIR /app 4 | COPY har-to-otel/har-to-otel.py /app 5 | COPY tracepusher.py /app 6 | COPY requirements.txt /app 7 | RUN pip install --no-cache-dir -r /app/requirements.txt 8 | 9 | 10 | # Stage 2: Use Google distroless and only copy in the app files 11 | FROM gcr.io/distroless/python3 12 | 13 | COPY --from=build-env /app /app 14 | # Copy site-packages (which contains the 'requests' module from 'build-env' into the new image) 15 | COPY --from=build-env /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages 16 | WORKDIR /app 17 | # Make sure Python knows where to find the 'requests' module 18 | ENV PYTHONPATH=/usr/local/lib/python3.12/site-packages 19 | 20 | ENTRYPOINT ["python", "./har-to-otel.py"] 21 | 22 | # Run from main directory (not har-to-otel) 23 | # docker buildx build --push --platform linux/arm64,linux/amd64 -t gardnera/har-to-otel:0.10.0 -f har-to-otel/Dockerfile . -------------------------------------------------------------------------------- /docs/usage/index.md: -------------------------------------------------------------------------------- 1 | ## Use tracepusher in various contexts 2 | 3 | ### Trace Kubernetes Jobs and CronJobs 4 | 5 | See [trace Kubernetes Jobs and Cronjobs](k8sjobs.md) for more information. 6 | 7 | ### Standalone Binary 8 | 9 | For most, this will be the preferred way to run tracepusher. 10 | 11 | Download the [relevant binary from the releases page on GitHub](https://github.com/agardnerit/tracepusher/releases/latest) and execute. 12 | 13 | See [run tracepusher as a standalone binary](standalone.md) for more information. 14 | 15 | ### Docker 16 | 17 | [Use tracepusher as a Docker image](docker.md) 18 | 19 | ### Docker CI-ready Image 20 | 21 | [tracepusher inside a Docker image w/ extra tools](ci.md) (eg. openssl). 22 | 23 | Useful for running CI pipelines in container images. 24 | 25 | ### Python 26 | 27 | [Run tracepusher as a Python script](python.md). 28 | 29 | ### HAR File to OpenTelemetry Converter 30 | 31 | This utility takes a `.har` file (HTTP Archive) as input, converts to OpenTelemetry and sends to the collector. 32 | 33 | Go here for the [HAR file to OpenTelemetry converter](har-to-otel.md) 34 | -------------------------------------------------------------------------------- /docs/usage/docker.md: -------------------------------------------------------------------------------- 1 | # Requirements and Prequisites 2 | - A running OpenTelemetry collector 3 | - Docker 4 | 5 | ## Basic Docker Usage 6 | 7 | ``` 8 | docker run gardnera/tracepusher:v0.8.0 \ 9 | -ep http(s)://OTEL-COLLECTOR-ENDPOINT:4318 \ 10 | -sen service_name \ 11 | -spn span_name \ 12 | -dur SPAN_TIME_IN_SECONDS 13 | ``` 14 | 15 | ### Optional Parameters 16 | 17 | ``` 18 | --duration-type ms|s (defaults to `s` > seconds) 19 | --dry-run True|False 20 | --debug True|False 21 | --time-shift True|False 22 | --parent-span-id <16 character hex id> 23 | --trace-id <32 character hex id> 24 | --span-id <16 character hex id> 25 | --span-attributes key=value key2=value2=type 26 | --span-events timeOffsetInMillis=EventName=AttributeKey=AttributeValue=type [event2...] [event3...] 27 | --span-kind UNSPECIFIED|INTERNAL|CLIENT|SERVER|CONSUMER|PRODUCER (defaults to `INTERNAL`) 28 | ``` 29 | 30 | For span atttribute types, see [Span Attribute Types](../reference/span-attribute-types.md). 31 | 32 | For span events, see [Span events](../reference/span-events.md) 33 | 34 | For multi-span traces, see [multi span traces](../reference/multi-span-traces.md) 35 | 36 | For duration type, see [duration type](../reference/duration-type.md) 37 | 38 | For span kind, see [span kind](../reference/span-kind.md) -------------------------------------------------------------------------------- /docs/reference/span-attribute-types.md: -------------------------------------------------------------------------------- 1 | ## Span Attribute Types 2 | 3 | The optional `-spnattrs` or equivalent long form version: `--span-attributes` exists to add span attributes to the spans that tracepusher creates. 4 | 5 | Add as many attributes as you like. 6 | 7 | ### Formatting Span Attributes 8 | 9 | Tracepusher will accept two possible inputs: 10 | 11 | - `--span-attributes foo=bar` 12 | - `--span-attributes foo=bar=` 13 | 14 | In the first, the value is assumed to be of type `stringValue`. 15 | 16 | In the second, **you** specify the value type. Possible types are: `stringValue`, `boolValue`, `intValue`, `doubleValue`, `arrayValue`, `kvlistValue` or `bytesValue`. 17 | 18 | Separate each attribute with a space. 19 | 20 | ``` 21 | python tracepusher.py \ 22 | --endpoint http(s)://OTEL-COLLECTOR-ENDPOINT:4318 23 | --service-name service_name \ 24 | --span-name spanA \ 25 | --duration 2 \ 26 | --span-attributes foo=bar foo2=23=intValue 27 | ``` 28 | 29 | ``` 30 | docker run gardnera/tracepusher:v0.8.0 \ 31 | -ep http(s)://OTEL-COLLECTOR-ENDPOINT:4318 \ 32 | -sen service_name \ 33 | -spn span_name \ 34 | -dur SPAN_TIME_IN_SECONDS \ 35 | -spnattrs foo=bar foo2=bar2=stringValue 36 | ``` 37 | 38 | ### Valid Types 39 | 40 | The following are all valid: 41 | 42 | - `stringValue` 43 | - `boolValue` 44 | - `intValue` 45 | - `doubleValue` 46 | - `arrayValue` 47 | - `kvlistValue` 48 | - `bytesValue` -------------------------------------------------------------------------------- /docs/usage/standalone.md: -------------------------------------------------------------------------------- 1 | # Download binary 2 | 3 | Download the relevant binary from the [GitHub releases page](https://github.com/agardnerit/tracepusher/releases/latest). 4 | 5 | ## Run tracepusher 6 | 7 | ``` 8 | ./tracepusher \ 9 | -ep http(s)://OTEL-COLLECTOR-ENDPOINT:4318 \ 10 | --insecure false \ 11 | -sen service_name \ 12 | -spn span_name \ 13 | -dur SPAN_TIME_IN_SECONDS 14 | ``` 15 | 16 | ### Optional Parameters 17 | 18 | ``` 19 | --dry-run True|False 20 | --debug True|False 21 | --time-shift True|False 22 | --duration-type ms|s (defaults to `s` > seconds) 23 | --parent-span-id <16 character hex id> 24 | --trace-id <32 character hex id> 25 | --span-id <16 character hex id> 26 | --span-attributes key=value key2=value2=type 27 | --span-events timeOffsetInMillis=EventName=AttributeKey=AttributeValue=type [event2...] [event3...] 28 | --span-kind UNSPECIFIED|INTERNAL|CLIENT|SERVER|CONSUMER|PRODUCER (defaults to `INTERNAL`) 29 | --span-status OK|ERROR (defaults to OK) 30 | ``` 31 | 32 | For span atttribute types, see [Span Attribute Types](../reference/span-attribute-types.md). 33 | 34 | For span events, see [Span events](../reference/span-events.md) 35 | 36 | For multi-span traces, see [multi span traces](../reference/multi-span-traces.md) 37 | 38 | For duration type, see [duration type](../reference/duration-type.md) 39 | 40 | For span kind, see [span kind](../reference/span-kind.md) 41 | -------------------------------------------------------------------------------- /docs/reference/span-status.md: -------------------------------------------------------------------------------- 1 | ## Span Status 2 | 3 | The optional flag `-ss` or `--span-status` allows users to specify the span status. 4 | 5 | If not specified, tracepusher assumes an `OK` status. 6 | 7 | For reference, these map to values of `0` (Unset), `1` (OK) or `2` (Error) according to the [OpenTelemetry specification](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto#L270-#L278). 8 | 9 | ### Valid Span Statuses 10 | 11 | These are case insensitive: 12 | 13 | - `OK` (default) 14 | - `ERROR` 15 | - `UNSET` (if you use anything other than "OK" or "ERROR") 16 | 17 | ### Examples 18 | 19 | #### Defaults to OK 20 | 21 | ```shell 22 | ./tracepusher \ 23 | -ep http://localhost:4318 \ 24 | -sen serviceA \ 25 | -spn span1 \ 26 | -dur 2 27 | ``` 28 | 29 | #### Explicitly set to OK 30 | 31 | ```shell 32 | ./tracepusher \ 33 | -ep http://localhost:4318 \ 34 | -sen serviceA \ 35 | -spn span1 \ 36 | -dur 2 \ 37 | --span-status OK 38 | ``` 39 | 40 | #### Explicitly set to Error 41 | ```shell 42 | ./tracepusher \ 43 | -ep http://localhost:4318 \ 44 | -sen serviceA \ 45 | -spn span1 \ 46 | -dur 2 \ 47 | --span-status ERROR 48 | ``` 49 | 50 | #### Invalid value (defaults to Unset) 51 | 52 | ```shell 53 | ./tracepusher \ 54 | -ep http://localhost:4318 \ 55 | -sen serviceA \ 56 | -spn span1 \ 57 | -dur 2 \ 58 | --span-status ABC123 59 | ``` -------------------------------------------------------------------------------- /docs/usage/python.md: -------------------------------------------------------------------------------- 1 | ## Requirements and Prequisites 2 | - A running OpenTelemetry collector 3 | - Python + Requests module 4 | 5 | ## Basic Python Usage 6 | 7 | `python3 tracepusher.py -h` or `python3 tracepusher.py --help` shows help text. 8 | 9 | ``` 10 | python tracepusher.py \ 11 | --endpoint http(s)://OTEL-COLLECTOR-ENDPOINT:4318 \ 12 | --service-name service_name \ 13 | --span-name spanA \ 14 | --duration 2 15 | ``` 16 | 17 | ### Optional Parameters 18 | 19 | ``` 20 | --duration-type ms|s (defaults to `s` > seconds) 21 | --dry-run True|False 22 | --debug True|False 23 | --time-shift True|False 24 | --parent-span-id <16 character hex id> 25 | --trace-id <32 character hex id> 26 | --span-id <16 character hex id> 27 | --span-attributes key=value [key2=value2...] 28 | --span-events timeOffsetInMillis=EventName=AttributeKey=AttributeValue=type [event2...] [event3...] 29 | --span-kind UNSPECIFIED|INTERNAL|CLIENT|SERVER|CONSUMER|PRODUCER (defaults to `INTERNAL`) 30 | ``` 31 | 32 | For information on span atttributes and span attribute types, see [Span Attribute Types](../reference/span-attribute-types.md). 33 | 34 | For information on span events, see [Span Events](../reference/span-events.md) 35 | 36 | For multi-span traces, see [multi span traces](../reference/multi-span-traces.md) 37 | 38 | For duration type, see [duration type](../reference/duration-type.md) 39 | 40 | For span kind, see [span kind](../reference/span-kind.md) -------------------------------------------------------------------------------- /docs/usage/har-to-otel.md: -------------------------------------------------------------------------------- 1 | # Chrome DevTools HAR File to OpenTelemetry Converter 2 | 3 | ![tracepusher HAR to OpenTelemetry Converter](assets/har-to-otel.jpg) 4 | 5 | This tool converts a `.har` file to OpenTelemetry traces and sends them to an OpenTelemetry collector using tracepusher. 6 | 7 | Download this tool from the [tracepusher releases assets on GitHub](https://github.com/agardnerIT/tracepusher/releases). 8 | 9 | ## Prerequisites 10 | 11 | This tool requires either: 12 | 13 | - A copy of the `tracepusher` binary >= `v0.10.0` in the `PATH` 14 | - A copy of [tracepusher.py](https://github.com/agardnerIT/tracepusher/blob/main/tracepusher.py) ( >= `v0.10.0`) in the same directory as `har-to-otel` 15 | 16 | If you can run `./tracepusher version` and get a version >= `0.10.0`, you're good to proceed. 17 | 18 | ## Usage 19 | 20 | ``` 21 | ./har-to-otel -f /path-to-file/YOUR-HAR-FILE.har -ep http://otel-collector-url:4318 --insecure true 22 | ``` 23 | 24 | ### Optional flags 25 | 26 | If set, these are added as span attributes: 27 | 28 | - `--timings [true|false]` (defaults to `true`) 29 | - `--request-headers [true|false]` (defaults to `false`) 30 | - `--response-headers [true|false]` (defaults to `false`) 31 | - `--request-cookies [true|false]` (defaults to `false`) 32 | - `--response-cookies [true|false]` (defaults to `false`) 33 | - `--debug [true|false]` (defaults to `false`) 34 | - `--dry-run [true|false]` (defaults to `false`) 35 | - `--version` (prints the `har-to-otel` version) -------------------------------------------------------------------------------- /docs/reference/insecure-flag.md: -------------------------------------------------------------------------------- 1 | ## Insecure Flag 2 | 3 | > Introduced in v0.9.0 4 | 5 | Default: `false` 6 | 7 | The optional `-insec [false|true]` or `--insecure [false|true]` flag exists to encourage "secure by default" practices by encouraging the sending of span only to `https://` endpoints. However, tracepusher **does** still work with `http://` endpoints. 8 | 9 | The `--insecure` flag affects whether or not tracepusher will connect to insecure `http://` endpoints or not. 10 | 11 | The `--insecure` flag operation differs by version. 12 | 13 | ### v0.8.* 14 | 15 | The `--insecure` is not available 16 | 17 | ### v0.9.* 18 | 19 | The `--insecure` flag defaults to `false` with the intention of meaning insecure endpoints are not allowed. However, to provide ample migration time for end users, the behaviour is as follows: 20 | 21 | #### `--insecure` flag is omitted 22 | 23 | This is the expected behaviour of everyone migrating from v0.8 to v0.9. 24 | 25 | The flag defaults to `false` BUT will still allow `http://` endpoints, just like before. 26 | 27 | Tracepusher will emit a soft `WARNING` message to inform users of the upcoming breaking change, like this: 28 | 29 | ``` 30 | WARN: --insecure flag is omitted or is set to false. Prior to v1.0 tracepusher still works as expected (span is sent). In v1.0 and above, you MUST set '--insecure true' if you want to send to an http:// endpoint. See https://github.com/agardnerIT/tracepusher/issues/78 31 | ``` 32 | 33 | #### `--insecure` flag is explicitly set to false 34 | 35 | From v0.9 upwards, users are encouraged to get into the best practice habit of explicitly setting this to `false` or `true`. 36 | 37 | Otherwise, for v0.9.*, the behaviour is as above. 38 | 39 | ### v1.0 40 | 41 | If the `--insecure` flag is omitted or explicitly set to `false`, calls to `http://` endpoints will be `BLOCKED`. 42 | 43 | Calls to `http://` endpoints MUST be accompanied with the `--insecure true` flag or calls will be blocked with this error: 44 | 45 | ``` 46 | ERROR: Endpoint is http:// (insecure). You MUST set '--insecure true'. Span has NOT been sent. 47 | ``` 48 | 49 | -------------------------------------------------------------------------------- /docs/reference/span-events.md: -------------------------------------------------------------------------------- 1 | ## Span Events 2 | 3 | The optional `-spnevnts` or equivalent long form version: `--span-events` exists to add span events to the spans that tracepusher creates. 4 | 5 | Add as many events as you like. 6 | 7 | ### Formatting Span Events 8 | 9 | Span events are formatted as follows. The first 4 parameters are mandatory. The fifth is optional. 10 | 11 | ``` 12 | === 13 | ``` 14 | 15 | In the first, the value is assumed to be of type `stringValue`. 16 | 17 | or 18 | 19 | ``` 20 | ==== 21 | ``` 22 | 23 | For example, to push an event that should be denoted at 100 milliseconds _after_ the span start time, where the event name is `eventA`, the key is `feature_flag.key`, the value is `hexColor` and the event value type (implied) is `stringValue`: 24 | 25 | ``` 26 | ./tracepusher \ 27 | --endpoint http://localhost:4318 \ 28 | --service-name serviceA \ 29 | --span-name span1 \ 30 | --duration 2 \ 31 | --span-events 0=eventA=feature_flag.key=hexColor 32 | ``` 33 | 34 | To send an event that should be attached at the beginning of the span, with a key of `userID` and a type set as an integer: 35 | 36 | ``` 37 | ./tracepusher \ 38 | --endpoint http://localhost:4318 \ 39 | --service-name serviceA \ 40 | --span-name span1 \ 41 | --duration 2 \ 42 | --span-events 0=eventA=userID=23=intValue 43 | ``` 44 | 45 | Tracepusher will accept two possible inputs: 46 | 47 | - `--span-events 0=eventName=key=vaue` 48 | - `--span-events 0=eventName=key=value=` 49 | 50 | ## Send Multiple Span Events 51 | 52 | Separate each event with a space. 53 | 54 | ``` 55 | ./tracepusher \ 56 | --endpoint http(s)://OTEL-COLLECTOR-ENDPOINT:4318 57 | --service-name service_name \ 58 | --span-name spanA \ 59 | --duration 2 \ 60 | --span-events 0=eventA=foo=bar 0=eventA=userID=23=intValue 61 | ``` 62 | 63 | ``` 64 | ./tracepusher \ 65 | -ep http(s)://OTEL-COLLECTOR-ENDPOINT:4318 \ 66 | -sen service_name \ 67 | -spn span_name \ 68 | -dur SPAN_TIME_IN_SECONDS \ 69 | -spnevnts 0=eventA=foo=bar 0=eventA=userID=23=intValue 70 | ``` 71 | 72 | ### Valid Types 73 | 74 | The following are all valid: 75 | 76 | - `stringValue` 77 | - `boolValue` 78 | - `intValue` 79 | - `doubleValue` 80 | - `arrayValue` 81 | - `kvlistValue` 82 | - `bytesValue` 83 | -------------------------------------------------------------------------------- /samples/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Download tracepusher binary 4 | # Do this ONCE, outside of the shell script! 5 | # This binary shown below is for MacOS 6 | # Change tracepusher_binary as appropriate for your environment 7 | # windows = tracepusher_${tracepusher_version}.exe 8 | # linux = tracepusher_linux_x64_${tracepusher_version} 9 | # 10 | # If you can't sudo, remove that line and run tracepusher 11 | # from the local directory 12 | # ie. change "tracepusher" to "./tracepusher" on lines 45 and 65 13 | # 14 | # == DOWNLOAD CODE - DO THIS ONCE == 15 | # tracepusher_version=0.10.0 16 | # tracepusher_binary=tracepusher_darwin_${tracepusher_version} 17 | # wget --quiet -O tracepusher https://github.com/agardnerIT/tracepusher/releases/download/${tracepusher_version}/${tracepusher_binary} 18 | # chmod +x tracepusher 19 | # sudo mv tracepusher /usr/local/bin 20 | 21 | trace_id=$(openssl rand -hex 16) 22 | span_id=$(openssl rand -hex 8) 23 | 24 | echo "trace_id: ${trace_id}" 25 | echo "span_id: ${span_id}" 26 | 27 | main_time_start=0 28 | echo "main time_start: ${main_time_start}" 29 | 30 | counter=1 31 | limit=3 32 | 33 | while [ $counter -le $limit ] 34 | do 35 | # This is unique to this span 36 | sub_span_id=$(openssl rand -hex 8) 37 | time_start=$SECONDS 38 | echo "loop: ${counter}" 39 | sleep 1 40 | time_end=$SECONDS 41 | duration=$(( $time_end - $time_start )) 42 | echo "loop time_start: ${time_start}. time_end: ${time_end}. duration: ${duration}" 43 | 44 | tracepusher \ 45 | --endpoint=http://localhost:4318 \ 46 | --service-name=serviceA \ 47 | --span-name="subspan${counter}" \ 48 | --duration=${duration} \ 49 | --trace-id=${trace_id} \ 50 | --parent-span-id=${span_id} \ 51 | --span-id=${sub_span_id} \ 52 | --time-shift=True 53 | echo "pushing subspan: ${sub_span_id} with span name: subspan${counter}. trace id: ${trace_id} and parent span id: ${span_id} and time shifted" 54 | 55 | counter=$(( $counter + 1 )) 56 | 57 | done 58 | 59 | time_end=$SECONDS 60 | 61 | echo "main time_start: ${main_time_start}. time_end: ${time_end}" 62 | 63 | echo "pushing main_trace with duration: ${time_end} and trace_id: ${trace_id} and span_id=${span_id} and time shifted" 64 | tracepusher \ 65 | --endpoint=http://localhost:4318 \ 66 | --service-name=serviceA \ 67 | --span-name="main_span" \ 68 | --duration=${time_end} \ 69 | --trace-id=${trace_id} \ 70 | --span-id=${span_id} \ 71 | --time-shift=True 72 | -------------------------------------------------------------------------------- /operator/crds.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: jobtracers.tracers.tracepusher.github.io 6 | spec: 7 | scope: Namespaced 8 | group: tracers.tracepusher.github.io 9 | names: 10 | kind: JobTracer 11 | plural: jobtracers 12 | singular: jobtracer 13 | shortNames: 14 | - jt 15 | versions: 16 | - name: v1 17 | served: true 18 | storage: true 19 | schema: 20 | openAPIV3Schema: 21 | type: object 22 | properties: 23 | spec: 24 | type: object 25 | default: {} 26 | properties: 27 | collectorEndpoint: 28 | type: string 29 | default: "" 30 | additionalPrinterColumns: 31 | - name: "Collector Endpoint" 32 | type: string 33 | description: "Default collector endpoint for this JobTracer" 34 | jsonPath: .spec.collectorEndpoint 35 | --- 36 | apiVersion: rbac.authorization.k8s.io/v1 37 | kind: ClusterRole 38 | metadata: 39 | name: jobtracer-operator-cluster-role 40 | rules: 41 | - apiGroups: ["apiextensions.k8s.io"] 42 | resources: ["customresourcedefinitions"] 43 | verbs: ["create"] 44 | - apiGroups: ["tracers.tracepusher.github.io"] 45 | resources: ["jobtracers"] 46 | verbs: ["*"] 47 | - apiGroups: ["batch"] 48 | resources: ["jobs"] 49 | verbs: ["get", "list", "watch"] 50 | --- 51 | apiVersion: v1 52 | kind: ServiceAccount 53 | metadata: 54 | name: jobtracer-service-account 55 | namespace: default 56 | --- 57 | apiVersion: rbac.authorization.k8s.io/v1 58 | kind: ClusterRoleBinding 59 | metadata: 60 | name: jobtracer-role-binding 61 | subjects: 62 | - kind: ServiceAccount 63 | name: jobtracer-service-account 64 | namespace: default 65 | roleRef: 66 | kind: ClusterRole 67 | name: jobtracer-operator-cluster-role 68 | apiGroup: rbac.authorization.k8s.io 69 | --- 70 | apiVersion: apps/v1 71 | kind: Deployment 72 | metadata: 73 | name: jobtracer-operator 74 | labels: 75 | app: jobtracer-operator 76 | spec: 77 | replicas: 1 78 | selector: 79 | matchLabels: 80 | app: jobtracer-operator 81 | template: 82 | metadata: 83 | labels: 84 | app: jobtracer-operator 85 | spec: 86 | serviceAccountName: jobtracer-service-account 87 | containers: 88 | - name: jobtracer-operator 89 | image: gardnera/tracepusher:operator-v0.1.0 -------------------------------------------------------------------------------- /docs/reference/otel-col.md: -------------------------------------------------------------------------------- 1 | This page shows sample configuration and instructions for creating an OpenTelemetry Collector. 2 | 3 | Specifically, the `config.yaml` shows how to send traces to Dynatrace as a backend. 4 | 5 | If you need Dynatrace tenant, [click here to signup for a free trial](https://dynatrace.com/trial). 6 | 7 | ## Download Collector 8 | 9 | The Python script will generate and push a trace to an OpenTelemetry collector. So of course, you need one available. 10 | 11 | If you have a collector already available, go on ahead to run the tool. If you **don't** have one already available, follow these steps. 12 | 13 | Download and extract the collector binary for your platform from [here](https://github.com/open-telemetry/opentelemetry-collector-releases/releases/tag/v0.78.0). 14 | 15 | For example, for windows: `https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.78.0/otelcol-contrib_0.78.0_windows_amd64.tar.gz` 16 | 17 | Unzip and extract so you have the binary (eg. `otelcol.exe`) 18 | 19 | ## Create config.yaml 20 | 21 | The OpenTelemetry collector needs a config file - this is how you decide which trace backend the traces will go to. 22 | 23 | Save this file alongside `otelcol.exe` as `config.yaml`. 24 | 25 | You will need to modify the `otlphttp` code for your backend. The example given is for Dynatrace trace ingest. 26 | For Dynatrace, the API token needs `Ingest OpenTelemetry traces` permissions. 27 | 28 | ``` 29 | receivers: 30 | otlp: 31 | protocols: 32 | grpc: 33 | http: 34 | 35 | processors: 36 | batch: 37 | send_batch_max_size: 1000 38 | timeout: 30s 39 | send_batch_size : 800 40 | 41 | memory_limiter: 42 | check_interval: 1s 43 | limit_percentage: 70 44 | spike_limit_percentage: 30 45 | 46 | exporters: 47 | logging: 48 | verbosity: detailed 49 | 50 | otlphttp: 51 | endpoint: https://abc12345.live.dynatrace.com/api/v2/otlp 52 | headers: 53 | Authorization: "Api-Token dt0c01.sample.secret" 54 | 55 | service: 56 | extensions: [] 57 | pipelines: 58 | traces: 59 | receivers: [otlp] 60 | processors: [batch] 61 | exporters: [otlphttp,logging] 62 | metrics: 63 | receivers: [otlp] 64 | processors: [memory_limiter,batch] 65 | exporters: [otlphttp] 66 | ``` 67 | 68 | ## Start The Collector 69 | 70 | Open a command / terminal window and run: 71 | 72 | ``` 73 | otelcol.exe --config config.yaml 74 | ``` 75 | 76 | Then run tracepusher: 77 | 78 | ``` 79 | python tracepusher.py --endpoint http://localhost:4318 --service-name tracepusher --span-name my-span --duration 2 80 | ``` 81 | -------------------------------------------------------------------------------- /docs/reference/multi-span-traces.md: -------------------------------------------------------------------------------- 1 | ![subspan schematic](../assets/subspan.schematic.excalidraw.png) 2 | ![complex trace](../assets/complex-trace.png) 3 | 4 | > TLDR: Prefer to read code? See the [GitLab sample pipeline](https://github.com/agardnerIT/tracepusher/blob/main/samples/script.sh) for a working example. 5 | 6 | tracepusher `v0.5.0` and above supports tracing any arbitrarily complex multi-span trace. It does so by allowing users to generate and pass in their own `trace-id` and `span-id`. 7 | 8 | Consider the following batch script: 9 | 10 | ``` 11 | #!/bin/bash 12 | 13 | main_time_start=0 14 | 15 | counter=1 16 | limit=5 17 | 18 | while [ $counter -le $limit ] 19 | do 20 | echo "in a loop. interation ${counter}" # 1, 2, 3, 4, 5 21 | done 22 | 23 | main_time_end=$SECONDS 24 | duration=$(( main_time_end - main_time_start )) 25 | 26 | ``` 27 | 28 | As a trace, this would be represented as `1` parent span (that lasts for `5` seconds). "Inside" that parent span would be `5` sub spans, each denoting "once around the loop". 29 | 30 | In the default mode, tracepusher will auto-generate trace and span IDs but you can generate your own and pass them in. For example: 31 | 32 | ``` 33 | # trace_id is 32 digits 34 | # span_id is 16 35 | trace_id=$(openssl rand -hex 16) 36 | span_id=$(openssl rand -hex 8) 37 | ``` 38 | 39 | The parent span would look like the following. Notice the `--time-shift=true` parameter is set. If this **was not** set, the timings would not make sense. 40 | 41 | For more information, see [time shifting](time-shifting.md) 42 | 43 | ### Parent Span Example 44 | 45 | ``` 46 | python3 tracepusher.py \ 47 | --endpoint http://localhost:4318 \ 48 | --service-name "serviceA" \ 49 | --span-name "main_span" \ 50 | --duration ${duration} \ 51 | --trace-id ${trace_id} \ 52 | --span-id ${span_id} \ 53 | --time-shift True 54 | ``` 55 | 56 | ### Sub Span Example 57 | 58 | ``` 59 | # Note: subspan time is tracked independently to "main" span time 60 | while [ $counter -lt $limit ] 61 | do 62 | # Calculate unique ID for this span 63 | sub_span_id=$(openssl rand -hex 8) 64 | sub_span_time_start=$SECONDS 65 | 66 | # Do real work here... 67 | sleep 1 68 | 69 | subspan_time_end=$SECONDS 70 | duration=$$(( $time_end - $time_start )) 71 | counter=$(( $counter + 1 )) 72 | 73 | python3 tracepusher.py \ 74 | --endpoint http://localhost:4318 \ 75 | --service-name serviceA \ 76 | --span-name "subspan${counter}" \ 77 | --duration ${duration} \ 78 | --trace-id ${trace_id} \ 79 | --parent-span-id ${span_id} \ 80 | --span-id ${subspan_id} \ 81 | --time-shift True 82 | done 83 | ``` 84 | -------------------------------------------------------------------------------- /docs/usage/k8sjobs.md: -------------------------------------------------------------------------------- 1 | # Trace Kubernetes Jobs and CronJobs with tracepusher 2 | 3 | ![Trace Kubernetes Jobs with tracepusher](assets/tracepusher-k8s-jobs.png) 4 | 5 | Install the tracepusher job operator to automatically generate OpenTelemetry traces for any `Job` and `CronJobs` on the cluster. 6 | 7 | ### Prerequisites 8 | 9 | First, make sure you have an OpenTelemetry collector running somewhere. 10 | 11 | ### 1. Install Custom Resource Definitions 12 | 13 | Install the tracepusher CRDs: 14 | 15 | ```shell 16 | kubectl apply -f https://raw.githubusercontent.com/agardnerIT/tracepusher/main/operator/crds.yml 17 | ``` 18 | 19 | ### 2. Create One (or more) JobTracer Objects 20 | 21 | A `JobTracer` should be created, one per namespace. 22 | 23 | This defines the defaults for that namespace. 24 | 25 | You can override the details on a per job basis. 26 | 27 | ```shell 28 | cat < 13 | 14 | Want to do a similar thing with logs? Check out [logpusher](https://agardnerit.github.io/logpusher). 15 | 16 | ## Uses 17 | 18 | - [DevTools .har file to OpenTelemetry Converter](usage/har-to-otel.md) 19 | - [Trace Kubernetes Jobs and CronJobs with OpenTelemetry](usage/k8sjobs.md) 20 | - [Trace CICD Pipelines with OpenTelemetry](https://github.com/agardnerIT/tracepusher/blob/main/samples/gitlab/README.md) 21 | - [Trace shell scripts with OpenTelemetry](https://github.com/agardnerIT/tracepusher/blob/main/samples/script.sh) 22 | - [Trace Helm with tracepusher](usage/helm.md) 23 | - [Use tracepusher in a CICD system](usage/ci.md) 24 | - [Logs to traces: Trace GitHub Codespace Creation](https://dev.to/agardnerit/transforming-github-codespace-log-files-to-opentelemetry-traces-23m3) 25 | - Trace anything with OpenTelemetry 26 | 27 | ## Try tracepusher 28 | See [try tracepusher](try.md) 29 | 30 | ## Quick Start 31 | 32 | Tracepusher is available as: 33 | 34 | - [Standalone binaries](usage/standalone.md) 35 | - [Python script](usage/python.md) 36 | - [Docker image](usage/docker.md) 37 | - [Kubernetes Operator](usage/k8sjobs.md) 38 | 39 | Download the binary [from the releases page](https://github.com/agardnerIT/tracepusher/releases/latest) then run: 40 | 41 | ``` 42 | ./tracepusher --endpoint http(s)://OTEL-COLLECTOR-ENDPOINT:4318 \ 43 | --service-name service_name \ 44 | --span-name span_name \ 45 | --duration SPAN_TIME_IN_SECONDS 46 | ``` 47 | 48 | ## Advanced Usage 49 | 50 | See the following pages for advanced usage and reference information for the flags: 51 | 52 | - [Standalone binary usage](usage/standalone.md) 53 | - [Docker usage](usage/docker.md) 54 | - [Python usage](usage/python.md) 55 | - [CI system usage](usage/ci.md) 56 | - [Complex (Multi Span) Traces](reference/multi-span-traces.md) 57 | - [Span time shifting](reference/time-shifting.md) 58 | - [Span attributes and span attribute types](reference/span-attribute-types.md) 59 | - [Span events](reference/span-events.md) 60 | - [Span status](reference/span-status.md) 61 | - [Insecure flag](reference/insecure-flag.md) 62 | - [Start time flag](reference/start-time.md) 63 | - [tracepusher flag reference pages](reference/index.md) 64 | -------------------------------------------------------------------------------- /samples/gitlab/.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: 2 | # Important: Specify the CI tagged image which provides a shell, and has no entrypoint. 3 | name: gardnera/tracepusher:v0.8.0-ci 4 | 5 | variables: 6 | # Set your OTEL collector endpoint address here... 7 | OTEL_COLLECTOR_ENDPOINT: "http://1.2.3.4:4318" 8 | # Debug the job execution steps 9 | #CI_DEBUG_TRACE: "true" 10 | 11 | stages: 12 | - preparation 13 | - build 14 | - verify 15 | 16 | # Do not change the before_script or after_script sections 17 | default: 18 | before_script: 19 | - echo subspan_start_time=$(date +%s) >> /tmp/vars.env 20 | - echo subspan_id=$(openssl rand -hex 8) >> /tmp/vars.env 21 | after_script: 22 | - source /tmp/vars.env . # reconsitute env vars from before_script 23 | - subspan_end_time=$(date +%s) 24 | - duration=$(( subspan_end_time - subspan_start_time )) 25 | - python3 /app/tracepusher.py 26 | --endpoint=$OTEL_COLLECTOR_ENDPOINT 27 | --service-name=$CI_PROJECT_NAME 28 | --span-name=$CI_JOB_NAME 29 | --duration=$duration 30 | --trace-id=$main_trace_id 31 | --span-id=$subspan_id 32 | --parent-span-id=$main_span_id 33 | --time-shift=True 34 | 35 | # Do not change anything in generate-trace-id 36 | # This job should run as the FIRST job in your pipelien 37 | # This generates a main trace id, main span id and starts the main trace timer 38 | # These variables are stored in variables.env 39 | # so that other jobs can use these details 40 | generate-trace-id: 41 | stage: preparation 42 | before_script: [] 43 | after_script: [] 44 | script: 45 | - echo "main_trace_id=$(openssl rand -hex 16)" >> variables.env 46 | - echo "main_span_id=$(openssl rand -hex 8)" >> variables.env 47 | - echo "main_trace_start_time=$(date +%s)" >> variables.env 48 | artifacts: 49 | reports: 50 | dotenv: variables.env # Use artifacts:reports:dotenv to expose the variables to other jobs 51 | 52 | # build-job is an example 53 | # of your actual work 54 | # This is the part you implement 55 | # Add as many jobs and stages as you like 56 | # The traces will be automatically generated for you 57 | build-job: 58 | stage: build 59 | script: 60 | # START: Do your real work here 61 | - echo "in build-job" 62 | - sleep 2 63 | # END: Do your real work here 64 | 65 | second-build-job: 66 | stage: build 67 | script: 68 | # START: Do your real work here 69 | - echo "in second-build-job" 70 | - sleep 1 71 | # END: Do your real work here 72 | 73 | verify-build: 74 | stage: verify 75 | script: 76 | # START: Do your real work here 77 | - echo "in verify-build" 78 | - sleep 3 79 | # END: Do your real work here 80 | 81 | # Do not change anything 82 | # push-main-trace runs in the .post stage 83 | # ie. after everything else 84 | # This utilises tracepusher for the final time 85 | # To push the main trace 86 | # After push-main-trace occurs, the backend system 87 | # has all spans and subspans necessary to build and display the trace 88 | push-main-trace: 89 | stage: .post 90 | after_script: [] 91 | script: 92 | - main_trace_end_time=$(date +%s) 93 | - duration=$(( main_trace_end_time - main_trace_start_time )) 94 | - python3 /app/tracepusher.py 95 | --endpoint=$OTEL_COLLECTOR_ENDPOINT 96 | --service-name=$CI_PROJECT_NAME 97 | --span-name=$CI_PROJECT_NAME-$CI_PIPELINE_ID 98 | --duration=$duration 99 | --trace-id=$main_trace_id 100 | --span-id=$main_span_id 101 | --time-shift=True -------------------------------------------------------------------------------- /.github/workflows/build_standalone_tracepusher_binaries.yml: -------------------------------------------------------------------------------- 1 | name: Build tracepusher Binaries 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | version: 6 | type: string 7 | description: "tracepusher version (eg. 0.8.0)" 8 | 9 | jobs: 10 | win_build_binary: 11 | name: "[WIN] Build Self Contained Binary" 12 | runs-on: windows-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v3 16 | with: 17 | ref: "main" 18 | - name: Configure AWS credentials 19 | uses: aws-actions/configure-aws-credentials@v2.2.0 20 | with: 21 | aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}" 22 | aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 23 | aws-region: "ap-southeast-2" 24 | - name: Install Python and Pip Packages 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: '3.11.4' 28 | cache: 'pip' 29 | - name: "Install Dependencies" 30 | run: pip install -r requirements.txt 31 | - name: "Build Binary" 32 | run: "python -m PyInstaller --onefile tracepusher.py" 33 | - name: "Upload Binary to S3" 34 | run: | 35 | aws s3 cp dist/tracepusher.exe s3://${{ secrets.BUCKET_NAME }}/tracepusher_${{ inputs.version }}.exe 36 | linux_x64_build_binary: 37 | name: "[LIN x64] Build Self Contained Binary" 38 | runs-on: ubuntu-20.04 39 | steps: 40 | - name: Checkout code 41 | uses: actions/checkout@v3 42 | with: 43 | ref: "main" 44 | - name: Configure AWS credentials 45 | uses: aws-actions/configure-aws-credentials@v2.2.0 46 | with: 47 | aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}" 48 | aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 49 | aws-region: "ap-southeast-2" 50 | - name: Install Python and Pip Packages 51 | uses: actions/setup-python@v4 52 | with: 53 | python-version: '3.11.4' 54 | cache: 'pip' 55 | - name: "Install Dependencies" 56 | run: pip install -r requirements.txt 57 | - name: "Build Binary" 58 | run: "python -m PyInstaller --onefile tracepusher.py" 59 | - name: "Rename file" 60 | run: "mv dist/tracepusher dist/tracepusher_linux_x64" 61 | - name: "Upload Binary to S3" 62 | run: | 63 | aws s3 cp dist/tracepusher_linux_x64 s3://${{ secrets.BUCKET_NAME }}/tracepusher_linux_x64_${{ inputs.version }} 64 | mac_os_build_binary: 65 | name: "[MacOS] Build Self Contained Binary" 66 | runs-on: macos-12 67 | steps: 68 | - name: Checkout code 69 | uses: actions/checkout@v3 70 | with: 71 | ref: "main" 72 | - name: Configure AWS credentials 73 | uses: aws-actions/configure-aws-credentials@v2.2.0 74 | with: 75 | aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}" 76 | aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 77 | aws-region: "ap-southeast-2" 78 | - name: Install Python and Pip Packages 79 | uses: actions/setup-python@v4 80 | with: 81 | python-version: '3.11.4' 82 | cache: 'pip' 83 | - name: "Install Dependencies" 84 | run: pip install -r requirements.txt 85 | - name: "Build Binary" 86 | run: "python -m PyInstaller --onefile tracepusher.py" 87 | - name: "Rename file" 88 | run: "mv dist/tracepusher dist/tracepusher_darwin_x64" 89 | - name: "Upload Binary to S3" 90 | run: | 91 | aws s3 cp dist/tracepusher_darwin_x64 s3://${{ secrets.BUCKET_NAME }}/tracepusher_darwin_x64_${{ inputs.version }} 92 | -------------------------------------------------------------------------------- /.github/workflows/build_standalone_har_to_otel_binaries.yml: -------------------------------------------------------------------------------- 1 | name: Build HAR to OpenTelemetry Converter Binaries 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | version: 6 | type: string 7 | description: "har-to-otel version (eg. 0.1.0)" 8 | 9 | jobs: 10 | win_build_binary: 11 | name: "[WIN] Build Self Contained Binary" 12 | runs-on: windows-latest 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v3 16 | with: 17 | ref: "main" 18 | - name: Configure AWS credentials 19 | uses: aws-actions/configure-aws-credentials@v2.2.0 20 | with: 21 | aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}" 22 | aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 23 | aws-region: "ap-southeast-2" 24 | - name: Install Python and Pip Packages 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: '3.11.4' 28 | cache: 'pip' 29 | - name: "Install Dependencies" 30 | run: pip install -r requirements.txt 31 | - name: "Build Binary" 32 | run: "python -m PyInstaller --onefile har-to-otel/har-to-otel.py" 33 | - name: "Upload Binary to S3" 34 | run: | 35 | aws s3 cp dist/har-to-otel.exe s3://${{ secrets.BUCKET_NAME }}/har-to-otel_${{ inputs.version }}.exe 36 | linux_x64_build_binary: 37 | name: "[LIN x64] Build Self Contained Binary" 38 | runs-on: ubuntu-20.04 39 | steps: 40 | - name: Checkout code 41 | uses: actions/checkout@v3 42 | with: 43 | ref: "main" 44 | - name: Configure AWS credentials 45 | uses: aws-actions/configure-aws-credentials@v2.2.0 46 | with: 47 | aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}" 48 | aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 49 | aws-region: "ap-southeast-2" 50 | - name: Install Python and Pip Packages 51 | uses: actions/setup-python@v4 52 | with: 53 | python-version: '3.11.4' 54 | cache: 'pip' 55 | - name: "Install Dependencies" 56 | run: pip install -r requirements.txt 57 | - name: "Build Binary" 58 | run: "python -m PyInstaller --onefile har-to-otel/har-to-otel.py" 59 | - name: "Rename file" 60 | run: "mv dist/har-to-otel dist/har-to-otel_linux_x64" 61 | - name: "Upload Binary to S3" 62 | run: | 63 | aws s3 cp dist/har-to-otel_linux_x64 s3://${{ secrets.BUCKET_NAME }}/har-to-otel_linux_x64_${{ inputs.version }} 64 | mac_os_build_binary: 65 | name: "[MacOS] Build Self Contained Binary" 66 | runs-on: macos-12 67 | steps: 68 | - name: Checkout code 69 | uses: actions/checkout@v3 70 | with: 71 | ref: "main" 72 | - name: Configure AWS credentials 73 | uses: aws-actions/configure-aws-credentials@v2.2.0 74 | with: 75 | aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}" 76 | aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" 77 | aws-region: "ap-southeast-2" 78 | - name: Install Python and Pip Packages 79 | uses: actions/setup-python@v4 80 | with: 81 | python-version: '3.11.4' 82 | cache: 'pip' 83 | - name: "Install Dependencies" 84 | run: pip install -r requirements.txt 85 | - name: "Build Binary" 86 | run: "python -m PyInstaller --onefile har-to-otel/har-to-otel.py" 87 | - name: "Rename file" 88 | run: "mv dist/har-to-otel dist/har-to-otel_darwin_x64" 89 | - name: "Upload Binary to S3" 90 | run: | 91 | aws s3 cp dist/har-to-otel_darwin_x64 s3://${{ secrets.BUCKET_NAME }}/har-to-otel_darwin_x64_${{ inputs.version }} -------------------------------------------------------------------------------- /samples/github_codespaces/creation_log_parser.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | import subprocess 3 | import secrets 4 | 5 | OTEL_COLLECTOR_URL_HTTP = "http://localhost:4318" 6 | SERVICE_NAME = "codespace-tracking" 7 | activity_list = [] 8 | #with open("creation.log", mode="r") as log: 9 | with open("/workspaces/.codespaces/.persistedshare/creation.log", mode="r") as log: 10 | loglines = log.readlines() 11 | 12 | for line in loglines: 13 | 14 | # All "starting activity" lines end with 3 dots 15 | # Except the final line which is a one-off exception 16 | # ================================================================================= 17 | if "..." in line or "Finished configuring codespace." in line: 18 | activity_list.append(line.strip()) 19 | 20 | log_start_time = activity_list[0][:24] 21 | 22 | trace_id = secrets.token_hex(16) 23 | main_span_id = secrets.token_hex(8) 24 | trace_start_time = None 25 | trace_end_time = None 26 | 27 | position = 0 28 | 29 | sub_spans_to_send = [] 30 | 31 | #print(f"Activity list len: {len(activity_list)}") 32 | for activity in activity_list: 33 | # We need to get two activities to create the span 34 | # Start and end 35 | # Imagine this: 36 | # 2024-06-03 11:24:45.950Z Configuration starting... 37 | # 2024-06-03 11:24:49.377Z Creating container... 38 | # 39 | # Here we need to know that "Configuration starting..." began at 11:24:45.950Z 40 | # Ended at 11:24:49.377Z (so duration was 573ms) 41 | # 42 | activity_start_left, activity_start_right = activity[:24], activity[26:] 43 | 44 | activity_start_left_dt = datetime.fromisoformat(activity_start_left) 45 | 46 | # Set trace start time 47 | if trace_start_time is None: trace_start_time = activity_start_left_dt 48 | if trace_end_time is None and position == len(activity_list)-1: trace_end_time = activity_start_left_dt + timedelta(seconds=1) 49 | 50 | try: 51 | activity_end_left = activity_list[position+1][:24] 52 | activity_end_left_dt = datetime.fromisoformat(activity_end_left) 53 | 54 | diff = activity_end_left_dt - activity_start_left_dt 55 | 56 | sub_spans_to_send.append({ 57 | "trace-id": trace_id, 58 | "span-id": secrets.token_hex(8), 59 | "parent-span-id": main_span_id, 60 | "start-time": activity_start_left_dt, 61 | "end-time": activity_end_left_dt, 62 | "span-name": activity_start_right, 63 | "duration-seconds": diff.total_seconds() 64 | }) 65 | except: 66 | main_span_length_seconds = (trace_end_time - trace_start_time).total_seconds() 67 | start_time = str(int(trace_start_time.timestamp()*1000000000)) 68 | duration = str(int(main_span_length_seconds*1000)) 69 | args = [ 70 | "tracepusher", 71 | "--endpoint", OTEL_COLLECTOR_URL_HTTP, 72 | "--insecure", "true", 73 | "--service-name", SERVICE_NAME, 74 | "--span-name", "codespace-creation", 75 | "--start-time", start_time, 76 | "--duration", duration, 77 | "--duration-type", "ms", 78 | "--trace-id", trace_id, 79 | "--span-id", secrets.token_hex(8) 80 | ] 81 | output = subprocess.run(args, capture_output=True, text=True) 82 | 83 | position += 1 84 | 85 | # Now send sub spans 86 | for span in sub_spans_to_send: 87 | start_time = str(int(span['start-time'].timestamp()*1000000000)) 88 | span_name = span['span-name'] 89 | span_parent_id = span['parent-span-id'] 90 | 91 | args = [ 92 | "tracepusher", 93 | "--endpoint", OTEL_COLLECTOR_URL_HTTP, 94 | "--insecure", "true", 95 | "--service-name", SERVICE_NAME, 96 | "--span-name", span_name, 97 | "--start-time", start_time, 98 | "--duration", str(int(span['duration-seconds']*1000)), 99 | "--duration-type", "ms", 100 | "--trace-id", span['trace-id'], 101 | "--span-id", span['span-id'], 102 | "--parent-span-id", span_parent_id 103 | ] 104 | 105 | # Send spans to OTEL collector 106 | subprocess.run(args, capture_output=True, text=True) 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenTelemetry tracepusher 2 | 3 | > OpenTelemetry traces for every usecase. 4 | 5 | ## [> View the tracepusher documentation <](https://agardnerit.github.io/tracepusher) 6 | 7 | Generate and push OpenTelemetry Trace data to an endpoint in JSON format. 8 | 9 | ![architecture](assets/architecture.png) 10 | ![trace](assets/complex-trace.png) 11 | 12 | # Watch: tracepusher in Action 13 | [YouTube: tracepusher tracing a GitLab pipeline](https://www.youtube.com/watch?v=zZDFQNHepyI). 14 | 15 | If you like tracepusher and want to do the same thing with logs, check out [logpusher](https://agardnerit.github.io/logpusher). 16 | 17 | # Uses 18 | 19 | - [.har file to OpenTelemetry Converter](https://agardnerit.github.io/tracepusher/usage/har-to-otel) 20 | - [Trace Kubernetes Jobs and Cronjobs with OpenTelemetry](https://agardnerit.github.io/tracepusher/usage/k8sjobs/) 21 | - [Trace CICD Pipelines with OpenTelemetry](samples/gitlab/README.md) 22 | - [Trace shell scripts with OpenTelemetry](samples/script.sh) 23 | - [Trace Helm with OpenTelemetry](https://github.com/agardnerit/helm-trace) 24 | - 🆕 [Tracking GitHub Codespace creation with OpenTelemetry](samples/github_codespaces/creation_log_parser.py) 25 | - Trace anything with OpenTelemetry! 26 | 27 | # Try In Browser 28 | 29 | See [try tracepusher in-browser without installation](https://agardnerit.github.io/tracepusher/try/). 30 | 31 | ## Standalone Binary 32 | 33 | See [download and use tracepusher as a standalone binary](https://agardnerit.github.io/tracepusher/usage/standalone.md) 34 | 35 | ## Python3 Usage 36 | 37 | See [use tracepusher as a Python script](https://agardnerit.github.io/tracepusher/usage/python). 38 | 39 | 40 | ## Docker Usage 41 | 42 | See [use tracepusher as a docker image](https://agardnerit.github.io/tracepusher/usage/docker/). 43 | 44 | ## CI Usage 45 | 46 | See [run a CI pipeline step as a docker image with tracepusher](https://agardnerit.github.io/tracepusher/usage/ci). 47 | 48 | ## Dry Run Mode 49 | 50 | See [dry run mode flag](https://agardnerit.github.io/tracepusher/reference/dry-run-mode/). 51 | 52 | ## Debug Mode 53 | 54 | See [debug mode flag](https://agardnerit.github.io/tracepusher/reference/debug-mode/). 55 | 56 | ## Time Shifting 57 | 58 | See [time shifting](https://agardnerit.github.io/tracepusher/reference/time-shifting/). 59 | 60 | ## Complex Tracing (Sub Span support) 61 | 62 | See [multi-span traces](https://agardnerit.github.io/tracepusher/reference/multi-span-traces/). 63 | 64 | ## Span Attributes 65 | 66 | > Only supported with `v0.6.0` and above. 67 | 68 | See [span attribute types](https://agardnerit.github.io/tracepusher/reference/span-attribute-types/). 69 | 70 | ## Span Events 71 | 72 | > Only supported with `v0.7.0` and above. 73 | 74 | See [span events](https://agardnerit.github.io/tracepusher/reference/span-events/). 75 | 76 | ## Span Kind 77 | 78 | > Only supported with `v0.8.0` and above. 79 | 80 | See [span kind](https://agardnerit.github.io/tracepusher/reference/span-kind/) 81 | 82 | ## Span Duration and Duration Type 83 | 84 | > Only supported with `v0.8.0` and above. 85 | 86 | tracepusher will generate spans of `n` seconds. 87 | 88 | This behaviour can be overridden by using the `--duration-type` parameter. 89 | 90 | See [duration type](https://agardnerit.github.io/tracepusher/reference/duration-type/) page. 91 | 92 | ## Span Status 93 | 94 | > Only supported with `v0.9.0` and above. 95 | 96 | tracepusher users can set the status of the span (`OK`, `ERROR` or `UNSET`). 97 | 98 | Default is `OK`. 99 | 100 | See [span status](https://agardnerit.github.io/tracepusher/reference/span-status) page. 101 | 102 | ## Insecure flag 103 | 104 | > Only supported with `v0.9.0` and above. 105 | 106 | tracepusher users can set `--insecure [false|true]` to allow sending spans to `http://` vs. `https://` endpoints. 107 | 108 | Defaults to `false` but behaviour differs by version. 109 | 110 | See [insecure flag](https://agardnerit.github.io/tracepusher/reference/insecure-flag) for more info. 111 | 112 | ## Start Time 113 | 114 | > Only supported with `v0.10.0` and above. 115 | 116 | tracepusher users can (optionally) set the span start time using the `--start-time` flag. If unset, defaults to `now`. 117 | 118 | See [start time flag](https://agardnerit.github.io/tracepusher/reference/start-time-flag) for more info. 119 | 120 | ## Spin up OpenTelemetry Collector 121 | 122 | See [OpenTelemetry Collector configuration](https://agardnerit.github.io/tracepusher/reference/otel-col) 123 | 124 | # Adopters 125 | 126 | Do you use tracepusher? Thanks and we'd love to know! 127 | 128 | Submit a PR and add your details to [ADOPTERS.md](ADOPTERS.md) 129 | 130 | # FAQs 131 | 132 | See [FAQ](https://agardnerit.github.io/tracepusher/faq). 133 | 134 | # Breaking Changes 135 | 136 | See [Breaking changes](https://agardnerit.github.io/tracepusher/breaking-changes) 137 | 138 | # Building Standalone Binaries 139 | 140 | Note: PyInstaller is platform dependent. You must build on whatever platform you wish to run tracepusher on. 141 | 142 | When tracepusher is released, the [build_standalone_binaries.yml workflow](.github/workflows/build_standalone_binaries.yml) completes this step and uploads the resulting binaries to S3 where we (currently manually) attach each generated binary to the release notes. 143 | 144 | ``` 145 | python -m PyInstaller --onefile tracepusher.py 146 | ``` 147 | 148 | # Building Docker Image 149 | 150 | Run all build commands from the root directory: 151 | 152 | ``` 153 | docker buildx build --platform linux/arm64,linux/amd64 --push -t tracepusher:dev-ci -f ./docker/ci/Dockerfile . 154 | docker buildx build --platform linux/arm64,linux/amd64 --push -t tracepusher:dev -f ./docker/standard/Dockerfile . 155 | ``` 156 | 157 | # Install Requirements 158 | 159 | For Non-Developers 160 | ``` 161 | pip install -r requirements.txt 162 | ``` 163 | 164 | For Developers 165 | ``` 166 | pip install -r requirements-dev.txt 167 | ``` 168 | 169 | # Testing 170 | 171 | Run the test suite: 172 | 173 | ``` 174 | pytest 175 | ``` 176 | 177 | ---------------------- 178 | 179 | # Contributing 180 | 181 | All contributions are most welcome! 182 | 183 | Get involved: 184 | - Tackle a [good first issue](https://github.com/agardnerIT/tracepusher/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) 185 | - Create an issue to suggest something new 186 | - File a PR to fix something 187 | 188 | 189 | 190 | 191 | 192 | 193 | Made with [contrib.rocks](https://contrib.rocks). 194 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /har-to-otel/har-to-otel.py: -------------------------------------------------------------------------------- 1 | import json 2 | import math 3 | import subprocess 4 | import secrets 5 | import argparse 6 | import sys 7 | 8 | HAR_TO_OTEL_VERSION="0.11.0" 9 | 10 | ##### Start input processing 11 | 12 | parser = argparse.ArgumentParser() 13 | 14 | parser.add_argument('-f','--file', required=True, default="") 15 | parser.add_argument('-ep','--endpoint', required=True, default="") 16 | parser.add_argument('-insec', '--insecure', required=False, default="False") 17 | parser.add_argument('-sen', '--service-name', required=False, default="har-to-otel") 18 | parser.add_argument('-t','--timings', required=False, default="True") 19 | parser.add_argument('-reqh','--request-headers', required=False, default="False") 20 | parser.add_argument('-resph','--response-headers', required=False, default="False") 21 | parser.add_argument('-reqc','--request-cookies', required=False, default="False") 22 | parser.add_argument('-respc','--response-cookies', required=False, default="False") 23 | parser.add_argument('-x','--debug', required=False, default="False") 24 | parser.add_argument('-dr','--dry-run','--dry', required=False, default="False") 25 | parser.add_argument('-v', '--version', action="version", version=HAR_TO_OTEL_VERSION) 26 | 27 | args = parser.parse_args() 28 | 29 | file_to_load = args.file 30 | endpoint = args.endpoint 31 | allow_insecure = args.insecure 32 | service_name = args.service_name 33 | add_timings = args.timings 34 | add_request_headers = args.request_headers 35 | add_response_headers = args.response_headers 36 | add_request_cookies = args.request_cookies 37 | add_response_cookies = args.response_cookies 38 | debug_mode = args.debug 39 | dry_run = args.dry_run 40 | 41 | # Check a file is provided 42 | if file_to_load == "": 43 | sys.exit("ERROR: You must provide the path to a .har file") 44 | 45 | # Check a file is provided 46 | if endpoint == "": 47 | sys.exit("ERROR: You must provide the OpenTelemetry collector endpoint eg. https://localhost:4318") 48 | 49 | ALLOW_INSECURE = False 50 | if allow_insecure.lower() == "true": 51 | ALLOW_INSECURE = True 52 | 53 | # If insecure collector endpoints are disallowed 54 | # and the collector endpoint IS insecure 55 | # fail with an error 56 | if not ALLOW_INSECURE and endpoint.startswith("http://"): 57 | sys.exit("ERROR: An insecure collector endpoint has been provided and the --insecure flag is not set OR set to --insecure false. To enable insecure endpoints, set --insecure true") 58 | # Add timings? 59 | ADD_TIMINGS = True 60 | if add_timings.lower() == "false": 61 | ADD_TIMINGS = False 62 | 63 | # Add request headers? 64 | ADD_REQUEST_HEADERS = False 65 | if add_request_headers.lower() == "true": 66 | ADD_REQUEST_HEADERS = True 67 | # Add response headers? 68 | ADD_RESPONSE_HEADERS = False 69 | if add_response_headers.lower() == "true": 70 | ADD_RESPONSE_HEADERS = True 71 | 72 | # Add request cookies? 73 | ADD_REQUEST_COOKIES = False 74 | if add_request_cookies.lower() == "true": 75 | ADD_REQUEST_COOKIES = True 76 | # Add response cookies? 77 | ADD_RESPONSE_COOKIES = False 78 | if add_response_cookies.lower() == "true": 79 | ADD_RESPONSE_COOKIES = True 80 | 81 | # Debug mode required? 82 | DEBUG_MODE = False 83 | if debug_mode.lower() == "true": 84 | print("> Debug mode is ON") 85 | DEBUG_MODE = True 86 | 87 | # Dry run mode enabled? 88 | DRY_RUN = False 89 | if dry_run.lower() == "true": 90 | print("> Dry run mode is ON. Nothing will actually be sent.") 91 | DRY_RUN = True 92 | 93 | ###### End input processing 94 | 95 | if DEBUG_MODE: 96 | print(f"HAR File: {file_to_load}") 97 | print(f"Endpoint: {endpoint}") 98 | print(f"Allow insecure collector endpoint: {ALLOW_INSECURE}") 99 | print(f"Add timings? {ADD_TIMINGS}") 100 | print(f"Add request headers? {ADD_REQUEST_HEADERS}") 101 | print(f"Add response headers? {ADD_RESPONSE_HEADERS}") 102 | print(f"Add request cookies? {ADD_REQUEST_COOKIES}") 103 | print(f"Add response cookies? {ADD_RESPONSE_COOKIES}") 104 | print(f"Debug mode: {DEBUG_MODE}") 105 | print(f"Service name: {service_name}") 106 | print(f"Dry run mode: {DRY_RUN}") 107 | 108 | # Credit: https://stackoverflow.com/a/14822210 109 | def convert_size(size_bytes): 110 | if size_bytes == 0: 111 | return "0B" 112 | size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") 113 | i = int(math.floor(math.log(size_bytes, 1000))) 114 | p = math.pow(1000, i) 115 | s = round(size_bytes / p, 2) 116 | return "%s %s" % (s, size_name[i]) 117 | 118 | def run_tracepusher(args=""): 119 | # Try to run tracepusher binary 120 | # Also ensure that tracepusher is at version 0.10.0 or above (required for --start-time flag) 121 | # If error occurs, tracepusher is not in the path 122 | # fallback to tracepusher.py in current directory 123 | 124 | print(f"ARGS TO TRACEPUSHER: {args}") 125 | 126 | tracepusher_output = "" 127 | 128 | # first try the local copy 129 | # if that fails or is the wrong version, try the path version 130 | TRACEPUSHER_TRY_PATH = False 131 | try: 132 | tracepusher_output = subprocess.run(f"python3 tracepusher.py --version", capture_output=True, shell=True, text=True) 133 | if tracepusher_output.returncode != 0: # This will happen if tracepusher is available but version < 0.10.0 ie. doesn't have --version flag 134 | TRACEPUSHER_TRY_PATH = True 135 | except: 136 | TRACEPUSHER_TRY_PATH = True 137 | 138 | if not TRACEPUSHER_TRY_PATH: # local tracepusher is the correct version 139 | print("Local tracepusher.py is the correct version. Use it.") 140 | try: 141 | tracepusher_output = subprocess.run(f"python3 tracepusher.py {args}" , capture_output=True, shell=True, text=True) 142 | except: 143 | print("Exception caught running local tracepusher.py") 144 | else: # trying standalone tracepusher 145 | # Local tracepusher.py missing or < 0.10.0. Try path... 146 | try: 147 | tracepusher_output = subprocess.run(f"tracepusher --version", capture_output=True, shell=True, text=True) 148 | 149 | if tracepusher_output.returncode == 0: 150 | # Path tracepusher is a suitable version. Use it 151 | tracepusher_output = subprocess.run(f"tracepusher {args}" , capture_output=True, shell=True, text=True) 152 | else: 153 | raise Exception("Cannot find a valid (>= v0.10.0) copy of tracepusher locally or in PATH. Nothing has been sent. Cannot continue. Exiting.") 154 | except: 155 | sys.exit("ERROR: Cannot find a valid (>= v0.10.0) copy of tracepusher locally or in PATH. Nothing has been sent. Cannot continue. Exiting.") 156 | 157 | return tracepusher_output 158 | 159 | with open(file=file_to_load, mode="r") as har_file: 160 | har_content = har_file.read() 161 | har_json = json.loads(har_content) 162 | 163 | page_and_items_array = [] 164 | 165 | for page in har_json['log']['pages']: 166 | 167 | page_name = page['title'] 168 | page_ref = page['id'] 169 | page_timings = page['pageTimings'] 170 | 171 | new_entry = { 172 | "page_ref": page_ref, 173 | "page_name": page_name, 174 | "pageTimings": page_timings, 175 | "items": [] 176 | } 177 | 178 | for loaded_item in har_json['log']['entries']: 179 | item_page_ref = loaded_item['pageref'] 180 | 181 | if item_page_ref == page_ref: 182 | new_entry['items'].append(loaded_item) 183 | 184 | page_and_items_array.append(new_entry) 185 | 186 | for page in page_and_items_array: 187 | 188 | # Generate a new trace_id for each page 189 | trace_id = secrets.token_hex(16) 190 | page_ref = page['page_ref'] 191 | page_name = page['page_name'] 192 | item_count_for_page = len(page['items']) 193 | 194 | if DEBUG_MODE: 195 | print("-- Processing page --") 196 | print(f"{page_ref} | {page_name} | Items: {item_count_for_page}") 197 | 198 | for loaded_item in page['items']: 199 | item_page_ref = loaded_item['pageref'] 200 | item_name = loaded_item['request']['url'] 201 | 202 | ui_time_dev_tools = round(loaded_item['time']) - round(loaded_item['timings']['blocked']) 203 | span_attributes = "" 204 | 205 | if page_ref == item_page_ref and item_name == page_name: 206 | if DEBUG_MODE: 207 | print(f"Adding pageTimings to {item_page_ref} which matches {page_ref} ({loaded_item['request']['url']})") 208 | 209 | for key in page['pageTimings']: 210 | span_attributes += f"{key}={page['pageTimings'][key]} " 211 | 212 | # Add items from top level 213 | span_attributes += f"serverIPAddress='{loaded_item['serverIPAddress']}' " 214 | # Add request details 215 | span_attributes += f"request.method={loaded_item['request']['method']} " 216 | span_attributes += f"request.httpVersion={loaded_item['request']['httpVersion']} " 217 | # Add request headers 218 | if ADD_REQUEST_HEADERS: 219 | if DEBUG_MODE: 220 | print(f"Adding request headers for {item_name}") 221 | 222 | for header in loaded_item['request']['headers']: 223 | dirty_header_value = header['value'] 224 | clean_header_value = dirty_header_value.replace('\"','') 225 | span_attributes += f"request.headers.{header['name']}=\"{clean_header_value}\" " 226 | # Add request cookies 227 | if ADD_REQUEST_COOKIES: 228 | if DEBUG_MODE: 229 | print(f"Adding request cookies for {item_name}") 230 | 231 | for cookie in loaded_item['request']['cookies']: 232 | cookie_name = cookie['name'] 233 | 234 | for key in cookie.keys(): 235 | span_attributes += f"request.cookies.cookie.{cookie_name}.{key}=\"{cookie[key]}\" " 236 | 237 | # Add response details 238 | try: 239 | span_attributes += f"response.status={loaded_item['response']['status']} " 240 | span_attributes += f"response.statusText=\"{loaded_item['response']['statusText']}\" " 241 | span_attributes += f"response.httpVersion={loaded_item['response']['httpVersion']} " 242 | span_attributes += f"response.content.size={loaded_item['response']['content']['size']} " 243 | span_attributes += f"response._transferSize={loaded_item['response']['_transferSize']} " 244 | except: 245 | pass 246 | 247 | # Add response headers 248 | if ADD_RESPONSE_HEADERS: 249 | if DEBUG_MODE: 250 | print(f"Adding response headers for {item_name}") 251 | 252 | for header in loaded_item['response']['headers']: 253 | dirty_header_value = header['value'] 254 | clean_header_value = dirty_header_value.replace('\"','') 255 | span_attributes += f"response.headers.{header['name']}=\"{clean_header_value}\" " 256 | 257 | # Add request cookies 258 | if ADD_RESPONSE_COOKIES: 259 | if DEBUG_MODE: 260 | print(f"Adding response cookies for {item_name}") 261 | 262 | for cookie in loaded_item['response']['cookies']: 263 | cookie_name = cookie['name'] 264 | 265 | for key in cookie.keys(): 266 | span_attributes += f"response.cookies.cookie.{cookie_name}.{key}=\"{cookie[key]}\" " 267 | 268 | # Add timings 269 | if ADD_TIMINGS: 270 | if DEBUG_MODE: 271 | print(f"Adding timings for {item_name}") 272 | for key in loaded_item['timings']: 273 | span_attributes += f"timings.{key}={loaded_item['timings'][key]} " 274 | 275 | span_id = secrets.token_hex(8) 276 | 277 | args = f"--endpoint {endpoint} --insecure {str(ALLOW_INSECURE).lower()} --service-name {service_name} --span-name '{loaded_item['request']['url']}' --duration {int(ui_time_dev_tools)} --duration-type 'ms' --trace-id {trace_id} --span-id {span_id} --start-time '{loaded_item['startedDateTime']}' --span-attributes {span_attributes} --debug {DEBUG_MODE} --dry-run {DRY_RUN}" 278 | 279 | if DEBUG_MODE: 280 | print(f"Args: {args}") 281 | # Run tracepusher 282 | tracepusher_response = run_tracepusher(args) 283 | 284 | if DEBUG_MODE: 285 | print(tracepusher_response) 286 | 287 | 288 | 289 | if DEBUG_MODE: 290 | print("-"*25) 291 | print(" > Top Level Stats < ") 292 | print(f"Pages in this HAR: {len(har_json['log']['pages'])}") 293 | print(f"page trace id: {trace_id[:7]} ({page_ref} | {page_name})") # Jaeger shows first 7 chars of trace id. Print for debugging. 294 | print("-"*25) 295 | print("Done...") -------------------------------------------------------------------------------- /tracepusher_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import subprocess 3 | 4 | def run_tracepusher(args=""): 5 | output = subprocess.run(f"python3 tracepusher.py {args}" , capture_output=True, shell=True, text=True) 6 | return output 7 | 8 | # Run tracepusher with no input params 9 | # Should error and so check error is present 10 | def test_run_no_params(): 11 | output = run_tracepusher() 12 | assert output.returncode > 0 13 | assert output.stderr != "" 14 | assert "error" in output.stderr 15 | 16 | def test_check_debug_mode(): 17 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 18 | output = run_tracepusher(args) 19 | assert output.returncode == 0 20 | assert "Debug mode is ON" in output.stdout 21 | 22 | def test_check_dry_run_mode(): 23 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 24 | output = run_tracepusher(args) 25 | assert output.returncode == 0 26 | assert "Dry run mode is ON" in output.stdout 27 | 28 | def test_check_collector_url(): 29 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 30 | output = run_tracepusher(args) 31 | assert output.returncode == 0 32 | assert "Collector URL: http://otelcollector:4317" in output.stdout 33 | 34 | def test_check_service_name(): 35 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 36 | output = run_tracepusher(args) 37 | assert output.returncode == 0 38 | assert "Service Name: serviceA" in output.stdout 39 | 40 | def test_check_span_name(): 41 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 42 | output = run_tracepusher(args) 43 | assert output.returncode == 0 44 | assert "Span Name: spanOne" in output.stdout 45 | 46 | def test_trace_length(): 47 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 48 | output = run_tracepusher(args) 49 | assert output.returncode == 0 50 | assert "Trace Length (s): 2" in output.stdout 51 | 52 | def test_check_time_shift_enabled(): 53 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --time-shift true" 54 | output = run_tracepusher(args) 55 | assert output.returncode == 0 56 | assert "Time shift enabled" in output.stdout 57 | assert "Time shifted? True" in output.stdout 58 | 59 | def test_check_time_shift_disabled(): 60 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 61 | output = run_tracepusher(args) 62 | assert output.returncode == 0 63 | assert "Time shifted? False" in output.stdout 64 | 65 | def test_span_kind_internal(): 66 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 67 | output = run_tracepusher(args) 68 | assert output.returncode == 0 69 | assert "'kind': 'SPAN_KIND_INTERNAL'" in output.stdout 70 | 71 | # Tracepusher should respect a span kind 72 | # set to INTERNAL and leave it as such 73 | def test_span_kind_set_to_internal(): 74 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind INTERNAL" 75 | output = run_tracepusher(args) 76 | assert output.returncode == 0 77 | assert "'kind': 'SPAN_KIND_INTERNAL'" in output.stdout 78 | 79 | # Tracepusher works with 80 | # span kind CLIENT 81 | def test_span_kind_set_to_client(): 82 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind CLIENT" 83 | output = run_tracepusher(args) 84 | assert output.returncode == 0 85 | assert "'kind': 'SPAN_KIND_CLIENT'" in output.stdout 86 | 87 | # Tracepusher works with 88 | # span kind SERVER 89 | def test_span_kind_set_to_server(): 90 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind SERVER" 91 | output = run_tracepusher(args) 92 | assert output.returncode == 0 93 | assert "'kind': 'SPAN_KIND_SERVER'" in output.stdout 94 | 95 | # Tracepusher works with 96 | # span kind CONSUMER 97 | def test_span_kind_set_to_consumer(): 98 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind CONSUMER" 99 | output = run_tracepusher(args) 100 | assert output.returncode == 0 101 | assert "'kind': 'SPAN_KIND_CONSUMER'" in output.stdout 102 | 103 | # Tracepusher works with 104 | # span kind PRODUCER 105 | def test_span_kind_set_to_producer(): 106 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind PRODUCER" 107 | output = run_tracepusher(args) 108 | assert output.returncode == 0 109 | assert "'kind': 'SPAN_KIND_PRODUCER'" in output.stdout 110 | 111 | # Tracepusher should transform a 112 | # span kind "UNSPECIFIED" to "INTERNAL" 113 | # automatically 114 | def test_span_kind_unspecified_to_internal(): 115 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind UNSPECIFIED" 116 | output = run_tracepusher(args) 117 | assert output.returncode == 0 118 | assert "'kind': 'SPAN_KIND_INTERNAL'" in output.stdout 119 | 120 | # An error should be thrown 121 | # If an invalid span kind is set 122 | def test_for_invalid_span_kind(): 123 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-kind SOME-INVALID_SPAN-KIND" 124 | output = run_tracepusher(args) 125 | assert output.returncode > 0 126 | assert "Error: invalid span kind provided." in output.stderr 127 | 128 | # Duration type should default to seconds 129 | def test_for_default_duration_type(): 130 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 131 | output = run_tracepusher(args) 132 | assert output.returncode == 0 133 | assert "Duration Type: s" in output.stdout 134 | 135 | # Duration type should be milliseconds 136 | def test_for_duration_type_milliseconds(): 137 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --duration-type ms" 138 | output = run_tracepusher(args) 139 | assert output.returncode == 0 140 | assert "Duration Type: ms" in output.stdout 141 | 142 | # An error should be thrown 143 | # If an invalid duration type is set 144 | def test_for_invalid_duration_type(): 145 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --duration-type hours" 146 | output = run_tracepusher(args) 147 | assert output.returncode > 0 148 | assert "Error: Duration Type invalid." in output.stderr 149 | 150 | # Check setting parent span works 151 | def test_for_parent_span_is_set(): 152 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --parent-span-id abc123" 153 | output = run_tracepusher(args) 154 | assert output.returncode == 0 155 | assert "> Pushing a child (sub) span with parent span id: abc123" in output.stdout 156 | 157 | # Check passing span events works 158 | def test_span_events_set_correctly(): 159 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-events 0=eventA=foo=bar" 160 | output = run_tracepusher(args) 161 | assert output.returncode == 0 162 | assert "'droppedEventsCount': 0" in output.stdout 163 | 164 | # Check passing span events works 165 | def test_sending_multiple_span_events(): 166 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-events 0=eventA=foo=bar 100=eventB=foo=bar" 167 | output = run_tracepusher(args) 168 | assert output.returncode == 0 169 | assert "'droppedEventsCount': 0" in output.stdout 170 | assert "{'key': 'foo', 'value': {'stringValue': 'bar'}}" in output.stdout 171 | assert "{'key': 'foo', 'value': {'stringValue': 'bar'}}" in output.stdout 172 | 173 | # Check passing an invalid 174 | # span event is dropped 175 | # This span event is missing a parameter 176 | def test_drop_invalid_span_event(): 177 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-events 0=foo=bar" 178 | output = run_tracepusher(args) 179 | assert output.returncode == 0 180 | assert "'droppedEventsCount': 1" in output.stdout 181 | 182 | # Check sending valid span attribute 183 | def test_check_valid_span_attribute(): 184 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-attributes foo=bar" 185 | output = run_tracepusher(args) 186 | assert output.returncode == 0 187 | assert "'droppedAttributesCount': 0" in output.stdout 188 | 189 | # Check sending valid span attribute 190 | def test_check_intValue_span_attribute(): 191 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-attributes userID=23=intValue" 192 | output = run_tracepusher(args) 193 | assert output.returncode == 0 194 | assert "'droppedAttributesCount': 0" in output.stdout 195 | assert "'intValue': '23'" in output.stdout 196 | 197 | # Check sending multiple valid span attribute 198 | def test_check_one_valid_one_invalid_span_attribute(): 199 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-attributes foo=bar=dsd=ds userID=23=intValue" 200 | output = run_tracepusher(args) 201 | assert output.returncode == 0 202 | assert "'droppedAttributesCount': 1" in output.stdout 203 | assert "'intValue': '23'" in output.stdout 204 | 205 | # Check that (by default) span 206 | # has status of OK 207 | def test_check_span_status_ok_when_not_set(): 208 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 209 | output = run_tracepusher(args) 210 | assert output.returncode == 0 211 | assert "'status': {'code': 1}" in output.stdout 212 | 213 | # Check that span has status of OK 214 | # When explicitly set 215 | def test_check_span_status_ok_when_set(): 216 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-status OK" 217 | output = run_tracepusher(args) 218 | assert output.returncode == 0 219 | assert "'status': {'code': 1}" in output.stdout 220 | 221 | # Check that span has status of ERROR 222 | # When explicitly set 223 | def test_check_span_status_error_when_set(): 224 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-status ERROR" 225 | output = run_tracepusher(args) 226 | assert output.returncode == 0 227 | assert "'status': {'code': 2}" in output.stdout 228 | 229 | # Check that span has status of UNSET 230 | # When set to something invalid / random 231 | def test_check_span_status_unset_when_set(): 232 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --span-status ABC123" 233 | output = run_tracepusher(args) 234 | assert output.returncode == 0 235 | assert "'status': {'code': 0}" in output.stdout 236 | 237 | # Check that --allow-insecure false 238 | # When flag is omitted 239 | # Also check that WARN message 240 | # TODO: Revisit this for v1.0 241 | def test_check_insecure_flag_false_when_unset(): 242 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true" 243 | output = run_tracepusher(args) 244 | assert output.returncode == 0 245 | assert "allow insecure endpoints: false" or "" in output.stdout.lower() 246 | assert "WARN: --insecure flag is omitted or is set to false. Prior to v1.0 tracepusher still works as expected (span is sent). In v1.0 and above, you MUST set '--insecure true' if you want to send to an http:// endpoint. See https://github.com/agardnerIT/tracepusher/issues/78" in output.stdout 247 | 248 | # Check that --allow-insecure flag false 249 | # When flag is explicitly set 250 | # TODO: Revisit this for v1.0 251 | def test_check_insecure_flag_false_when_set(): 252 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --insecure false" 253 | output = run_tracepusher(args) 254 | assert output.returncode == 0 255 | assert "allow insecure endpoints: false" in output.stdout.lower() 256 | assert "WARN: --insecure flag is omitted or is set to false. Prior to v1.0 tracepusher still works as expected (span is sent). In v1.0 and above, you MUST set '--insecure true' if you want to send to an http:// endpoint. See https://github.com/agardnerIT/tracepusher/issues/78" in output.stdout 257 | 258 | # Check that --allow-insecure flag false 259 | # When flag is explicitly set 260 | # TODO: Revisit this for v1.0 261 | def test_check_insecure_flag_true_when_set(): 262 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --insecure True" 263 | output = run_tracepusher(args) 264 | assert output.returncode == 0 265 | assert "allow insecure endpoints: true" in output.stdout.lower() 266 | 267 | # Check that --trace-id abc123 268 | # Errors (as it should) 269 | # Because trace id "abc123" is too short 270 | def test_check_invalid_trace_length_fails_when_set(): 271 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --insecure True --trace-id abc123" 272 | output = run_tracepusher(args) 273 | assert output.returncode > 0 274 | assert "Error: trace_id should be 32 characters long!" in output.stderr 275 | 276 | # Check that --span-id abc123 277 | # Errors (as it should) 278 | # Because span id "abc123" is too short 279 | def test_check_invalid_span_length_fails_when_set(): 280 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --insecure True --span-id abc123" 281 | output = run_tracepusher(args) 282 | assert output.returncode > 0 283 | assert "Error: span_id should be 16 characters long!" in output.stderr 284 | 285 | # Check that an invalid start_time duration 286 | # warns and defaults to now 287 | def test_check_invalid_too_short_start_time_fails_when_set(): 288 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --start-time abc123" 289 | output = run_tracepusher(args) 290 | assert output.returncode == 0 291 | assert "Got an explicit start time" in output.stdout 292 | assert "WARN: --start-time was in an incorrect format. If providing a date/time it must be in UTC and end with uppercase 'Z'. eg. '2023-11-26T03:05:16.844Z'. Trace will be sent with start_time of now." in output.stdout 293 | 294 | # check that a 19 digit (invalid) start time 295 | # errors 296 | # eg. "2023-11-26T03:05:16" 297 | def test_check_invalid_19_digit_start_time_fails_when_set(): 298 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --start-time 2023-11-26T03:05:16" 299 | output = run_tracepusher(args) 300 | assert output.returncode == 0 301 | assert "Got an explicit start time" in output.stdout 302 | assert "WARN: --start-time was in an invalid format. Trace will be sent but start time will default to 'now'." in output.stdout 303 | assert "Provided start time: 2023-11-26T03:05:16" in output.stdout 304 | #assert "WARN: --start-time value was in an incorrect format. Valid formats: 1) 19 digit integer representing millis since epoch 2) '%Y-%m-%dT%H:%M:%S.%fZ' eg. '2023-11-26T03:05:16.844Z'. Trace will be send with start_time of now." in output.stdout 305 | 306 | # Check that a valid 19 digit start time succeeds 307 | def test_check_valid_19_digit_start_time_success(): 308 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --start-time 1704590804392761000" 309 | output = run_tracepusher(args) 310 | assert output.returncode == 0 311 | assert "Got an explicit start time" in output.stdout 312 | assert "Provided start time: 1704590804392761000" in output.stdout 313 | assert "Start time: 1704590804392761000" in output.stdout 314 | 315 | # Check that an invalid datetime warns 316 | def test_check_invalid_19_digit_start_time_warns(): 317 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --start-time 2023-11-26T03:05:16AEST" 318 | output = run_tracepusher(args) 319 | assert output.returncode == 0 320 | assert "Got an explicit start time" in output.stdout 321 | assert "Provided start time: 2023-11-26T03:05:16AEST" in output.stdout 322 | assert "WARN: --start-time was in an incorrect format. If providing a date/time it must be in UTC and end with uppercase 'Z'. eg. '2023-11-26T03:05:16.844Z'. Trace will be sent with start_time of now." in output.stdout 323 | 324 | # Check that a 19 digit (valid) start time 325 | # that also ends with Z (valid) 326 | # but doesn't logically make sense 327 | # warns and defaults to now() 328 | def test_check_invalid_19_digit_with_end_z_warns(): 329 | args = "-ep http://otelcollector:4317 -sen serviceA -spn spanOne -dur 2 --dry-run true --debug true --start-time 123456789012345678Z" 330 | output = run_tracepusher(args) 331 | assert output.returncode == 0 332 | assert "Got an explicit start time" in output.stdout 333 | assert "Provided start time: 123456789012345678Z" in output.stdout 334 | assert "WARN: --start-time was in an invalid format. Trace will be sent but start time will default to 'now'." in output.stdout 335 | 336 | def test_tracepusher_version(): 337 | args = "--version" 338 | output = run_tracepusher(args) 339 | assert output.returncode == 0 340 | assert "0.10.0" in output.stdout 341 | -------------------------------------------------------------------------------- /tracepusher.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import requests 3 | import time 4 | import secrets 5 | import argparse 6 | import datetime 7 | 8 | TRACEPUSHER_VERSION="0.10.0" 9 | 10 | # This script is very simple. It does the equivalent of: 11 | # curl -i -X POST http(s)://endpoint/v1/traces \ 12 | # -H "Content-Type: application/json" \ 13 | # -d @trace.json 14 | 15 | ############################################################################# 16 | # USAGE 17 | # python tracepusher.py -ep=http(s)://localhost:4318 -sen=serviceNameA -spn=spanX -dur=2 18 | ############################################################################# 19 | 20 | # Modified from: https://stackoverflow.com/a/11111177/9997385 21 | def unix_time_millis(dt): 22 | epoch = datetime.datetime.utcfromtimestamp(0) 23 | # This looks odd but the .0f means no decimal places 24 | # Then appending 000 makes the string 19 chars long like: 25 | # 1700967916494000000 26 | return '{:.0f}000'.format((dt - epoch).total_seconds() * 1000000) 27 | 28 | # Minimum 29 | # offsetInMillis=name=key=value 30 | # In which case: 31 | # - timestamp of event is + offset by milliseconds given in input from start_time_nanos_since_epoch 32 | # - value type is assumed to be string 33 | # 34 | # offsetInMillis=name=key=value=valueType 35 | # 0=event1=userID=23=intValue 36 | def get_span_events_list(args, start_time_nanos_since_epoch): 37 | 38 | event_list = [] 39 | dropped_event_count = 0 40 | 41 | if args == None or len(args) < 1: 42 | return event_list, dropped_event_count 43 | 44 | for item in args: 45 | # How many = are in the item? 46 | # <3 = invalid item. Ignore 47 | # 3 = offsetInMillis=name=key=value (tracepusher assumes type=stringValue) 48 | # 4 = offsetInMillis=name=key=value=type (user is explicitly telling us the type. tracepusher uses it) 49 | # >4 = invalid item. Ignore 50 | number_of_equals = item.count("=") 51 | if number_of_equals != 3 and number_of_equals != 4: 52 | dropped_event_count += 1 53 | continue 54 | 55 | offset_millis = 0 56 | event_name = "" 57 | event_key = "" 58 | event_value = "" 59 | event_type = "stringValue" 60 | dropped_event_count = 0 61 | 62 | if number_of_equals != 3 and number_of_equals != 4: 63 | dropped_event_count += 1 64 | 65 | if number_of_equals == 3: 66 | offset_millis_string, event_name, event_key, event_value = item.split("=", maxsplit=3) 67 | offset_millis = int(offset_millis_string) 68 | # User did not pass a type. Assuming type == 'stringValue' 69 | 70 | if number_of_equals == 4: 71 | offset_millis_string, event_name, event_key, event_value, event_type = item.split('=',maxsplit=4) 72 | offset_millis = int(offset_millis_string) 73 | # User passed an explicit type. Tracepusher will use it. 74 | 75 | # calculate event time 76 | # millis to nanos 77 | offset_nanos = offset_millis * 1000000 78 | event_time = start_time_nanos_since_epoch + offset_nanos 79 | 80 | event_list.append({ 81 | "timeUnixNano": event_time, 82 | "name": event_name, 83 | "attributes": [{ 84 | "key": event_key, 85 | "value": { 86 | event_type: event_value 87 | } 88 | }] 89 | }) 90 | 91 | return event_list, dropped_event_count 92 | 93 | # Returns attributes list: 94 | # From spec: https://opentelemetry.io/docs/concepts/signals/traces/#attributes 95 | # Syntax: { 96 | # "key": "my.scope.attribute", 97 | # "value": { 98 | # "stringValue": "some scope attribute" 99 | # } 100 | # } 101 | # Ref: https://github.com/open-telemetry/opentelemetry-proto/blob/9876ebfc5bcf629d1438d1cf1ee8a1a4ec21676c/examples/trace.json#L20-L56 102 | # Values must be a non-null string, boolean, floating point value, integer, or an array of these values 103 | # stringValue, boolValue, intValue, doubleValue, arrayValue, kvlistValue, bytesValue are all valid 104 | def get_span_attributes_list(args): 105 | 106 | arg_list = [] 107 | dropped_attribute_count = 0 108 | 109 | if args == None or len(args) < 1: 110 | return arg_list, dropped_attribute_count 111 | 112 | for item in args: 113 | # How many = are in the item? 114 | # 0 = invalid item. Ignore 115 | # 1 = key=value (tracepusher assumes type=stringValue) 116 | # 2 = key=value=type (user is explicitly telling us the type. tracepusher uses it) 117 | # >3 = invalid item. tracepusher does not support span keys and value containing equals. Ignore. 118 | number_of_equals = item.count("=") 119 | if number_of_equals == 0 or number_of_equals > 2: 120 | dropped_attribute_count += 1 121 | continue 122 | 123 | key = "" 124 | value = "" 125 | type = "stringValue" 126 | 127 | if number_of_equals == 1: 128 | key, value = item.split("=", maxsplit=1) 129 | # User did not pass a type. Assuming type == 'stringValue' 130 | 131 | if number_of_equals == 2: 132 | key, value, type = item.split('=',maxsplit=2) 133 | # User passed an explicit type. Tracepusher will use it. 134 | 135 | arg_list.append({"key": key, "value": { type: value}}) 136 | 137 | return arg_list, dropped_attribute_count 138 | 139 | def process_span_kind(input): 140 | valid_values = [ 141 | "UNSPECIFIED", 142 | "INTERNAL", 143 | "CLIENT", 144 | "SERVER", 145 | "CONSUMER", 146 | "PRODUCER" 147 | ] 148 | output = "" 149 | output = input.upper() 150 | # If span kind is not valid 151 | # Maintain backwards compatibility 152 | # Default to SPAN_KIND_INTERNAL 153 | # If span kind is set to unspecified 154 | # Default (as per OTEL spec) to INTERNAL 155 | if output not in valid_values: 156 | output = "" 157 | elif output == "UNSPECIFIED": 158 | output = "SPAN_KIND_INTERNAL" 159 | else: 160 | output = f"SPAN_KIND_{output}" 161 | 162 | return output 163 | 164 | def check_duration_type(input): 165 | valid_values = [ "ms", "s" ] 166 | 167 | if input.lower() in valid_values: 168 | return True 169 | else: 170 | return False 171 | 172 | def get_span_status_int(input): 173 | if input.lower() == "ok": 174 | return 1 175 | if input.lower() == "error": 176 | return 2 177 | return 0 178 | 179 | parser = argparse.ArgumentParser() 180 | 181 | # Notes: 182 | # You can use either short or long (mix and match is OK) 183 | # Hyphens are replaced with underscores hence for retrieval 184 | # and leading hyphens are trimmed 185 | # --span-name becomes args.span_name 186 | # Retrieval also uses the second parameter 187 | # Hence args.dry_run will work but args.d won't 188 | parser.add_argument('-ep', '--endpoint', required=True) 189 | parser.add_argument('-sen','--service-name', required=True) 190 | parser.add_argument('-spn', '--span-name', required=True) 191 | parser.add_argument('-dur', '--duration', required=True, type=int) 192 | parser.add_argument('-dt', '--duration-type', required=False, default="s") 193 | parser.add_argument('-dr','--dry-run','--dry', required=False, default="False") 194 | parser.add_argument('-x', '--debug', required=False, default="False") 195 | parser.add_argument('-ts', '--time-shift', required=False, default="False") 196 | parser.add_argument('-psid','--parent-span-id', required=False, default="") 197 | parser.add_argument('-tid', '--trace-id', required=False, default="") 198 | parser.add_argument('-sid', '--span-id', required=False, default="") 199 | parser.add_argument('-spnattrs', '--span-attributes', required=False, nargs='*') 200 | parser.add_argument('-spnevnts', '--span-events', required=False, nargs='*') 201 | parser.add_argument('-sk', '--span-kind', required=False, default="INTERNAL") 202 | parser.add_argument('-ss', '--span-status', required=False, default="OK") 203 | parser.add_argument('-insec', '--insecure', required=False, default="False") 204 | parser.add_argument('-st', '--start-time', required=False, default="") 205 | parser.add_argument('-v', '--version', action="version", version=TRACEPUSHER_VERSION) 206 | 207 | args = parser.parse_args() 208 | 209 | endpoint = args.endpoint 210 | service_name = args.service_name 211 | span_name = args.span_name 212 | duration = args.duration 213 | duration_type = args.duration_type 214 | dry_run = args.dry_run 215 | debug_mode = args.debug 216 | time_shift = args.time_shift 217 | parent_span_id = args.parent_span_id 218 | trace_id = args.trace_id 219 | span_id = args.span_id 220 | span_kind = args.span_kind 221 | span_status = get_span_status_int(args.span_status) 222 | allow_insecure = args.insecure 223 | start_time = args.start_time 224 | 225 | span_attributes_list, dropped_attribute_count = get_span_attributes_list(args.span_attributes) 226 | span_kind = process_span_kind(span_kind) 227 | if span_kind == "": 228 | sys.exit("Error: invalid span kind provided.") 229 | 230 | duration_type_valid = check_duration_type(duration_type) 231 | if not duration_type_valid: 232 | sys.exit("Error: Duration Type invalid. Try `ms` for milliseconds or `s` for seconds") 233 | 234 | # Debug mode required? 235 | DEBUG_MODE = False 236 | if debug_mode.lower() == "true": 237 | print("> Debug mode is ON") 238 | DEBUG_MODE = True 239 | 240 | DRY_RUN = False 241 | if dry_run.lower() == "true": 242 | print("> Dry run mode is ON. Nothing will actually be sent.") 243 | DRY_RUN = True 244 | 245 | TIME_SHIFT = False 246 | if time_shift.lower() == "true": 247 | print("> Time shift enabled. Will shift the start and end time back in time by DURATION seconds.") 248 | TIME_SHIFT = True 249 | 250 | HAS_PARENT_SPAN = False 251 | if parent_span_id != "": 252 | print(f"> Pushing a child (sub) span with parent span id: {parent_span_id}") 253 | HAS_PARENT_SPAN = True 254 | 255 | # Prior to v1.0 256 | # This flag will ONLY print a soft WARNING 257 | # If the flag is False (explicitly or omitted) 258 | # a warning is given that in v1.0 calls to http:// endpoints 259 | # will FAIL if "--insecure true" is NOT set 260 | # 261 | # In other words, prior to v1.0 no breaking change 262 | # v1.0 and above, if a user wishes to send to an http:// endpoint 263 | # --insecure true MUST be set 264 | # 265 | # Best practice: Start setting this flag now! 266 | 267 | # First convert to boolean 268 | ALLOW_INSECURE = False 269 | if allow_insecure.lower() == "true": 270 | ALLOW_INSECURE = True 271 | 272 | # TODO: Adjust this error message for >=v1.0 273 | # From v1.0 make this WARN only appear in DEBUG_MODE 274 | if not ALLOW_INSECURE: 275 | print("WARN: --insecure flag is omitted or is set to false. Prior to v1.0 tracepusher still works as expected (span is sent). In v1.0 and above, you MUST set '--insecure true' if you want to send to an http:// endpoint. See https://github.com/agardnerIT/tracepusher/issues/78") 276 | 277 | # If user has explicitly provided a start time: 278 | # 1) It must be in the following format: 279 | # 2023-11-26T03:05:16.494Z 280 | # yyyy-mm-ddThh:mm:ss.mssTZ 281 | # 3) Convert to millis since epoch 282 | # eg. 1700931916494 283 | # 4) Further converter to nanos (eg. + 9 zeros) 284 | # eg. 1700931916494000000 285 | 286 | HAS_START_TIME = False 287 | start_time_nanos_since_epoch = 0 288 | 289 | if start_time != "": 290 | HAS_START_TIME = True 291 | print("Got an explicit start time") 292 | # The ONLY input formats tracepusher currently supports are: 293 | # 1) A 19 digit string representing milliseconds since the epoch: eg. 1700967916494000000 294 | # 2) "%Y-%m-%dT%H:%M:%S.%fZ" eg. "2023-11-26T03:05:16.844Z" 295 | 296 | if len(start_time) == 19: 297 | # Try to cast input to an int 298 | # If an exception is caught, the user maybe tried to pass another 19 digit string like... 299 | # "2023-11-26T03:05:16". This is NOT supported! 300 | try: 301 | start_time_nanos_since_epoch = int(start_time) 302 | except: 303 | HAS_START_TIME = False 304 | print("WARN: --start-time was in an invalid format. Trace will be sent but start time will default to 'now'.") 305 | else: 306 | try: 307 | if not start_time.endswith("Z"): 308 | print("WARN: --start-time was in an incorrect format. If providing a date/time it must be in UTC and end with uppercase 'Z'. eg. '2023-11-26T03:05:16.844Z'. Trace will be sent with start_time of now.") 309 | # The provided start time had an invalid value, so fallback to the trace having a start time of "now" 310 | HAS_START_TIME = False 311 | else: 312 | start_time_nanos_since_epoch = unix_time_millis(datetime.datetime.strptime(start_time, '%Y-%m-%dT%H:%M:%S.%fZ')) 313 | except: 314 | print("WARN: --start-time value was in an incorrect format. Valid formats: 1) 19 digit integer representing millis since epoch 2) '%Y-%m-%dT%H:%M:%S.%fZ' eg. '2023-11-26T03:05:16.844Z'. Trace will be sent with start_time of now.") 315 | # The provided start time had an invalid value, so fallback to the trace having a start time of "now" 316 | HAS_START_TIME = False 317 | 318 | if DEBUG_MODE: 319 | print(f"Endpoint: {endpoint}") 320 | print(f"Service Name: {service_name}") 321 | print(f"Span Name: {span_name}") 322 | print(f"Duration: {duration}") 323 | print(f"Duration Type: {duration_type}") 324 | print(f"Dry Run: {type(dry_run)} = {dry_run}") 325 | print(f"Debug: {type(debug_mode)} = {debug_mode}") 326 | print(f"Time Shift: {type(time_shift)} = {time_shift}") 327 | print(f"Parent Span ID: {parent_span_id}") 328 | print(f"Trace ID: {trace_id}") 329 | print(f"Span ID: {span_id}") 330 | print(f"Dropped Attribute Count: {dropped_attribute_count}") 331 | print(f"Span Kind: {span_kind}") 332 | print(f"Span Status: {span_status}") 333 | print(f"Allow insecure endpoints: {allow_insecure}") 334 | print(f"Provided start time: {start_time}. Nanos since epoch: {start_time_nanos_since_epoch}") 335 | 336 | # disable until v1.0 337 | #if endpoint.startswith("http://") and not ALLOW_INSECURE: 338 | # print("ERROR: Endpoint is http:// (insecure). You MUST set '--insecure true'. Span has NOT been sent.") 339 | # sys.exit(1) 340 | 341 | # Generate random chars for trace and span IDs 342 | # of 32 chars and 16 chars respectively 343 | # per secrets documentation 344 | # each byte is converted to two hex digits 345 | # hence this "appears" wrong by half but isn't 346 | # If this is a child span, we already have a trace_id and parent_span_id 347 | # So do not generate 348 | 349 | if trace_id == "": 350 | trace_id = secrets.token_hex(16) 351 | if len(trace_id) != 32: 352 | sys.exit("Error: trace_id should be 32 characters long!") 353 | 354 | if span_id == "": 355 | span_id = secrets.token_hex(8) 356 | if len(span_id) != 16: 357 | sys.exit("Error: span_id should be 16 characters long!") 358 | 359 | if DEBUG_MODE: 360 | print(f"Trace ID: {trace_id}") 361 | print(f"Span ID: {span_id}") 362 | print(f"Parent Span ID: {parent_span_id}") 363 | 364 | duration_nanos = 0 365 | if duration_type == "ms": 366 | duration_nanos = duration * 1000000 # ms to ns 367 | elif duration_type == "s": 368 | duration_nanos = duration * 1000000000 # s to ns 369 | 370 | if not HAS_START_TIME: 371 | # get time now 372 | start_time_nanos_since_epoch = time.time_ns() 373 | 374 | # calculate future time by adding that many nanoseconds 375 | time_future = int(start_time_nanos_since_epoch) + duration_nanos 376 | 377 | # shift start_time_nanos_since_epoch and time_future back by duration 378 | if not HAS_START_TIME and TIME_SHIFT: 379 | start_time_nanos_since_epoch = start_time_nanos_since_epoch - duration_nanos 380 | time_future = time_future - duration_nanos 381 | 382 | if DEBUG_MODE: 383 | print(f"Time shifted? {TIME_SHIFT}") 384 | print(f"Start time: {start_time_nanos_since_epoch}") 385 | print(f"Time future: {time_future}") 386 | 387 | # Now that the right start / end time is available 388 | # process any span events 389 | span_events_list, dropped_event_count = get_span_events_list(args.span_events, start_time_nanos_since_epoch) 390 | 391 | if DEBUG_MODE: 392 | print("Printing Span Events List:") 393 | print(span_events_list) 394 | print("-----") 395 | print(f"Dropped Span Events Count: {dropped_event_count}") 396 | 397 | trace = { 398 | "resourceSpans": [ 399 | { 400 | "resource": { 401 | "attributes": [ 402 | { 403 | "key": "service.name", 404 | "value": { 405 | "stringValue": service_name 406 | } 407 | } 408 | ] 409 | }, 410 | "scopeSpans": [ 411 | { 412 | "scope": { 413 | "name": "manual-test" 414 | }, 415 | "spans": [ 416 | { 417 | "traceId": trace_id, 418 | "spanId": span_id, 419 | "name": span_name, 420 | "kind": span_kind, 421 | "start_time_unix_nano": start_time_nanos_since_epoch, 422 | "end_time_unix_nano": time_future, 423 | "droppedAttributesCount": dropped_attribute_count, 424 | "attributes": span_attributes_list, 425 | "events": span_events_list, 426 | "droppedEventsCount": dropped_event_count, 427 | "status": { 428 | "code": span_status 429 | } 430 | } 431 | ] 432 | } 433 | ] 434 | } 435 | ] 436 | } 437 | 438 | if HAS_PARENT_SPAN: 439 | # Add parent_span_id field 440 | trace['resourceSpans'][0]['scopeSpans'][0]['spans'][0]['parentSpanId'] = parent_span_id 441 | 442 | if DEBUG_MODE: 443 | print("Trace:") 444 | print(trace) 445 | 446 | if DRY_RUN: 447 | print(f"Collector URL: {endpoint}. Service Name: {service_name}. Span Name: {span_name}. Trace Length ({duration_type}): {duration}") 448 | # Only print if also not running in DEBUG_MODE 449 | # Otherwise we get a double print 450 | if not DEBUG_MODE: 451 | print("Trace:") 452 | print(trace) 453 | 454 | if not DRY_RUN: 455 | resp = requests.post( 456 | f"{endpoint}/v1/traces", 457 | headers={ "Content-Type": "application/json" }, 458 | json=trace, 459 | timeout=5 460 | ) 461 | print(resp) 462 | -------------------------------------------------------------------------------- /operator/operator-logic.py: -------------------------------------------------------------------------------- 1 | import kopf 2 | from kubernetes import client, config 3 | from kubernetes.client.rest import ApiException 4 | from datetime import datetime, timezone 5 | import subprocess 6 | import secrets 7 | 8 | # Job names are listed in here while they're being traced 9 | jobs_to_process = [] 10 | 11 | # Cache built from JobTracer objects 12 | # Which tracks which namespaces to trace 13 | namespaces_to_trace = [] 14 | 15 | # Initialise the Operator 16 | # The operator is starting 17 | # Read each JobTracer in the cluster 18 | # Populate the namespaces_to_trace list 19 | @kopf.on.startup() 20 | def initialise_operator(logger, **kwargs): 21 | 22 | logger.info("#"*30) 23 | 24 | logger.info(f"Tracepusher Version: {call_tracepusher(tracepusher_args=['--version'], logger=logger)}") 25 | 26 | try: 27 | configuration = config.load_incluster_config() 28 | except Exception: 29 | logger.info("Cannot load config from within cluster. Loading local config. This is OK if you are a developer and running locally.") 30 | configuation = config.load_config() 31 | 32 | with client.ApiClient() as api_client: 33 | api_instance = client.CustomObjectsApi(api_client) 34 | try: 35 | api_response = api_instance.list_cluster_custom_object( 36 | group="tracers.tracepusher.github.io", 37 | version="v1", 38 | plural="jobtracers", 39 | pretty=True, 40 | timeout_seconds=10 41 | ) 42 | 43 | for jt in api_response.get('items'): 44 | 45 | namespaces_to_trace.append({ 46 | "namespace": jt['metadata']['namespace'], 47 | "spec": jt['spec'] 48 | }) 49 | 50 | logger.info(f"Print List: {namespaces_to_trace}") 51 | logger.info('#'*30) 52 | except ApiException as e: 53 | logger.info("Exception when calling CustomObjectsApi->list_cluster_custom_object: %s\n" % e) 54 | 55 | def get_jobs_to_track_keys(): 56 | return [ item['key'] for item in jobs_to_process ] 57 | 58 | def get_namespace_object(namespace): 59 | return next((item for item in namespaces_to_trace if item['namespace']==namespace), None) 60 | 61 | # Given a job_key in the form "jobName/Namespace" 62 | # Returns the job object to process 63 | # or None 64 | def get_job_to_process_object(job_key): 65 | if "/" not in job_key: 66 | logger.error(f"Invalid job_key syntax for: {job_key}. Must be 'jobName/Namespace'. Investigate!") 67 | return None 68 | return next((item for item in jobs_to_process if item['key']==job_key), None) 69 | 70 | def add_job_to_toprocess_list(name, namespace, collector_endpoint, logger): 71 | # Generate main trace ID and span ID 72 | # The trace_id will be common for all traces in this job 73 | job_name_key = f"{name}/{namespace}" 74 | main_trace_id = secrets.token_hex(16) 75 | main_span_id = secrets.token_hex(8) 76 | 77 | jobs_to_process.append({ 78 | "key": job_name_key, 79 | "job_name": name, 80 | "namespace": namespace, 81 | "collector_endpoint": collector_endpoint, 82 | "main_trace_id": main_trace_id, 83 | "main_span_id": main_span_id 84 | }) 85 | logger.info(f"Added new job: {job_name_key} to list. List is now: {jobs_to_process}") 86 | 87 | def update_job(name, namespace, collector_endpoint, logger): 88 | job_name_key = f"{name}/{namespace}" 89 | job_to_process = get_job_to_process_object(job_key=job_name_key) 90 | if job_to_process is not None: 91 | job_to_process['collector_endpoint'] = collector_endpoint 92 | 93 | logger.info(f"Updated Job: {job_name_key}. List is now: {jobs_to_process}") 94 | 95 | def delete_job(name, namespace): 96 | job_name_key = f"{name}/{namespace}" 97 | job_to_delete = get_job_to_process_object(job_key=job_name_key) 98 | if job_to_delete is not None: 99 | jobs_to_process.remove(job_to_delete) 100 | 101 | def call_tracepusher(tracepusher_args, logger): 102 | 103 | full_cmd = ['python3', 'tracepusher.py'] + tracepusher_args 104 | 105 | logger.info(f"TP ARGS: {tracepusher_args}") 106 | 107 | # Call tracepusher 108 | subprocess.call(full_cmd) 109 | 110 | # Kopf Filter 111 | # Custom filter to only react to new events 112 | # Ignoring those which happened prior to operator 113 | # Startup 114 | def is_new_event(event, **_): 115 | if event['type'] is None: return False 116 | else: return True 117 | 118 | # Kopf filter 119 | # Return true if Pod was spawned by a job 120 | # ie. has a "job-name" label 121 | def is_pod_spawned_by_job(event, **_): 122 | if event['object']['kind'] == "Pod" and "job-name" in event['object']['metadata']['labels']: return True 123 | return False 124 | 125 | # Kopf filter 126 | # Only process events without finalizers 127 | def has_no_finalizers(event, **_): 128 | if "finalizers" in event['object']['metadata']: return False 129 | return True 130 | 131 | # type = "job" | "container" 132 | def get_tz_aware_start_finish_times(object, type): 133 | 134 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 135 | 136 | workload_succeeded = False 137 | start_time_str = "" 138 | finish_time_str = "" 139 | workload_start_time = None 140 | workload_end_time = None 141 | 142 | if type == "job": 143 | if "succeeded" in object['status']: workload_status = True 144 | 145 | start_time_str = object['status']['startTime'] 146 | 147 | if workload_succeeded: 148 | finish_time_str = object['status']['completionTime'] 149 | else: 150 | finish_time_str = object['status']['conditions'][0]['lastTransitionTime'] 151 | 152 | if type == "container": 153 | start_time_str = object['state']['terminated']['startedAt'] 154 | finish_time_str = object['state']['terminated']['finishedAt'] 155 | 156 | # Do processing in case either field starts with a 157 | # date of 1970 158 | # If start time is empty 159 | # set to be the same as the end time 160 | if start_time_str.startswith('1970'): 161 | start_time_str = finish_time_str 162 | 163 | # Make times timezone aware for UTC 164 | workload_start_time = datetime.strptime(start_time_str, DATE_FORMAT) 165 | workload_start_time = workload_start_time.replace(tzinfo=timezone.utc) 166 | workload_end_time = datetime.strptime(finish_time_str, DATE_FORMAT) 167 | workload_end_time = workload_end_time.replace(tzinfo=timezone.utc) 168 | 169 | duration_seconds = int((workload_end_time - workload_start_time).total_seconds()) 170 | 171 | return workload_start_time, workload_end_time, duration_seconds 172 | 173 | 174 | @kopf.on.event( 175 | 'pods', 176 | annotations={"tracepusher/ignore": kopf.ABSENT}, 177 | when=kopf.all_([is_new_event, is_pod_spawned_by_job, has_no_finalizers])) 178 | def on_update_pod(event, spec, name, namespace, logger, **kwargs): 179 | 180 | # If pod is being deleted, ignore 181 | # Do not send a trace for a pod deletion 182 | if event["type"] == "DELETED": return 183 | 184 | job_name = event['object']['metadata']['labels']['job-name'] 185 | 186 | # Make composite key of job name + namespace 187 | # This is used to lookup the jobs the Operator SHOULD track 188 | job_key = f"{job_name}/{namespace}" 189 | 190 | # Skip jobs that should not be processed 191 | job_keys_to_track = get_jobs_to_track_keys() 192 | logger.info(f"Job Keys: {job_keys_to_track}") 193 | if job_key not in job_keys_to_track: return 194 | 195 | if event['object']['status']['phase'] == "Succeeded" or event['object']['status']['phase'] == "Failed": 196 | 197 | job_obj_from_tracker_list = get_job_to_process_object(job_key) 198 | if job_obj_from_tracker_list is None: 199 | logger.error(f"No job in jobs_to_process list for job_key {job_key}. Error. Investigate!") 200 | return 201 | job_main_trace_id = job_obj_from_tracker_list['main_trace_id'] 202 | job_main_span_id = job_obj_from_tracker_list['main_span_id'] 203 | 204 | job_name = event['object']['metadata']['labels']['job-name'] 205 | 206 | container_statuses = event['object']['status']['containerStatuses'] 207 | for container_status in container_statuses: 208 | 209 | container_span_status = "OK" 210 | 211 | container_name = container_status['name'] 212 | container_exit_code = container_status['state']['terminated']['exitCode'] 213 | container_message = "" 214 | # TODO: This is too presumptive 215 | # Let users annotate to give their own success code 216 | if container_exit_code != 0: container_span_status = "Error" 217 | 218 | container_exit_reason = container_status['state']['terminated']['reason'] 219 | container_start_time, container_end_time, container_duration = get_tz_aware_start_finish_times(container_status, "container") 220 | 221 | if "message" in container_status['state']['terminated']: 222 | container_message = container_status['state']['terminated']['message'] 223 | 224 | # Generate a unique span ID for this container 225 | # Link to main using trace_id and parent_span_id 226 | container_span_id = secrets.token_hex(8) 227 | 228 | # Call tracepusher 229 | tracepusher_args = [ 230 | "-ep", 231 | job_obj_from_tracker_list['collector_endpoint'], 232 | "-sen", 233 | "k8s", 234 | "-spn", 235 | container_name, 236 | "-dur", 237 | f"{container_duration}", 238 | "--span-status", 239 | container_span_status, 240 | "--time-shift", 241 | "true", 242 | "--trace-id", 243 | job_main_trace_id, 244 | "--parent-span-id", 245 | job_main_span_id, 246 | "-spnattrs", 247 | "type=container", 248 | f"namespace={namespace}", 249 | f"exit_code={container_exit_code}", 250 | f"reason={container_exit_reason}", 251 | f"message={container_message}" 252 | ] 253 | call_tracepusher(tracepusher_args=tracepusher_args, logger=logger) 254 | 255 | # This happens on every change event for jobs 256 | # That are NOT annotated with "tracepusher/ignore" 257 | @kopf.on.event( 258 | 'job', 259 | annotations={"tracepusher/ignore": kopf.ABSENT}, 260 | when=is_new_event) 261 | def on_event_job(event, spec, name, namespace, logger, **kwargs): 262 | 263 | tracker_details = get_namespace_object(namespace) 264 | if tracker_details is None: 265 | # Have not been asked to trace this namespace 266 | # Return quickly and do not process further 267 | logger.info(f"Did not find tracker_details in list. We are not configured to track: {namespace}") 268 | logger.info(f"tracker list is: {namespaces_to_trace}") 269 | return 270 | 271 | # If job is deleted 272 | # Remove from list 273 | # Note: jobs are deleted elsewhere during normal operation 274 | # This logic catches a side-issue 275 | # Where a job exists, the operator restarts and THEN 276 | # the job is deleted 277 | if event['type'] == "DELETED": 278 | logger.info(f">> DELETED A JOB: {name}/{namespace}") 279 | delete_job(name, namespace) 280 | # Do not re-trace a deleted job 281 | return 282 | 283 | # Get the collector endpoint 284 | namespace_default_collector_endpoint = tracker_details['spec']['collectorEndpoint'] 285 | collector_endpoint_to_set = namespace_default_collector_endpoint 286 | logger.info(f"Got collector endpoint for the namespace from tracker_details: {namespace_default_collector_endpoint}") 287 | 288 | # Check for an annotation on the Job which overrides JobTracer global 289 | # collector endpoint 290 | if "tracepusher/collector" in event['object']['metadata']['annotations']: 291 | collector_endpoint_to_set = event['object']['metadata']['annotations']['tracepusher/collector'] 292 | logger.info(f"Got a collector endpoint Override for job: {name} in namespace: {namespace}. Overridden collector URL is: {collector_endpoint_to_set}") 293 | 294 | # If a collector endpoint 295 | # is not set 296 | # stop immediately 297 | # no point in continuing 298 | if collector_endpoint_to_set == "": 299 | logger.info("Collector endpoint is not set. Will not track this job.") 300 | return 301 | else: 302 | # Get existing keys and only add once 303 | # Create a composite key of job name + namespace 304 | # As there could be two identical job names in different namespaces 305 | job_name_key = f"{name}/{namespace}" 306 | 307 | logger.info(f"Tracking job: {name} (job_name_key is {job_name_key}) with collector endpoint: {collector_endpoint_to_set}") 308 | # Update the collector URL on the global list now 309 | job_to_update = get_job_to_process_object(job_key=job_name_key) 310 | # If this is None, the Job existed before the operator started 311 | # So the list could be empty. Add it now 312 | if job_to_update is None: 313 | logger.info("Adding job: {name} to WILL PROCESS list.") 314 | add_job_to_toprocess_list(name=name,namespace=namespace,collector_endpoint=collector_endpoint_to_set, logger=logger) 315 | else: 316 | update_job(name=name, namespace=namespace, collector_endpoint=collector_endpoint_to_set, logger=logger) 317 | logger.info(f"Need to update {name}/{namespace} with new collector URL: {collector_endpoint_to_set}") 318 | 319 | logger.info(f"Printing new list: {jobs_to_process}") 320 | 321 | # A valid job to track 322 | # If it isn't already in the list 323 | if event['type'] == "ADDED" or event['type'] == "MODIFIED": 324 | # Get existing keys and only add once 325 | # Create a composite key of job name + namespace 326 | # As there could be two identical job names in different namespaces 327 | job_name_key = f"{name}/{namespace}" 328 | keys = [ item['key'] for item in jobs_to_process ] 329 | if job_name_key not in keys: 330 | add_job_to_toprocess_list(name, namespace, collector_endpoint_to_set) 331 | else: # job is already in the list, alter it 332 | job_to_update = get_job_to_process_object(job_key=job_name_key) 333 | logger.info(f"Need to update {job_to_update['key']} with new collector URL: {collector_endpoint_to_set}") 334 | job_to_update['collector_endpoint'] = collector_endpoint_to_set 335 | logger.info(f"Printing new list: {jobs_to_process}") 336 | 337 | # Job finishes when all of the these conditions are true: 338 | if event['type'] == "MODIFIED" and event['object']['status'] is not None: 339 | 340 | # if event.object.status has either 'succeeded' or 'failed' fields 341 | # it has completed 342 | job_status = None 343 | job_has_finished = False 344 | job_reason = "" 345 | job_message = "" 346 | if "succeeded" in event['object']['status']: 347 | job_status = "OK" 348 | job_has_finished = True 349 | elif "failed" in event['object']['status']: 350 | job_status = "ERROR" 351 | job_has_finished = True 352 | job_reason = event['object']['status']['conditions'][0]['reason'] 353 | job_message = event['object']['status']['conditions'][0]['message'] 354 | 355 | # Job hasn't finished 356 | # Do not process further 357 | if not job_has_finished: 358 | return 359 | 360 | logger.info(f"Job has finished") 361 | 362 | job_obj_from_tracker_list = get_job_to_process_object(job_key=job_name_key) 363 | 364 | job_main_trace_id = job_obj_from_tracker_list['main_trace_id'] 365 | job_main_span_id = job_obj_from_tracker_list['main_span_id'] 366 | job_collector_url = job_obj_from_tracker_list['collector_endpoint'] 367 | 368 | #logger.info(f"List before removal: {jobs_to_process}") 369 | if job_obj_from_tracker_list is not None: 370 | delete_job(name=name,namespace=namespace) 371 | #logger.info(f"List after removal: {jobs_to_process}") 372 | 373 | job_start_time, job_end_time, job_duration = get_tz_aware_start_finish_times(event['object'], "job") 374 | logger.info(f"JOB Details. JST: {job_start_time} - JET: {job_end_time} - JD: {job_duration}") 375 | 376 | # Call tracepusher 377 | tracepusher_args = [ 378 | "-ep", 379 | job_obj_from_tracker_list['collector_endpoint'], 380 | "-sen", 381 | "k8s", 382 | "-spn", 383 | name, 384 | "-dur", 385 | f"{job_duration}", 386 | "--span-status", 387 | job_status, 388 | "--time-shift", 389 | "true", 390 | "--trace-id", 391 | job_main_trace_id, 392 | "--span-id", 393 | job_main_span_id, 394 | "-spnattrs", 395 | "type=job", 396 | f"namespace={namespace}", 397 | f"job_message={job_message}", 398 | f"job_reason={job_reason}" 399 | ] 400 | call_tracepusher(tracepusher_args=tracepusher_args, logger=logger) 401 | 402 | 403 | # Whenever a JobTracer object 404 | # is created or updated 405 | @kopf.on.create('jobtracers') 406 | @kopf.on.update('jobtracers') 407 | def on_create_jobtracer(spec, name, namespace, logger, **kwargs): 408 | 409 | logger.info("Created or updating a JobTracer ") 410 | 411 | existing_tracker_object = get_namespace_object(namespace) 412 | 413 | # If no existing item 414 | # This is a new namespace to track request 415 | # Add to the list 416 | if existing_tracker_object is None: 417 | logger.info(f"Adding new namespace: {namespace} to list. Will track jobs.") 418 | namespaces_to_trace.append({ 419 | "namespace": namespace, 420 | "spec": spec 421 | }) 422 | else: 423 | logger.info(f"A JobTracer already exists for namespace: {namespace}. Will replace config") 424 | logger.info(f"JobTracer for {namespace} replaced {existing_tracker_object['spec']} with {spec}") 425 | existing_tracker_object['spec'] = spec 426 | 427 | logger.info(namespaces_to_trace) 428 | logger.info("-"*30) 429 | 430 | # A JobTracer has been deleted 431 | # Remove from the list 432 | @kopf.on.delete('jobtracers') 433 | def on_delete_jobtracer(spec, name, namespace, logger, **kwargs): 434 | logger.info(f"Deleted a jobtracer. Namespace is {namespace}") 435 | 436 | existing_tracker_object = get_namespace_object(namespace) 437 | 438 | if existing_tracker_object is not None: 439 | logger.info(f"tracking list before remove: {namespaces_to_trace}") 440 | namespaces_to_trace.remove(existing_tracker_object) 441 | logger.info(f"tracking list after remove: {namespaces_to_trace}") 442 | logger.info("+"*30) --------------------------------------------------------------------------------