├── .dockerignore ├── .editorconfig ├── .envrc ├── .gitignore ├── .golangci.yaml ├── .license-scan-overrides.jsonl ├── .license-scan-rules.json ├── CODEOWNERS ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── LICENSES └── Apache-2.0.txt ├── Makefile ├── Makefile.maker.yaml ├── README.md ├── REUSE.toml ├── Tiltfile ├── build └── .gitignore ├── commands ├── checks │ └── checks.go ├── error │ └── error.go ├── fillup │ └── fillup.go └── replay │ └── replay.go ├── cortex.secrets.example.yaml ├── docs ├── adrs │ └── 001-event-based-feature-extraction.md ├── architecture.md ├── develop.md ├── features.md └── readme.md ├── go.mod ├── go.sum ├── helm ├── cortex │ ├── .helmignore │ ├── Chart.lock │ ├── Chart.yaml │ ├── charts │ │ └── .gitignore │ ├── templates │ │ ├── _helpers.tpl │ │ ├── cli.yaml │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ └── service.yaml │ └── values.yaml ├── postgres │ ├── .helmignore │ ├── Chart.lock │ ├── Chart.yaml │ ├── charts │ │ └── .gitignore │ ├── templates │ │ └── .gitkeep │ └── values.yaml ├── prometheus-deps │ ├── .helmignore │ ├── Chart.lock │ ├── Chart.yaml │ ├── charts │ │ └── .gitignore │ ├── templates │ │ ├── alertmanager.yaml │ │ ├── prometheus.yaml │ │ └── rbac.yaml │ └── values.yaml └── prometheus │ ├── .helmignore │ ├── Chart.lock │ ├── Chart.yaml │ ├── charts │ └── .gitignore │ ├── templates │ ├── alerts.yaml │ └── servicemonitor.yaml │ └── values.yaml ├── internal ├── conf │ ├── conf.go │ ├── conf_test.go │ ├── graph.go │ ├── graph_test.go │ ├── logging.go │ ├── logging_test.go │ ├── opts.go │ ├── opts_test.go │ ├── validation.go │ └── validation_test.go ├── db │ ├── db.go │ ├── db_test.go │ ├── migrations.go │ ├── migrations │ │ └── 001_add_openstack_servers_flavorid.sql │ └── migrations_test.go ├── features │ ├── monitor.go │ ├── monitor_test.go │ ├── pipeline.go │ ├── pipeline_test.go │ └── plugins │ │ ├── base.go │ │ ├── base_test.go │ │ ├── interface.go │ │ ├── kvm │ │ ├── node_exporter_host_cpu_usage.go │ │ ├── node_exporter_host_cpu_usage.sql │ │ ├── node_exporter_host_cpu_usage_test.go │ │ ├── node_exporter_host_memory_active.go │ │ ├── node_exporter_host_memory_active.sql │ │ └── node_exporter_host_memory_active_test.go │ │ ├── shared │ │ ├── flavor_host_space.go │ │ ├── flavor_host_space.sql │ │ ├── flavor_host_space_test.go │ │ ├── vm_host_residency.go │ │ ├── vm_host_residency.sql │ │ ├── vm_host_residency_test.go │ │ ├── vm_life_span.go │ │ ├── vm_life_span.sql │ │ └── vm_life_span_test.go │ │ └── vmware │ │ ├── vrops_hostsystem_contention.go │ │ ├── vrops_hostsystem_contention.sql │ │ ├── vrops_hostsystem_contention_test.go │ │ ├── vrops_hostsystem_resolver.go │ │ ├── vrops_hostsystem_resolver.sql │ │ ├── vrops_hostsystem_resolver_test.go │ │ ├── vrops_project_noisiness.go │ │ ├── vrops_project_noisiness.sql │ │ └── vrops_project_noisiness_test.go ├── kpis │ ├── pipeline.go │ ├── pipeline_test.go │ └── plugins │ │ ├── base.go │ │ ├── base_test.go │ │ ├── histogram.go │ │ ├── histogram_test.go │ │ ├── interface.go │ │ ├── shared │ │ ├── host_utilization.go │ │ ├── host_utilization_test.go │ │ ├── vm_life_span.go │ │ ├── vm_life_span_test.go │ │ ├── vm_migration_statistics.go │ │ └── vm_migration_statistics_test.go │ │ └── vmware │ │ ├── host_contention.go │ │ ├── host_contention_test.go │ │ ├── project_noisiness.go │ │ └── project_noisiness_test.go ├── monitoring │ ├── monitoring.go │ └── monitoring_test.go ├── mqtt │ ├── mqtt.go │ └── mqtt_test.go ├── scheduler │ ├── api │ │ ├── http │ │ │ ├── api.go │ │ │ ├── api_test.go │ │ │ ├── messages.go │ │ │ └── monitor.go │ │ ├── interface.go │ │ ├── messages.go │ │ └── messages_test.go │ ├── monitor.go │ ├── monitor_test.go │ ├── pipeline.go │ ├── pipeline_test.go │ ├── plugins │ │ ├── activation.go │ │ ├── activation_test.go │ │ ├── base.go │ │ ├── base_test.go │ │ ├── interface.go │ │ ├── kvm │ │ │ ├── avoid_overloaded_hosts_cpu.go │ │ │ ├── avoid_overloaded_hosts_cpu_test.go │ │ │ ├── avoid_overloaded_hosts_memory.go │ │ │ └── avoid_overloaded_hosts_memory_test.go │ │ ├── shared │ │ │ ├── flavor_binpacking.go │ │ │ └── flavor_binpacking_test.go │ │ └── vmware │ │ │ ├── anti_affinity_noisy_projects.go │ │ │ ├── anti_affinity_noisy_projects_test.go │ │ │ ├── avoid_contended_hosts.go │ │ │ └── avoid_contended_hosts_test.go │ ├── validation.go │ └── validation_test.go └── sync │ ├── datasource.go │ ├── monitor.go │ ├── monitor_test.go │ ├── openstack │ ├── keystone.go │ ├── keystone_test.go │ ├── nova_api.go │ ├── nova_api_test.go │ ├── nova_sync.go │ ├── nova_sync_test.go │ ├── nova_types.go │ ├── nova_types_test.go │ ├── placement_api.go │ ├── placement_api_test.go │ ├── placement_sync.go │ ├── placement_sync_test.go │ ├── placement_types.go │ ├── placement_types_test.go │ ├── sync.go │ ├── sync_test.go │ └── triggers.go │ ├── pipeline.go │ ├── pipeline_test.go │ ├── prometheus │ ├── prometheus.go │ ├── prometheus_test.go │ ├── sync.go │ ├── sync_test.go │ ├── triggers.go │ ├── types.go │ └── types_test.go │ ├── sso.go │ └── sso_test.go ├── main.go ├── plutono ├── Dockerfile ├── app.yaml ├── plutono.ini └── provisioning │ ├── dashboards │ ├── cortex.json │ ├── cortex.json.license │ └── dashboards.yml │ └── datasources │ └── datasources.yml ├── shell.nix ├── testlib ├── db │ ├── containers │ │ ├── postgres.go │ │ └── postgres_test.go │ └── env.go ├── monitoring │ └── mock.go ├── mqtt │ ├── containers │ │ ├── rabbitmq-entrypoint.sh │ │ ├── rabbitmq.conf │ │ ├── rabbitmq.go │ │ └── rabbitmq_test.go │ └── mock.go └── scheduler │ ├── api │ └── interface.go │ └── plugins │ └── interface.go └── visualizer ├── Dockerfile ├── app.yaml ├── index.html └── vendor ├── mqtt.min.js └── mqtt.min.js.license /.dockerignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | /.dockerignore 5 | .DS_Store 6 | # TODO: uncomment when applications no longer use git to get version information 7 | #.git/ 8 | /.github/ 9 | /.gitignore 10 | /.goreleaser.yml 11 | /*.env* 12 | /.golangci.yaml 13 | /.vscode/ 14 | /build/ 15 | /CONTRIBUTING.md 16 | /Dockerfile 17 | /docs/ 18 | /LICENSE* 19 | /Makefile.maker.yaml 20 | /README.md 21 | /report.html 22 | /shell.nix 23 | /testing/ 24 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | root = true 5 | 6 | [*] 7 | insert_final_newline = true 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | indent_style = space 11 | indent_size = 2 12 | 13 | [{Makefile,go.mod,go.sum,*.go}] 14 | indent_style = tab 15 | indent_size = 2 16 | 17 | [*.md] 18 | trim_trailing_whitespace = false 19 | 20 | [{LICENSE,LICENSES/*,vendor/**}] 21 | charset = unset 22 | end_of_line = unset 23 | indent_size = unset 24 | indent_style = unset 25 | insert_final_newline = unset 26 | trim_trailing_whitespace = unset 27 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-FileCopyrightText: Copyright 2019–2020 Target, Copyright 2021 The Nix Community 3 | # SPDX-License-Identifier: Apache-2.0 4 | if type -P lorri &>/dev/null; then 5 | eval "$(lorri direnv)" 6 | elif type -P nix &>/dev/null; then 7 | use nix 8 | else 9 | echo "Found no nix binary. Skipping activating nix-shell..." 10 | fi 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | build/** 8 | 9 | # Test binary, built with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Dependency directories (remove the comment below to include it) 16 | vendor/** 17 | 18 | # Go workspace file 19 | go.work 20 | go.work.sum 21 | 22 | # env file 23 | .env 24 | 25 | # Just in case someone decides to put the secrets into the project dir. 26 | cortex.secrets.yaml 27 | 28 | # macOS specific files 29 | .DS_Store -------------------------------------------------------------------------------- /.license-scan-overrides.jsonl: -------------------------------------------------------------------------------- 1 | {"name": "github.com/chzyer/logex", "licenceType": "MIT"} 2 | {"name": "github.com/hashicorp/vault/api/auth/approle", "licenceType": "MPL-2.0"} 3 | {"name": "github.com/jpillora/longestcommon", "licenceType": "MIT"} 4 | {"name": "github.com/mattn/go-localereader", "licenceType": "MIT"} 5 | {"name": "github.com/miekg/dns", "licenceType": "BSD-3-Clause"} 6 | {"name": "github.com/spdx/tools-golang", "licenceTextOverrideFile": "vendor/github.com/spdx/tools-golang/LICENSE.code"} 7 | {"name": "github.com/xeipuuv/gojsonpointer", "licenceType": "Apache-2.0"} 8 | {"name": "github.com/xeipuuv/gojsonreference", "licenceType": "Apache-2.0"} 9 | {"name": "github.com/xeipuuv/gojsonschema", "licenceType": "Apache-2.0"} 10 | {"name": "github.wdf.sap.corp/cc/nxos-gnmi-go", "licenceType": "Apache-2.0"} 11 | -------------------------------------------------------------------------------- /.license-scan-rules.json: -------------------------------------------------------------------------------- 1 | { 2 | "allowlist": [ 3 | "Apache-2.0", 4 | "BSD-2-Clause", 5 | "BSD-2-Clause-FreeBSD", 6 | "BSD-3-Clause", 7 | "EPL-2.0", 8 | "ISC", 9 | "MIT", 10 | "MPL-2.0", 11 | "Unlicense", 12 | "Zlib" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * p.matthes@sap.com arno.uhlig@sap.com malte.viering@sap.com max.hofmann@sap.com marcel.bloecher@sap.com markus.wieland@sap.com julius.clausnitzer@sap.com 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Contributing 8 | 9 | ## Code of Conduct 10 | 11 | All members of the project community must abide by the [SAP Open Source Code of Conduct](https://github.com/SAP/.github/blob/main/CODE_OF_CONDUCT.md). 12 | Only by respecting each other we can develop a productive, collaborative community. 13 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting [a project maintainer](.reuse/dep5). 14 | 15 | ## Engaging in Our Project 16 | 17 | We use GitHub to manage reviews of pull requests. 18 | 19 | * If you are a new contributor, see: [Steps to Contribute](#steps-to-contribute) 20 | 21 | * Before implementing your change, create an issue that describes the problem you would like to solve or the code that should be enhanced. Please note that you are willing to work on that issue. 22 | 23 | * The team will review the issue and decide whether it should be implemented as a pull request. In that case, they will assign the issue to you. If the team decides against picking up the issue, the team will post a comment with an explanation. 24 | 25 | ## Steps to Contribute 26 | 27 | Should you wish to work on an issue, please claim it first by commenting on the GitHub issue that you want to work on. This is to prevent duplicated efforts from other contributors on the same issue. 28 | 29 | If you have questions about one of the issues, please comment on them, and one of the maintainers will clarify. 30 | 31 | ## Contributing Code or Documentation 32 | 33 | You are welcome to contribute code in order to fix a bug or to implement a new feature that is logged as an issue. 34 | 35 | The following rule governs code contributions: 36 | 37 | * Contributions must be licensed under the [Apache 2.0 License](./LICENSE) 38 | * Due to legal reasons, contributors will be asked to accept a Developer Certificate of Origin (DCO) when they create the first pull request to this project. This happens in an automated fashion during the submission process. SAP uses [the standard DCO text of the Linux Foundation](https://developercertificate.org/). 39 | 40 | ## Issues and Planning 41 | 42 | * We use GitHub issues to track bugs and enhancement requests. 43 | 44 | * Please provide as much context as possible when you open an issue. The information you provide must be comprehensive enough to reproduce that issue for the assignee. 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24.3-alpine3.21 AS builder 2 | 3 | RUN apk add --no-cache --no-progress ca-certificates gcc git make musl-dev 4 | 5 | COPY . /src 6 | ARG BININFO_BUILD_DATE BININFO_COMMIT_HASH BININFO_VERSION # provided to 'make install' 7 | RUN make -C /src install PREFIX=/pkg GOTOOLCHAIN=local 8 | 9 | ################################################################################ 10 | 11 | FROM alpine:3.21 12 | 13 | RUN addgroup -g 4200 appgroup \ 14 | && adduser -h /home/appuser -s /sbin/nologin -G appgroup -D -u 4200 appuser 15 | 16 | # upgrade all installed packages to fix potential CVEs in advance 17 | # also remove apk package manager to hopefully remove dependency on OpenSSL 🤞 18 | RUN apk upgrade --no-cache --no-progress \ 19 | && apk del --no-cache --no-progress apk-tools alpine-keys alpine-release libc-utils 20 | 21 | COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs/ 22 | COPY --from=builder /etc/ssl/cert.pem /etc/ssl/cert.pem 23 | COPY --from=builder /pkg/ /usr/ 24 | # make sure all binaries can be executed 25 | RUN cortex --version 2>/dev/null 26 | 27 | ARG BININFO_BUILD_DATE BININFO_COMMIT_HASH BININFO_VERSION 28 | LABEL source_repository="https://github.com/trickyteache/cortex" \ 29 | org.opencontainers.image.url="https://github.com/trickyteache/cortex" \ 30 | org.opencontainers.image.created=${BININFO_BUILD_DATE} \ 31 | org.opencontainers.image.revision=${BININFO_COMMIT_HASH} \ 32 | org.opencontainers.image.version=${BININFO_VERSION} 33 | 34 | USER 4200:4200 35 | WORKDIR /home/appuser 36 | ENTRYPOINT [ "/usr/bin/cortex" ] 37 | -------------------------------------------------------------------------------- /Makefile.maker.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Configuration file for 5 | 6 | metadata: 7 | url: https://github.com/trickyteache/cortex 8 | 9 | binaries: 10 | - name: cortex 11 | fromPackage: . 12 | installTo: bin/ 13 | 14 | golang: 15 | setGoModVersion: true 16 | 17 | golangciLint: 18 | # We customized the config, so we don't want to overwrite it. 19 | createConfig: false 20 | 21 | githubWorkflow: 22 | ci: 23 | enabled: true 24 | coveralls: false 25 | 26 | renovate: 27 | enabled: true 28 | assignees: 29 | - auhlig 30 | - PhilippMatthes 31 | 32 | dockerfile: 33 | enabled: true 34 | 35 | # Don't override REUSE.toml since it contains custom information. 36 | reuse: 37 | enabled: false 38 | 39 | verbatim: | 40 | run: build/cortex 41 | ./build/cortex -------------------------------------------------------------------------------- /REUSE.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | SPDX-PackageName = "Cortex" 3 | SPDX-PackageSupplier = "Philipp Matthes " 4 | SPDX-PackageDownloadLocation = "https://github.com/trickyteache/cortex" 5 | SPDX-PackageComment = "The code in this project may include calls to APIs (\"API Calls\") of\n SAP or third-party products or services developed outside of this project\n (\"External Products\").\n \"APIs\" means application programming interfaces, as well as their respective\n specifications and implementing code that allows software to communicate with\n other software.\n API Calls to External Products are not licensed under the open source license\n that governs this project. The use of such API Calls and related External\n Products are subject to applicable additional agreements with the relevant\n provider of the External Products. In no event shall the open source license\n that governs this project grant any rights in or to any External Products,or\n alter, expand or supersede any terms of the applicable additional agreements.\n If you have a valid license agreement with SAP for the use of a particular SAP\n External Product, then you may make use of any API Calls included in this\n project's code for that SAP External Product, subject to the terms of such\n license agreement. If you do not have a valid license agreement for the use of\n a particular SAP External Product, then you may only make use of any API Calls\n in this project for that SAP External Product for your internal, non-productive\n and non-commercial test and evaluation of such API Calls. Nothing herein grants\n you any rights to use or access any SAP External Product, or provide any third\n parties the right to use of access any SAP External Product, through API Calls." 6 | 7 | [[annotations]] 8 | path = "**" 9 | precedence = "aggregate" 10 | SPDX-FileCopyrightText = "2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors" 11 | SPDX-License-Identifier = "Apache-2.0" 12 | -------------------------------------------------------------------------------- /build/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | * 5 | !.gitignore -------------------------------------------------------------------------------- /commands/error/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "encoding/json" 10 | "log/slog" 11 | "net/http" 12 | ) 13 | 14 | // Simulate a request error by sending a malformed payload to the scheduler. 15 | func main() { 16 | request := struct { 17 | HereBeDragons string `json:"here_be_dragons"` 18 | }{ 19 | HereBeDragons: "123", 20 | } 21 | 22 | url := "http://localhost:8080/scheduler/nova/external" 23 | slog.Info("sending POST request", "url", url) 24 | requestBody, err := json.Marshal(request) 25 | if err != nil { 26 | slog.Error("failed to marshal request", "error", err) 27 | return 28 | } 29 | ctx := context.Background() 30 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(requestBody)) 31 | if err != nil { 32 | slog.Error("failed to create request", "error", err) 33 | return 34 | } 35 | req.Header.Set("Content-Type", "application/json") 36 | 37 | resp, err := http.DefaultClient.Do(req) 38 | if err != nil { 39 | slog.Error("failed to send POST request", "error", err) 40 | return 41 | } 42 | defer resp.Body.Close() 43 | 44 | // Print out response json (without unmarshalling it) 45 | buf := new(bytes.Buffer) 46 | if _, err := buf.ReadFrom(resp.Body); err != nil { 47 | slog.Error("failed to read response", "error", err) 48 | return 49 | } 50 | slog.Info("received response", "status", resp.Status, "body", buf.String()) 51 | } 52 | -------------------------------------------------------------------------------- /commands/replay/replay.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "encoding/json" 10 | "flag" 11 | "fmt" 12 | "io" 13 | "math/rand" 14 | "net/http" 15 | "os" 16 | 17 | mqtt "github.com/eclipse/paho.mqtt.golang" 18 | "github.com/sapcc/go-bits/must" 19 | ) 20 | 21 | // Replay Nova messages retrieved from the telemetry mqtt broker to a local Cortex instance. 22 | // Use together with: `kubectl port-forward cortex-mqtt-0 18830:1883` 23 | func main() { 24 | // Parse command-line arguments 25 | host := flag.String("h", "tcp://localhost:18830", "The cortex MQTT broker to connect to") 26 | username := flag.String("u", "cortex", "The username to use for the MQTT connection") 27 | password := flag.String("p", "cortex", "The password to use for the MQTT connection") 28 | cortexURL := flag.String("c", "http://localhost:8080", "The Cortex instance to forward to") 29 | help := flag.Bool("help", false, "Show this help message") 30 | flag.Usage = func() { 31 | fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [options]\n", os.Args[0]) 32 | flag.PrintDefaults() 33 | } 34 | flag.Parse() 35 | if *help { 36 | flag.Usage() 37 | os.Exit(0) 38 | } 39 | 40 | opts := mqtt.NewClientOptions() 41 | opts.AddBroker(*host) 42 | opts.SetUsername(*username) 43 | opts.SetPassword(*password) 44 | //nolint:gosec // We don't care if the client id is cryptographically secure. 45 | opts.SetClientID(fmt.Sprintf("cortex-replay-%d", rand.Intn(1_000_000))) 46 | 47 | client := mqtt.NewClient(opts) 48 | if conn := client.Connect(); conn.Wait() && conn.Error() != nil { 49 | fmt.Fprintf(os.Stderr, "Failed to connect to MQTT broker: %v\n", conn.Error()) 50 | os.Exit(1) 51 | } 52 | defer client.Disconnect(1000) 53 | 54 | topic := "cortex/scheduler/pipeline/finished" 55 | client.Subscribe(topic, 2, func(client mqtt.Client, msg mqtt.Message) { 56 | // Unwrap the "request" from the message 57 | var payload map[string]any 58 | must.Succeed(json.Unmarshal(msg.Payload(), &payload)) 59 | request, ok := payload["request"] 60 | if !ok { 61 | fmt.Fprintf(os.Stderr, "Message does not contain a 'request' field\n") 62 | return 63 | } 64 | // Forward the request to the local Cortex instance 65 | requestBody := must.Return(json.Marshal(request)) 66 | url := *cortexURL + "/scheduler/nova/external" 67 | req := must.Return(http.NewRequestWithContext(context.Background(), http.MethodPost, url, bytes.NewBuffer(requestBody))) 68 | req.Header.Set("Content-Type", "application/json") 69 | resp, err := http.DefaultClient.Do(req) 70 | must.Succeed(err) 71 | defer resp.Body.Close() 72 | if resp.StatusCode != http.StatusOK { 73 | body := must.Return(io.ReadAll(resp.Body)) 74 | fmt.Fprintf(os.Stderr, "Cortex responded with status %d: %s\n", resp.StatusCode, string(body)) 75 | return 76 | } 77 | fmt.Printf("Successfully forwarded message received on topic %s to Cortex.\n", msg.Topic()) 78 | }) 79 | 80 | // Block the main thread to keep the program running 81 | select {} 82 | } 83 | -------------------------------------------------------------------------------- /cortex.secrets.example.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Override config values that contain sensitive information or 5 | # are specific to your environment. These values can be used in the Tiltfile. 6 | 7 | # SSO certificate to use. 8 | sharedSSOCert: &sharedSSOCert 9 | # Certificate "public key". (Optional, remove this key if not needed) 10 | cert: | 11 | -----BEGIN CERTIFICATE----- 12 | Your certificate here 13 | -----END CERTIFICATE----- 14 | # Certificate private key. (Optional, remove this key if not needed) 15 | certKey: | 16 | -----BEGIN PRIVATE KEY----- 17 | Your private key here 18 | -----END PRIVATE KEY 19 | # Whether the certificate is self-signed. 20 | # If true, the certificate is not verified. 21 | selfSigned: false 22 | 23 | conf: 24 | sync: 25 | # Override the endpoints to your Prometheus instances. 26 | prometheus: 27 | hosts: 28 | # See: https://github.com/sapcc/vrops-exporter 29 | - name: vmware_prometheus 30 | url: https://path-to-your-vrops-prometheus 31 | sso: *sharedSSOCert 32 | provides: [vrops_vm_metric, vrops_host_metric] 33 | # See: https://github.com/prometheus/node_exporter 34 | - name: kvm_prometheus 35 | url: https://path-to-your-node-exporter 36 | sso: *sharedSSOCert 37 | provides: [node_exporter_metric] 38 | # Override the endpoints and credentials to your OpenStack. 39 | openstack: 40 | keystone: 41 | url: https://path-to-keystone/v3 42 | sso: *sharedSSOCert 43 | username: openstack-user-with-all-project-read-access 44 | password: openstack-user-password 45 | projectName: openstack-project-of-user 46 | userDomainName: openstack-domain-of-user 47 | projectDomainName: openstack-domain-of-project-scoped-to 48 | -------------------------------------------------------------------------------- /docs/readme.md: -------------------------------------------------------------------------------- 1 | # What is Cortex? 2 | 3 | - For a deep understanding of the ideas and concepts behind cortex, see [the architecture docs](architecture.md). 4 | - For a guide on how to develop with cortex, see [the development docs](develop.md). 5 | - For a list of current and future scheduling features, see [the feature docs](features.md). 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/trickyteache/cortex 2 | 3 | go 1.24 4 | 5 | replace github.com/trickyteache/cortex/commands => ./commands 6 | 7 | replace github.com/trickyteache/cortex/testlib => ./testlib 8 | 9 | require ( 10 | github.com/dlmiddlecote/sqlstats v1.0.2 11 | github.com/eclipse/paho.mqtt.golang v1.5.0 12 | github.com/go-gorp/gorp v2.2.0+incompatible 13 | github.com/gophercloud/gophercloud/v2 v2.7.0 14 | github.com/lib/pq v1.10.9 15 | github.com/mattn/go-sqlite3 v1.14.28 16 | github.com/ory/dockertest v3.3.5+incompatible 17 | github.com/prometheus/client_golang v1.22.0 18 | github.com/prometheus/client_model v0.6.2 19 | github.com/sapcc/go-api-declarations v1.15.0 20 | github.com/sapcc/go-bits v0.0.0-20250506185838-b658155ceaf5 21 | go.uber.org/automaxprocs v1.6.0 22 | ) 23 | 24 | require ( 25 | github.com/Microsoft/go-winio v0.6.2 // indirect 26 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect 27 | github.com/cenkalti/backoff v2.2.1+incompatible // indirect 28 | github.com/containerd/continuity v0.4.5 // indirect 29 | github.com/docker/go-connections v0.5.0 // indirect 30 | github.com/docker/go-units v0.5.0 // indirect 31 | github.com/go-sql-driver/mysql v1.9.2 // indirect 32 | github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect 33 | github.com/moby/sys/user v0.4.0 // indirect 34 | github.com/opencontainers/go-digest v1.0.0 // indirect 35 | github.com/opencontainers/image-spec v1.1.1 // indirect 36 | github.com/opencontainers/runc v1.3.0 // indirect 37 | github.com/pkg/errors v0.9.1 // indirect 38 | github.com/poy/onpar v1.1.2 // indirect 39 | github.com/sirupsen/logrus v1.9.3 // indirect 40 | github.com/ziutek/mymysql v1.5.4 // indirect 41 | golang.org/x/net v0.40.0 // indirect 42 | gotest.tools v2.2.0+incompatible // indirect 43 | ) 44 | 45 | require ( 46 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect 47 | github.com/beorn7/perks v1.0.1 // indirect 48 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 49 | github.com/golang-migrate/migrate/v4 v4.18.3 // indirect 50 | github.com/gorilla/websocket v1.5.3 // indirect 51 | github.com/hashicorp/errwrap v1.1.0 // indirect 52 | github.com/hashicorp/go-multierror v1.1.1 // indirect 53 | github.com/kylelemons/godebug v1.1.0 // indirect 54 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 55 | github.com/prometheus/common v0.63.0 // indirect 56 | github.com/prometheus/procfs v0.16.1 // indirect 57 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 58 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 59 | go.uber.org/atomic v1.11.0 // indirect 60 | golang.org/x/sync v0.14.0 // indirect 61 | golang.org/x/sys v0.33.0 // indirect 62 | google.golang.org/protobuf v1.36.6 // indirect 63 | ) 64 | -------------------------------------------------------------------------------- /helm/cortex/.helmignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Patterns to ignore when building packages. 5 | # This supports shell glob matching, relative path matching, and 6 | # negation (prefixed with !). Only one pattern per line. 7 | .DS_Store 8 | # Common VCS dirs 9 | .git/ 10 | .gitignore 11 | charts/.gitignore 12 | .bzr/ 13 | .bzrignore 14 | .hg/ 15 | .hgignore 16 | .svn/ 17 | # Common backup files 18 | *.swp 19 | *.bak 20 | *.tmp 21 | *.orig 22 | *~ 23 | # Various IDEs 24 | .project 25 | .idea/ 26 | *.tmproj 27 | .vscode/ 28 | -------------------------------------------------------------------------------- /helm/cortex/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: owner-info 3 | repository: oci://ghcr.io/sapcc/helm-charts 4 | version: 1.0.0 5 | - name: rabbitmq 6 | repository: oci://registry-1.docker.io/bitnamicharts 7 | version: 16.0.2 8 | digest: sha256:f79ad0d4a3ab8669dbf47a9b1c6c572bdcf53c3384f32a15bbc0663c0bcce68f 9 | generated: "2025-05-12T15:02:03.711284+02:00" 10 | -------------------------------------------------------------------------------- /helm/cortex/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v2 5 | name: cortex 6 | description: A Helm chart for deploying Cortex. 7 | 8 | # A chart can be either an 'application' or a 'library' chart. 9 | # 10 | # Application charts are a collection of templates that can be packaged into versioned archives 11 | # to be deployed. 12 | # 13 | # Library charts provide useful utilities or functions for the chart developer. They're included as 14 | # a dependency of application charts to inject those utilities and functions into the rendering 15 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 16 | type: application 17 | 18 | # This is the chart version. This version number should be incremented each time you make changes 19 | # to the chart and its templates, including the app version. 20 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 21 | version: 0.15.3 22 | 23 | # This is the version number of the application being deployed. This version number should be 24 | # incremented each time you make changes to the application. Versions are not expected to 25 | # follow Semantic Versioning. They should reflect the version the application is using. 26 | # It is recommended to use it with quotes. 27 | appVersion: "0.1.0" 28 | 29 | dependencies: 30 | # Owner info adds a configmap to the kubernetes cluster with information on 31 | # the service owner. This makes it easier to find out who to contact in case 32 | # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info 33 | - name: owner-info 34 | repository: oci://ghcr.io/sapcc/helm-charts 35 | version: 1.0.0 36 | # RabbitMQ is a message broker, see: https://artifacthub.io/packages/helm/bitnami/rabbitmq 37 | - name: rabbitmq 38 | repository: oci://registry-1.docker.io/bitnamicharts 39 | version: 16.0.2 40 | -------------------------------------------------------------------------------- /helm/cortex/charts/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | * 5 | !.gitignore -------------------------------------------------------------------------------- /helm/cortex/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | {{/* 5 | Expand the name of the chart. 6 | */}} 7 | {{- define "cortex.name" -}} 8 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 9 | {{- end }} 10 | 11 | {{/* 12 | Create a default fully qualified app name. 13 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 14 | If release name contains chart name it will be used as a full name. 15 | */}} 16 | {{- define "cortex.fullname" -}} 17 | {{- if .Values.fullnameOverride }} 18 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 19 | {{- else }} 20 | {{- $name := default .Chart.Name .Values.nameOverride }} 21 | {{- if contains $name .Release.Name }} 22 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 23 | {{- else }} 24 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 25 | {{- end }} 26 | {{- end }} 27 | {{- end }} 28 | 29 | {{/* 30 | Create chart name and version as used by the chart label. 31 | */}} 32 | {{- define "cortex.chart" -}} 33 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 34 | {{- end }} 35 | 36 | {{/* 37 | Common labels 38 | */}} 39 | {{- define "cortex.labels" -}} 40 | helm.sh/chart: {{ include "cortex.chart" . }} 41 | {{ include "cortex.selectorLabels" . }} 42 | {{- if .Chart.AppVersion }} 43 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 44 | {{- end }} 45 | app.kubernetes.io/managed-by: {{ .Release.Service }} 46 | {{- end }} 47 | 48 | {{/* 49 | Selector labels 50 | */}} 51 | {{- define "cortex.selectorLabels" -}} 52 | app.kubernetes.io/name: {{ include "cortex.name" . }} 53 | app.kubernetes.io/instance: {{ .Release.Name }} 54 | {{- end }} 55 | 56 | {{/* 57 | Create the name of the service account to use 58 | */}} 59 | {{- define "cortex.serviceAccountName" -}} 60 | {{- if .Values.serviceAccount.create }} 61 | {{- default (include "cortex.fullname" .) .Values.serviceAccount.name }} 62 | {{- else }} 63 | {{- default "default" .Values.serviceAccount.name }} 64 | {{- end }} 65 | {{- end }} 66 | -------------------------------------------------------------------------------- /helm/cortex/templates/cli.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v1 5 | kind: Pod 6 | metadata: 7 | name: {{ $.Chart.Name }}-cli 8 | labels: 9 | app: {{ $.Chart.Name }}-cli 10 | {{- include "cortex.labels" $ | nindent 4 }} 11 | spec: 12 | {{- with $.Values.imagePullSecrets }} 13 | imagePullSecrets: 14 | {{- toYaml . | nindent 4 }} 15 | {{- end }} 16 | securityContext: 17 | {{- toYaml $.Values.podSecurityContext | nindent 4 }} 18 | containers: 19 | - name: {{ $.Chart.Name }}-cli 20 | command: 21 | - "/bin/sh" 22 | - "-c" 23 | - "echo 'Waiting for commands...' && sleep infinity" 24 | securityContext: 25 | {{- toYaml $.Values.securityContext | nindent 8 }} 26 | image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}" 27 | imagePullPolicy: {{ $.Values.image.pullPolicy }} 28 | resources: 29 | {{- toYaml $.Values.resources | nindent 8 }} 30 | volumeMounts: 31 | - name: {{ include "cortex.fullname" $ }}-config-volume 32 | mountPath: /etc/config 33 | {{- with $.Values.volumeMounts }} 34 | {{- toYaml . | nindent 8 }} 35 | {{- end }} 36 | volumes: 37 | - name: {{ include "cortex.fullname" $ }}-config-volume 38 | configMap: 39 | name: {{ include "cortex.fullname" $ }}-config 40 | {{- with $.Values.volumes }} 41 | {{- toYaml . | nindent 4 }} 42 | {{- end }} 43 | {{- with $.Values.nodeSelector }} 44 | nodeSelector: 45 | {{- toYaml . | nindent 4 }} 46 | {{- end }} 47 | {{- with $.Values.affinity }} 48 | affinity: 49 | {{- toYaml . | nindent 4 }} 50 | {{- end }} 51 | {{- with $.Values.tolerations }} 52 | tolerations: 53 | {{- toYaml . | nindent 4 }} 54 | {{- end }} -------------------------------------------------------------------------------- /helm/cortex/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v1 5 | kind: ConfigMap 6 | metadata: 7 | name: {{ include "cortex.fullname" . }}-config 8 | data: 9 | conf.json: |- 10 | {{- if .Values.conf }} 11 | {{ toJson .Values.conf }} 12 | {{- else }} 13 | {} 14 | {{- end }} -------------------------------------------------------------------------------- /helm/cortex/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | {{- range .Values.modes }} 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: {{ $.Chart.Name }}-{{ .name }} 10 | annotations: 11 | # Roll the service when its configmap changes. 12 | checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $ | sha256sum }} 13 | labels: 14 | app: {{ $.Chart.Name }}-{{ .name }} 15 | {{- include "cortex.labels" $ | nindent 4 }} 16 | spec: 17 | replicas: {{ .replicas }} 18 | selector: 19 | matchLabels: 20 | {{- include "cortex.selectorLabels" $ | nindent 6 }} 21 | template: 22 | metadata: 23 | {{- with $.Values.podAnnotations }} 24 | annotations: 25 | {{- toYaml . | nindent 8 }} 26 | {{- end }} 27 | labels: 28 | app: {{ $.Chart.Name }}-{{ .name }} 29 | {{- include "cortex.labels" $ | nindent 8 }} 30 | {{- with $.Values.podLabels }} 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | spec: 34 | {{- with $.Values.imagePullSecrets }} 35 | imagePullSecrets: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | securityContext: 39 | {{- toYaml $.Values.podSecurityContext | nindent 8 }} 40 | containers: 41 | - name: {{ $.Chart.Name }}-{{ .name }} 42 | args: {{ .args | toJson }} 43 | securityContext: 44 | {{- toYaml $.Values.securityContext | nindent 12 }} 45 | image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}" 46 | imagePullPolicy: {{ $.Values.image.pullPolicy }} 47 | ports: 48 | - name: api 49 | containerPort: {{ $.Values.conf.api.port }} 50 | protocol: TCP 51 | - name: metrics 52 | containerPort: {{ $.Values.conf.monitoring.port }} 53 | protocol: TCP 54 | livenessProbe: 55 | {{- toYaml $.Values.livenessProbe | nindent 12 }} 56 | readinessProbe: 57 | {{- toYaml $.Values.readinessProbe | nindent 12 }} 58 | resources: 59 | {{- toYaml $.Values.resources | nindent 12 }} 60 | volumeMounts: 61 | - name: {{ include "cortex.fullname" $ }}-config-volume 62 | mountPath: /etc/config 63 | {{- with $.Values.volumeMounts }} 64 | {{- toYaml . | nindent 12 }} 65 | {{- end }} 66 | volumes: 67 | - name: {{ include "cortex.fullname" $ }}-config-volume 68 | configMap: 69 | name: {{ include "cortex.fullname" $ }}-config 70 | {{- with $.Values.volumes }} 71 | {{- toYaml . | nindent 8 }} 72 | {{- end }} 73 | {{- with $.Values.nodeSelector }} 74 | nodeSelector: 75 | {{- toYaml . | nindent 8 }} 76 | {{- end }} 77 | {{- with $.Values.affinity }} 78 | affinity: 79 | {{- toYaml . | nindent 8 }} 80 | {{- end }} 81 | {{- with $.Values.tolerations }} 82 | tolerations: 83 | {{- toYaml . | nindent 8 }} 84 | {{- end }} 85 | {{- end }} -------------------------------------------------------------------------------- /helm/cortex/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | {{- range .Values.modes }} 5 | --- 6 | apiVersion: v1 7 | kind: Service 8 | metadata: 9 | name: {{ $.Chart.Name }}-{{ .name }} 10 | labels: 11 | {{- include "cortex.labels" $ | nindent 4 }} 12 | # This label will be added to the prometheus metrics. 13 | component: {{ $.Chart.Name }}-{{ .name }} 14 | spec: 15 | type: ClusterIP 16 | ports: 17 | - port: {{ $.Values.conf.api.port }} 18 | targetPort: api 19 | protocol: TCP 20 | name: api 21 | - port: {{ $.Values.conf.monitoring.port }} 22 | targetPort: metrics 23 | protocol: TCP 24 | name: metrics 25 | selector: 26 | app: {{ $.Chart.Name }}-{{ .name }} 27 | {{- end }} -------------------------------------------------------------------------------- /helm/postgres/.helmignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Patterns to ignore when building packages. 5 | # This supports shell glob matching, relative path matching, and 6 | # negation (prefixed with !). Only one pattern per line. 7 | .DS_Store 8 | # Common VCS dirs 9 | .git/ 10 | .gitignore 11 | charts/.gitignore 12 | .bzr/ 13 | .bzrignore 14 | .hg/ 15 | .hgignore 16 | .svn/ 17 | # Common backup files 18 | *.swp 19 | *.bak 20 | *.tmp 21 | *.orig 22 | *~ 23 | # Various IDEs 24 | .project 25 | .idea/ 26 | *.tmproj 27 | .vscode/ 28 | -------------------------------------------------------------------------------- /helm/postgres/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: postgresql 3 | repository: oci://registry-1.docker.io/bitnamicharts 4 | version: 15.5.27 5 | - name: owner-info 6 | repository: oci://ghcr.io/sapcc/helm-charts 7 | version: 1.0.0 8 | digest: sha256:26cb9c33b10bfaec7d04e718c321935eb031c3d0e426902afc12b99541a20ad7 9 | generated: "2025-04-23T08:52:04.316499+02:00" 10 | -------------------------------------------------------------------------------- /helm/postgres/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v2 5 | description: Postgres setup for Cortex. 6 | name: cortex-postgres 7 | version: 0.1.0 8 | dependencies: 9 | - name: postgresql 10 | repository: oci://registry-1.docker.io/bitnamicharts 11 | # Use a specific version of this chart which contains compliant images. 12 | version: 15.5.27 13 | # Owner info adds a configmap to the kubernetes cluster with information on 14 | # the service owner. This makes it easier to find out who to contact in case 15 | # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info 16 | - name: owner-info 17 | repository: oci://ghcr.io/sapcc/helm-charts 18 | version: 1.0.0 -------------------------------------------------------------------------------- /helm/postgres/charts/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | * 5 | !.gitignore -------------------------------------------------------------------------------- /helm/postgres/templates/.gitkeep: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | In case we want to customize the postgres setup, this would be the place to do that. -------------------------------------------------------------------------------- /helm/postgres/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | postgresql: 5 | fullnameOverride: cortex-postgresql 6 | volumePermissions: 7 | enabled: true 8 | auth: 9 | postgresPassword: secret 10 | service: 11 | ports: 12 | postgresql: 5432 13 | 14 | owner-info: 15 | helm-chart-url: "https://github.com/trickyteache/cortex/helm/postgres" 16 | maintainers: 17 | - "p.matthes@sap.com" 18 | support-group: "cobaltcore-dev" 19 | enabled: true -------------------------------------------------------------------------------- /helm/prometheus-deps/.helmignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Patterns to ignore when building packages. 5 | # This supports shell glob matching, relative path matching, and 6 | # negation (prefixed with !). Only one pattern per line. 7 | .DS_Store 8 | # Common VCS dirs 9 | .git/ 10 | .gitignore 11 | charts/.gitignore 12 | .bzr/ 13 | .bzrignore 14 | .hg/ 15 | .hgignore 16 | .svn/ 17 | # Common backup files 18 | *.swp 19 | *.bak 20 | *.tmp 21 | *.orig 22 | *~ 23 | # Various IDEs 24 | .project 25 | .idea/ 26 | *.tmproj 27 | .vscode/ 28 | -------------------------------------------------------------------------------- /helm/prometheus-deps/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: kube-prometheus-stack 3 | repository: https://prometheus-community.github.io/helm-charts 4 | version: 72.2.0 5 | - name: owner-info 6 | repository: oci://ghcr.io/sapcc/helm-charts 7 | version: 1.0.0 8 | digest: sha256:6c553922fc886dc038dc25fb18e02a8087c94885e43c3f62e964a5596e76c19c 9 | generated: "2025-05-12T14:31:46.394636+02:00" 10 | -------------------------------------------------------------------------------- /helm/prometheus-deps/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v2 5 | description: Prometheus alerts dependencies for Cortex. 6 | name: cortex-alerts-deps 7 | version: 0.2.0 8 | dependencies: 9 | # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. 10 | - name: kube-prometheus-stack 11 | repository: https://prometheus-community.github.io/helm-charts 12 | version: 72.2.0 13 | # Owner info adds a configmap to the kubernetes cluster with information on 14 | # the service owner. This makes it easier to find out who to contact in case 15 | # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info 16 | - name: owner-info 17 | repository: oci://ghcr.io/sapcc/helm-charts 18 | version: 1.0.0 19 | -------------------------------------------------------------------------------- /helm/prometheus-deps/charts/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | * 5 | !.gitignore -------------------------------------------------------------------------------- /helm/prometheus-deps/templates/alertmanager.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: monitoring.coreos.com/v1alpha1 5 | kind: AlertmanagerConfig 6 | metadata: 7 | name: cortex-alertmanager 8 | labels: 9 | alertmanagerConfig: cortex-alertmanager 10 | spec: 11 | route: 12 | groupBy: ['job'] 13 | groupWait: 30s 14 | groupInterval: 5m 15 | repeatInterval: 12h 16 | receiver: 'log' 17 | receivers: 18 | - name: 'log' 19 | webhookConfigs: 20 | - url: 'http://cortex-alertmanager-logger:9094/' 21 | sendResolved: true 22 | --- 23 | apiVersion: monitoring.coreos.com/v1 24 | kind: Alertmanager 25 | metadata: 26 | name: cortex-alertmanager 27 | spec: 28 | replicas: 1 29 | alertmanagerConfigSelector: 30 | matchLabels: 31 | alertmanagerConfig: cortex-alertmanager 32 | -------------------------------------------------------------------------------- /helm/prometheus-deps/templates/prometheus.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: monitoring.coreos.com/v1 5 | kind: Prometheus 6 | metadata: 7 | name: cortex-prometheus 8 | spec: 9 | serviceAccountName: cortex-prometheus 10 | serviceMonitorNamespaceSelector: {} 11 | serviceMonitorSelector: 12 | matchLabels: 13 | name: cortex-prometheus 14 | podMonitorSelector: {} 15 | enableAdminAPI: true 16 | resources: 17 | requests: 18 | memory: 400Mi 19 | alerting: 20 | alertmanagers: 21 | - namespace: default 22 | name: alertmanager-operated 23 | port: web 24 | scrapeInterval: 60s 25 | ruleSelector: 26 | matchLabels: 27 | type: alerting-rules 28 | -------------------------------------------------------------------------------- /helm/prometheus-deps/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v1 5 | kind: ServiceAccount 6 | metadata: 7 | name: cortex-prometheus 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | kind: ClusterRole 11 | metadata: 12 | name: cortex-prometheus 13 | rules: 14 | - apiGroups: [""] 15 | resources: 16 | - nodes 17 | - nodes/metrics 18 | - services 19 | - endpoints 20 | - pods 21 | verbs: ["get", "list", "watch"] 22 | - apiGroups: [""] 23 | resources: 24 | - configmaps 25 | verbs: ["get"] 26 | - apiGroups: 27 | - networking.k8s.io 28 | resources: 29 | - ingresses 30 | verbs: ["get", "list", "watch"] 31 | - nonResourceURLs: ["/metrics"] 32 | verbs: ["get"] 33 | --- 34 | apiVersion: rbac.authorization.k8s.io/v1 35 | kind: ClusterRoleBinding 36 | metadata: 37 | name: cortex-prometheus 38 | roleRef: 39 | apiGroup: rbac.authorization.k8s.io 40 | kind: ClusterRole 41 | name: cortex-prometheus 42 | subjects: 43 | - kind: ServiceAccount 44 | name: cortex-prometheus 45 | namespace: default 46 | -------------------------------------------------------------------------------- /helm/prometheus-deps/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | kube-prometheus-stack: 5 | # kube-prometheus-stack deploys a bunch of junk we don't need. 6 | # Therefore, we disable all components and only enable the prometheus operator. 7 | # See: https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/ci/01-provision-crds-values.yaml 8 | coreDns: 9 | enabled: false 10 | kubeApiServer: 11 | enabled: false 12 | kubeControllerManager: 13 | enabled: false 14 | kubeDns: 15 | enabled: false 16 | kubeEtcd: 17 | enabled: false 18 | kubeProxy: 19 | enabled: false 20 | kubeScheduler: 21 | enabled: false 22 | kubeStateMetrics: 23 | enabled: false 24 | kubelet: 25 | enabled: false 26 | nodeExporter: 27 | enabled: false 28 | grafana: 29 | enabled: false 30 | defaultRules: 31 | create: false 32 | alertmanager: 33 | enabled: false 34 | prometheus: 35 | enabled: false 36 | fullnameOverride: "cortex" 37 | prometheusOperator: 38 | enabled: true 39 | fullnameOverride: cortex-prometheus-operator 40 | serviceMonitor: 41 | selfMonitor: false 42 | tls: 43 | enabled: false 44 | admissionWebhooks: 45 | enabled: false 46 | namespaces: 47 | releaseNamespace: true 48 | additional: 49 | - kube-system 50 | 51 | owner-info: 52 | helm-chart-url: "https://github.com/trickyteache/cortex/helm/prometheus" 53 | maintainers: 54 | - "p.matthes@sap.com" 55 | support-group: "cobaltcore-dev" 56 | enabled: true -------------------------------------------------------------------------------- /helm/prometheus/.helmignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Patterns to ignore when building packages. 5 | # This supports shell glob matching, relative path matching, and 6 | # negation (prefixed with !). Only one pattern per line. 7 | .DS_Store 8 | # Common VCS dirs 9 | .git/ 10 | .gitignore 11 | charts/.gitignore 12 | .bzr/ 13 | .bzrignore 14 | .hg/ 15 | .hgignore 16 | .svn/ 17 | # Common backup files 18 | *.swp 19 | *.bak 20 | *.tmp 21 | *.orig 22 | *~ 23 | # Various IDEs 24 | .project 25 | .idea/ 26 | *.tmproj 27 | .vscode/ 28 | -------------------------------------------------------------------------------- /helm/prometheus/Chart.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: owner-info 3 | repository: oci://ghcr.io/sapcc/helm-charts 4 | version: 1.0.0 5 | digest: sha256:7643f231cc4ebda347fd12ec62fe4445c280e2b71d27eec555f3025290f5038f 6 | generated: "2025-05-12T14:35:31.273371+02:00" 7 | -------------------------------------------------------------------------------- /helm/prometheus/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: v2 5 | description: Prometheus alerts setup for Cortex. 6 | name: cortex-alerts 7 | version: 0.2.1 8 | dependencies: 9 | # Owner info adds a configmap to the kubernetes cluster with information on 10 | # the service owner. This makes it easier to find out who to contact in case 11 | # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info 12 | - name: owner-info 13 | repository: oci://ghcr.io/sapcc/helm-charts 14 | version: 1.0.0 15 | -------------------------------------------------------------------------------- /helm/prometheus/charts/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | * 5 | !.gitignore -------------------------------------------------------------------------------- /helm/prometheus/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: monitoring.coreos.com/v1 5 | kind: ServiceMonitor 6 | metadata: 7 | name: cortex-prometheus 8 | labels: 9 | name: cortex-prometheus 10 | prometheus: openstack 11 | spec: 12 | selector: 13 | matchLabels: 14 | # Select all services from the Cortex helm chart. 15 | app.kubernetes.io/name: cortex 16 | namespaceSelector: 17 | any: true 18 | targetLabels: 19 | - component 20 | endpoints: 21 | - port: metrics 22 | -------------------------------------------------------------------------------- /helm/prometheus/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | global: 5 | prometheus: cortex-prometheus 6 | 7 | alerts: 8 | # Labels applied to all alerts. 9 | labels: 10 | # Labels applied to all alert levels. 11 | global: 12 | service: cortex 13 | # Labels applied to specific alerts. 14 | cortexSchedulerDown: 15 | severity: warning 16 | cortexSyncerDown: 17 | severity: info 18 | cortexExtractorDown: 19 | severity: info 20 | cortexHttpRequest400sTooHigh: 21 | severity: info 22 | cortexHttpRequest500sTooHigh: 23 | severity: info 24 | cortexHighMemoryUsage: 25 | severity: info 26 | cortexHighCPUUsage: 27 | severity: info 28 | cortexLongPipelineRun: 29 | severity: info 30 | cortexSyncNotSuccessful: 31 | severity: info 32 | cortexSyncObjectsDroppedToZero: 33 | severity: info 34 | cortexSyncObjectsTooHigh: 35 | severity: info 36 | 37 | owner-info: 38 | helm-chart-url: "https://github.com/trickyteache/cortex/helm/prometheus" 39 | maintainers: 40 | - "p.matthes@sap.com" 41 | support-group: "cobaltcore-dev" 42 | enabled: true -------------------------------------------------------------------------------- /internal/conf/logging.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package conf 5 | 6 | import ( 7 | "log/slog" 8 | "os" 9 | ) 10 | 11 | // Conform to the slog.Leveler interface. 12 | func (c LoggingConfig) Level() slog.Level { 13 | switch c.LevelStr { 14 | case "debug": 15 | return slog.LevelDebug 16 | case "info": 17 | return slog.LevelInfo 18 | case "warn": 19 | return slog.LevelWarn 20 | case "error": 21 | return slog.LevelError 22 | default: 23 | return slog.LevelInfo 24 | } 25 | } 26 | 27 | // Set the structured logger as given in the config. 28 | func (c LoggingConfig) SetDefaultLogger() { 29 | opts := &slog.HandlerOptions{Level: c} 30 | var handler slog.Handler 31 | switch c.Format { 32 | case "json": 33 | handler = slog.NewJSONHandler(os.Stdout, opts) 34 | default: 35 | handler = slog.NewTextHandler(os.Stdout, opts) 36 | } 37 | slog.SetDefault(slog.New(handler)) 38 | slog.Info("logging: set default logger", "level", c.LevelStr, "format", c.Format) 39 | } 40 | -------------------------------------------------------------------------------- /internal/conf/logging_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package conf 5 | 6 | import ( 7 | "bytes" 8 | "io" 9 | "log/slog" 10 | "os" 11 | "strings" 12 | "testing" 13 | ) 14 | 15 | func TestLoggingConfig_Level(t *testing.T) { 16 | tests := []struct { 17 | levelStr string 18 | expected slog.Level 19 | }{ 20 | {"debug", slog.LevelDebug}, 21 | {"info", slog.LevelInfo}, 22 | {"warn", slog.LevelWarn}, 23 | {"error", slog.LevelError}, 24 | {"unknown", slog.LevelInfo}, // default case 25 | } 26 | 27 | for _, tt := range tests { 28 | t.Run(tt.levelStr, func(t *testing.T) { 29 | config := LoggingConfig{LevelStr: tt.levelStr} 30 | level := config.Level() 31 | if level != tt.expected { 32 | t.Errorf("expected %v, got %v", tt.expected, level) 33 | } 34 | }) 35 | } 36 | } 37 | 38 | func TestLoggingConfig_SetDefaultLogger(t *testing.T) { 39 | tests := []struct { 40 | format string 41 | expected string 42 | }{ 43 | {"json", `"level":"INFO","msg":"logging: set default logger","level":"info","format":"json"`}, 44 | {"text", `level=INFO msg="logging: set default logger" level=info format=text`}, 45 | } 46 | 47 | for _, tt := range tests { 48 | t.Run(tt.format, func(t *testing.T) { 49 | config := LoggingConfig{LevelStr: "info", Format: tt.format} 50 | 51 | // Capture the output 52 | r, w, err := os.Pipe() 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | stdout := os.Stdout 57 | defer func() { os.Stdout = stdout }() 58 | os.Stdout = w 59 | 60 | config.SetDefaultLogger() 61 | 62 | // Close the writer and read the output 63 | w.Close() 64 | var buf bytes.Buffer 65 | if _, err := io.Copy(&buf, r); err != nil { 66 | t.Fatal(err) 67 | } 68 | 69 | // Check the output 70 | output := buf.String() 71 | if !strings.Contains(output, tt.expected) { 72 | t.Errorf("expected output to contain %q, got %q", tt.expected, output) 73 | } 74 | }) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /internal/conf/opts.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package conf 5 | 6 | import ( 7 | "encoding/json" 8 | "log/slog" 9 | ) 10 | 11 | // Raw options that are not directly unmarshalled when loading from json. 12 | // Usage: call Unmarshal to unmarshal the options into a struct. 13 | type RawOpts struct { 14 | // Postponed unmarshal function. 15 | unmarshal func(any) error 16 | } 17 | 18 | // Create a new RawOpts instance with the given json string. 19 | func NewRawOpts(rawJson string) RawOpts { 20 | return RawOpts{unmarshal: func(v any) error { 21 | return json.Unmarshal([]byte(rawJson), v) 22 | }} 23 | } 24 | 25 | // Call the postponed unmarshal function and unmarshal the options into a struct. 26 | func (msg *RawOpts) Unmarshal(v any) error { 27 | if msg.unmarshal == nil { 28 | // No unmarshal function set (e.g. empty json), return nil. 29 | return nil 30 | } 31 | return msg.unmarshal(v) 32 | } 33 | 34 | // Override the default json unmarshal behavior to postpone the unmarshal. 35 | func (msg *RawOpts) UnmarshalJSON(data []byte) error { 36 | msg.unmarshal = func(v any) error { 37 | return json.Unmarshal(data, v) 38 | } 39 | return nil 40 | } 41 | 42 | // Mixin that adds the ability to load options from a json map. 43 | // Usage: type StructUsingOpts struct { conf.JsonOpts[MyOpts] } 44 | type JsonOpts[Options any] struct { 45 | // Options loaded from a json config using the Load method. 46 | Options Options 47 | } 48 | 49 | // Set the options contained in the opts json map. 50 | func (s *JsonOpts[Options]) Load(opts RawOpts) error { 51 | var o Options 52 | if err := opts.Unmarshal(&o); err != nil { 53 | return err 54 | } 55 | slog.Info("jsonopts: loaded", "options", o) 56 | s.Options = o 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /internal/conf/opts_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package conf 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | type MockOptions struct { 11 | Option1 string `json:"option1"` 12 | Option2 int `json:"option2"` 13 | } 14 | 15 | func TestJsonOpts(t *testing.T) { 16 | opts := NewRawOpts(`{ 17 | "option1": "value1", 18 | "option2": 2 19 | }`) 20 | 21 | jsonOpts := JsonOpts[MockOptions]{} 22 | err := jsonOpts.Load(opts) 23 | if err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | if jsonOpts.Options.Option1 != "value1" { 27 | t.Errorf("expected option1 to be 'value1', got %v", jsonOpts.Options.Option1) 28 | } 29 | if jsonOpts.Options.Option2 != 2 { 30 | t.Errorf("expected option2 to be 2, got %v", jsonOpts.Options.Option2) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/db/migrations/001_add_openstack_servers_flavorid.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2025 SAP SE 2 | -- SPDX-License-Identifier: Apache-2.0 3 | 4 | -- Add the new column with a default value. 5 | ALTER TABLE IF EXISTS openstack_servers 6 | ADD COLUMN IF NOT EXISTS flavor_id VARCHAR(255) DEFAULT 'flavor'; 7 | -------------------------------------------------------------------------------- /internal/features/monitor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | package features 4 | 5 | import ( 6 | "strings" 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/monitoring" 11 | "github.com/prometheus/client_golang/prometheus" 12 | "github.com/prometheus/client_golang/prometheus/testutil" 13 | ) 14 | 15 | func TestFeatureExtractorMonitor(t *testing.T) { 16 | registry := &monitoring.Registry{Registry: prometheus.NewRegistry()} 17 | monitor := NewPipelineMonitor(registry) 18 | 19 | // Mock feature extractor 20 | mockExtractor := &mockFeatureExtractor{ 21 | name: "mock_extractor", 22 | // Usually the features are a struct, but it doesn't matter for this test 23 | extractFunc: func() ([]plugins.Feature, error) { 24 | return []plugins.Feature{"1", "2"}, nil 25 | }, 26 | } 27 | 28 | // Wrap the mock extractor with the monitor 29 | extractorMonitor := monitorFeatureExtractor(mockExtractor, monitor) 30 | 31 | // Test stepRunTimer 32 | expectedStepRunTimer := ` 33 | # HELP cortex_feature_pipeline_step_run_duration_seconds Duration of feature pipeline step run 34 | # TYPE cortex_feature_pipeline_step_run_duration_seconds histogram 35 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.005"} 1 36 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.01"} 1 37 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.025"} 1 38 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.05"} 1 39 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.1"} 1 40 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.25"} 1 41 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="0.5"} 1 42 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="1"} 1 43 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="2.5"} 1 44 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="5"} 1 45 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="10"} 1 46 | cortex_feature_pipeline_step_run_duration_seconds_bucket{step="mock_extractor",le="+Inf"} 1 47 | cortex_feature_pipeline_step_run_duration_seconds_sum{step="mock_extractor"} 0 48 | cortex_feature_pipeline_step_run_duration_seconds_count{step="mock_extractor"} 1 49 | ` 50 | extractorMonitor.runTimer.Observe(0) 51 | err := testutil.GatherAndCompare(registry, strings.NewReader(expectedStepRunTimer), "cortex_feature_pipeline_step_run_duration_seconds") 52 | if err != nil { 53 | t.Fatalf("stepRunTimer test failed: %v", err) 54 | } 55 | 56 | // Test stepFeatureCounter 57 | expectedStepFeatureCounter := ` 58 | # HELP cortex_feature_pipeline_step_features Number of features extracted by a feature pipeline step 59 | # TYPE cortex_feature_pipeline_step_features gauge 60 | cortex_feature_pipeline_step_features{step="mock_extractor"} 2 61 | ` 62 | features, err := extractorMonitor.Extract() 63 | if err != nil { 64 | t.Fatalf("Extract() error = %v, want nil", err) 65 | } 66 | if len(features) != 2 { 67 | t.Fatalf("Extract() returned %d features, want 2", len(features)) 68 | } 69 | err = testutil.GatherAndCompare(registry, strings.NewReader(expectedStepFeatureCounter), "cortex_feature_pipeline_step_features") 70 | if err != nil { 71 | t.Fatalf("stepFeatureCounter test failed: %v", err) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /internal/features/plugins/base.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "log/slog" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | ) 12 | 13 | // Common base for all extractors that provides some functionality 14 | // that would otherwise be duplicated across all extractors. 15 | type BaseExtractor[Opts any, Feature db.Table] struct { 16 | // Options to pass via yaml to this step. 17 | conf.JsonOpts[Opts] 18 | // Database connection. 19 | DB db.DB 20 | } 21 | 22 | // Init the extractor with the database and options. 23 | func (e *BaseExtractor[Opts, Feature]) Init(db db.DB, opts conf.RawOpts) error { 24 | if err := e.Load(opts); err != nil { 25 | return err 26 | } 27 | e.DB = db 28 | var f Feature 29 | return db.CreateTable(db.AddTable(f)) 30 | } 31 | 32 | // Extract the features directly from an sql query. 33 | func (e *BaseExtractor[Opts, F]) ExtractSQL(query string) ([]Feature, error) { 34 | var features []F 35 | if _, err := e.DB.Select(&features, query); err != nil { 36 | return nil, err 37 | } 38 | return e.Extracted(features) 39 | } 40 | 41 | // Replace all features of the given model in the database and 42 | // return them as a slice of generic features for counting. 43 | func (e *BaseExtractor[Opts, F]) Extracted(fs []F) ([]Feature, error) { 44 | if err := db.ReplaceAll(e.DB, fs...); err != nil { 45 | return nil, err 46 | } 47 | output := make([]Feature, len(fs)) 48 | for i, f := range fs { 49 | output[i] = f 50 | } 51 | var model F 52 | slog.Info("features: extracted", model.TableName(), len(output)) 53 | return output, nil 54 | } 55 | -------------------------------------------------------------------------------- /internal/features/plugins/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/trickyteache/cortex/internal/db" 9 | ) 10 | 11 | // Each feature extractor must conform to this interface. 12 | type FeatureExtractor interface { 13 | // Configure the feature extractor with a database and options. 14 | // This function should also create the needed database structures. 15 | Init(db db.DB, opts conf.RawOpts) error 16 | // Extract features from the given data. 17 | Extract() ([]Feature, error) 18 | // Get the name of this feature extractor. 19 | // This name is used to identify the extractor in metrics, config, logs, etc. 20 | // Should be something like: "my_cool_feature_extractor". 21 | GetName() string 22 | // Get message topics that trigger a re-execution of this extractor. 23 | Triggers() []string 24 | } 25 | 26 | type Feature any 27 | -------------------------------------------------------------------------------- /internal/features/plugins/kvm/node_exporter_host_cpu_usage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/prometheus" 11 | ) 12 | 13 | // Feature that maps CPU usage of kvm hosts. 14 | type NodeExporterHostCPUUsage struct { 15 | ComputeHost string `db:"compute_host"` 16 | AvgCPUUsage float64 `db:"avg_cpu_usage"` 17 | MaxCPUUsage float64 `db:"max_cpu_usage"` 18 | } 19 | 20 | // Table under which the feature is stored. 21 | func (NodeExporterHostCPUUsage) TableName() string { 22 | return "feature_host_cpu_usage" 23 | } 24 | 25 | // Extractor that extracts CPU usage of kvm hosts and stores 26 | // it as a feature into the database. 27 | type NodeExporterHostCPUUsageExtractor struct { 28 | // Common base for all extractors that provides standard functionality. 29 | plugins.BaseExtractor[ 30 | struct{}, // No options passed through yaml config 31 | NodeExporterHostCPUUsage, // Feature model 32 | ] 33 | } 34 | 35 | // Name of this feature extractor that is used in the yaml config, for logging etc. 36 | func (*NodeExporterHostCPUUsageExtractor) GetName() string { 37 | return "node_exporter_host_cpu_usage_extractor" 38 | } 39 | 40 | // Get message topics that trigger a re-execution of this extractor. 41 | func (NodeExporterHostCPUUsageExtractor) Triggers() []string { 42 | return []string{ 43 | prometheus.TriggerMetricAliasSynced("node_exporter_cpu_usage_pct"), 44 | } 45 | } 46 | 47 | //go:embed node_exporter_host_cpu_usage.sql 48 | var nodeExporterHostCPUUsageSQL string 49 | 50 | // Extract CPU usage of kvm hosts. 51 | func (e *NodeExporterHostCPUUsageExtractor) Extract() ([]plugins.Feature, error) { 52 | return e.ExtractSQL(nodeExporterHostCPUUsageSQL) 53 | } 54 | -------------------------------------------------------------------------------- /internal/features/plugins/kvm/node_exporter_host_cpu_usage.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | node AS compute_host, 3 | AVG(value) AS avg_cpu_usage, 4 | MAX(value) AS max_cpu_usage 5 | FROM node_exporter_metrics 6 | WHERE name = 'node_exporter_cpu_usage_pct' 7 | GROUP BY node; -------------------------------------------------------------------------------- /internal/features/plugins/kvm/node_exporter_host_cpu_usage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/sync/prometheus" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | ) 14 | 15 | func TestNodeExporterHostCPUUsageExtractor_Init(t *testing.T) { 16 | dbEnv := testlibDB.SetupDBEnv(t) 17 | testDB := db.DB{DbMap: dbEnv.DbMap} 18 | defer testDB.Close() 19 | defer dbEnv.Close() 20 | 21 | extractor := &NodeExporterHostCPUUsageExtractor{} 22 | if err := extractor.Init(testDB, conf.NewRawOpts("{}")); err != nil { 23 | t.Fatalf("expected no error, got %v", err) 24 | } 25 | 26 | if !testDB.TableExists(NodeExporterHostCPUUsage{}) { 27 | t.Error("expected table to be created") 28 | } 29 | } 30 | 31 | func TestNodeExporterHostCPUUsageExtractor_Extract(t *testing.T) { 32 | dbEnv := testlibDB.SetupDBEnv(t) 33 | testDB := db.DB{DbMap: dbEnv.DbMap} 34 | defer testDB.Close() 35 | defer dbEnv.Close() 36 | 37 | // Create dependency tables 38 | if err := testDB.CreateTable( 39 | testDB.AddTable(prometheus.NodeExporterMetric{}), 40 | ); err != nil { 41 | t.Fatalf("expected no error, got %v", err) 42 | } 43 | 44 | // Insert mock data into the node_exporter_metrics table 45 | _, err := testDB.Exec(` 46 | INSERT INTO node_exporter_metrics (node, name, value) 47 | VALUES 48 | ('node1', 'node_exporter_cpu_usage_pct', 20.0), 49 | ('node2', 'node_exporter_cpu_usage_pct', 30.0), 50 | ('node1', 'node_exporter_cpu_usage_pct', 40.0) 51 | `) 52 | if err != nil { 53 | t.Fatalf("expected no error, got %v", err) 54 | } 55 | 56 | extractor := &NodeExporterHostCPUUsageExtractor{} 57 | if err := extractor.Init(testDB, conf.NewRawOpts("{}")); err != nil { 58 | t.Fatalf("expected no error, got %v", err) 59 | } 60 | if _, err = extractor.Extract(); err != nil { 61 | t.Fatalf("expected no error, got %v", err) 62 | } 63 | 64 | // Verify the data was inserted into the feature_host_cpu_usage table 65 | var usages []NodeExporterHostCPUUsage 66 | _, err = testDB.Select(&usages, "SELECT * FROM feature_host_cpu_usage") 67 | if err != nil { 68 | t.Fatalf("expected no error, got %v", err) 69 | } 70 | 71 | if len(usages) != 2 { 72 | t.Errorf("expected 2 rows, got %d", len(usages)) 73 | } 74 | expected := map[string]struct { 75 | AvgCPUUsage float64 76 | MaxCPUUsage float64 77 | }{ 78 | "node1": {AvgCPUUsage: 30.0, MaxCPUUsage: 40.0}, // Average of 20.0 and 40.0, Max of 40.0 79 | "node2": {AvgCPUUsage: 30.0, MaxCPUUsage: 30.0}, // Single value of 30.0 80 | } 81 | for _, u := range usages { 82 | if expected[u.ComputeHost].AvgCPUUsage != u.AvgCPUUsage { 83 | t.Errorf( 84 | "expected avg_cpu_usage for compute_host %s to be %f, got %f", 85 | u.ComputeHost, expected[u.ComputeHost].AvgCPUUsage, u.AvgCPUUsage, 86 | ) 87 | } 88 | if expected[u.ComputeHost].MaxCPUUsage != u.MaxCPUUsage { 89 | t.Errorf( 90 | "expected max_cpu_usage for compute_host %s to be %f, got %f", 91 | u.ComputeHost, expected[u.ComputeHost].MaxCPUUsage, u.MaxCPUUsage, 92 | ) 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /internal/features/plugins/kvm/node_exporter_host_memory_active.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/prometheus" 11 | ) 12 | 13 | // Feature that maps memory active percentage of kvm hosts. 14 | type NodeExporterHostMemoryActive struct { 15 | ComputeHost string `db:"compute_host"` 16 | AvgMemoryActive float64 `db:"avg_memory_active"` 17 | MaxMemoryActive float64 `db:"max_memory_active"` 18 | } 19 | 20 | // Table under which the feature is stored. 21 | func (NodeExporterHostMemoryActive) TableName() string { 22 | return "feature_host_memory_active" 23 | } 24 | 25 | // Extractor that extracts how much memory of kvm hosts is active and stores 26 | // it as a feature into the database. 27 | type NodeExporterHostMemoryActiveExtractor struct { 28 | // Common base for all extractors that provides standard functionality. 29 | plugins.BaseExtractor[ 30 | struct{}, // No options passed through yaml config 31 | NodeExporterHostMemoryActive, // Feature model 32 | ] 33 | } 34 | 35 | // Name of this feature extractor that is used in the yaml config, for logging etc. 36 | func (*NodeExporterHostMemoryActiveExtractor) GetName() string { 37 | return "node_exporter_host_memory_active_extractor" 38 | } 39 | 40 | // Get message topics that trigger a re-execution of this extractor. 41 | func (NodeExporterHostMemoryActiveExtractor) Triggers() []string { 42 | return []string{ 43 | prometheus.TriggerMetricAliasSynced("node_exporter_memory_active_pct"), 44 | } 45 | } 46 | 47 | //go:embed node_exporter_host_memory_active.sql 48 | var nodeExporterHostMemoryActiveSQL string 49 | 50 | // Extract how much memory of kvm hosts is active. 51 | func (e *NodeExporterHostMemoryActiveExtractor) Extract() ([]plugins.Feature, error) { 52 | return e.ExtractSQL(nodeExporterHostMemoryActiveSQL) 53 | } 54 | -------------------------------------------------------------------------------- /internal/features/plugins/kvm/node_exporter_host_memory_active.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | node AS compute_host, 3 | AVG(value) AS avg_memory_active, 4 | MAX(value) AS max_memory_active 5 | FROM node_exporter_metrics 6 | WHERE name = 'node_exporter_memory_active_pct' 7 | GROUP BY node; -------------------------------------------------------------------------------- /internal/features/plugins/kvm/node_exporter_host_memory_active_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/sync/prometheus" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | ) 14 | 15 | func TestNodeExporterHostMemoryActiveExtractor_Init(t *testing.T) { 16 | dbEnv := testlibDB.SetupDBEnv(t) 17 | testDB := db.DB{DbMap: dbEnv.DbMap} 18 | defer testDB.Close() 19 | defer dbEnv.Close() 20 | 21 | extractor := &NodeExporterHostMemoryActiveExtractor{} 22 | if err := extractor.Init(testDB, conf.NewRawOpts("{}")); err != nil { 23 | t.Fatalf("expected no error, got %v", err) 24 | } 25 | 26 | if !testDB.TableExists(NodeExporterHostMemoryActive{}) { 27 | t.Error("expected table to be created") 28 | } 29 | } 30 | 31 | func TestNodeExporterHostMemoryActiveExtractor_Extract(t *testing.T) { 32 | dbEnv := testlibDB.SetupDBEnv(t) 33 | testDB := db.DB{DbMap: dbEnv.DbMap} 34 | defer testDB.Close() 35 | defer dbEnv.Close() 36 | 37 | // Create dependency tables 38 | if err := testDB.CreateTable( 39 | testDB.AddTable(prometheus.NodeExporterMetric{}), 40 | ); err != nil { 41 | t.Fatalf("expected no error, got %v", err) 42 | } 43 | 44 | // Insert mock data into the node_exporter_metrics table 45 | _, err := testDB.Exec(` 46 | INSERT INTO node_exporter_metrics (node, name, value) 47 | VALUES 48 | ('node1', 'node_exporter_memory_active_pct', 20.0), 49 | ('node2', 'node_exporter_memory_active_pct', 30.0), 50 | ('node1', 'node_exporter_memory_active_pct', 40.0) 51 | `) 52 | if err != nil { 53 | t.Fatalf("expected no error, got %v", err) 54 | } 55 | 56 | extractor := &NodeExporterHostMemoryActiveExtractor{} 57 | if err := extractor.Init(testDB, conf.NewRawOpts("{}")); err != nil { 58 | t.Fatalf("expected no error, got %v", err) 59 | } 60 | if _, err = extractor.Extract(); err != nil { 61 | t.Fatalf("expected no error, got %v", err) 62 | } 63 | 64 | // Verify the data was inserted into the feature_host_memory_active table 65 | var usages []NodeExporterHostMemoryActive 66 | _, err = testDB.Select(&usages, "SELECT * FROM feature_host_memory_active") 67 | if err != nil { 68 | t.Fatalf("expected no error, got %v", err) 69 | } 70 | 71 | if len(usages) != 2 { 72 | t.Errorf("expected 2 rows, got %d", len(usages)) 73 | } 74 | expected := map[string]struct { 75 | AvgMemoryActive float64 76 | MaxMemoryActive float64 77 | }{ 78 | "node1": {AvgMemoryActive: 30.0, MaxMemoryActive: 40.0}, // Average of 20.0 and 40.0, Max of 40.0 79 | "node2": {AvgMemoryActive: 30.0, MaxMemoryActive: 30.0}, // Single value of 30.0 80 | } 81 | for _, u := range usages { 82 | if expected[u.ComputeHost].AvgMemoryActive != u.AvgMemoryActive { 83 | t.Errorf( 84 | "expected avg_cpu_usage for compute_host %s to be %f, got %f", 85 | u.ComputeHost, expected[u.ComputeHost].AvgMemoryActive, u.AvgMemoryActive, 86 | ) 87 | } 88 | if expected[u.ComputeHost].MaxMemoryActive != u.MaxMemoryActive { 89 | t.Errorf( 90 | "expected max_cpu_usage for compute_host %s to be %f, got %f", 91 | u.ComputeHost, expected[u.ComputeHost].MaxMemoryActive, u.MaxMemoryActive, 92 | ) 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /internal/features/plugins/shared/flavor_host_space.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/openstack" 11 | ) 12 | 13 | // Feature that maps the space left on a compute host after the placement of a flavor. 14 | type FlavorHostSpace struct { 15 | // ID of the OpenStack flavor. 16 | FlavorID string `db:"flavor_id"` 17 | // Name of the OpenStack compute host. 18 | ComputeHost string `db:"compute_host"` 19 | // RAM left after the placement of the flavor. 20 | RAMLeftMB int `db:"ram_left_mb"` 21 | // CPU left after the placement of the flavor. 22 | VCPUsLeft int `db:"vcpus_left"` 23 | // Disk left after the placement of the flavor. 24 | DiskLeftGB int `db:"disk_left_gb"` 25 | } 26 | 27 | // Table under which the feature is stored. 28 | func (FlavorHostSpace) TableName() string { 29 | return "feature_flavor_host_space" 30 | } 31 | 32 | // Extractor that extracts the space left on a compute host after the placement 33 | // of a flavor and stores it as a feature into the database. 34 | type FlavorHostSpaceExtractor struct { 35 | // Common base for all extractors that provides standard functionality. 36 | plugins.BaseExtractor[ 37 | struct{}, // No options passed through yaml config 38 | FlavorHostSpace, // Feature model 39 | ] 40 | } 41 | 42 | // Name of this feature extractor that is used in the yaml config, for logging etc. 43 | func (*FlavorHostSpaceExtractor) GetName() string { 44 | return "flavor_host_space_extractor" 45 | } 46 | 47 | // Get message topics that trigger a re-execution of this extractor. 48 | func (FlavorHostSpaceExtractor) Triggers() []string { 49 | return []string{ 50 | openstack.TriggerNovaFlavorsSynced, 51 | openstack.TriggerNovaHypervisorsSynced, 52 | } 53 | } 54 | 55 | //go:embed flavor_host_space.sql 56 | var flavorHostSpaceQuery string 57 | 58 | // Extract the space left on a compute host after the placement of a flavor. 59 | // Depends on the OpenStack flavors and hypervisors to be synced. 60 | func (e *FlavorHostSpaceExtractor) Extract() ([]plugins.Feature, error) { 61 | return e.ExtractSQL(flavorHostSpaceQuery) 62 | } 63 | -------------------------------------------------------------------------------- /internal/features/plugins/shared/flavor_host_space.sql: -------------------------------------------------------------------------------- 1 | WITH flavor_host_space AS ( 2 | SELECT 3 | flavors.id AS flavor_id, 4 | hypervisors.service_host AS compute_host, 5 | hypervisors.free_ram_mb - flavors.ram AS ram_left_mb, 6 | hypervisors.vcpus - hypervisors.vcpus_used - flavors.vcpus AS vcpus_left, 7 | hypervisors.free_disk_gb - flavors.disk AS disk_left_gb 8 | FROM openstack_flavors AS flavors 9 | CROSS JOIN openstack_hypervisors AS hypervisors 10 | ) 11 | SELECT * FROM flavor_host_space; -------------------------------------------------------------------------------- /internal/features/plugins/shared/vm_host_residency.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/openstack" 11 | ) 12 | 13 | // Feature that describes how long a vm was running on a host until it needed 14 | // to move out, and the reason for the move (i.e., who forced it to move). 15 | type VMHostResidency struct { 16 | // Time the vm stayed on the host in seconds. 17 | Duration int `db:"duration"` 18 | // Flavor id of the virtual machine. 19 | FlavorID string `db:"flavor_id"` 20 | // Flavor name of the virtual machine. 21 | FlavorName string `db:"flavor_name"` 22 | // The UUID of the virtual machine. 23 | InstanceUUID string `db:"instance_uuid"` 24 | // The migration uuid. 25 | MigrationUUID string `db:"migration_uuid"` 26 | // The host the VM was running on and needed to move out. 27 | SourceHost string `db:"source_host"` 28 | // The host the VM was moved to. 29 | TargetHost string `db:"target_host"` 30 | // The source node the VM was running on and needed to move out. 31 | SourceNode string `db:"source_node"` 32 | // The target node the VM was moved to. 33 | TargetNode string `db:"target_node"` 34 | // Who forced the VM to move out. 35 | UserID string `db:"user_id"` 36 | // To which project the user belongs that forced the VM to move out. 37 | ProjectID string `db:"project_id"` 38 | // Migration type (live-migration or resize). 39 | Type string `db:"type"` 40 | // Time when the migration was triggered in seconds since epoch. 41 | Time int `db:"time"` 42 | } 43 | 44 | // Table under which the feature is stored. 45 | func (VMHostResidency) TableName() string { 46 | return "feature_vm_host_residency" 47 | } 48 | 49 | // Extractor that extracts the time elapsed until the first migration of a virtual machine. 50 | type VMHostResidencyExtractor struct { 51 | // Common base for all extractors that provides standard functionality. 52 | plugins.BaseExtractor[ 53 | struct{}, // No options passed through yaml config 54 | VMHostResidency, // Feature model 55 | ] 56 | } 57 | 58 | // Name of this feature extractor that is used in the yaml config, for logging etc. 59 | func (*VMHostResidencyExtractor) GetName() string { 60 | return "vm_host_residency_extractor" 61 | } 62 | 63 | // Get message topics that trigger a re-execution of this extractor. 64 | func (VMHostResidencyExtractor) Triggers() []string { 65 | return []string{ 66 | openstack.TriggerNovaServersSynced, 67 | openstack.TriggerNovaMigrationsSynced, 68 | openstack.TriggerNovaFlavorsSynced, 69 | } 70 | } 71 | 72 | //go:embed vm_host_residency.sql 73 | var vmHostResidencyQuery string 74 | 75 | // Extract the time elapsed until the first migration of a virtual machine. 76 | // Depends on the OpenStack servers and migrations to be synced. 77 | func (e *VMHostResidencyExtractor) Extract() ([]plugins.Feature, error) { 78 | return e.ExtractSQL(vmHostResidencyQuery) 79 | } 80 | -------------------------------------------------------------------------------- /internal/features/plugins/shared/vm_host_residency.sql: -------------------------------------------------------------------------------- 1 | WITH durations AS ( 2 | SELECT 3 | migrations.instance_uuid, 4 | migrations.uuid AS migration_uuid, 5 | servers.flavor_id AS flavor_id, 6 | flavors.name AS flavor_name, 7 | COALESCE( 8 | CAST(EXTRACT(EPOCH FROM ( 9 | -- Use the LAG window function to get the timestamp of 10 | -- the previous migration 11 | migrations.created_at::timestamp - 12 | LAG(migrations.created_at) OVER ( 13 | PARTITION BY migrations.instance_uuid 14 | ORDER BY migrations.created_at 15 | )::timestamp 16 | )) AS BIGINT), 17 | -- Use the duration since the server was created if there is no 18 | -- previous migration 19 | CAST(EXTRACT(EPOCH FROM ( 20 | migrations.created_at::timestamp - 21 | servers.created::timestamp 22 | )) AS BIGINT) 23 | ) AS duration 24 | FROM openstack_migrations AS migrations 25 | LEFT JOIN openstack_servers AS servers ON servers.id = migrations.instance_uuid 26 | LEFT JOIN openstack_flavors AS flavors ON flavors.id = servers.flavor_id 27 | ) 28 | SELECT 29 | -- Sometimes the server can be vanished already, set default 30 | -- values for that case. 31 | COALESCE(durations.duration, 0) AS duration, 32 | COALESCE(durations.flavor_id, 'unknown') AS flavor_id, 33 | COALESCE(durations.flavor_name, 'unknown') AS flavor_name, 34 | migrations.instance_uuid AS instance_uuid, 35 | migrations.uuid AS migration_uuid, 36 | migrations.source_compute AS source_host, 37 | migrations.dest_compute AS target_host, 38 | migrations.source_node AS source_node, 39 | migrations.dest_node AS target_node, 40 | COALESCE(migrations.user_id, 'unknown') AS user_id, 41 | COALESCE(migrations.project_id, 'unknown') AS project_id, 42 | migrations.migration_type AS type, 43 | CAST(EXTRACT(EPOCH FROM (migrations.created_at::timestamp)) AS BIGINT) AS time 44 | FROM openstack_migrations AS migrations 45 | LEFT JOIN durations ON migrations.uuid = durations.migration_uuid; -------------------------------------------------------------------------------- /internal/features/plugins/shared/vm_life_span.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/openstack" 11 | ) 12 | 13 | // Feature that describes how long a vm existed before it was deleted. 14 | type VMLifeSpan struct { 15 | // Time the vm stayed on the host in seconds. 16 | Duration int `db:"duration"` 17 | // Flavor id of the virtual machine. 18 | FlavorID string `db:"flavor_id"` 19 | // Flavor name of the virtual machine. 20 | FlavorName string `db:"flavor_name"` 21 | // The UUID of the virtual machine. 22 | InstanceUUID string `db:"instance_uuid"` 23 | } 24 | 25 | // Table under which the feature is stored. 26 | func (VMLifeSpan) TableName() string { 27 | return "feature_vm_life_span" 28 | } 29 | 30 | // Extractor that extracts the time elapsed until the vm was deleted. 31 | type VMLifeSpanExtractor struct { 32 | // Common base for all extractors that provides standard functionality. 33 | plugins.BaseExtractor[ 34 | struct{}, // No options passed through yaml config 35 | VMLifeSpan, // Feature model 36 | ] 37 | } 38 | 39 | // Name of this feature extractor that is used in the yaml config, for logging etc. 40 | func (*VMLifeSpanExtractor) GetName() string { 41 | return "vm_life_span_extractor" 42 | } 43 | 44 | // Get message topics that trigger a re-execution of this extractor. 45 | func (VMLifeSpanExtractor) Triggers() []string { 46 | return []string{ 47 | openstack.TriggerNovaServersSynced, 48 | openstack.TriggerNovaFlavorsSynced, 49 | } 50 | } 51 | 52 | //go:embed vm_life_span.sql 53 | var vmLifeSpanQuery string 54 | 55 | // Extract the time elapsed until the first migration of a virtual machine. 56 | // Depends on the OpenStack servers and migrations to be synced. 57 | func (e *VMLifeSpanExtractor) Extract() ([]plugins.Feature, error) { 58 | return e.ExtractSQL(vmLifeSpanQuery) 59 | } 60 | -------------------------------------------------------------------------------- /internal/features/plugins/shared/vm_life_span.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CAST(EXTRACT(EPOCH FROM ( 3 | servers.updated::timestamp - servers.created::timestamp 4 | )) AS BIGINT) AS duration, 5 | COALESCE(servers.flavor_id, 'unknown') AS flavor_id, 6 | COALESCE(flavors.name, 'unknown') AS flavor_name, 7 | servers.id AS instance_uuid 8 | FROM openstack_servers servers 9 | LEFT JOIN openstack_flavors flavors ON flavors.id = servers.flavor_id 10 | WHERE servers.status = 'DELETED'; -------------------------------------------------------------------------------- /internal/features/plugins/shared/vm_life_span_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | // Copyright 2025 SAP SE 5 | // SPDX-License-Identifier: Apache-2.0 6 | 7 | package shared 8 | 9 | import ( 10 | "os" 11 | "testing" 12 | 13 | "github.com/trickyteache/cortex/internal/conf" 14 | "github.com/trickyteache/cortex/internal/db" 15 | "github.com/trickyteache/cortex/internal/sync/openstack" 16 | testlibDB "github.com/trickyteache/cortex/testlib/db" 17 | ) 18 | 19 | func TestVMLifeSpanExtractor_Init(t *testing.T) { 20 | dbEnv := testlibDB.SetupDBEnv(t) 21 | testDB := db.DB{DbMap: dbEnv.DbMap} 22 | defer testDB.Close() 23 | defer dbEnv.Close() 24 | 25 | extractor := &VMLifeSpanExtractor{} 26 | if err := extractor.Init(testDB, conf.NewRawOpts("{}")); err != nil { 27 | t.Fatalf("expected no error during initialization, got %v", err) 28 | } 29 | 30 | if !testDB.TableExists(VMLifeSpan{}) { 31 | t.Error("expected table to be created") 32 | } 33 | } 34 | 35 | func TestVMLifeSpanExtractor_Extract(t *testing.T) { 36 | // We're using postgres specific syntax here. 37 | if os.Getenv("POSTGRES_CONTAINER") != "1" { 38 | t.Skip("skipping test; set POSTGRES_CONTAINER=1 to run") 39 | } 40 | 41 | dbEnv := testlibDB.SetupDBEnv(t) 42 | testDB := db.DB{DbMap: dbEnv.DbMap} 43 | defer testDB.Close() 44 | defer dbEnv.Close() 45 | 46 | // Create dependency tables 47 | if err := testDB.CreateTable( 48 | testDB.AddTable(openstack.Server{}), 49 | testDB.AddTable(openstack.Flavor{}), 50 | ); err != nil { 51 | t.Fatalf("failed to create dependency tables: %v", err) 52 | } 53 | 54 | // Insert mock data into the servers and flavors tables 55 | if _, err := testDB.Exec(` 56 | INSERT INTO openstack_servers (id, flavor_id, created, updated, status) 57 | VALUES 58 | ('server1', 'flavor1', '2025-01-01T00:00:00Z', '2025-01-03T00:00:00Z', 'DELETED'), 59 | ('server2', 'flavor2', '2025-01-02T00:00:00Z', '2025-01-04T00:00:00Z', 'DELETED') 60 | `); err != nil { 61 | t.Fatalf("failed to insert servers: %v", err) 62 | } 63 | 64 | flavors := []any{ 65 | &openstack.Flavor{ID: "flavor1", Name: "small"}, 66 | &openstack.Flavor{ID: "flavor2", Name: "medium"}, 67 | } 68 | if err := testDB.Insert(flavors...); err != nil { 69 | t.Fatalf("failed to insert flavors: %v", err) 70 | } 71 | 72 | extractor := &VMLifeSpanExtractor{} 73 | if err := extractor.Init(testDB, conf.NewRawOpts("{}")); err != nil { 74 | t.Fatalf("expected no error during initialization, got %v", err) 75 | } 76 | 77 | features, err := extractor.Extract() 78 | if err != nil { 79 | t.Fatalf("expected no error during extraction, got %v", err) 80 | } 81 | 82 | if len(features) != 2 { 83 | t.Errorf("expected 2 features, got %d", len(features)) 84 | } 85 | 86 | expected := map[string]VMLifeSpan{ 87 | "server1": {Duration: 172800, FlavorID: "flavor1", FlavorName: "small", InstanceUUID: "server1"}, 88 | "server2": {Duration: 172800, FlavorID: "flavor2", FlavorName: "medium", InstanceUUID: "server2"}, 89 | } 90 | 91 | for _, feature := range features { 92 | vmLifeSpan := feature.(VMLifeSpan) 93 | expectedFeature := expected[vmLifeSpan.InstanceUUID] 94 | 95 | if vmLifeSpan != expectedFeature { 96 | t.Errorf("unexpected feature for instance %s: got %+v, expected %+v", vmLifeSpan.InstanceUUID, vmLifeSpan, expectedFeature) 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /internal/features/plugins/vmware/vrops_hostsystem_contention.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/prometheus" 11 | ) 12 | 13 | // Feature that maps CPU contention of vROps hostsystems. 14 | type VROpsHostsystemContention struct { 15 | ComputeHost string `db:"compute_host"` 16 | AvgCPUContention float64 `db:"avg_cpu_contention"` 17 | MaxCPUContention float64 `db:"max_cpu_contention"` 18 | } 19 | 20 | // Table under which the feature is stored. 21 | func (VROpsHostsystemContention) TableName() string { 22 | return "feature_vrops_hostsystem_contention" 23 | } 24 | 25 | // Extractor that extracts CPU contention of vROps hostsystems and stores 26 | // it as a feature into the database. 27 | type VROpsHostsystemContentionExtractor struct { 28 | // Common base for all extractors that provides standard functionality. 29 | plugins.BaseExtractor[ 30 | struct{}, // No options passed through yaml config 31 | VROpsHostsystemContention, // Feature model 32 | ] 33 | } 34 | 35 | // Name of this feature extractor that is used in the yaml config, for logging etc. 36 | func (*VROpsHostsystemContentionExtractor) GetName() string { 37 | return "vrops_hostsystem_contention_extractor" 38 | } 39 | 40 | // Get message topics that trigger a re-execution of this extractor. 41 | func (VROpsHostsystemContentionExtractor) Triggers() []string { 42 | return []string{ 43 | prometheus.TriggerMetricAliasSynced("vrops_hostsystem_cpu_contention_percentage"), 44 | } 45 | } 46 | 47 | //go:embed vrops_hostsystem_contention.sql 48 | var vropsHostsystemContentionSQL string 49 | 50 | // Extract CPU contention of hostsystems. 51 | // Depends on resolved vROps hostsystems (feature_vrops_resolved_hostsystem). 52 | func (e *VROpsHostsystemContentionExtractor) Extract() ([]plugins.Feature, error) { 53 | return e.ExtractSQL(vropsHostsystemContentionSQL) 54 | } 55 | -------------------------------------------------------------------------------- /internal/features/plugins/vmware/vrops_hostsystem_contention.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | h.nova_compute_host AS compute_host, 3 | AVG(m.value) AS avg_cpu_contention, 4 | MAX(m.value) AS max_cpu_contention 5 | FROM vrops_host_metrics m 6 | JOIN feature_vrops_resolved_hostsystem h ON m.hostsystem = h.vrops_hostsystem 7 | WHERE m.name = 'vrops_hostsystem_cpu_contention_percentage' 8 | GROUP BY h.nova_compute_host; -------------------------------------------------------------------------------- /internal/features/plugins/vmware/vrops_hostsystem_resolver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/openstack" 11 | "github.com/trickyteache/cortex/internal/sync/prometheus" 12 | ) 13 | 14 | // Feature that resolves the vROps metrics hostsystem label to the 15 | // corresponding Nova compute host. 16 | type ResolvedVROpsHostsystem struct { 17 | VROpsHostsystem string `db:"vrops_hostsystem"` 18 | NovaComputeHost string `db:"nova_compute_host"` 19 | } 20 | 21 | // Table under which the feature is stored. 22 | func (ResolvedVROpsHostsystem) TableName() string { 23 | return "feature_vrops_resolved_hostsystem" 24 | } 25 | 26 | // Extractor that resolves the vROps metrics hostsystem label to the 27 | // corresponding Nova compute host and stores it as a feature into the database. 28 | type VROpsHostsystemResolver struct { 29 | // Common base for all extractors that provides standard functionality. 30 | plugins.BaseExtractor[ 31 | struct{}, // No options passed through yaml config 32 | ResolvedVROpsHostsystem, // Feature model 33 | ] 34 | } 35 | 36 | // Get message topics that trigger a re-execution of this extractor. 37 | func (VROpsHostsystemResolver) Triggers() []string { 38 | return []string{ 39 | openstack.TriggerNovaServersSynced, 40 | openstack.TriggerNovaHypervisorsSynced, 41 | prometheus.TriggerMetricTypeSynced("vrops_vm_metrics"), 42 | } 43 | } 44 | 45 | // Name of this feature extractor that is used in the yaml config, for logging etc. 46 | func (e *VROpsHostsystemResolver) GetName() string { 47 | return "vrops_hostsystem_resolver" 48 | } 49 | 50 | //go:embed vrops_hostsystem_resolver.sql 51 | var vropsHostsystemSQL string 52 | 53 | // Resolve vROps hostsystems to Nova compute hosts. 54 | func (e *VROpsHostsystemResolver) Extract() ([]plugins.Feature, error) { 55 | return e.ExtractSQL(vropsHostsystemSQL) 56 | } 57 | -------------------------------------------------------------------------------- /internal/features/plugins/vmware/vrops_hostsystem_resolver.sql: -------------------------------------------------------------------------------- 1 | -- Resolve hostsystem names from vROps to Nova compute hosts 2 | SELECT 3 | m.hostsystem AS vrops_hostsystem, 4 | h.service_host AS nova_compute_host 5 | FROM vrops_vm_metrics m 6 | JOIN openstack_servers s ON m.instance_uuid = s.id 7 | JOIN openstack_hypervisors h ON s.os_ext_srv_attr_hypervisor_hostname = h.hostname 8 | GROUP BY m.hostsystem, h.service_host; -------------------------------------------------------------------------------- /internal/features/plugins/vmware/vrops_project_noisiness.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | _ "embed" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins" 10 | "github.com/trickyteache/cortex/internal/sync/openstack" 11 | "github.com/trickyteache/cortex/internal/sync/prometheus" 12 | ) 13 | 14 | // Feature that calculates the noisiness of projects and on which 15 | // compute hosts they are currently running. 16 | type VROpsProjectNoisiness struct { 17 | Project string `db:"project"` 18 | ComputeHost string `db:"compute_host"` 19 | AvgCPUOfProject float64 `db:"avg_cpu_of_project"` 20 | } 21 | 22 | // Table under which the feature is stored. 23 | func (VROpsProjectNoisiness) TableName() string { 24 | return "feature_vrops_project_noisiness" 25 | } 26 | 27 | // Extractor that extracts the noisiness of projects and on which compute 28 | // hosts they are currently running and stores it as a feature into the database. 29 | type VROpsProjectNoisinessExtractor struct { 30 | // Common base for all extractors that provides standard functionality. 31 | plugins.BaseExtractor[ 32 | struct{}, // No options passed through yaml config 33 | VROpsProjectNoisiness, // Feature model 34 | ] 35 | } 36 | 37 | // Name of this feature extractor that is used in the yaml config, for logging etc. 38 | func (e *VROpsProjectNoisinessExtractor) GetName() string { 39 | return "vrops_project_noisiness_extractor" 40 | } 41 | 42 | // Get message topics that trigger a re-execution of this extractor. 43 | func (VROpsProjectNoisinessExtractor) Triggers() []string { 44 | return []string{ 45 | openstack.TriggerNovaServersSynced, 46 | openstack.TriggerNovaHypervisorsSynced, 47 | prometheus.TriggerMetricAliasSynced("vrops_virtualmachine_cpu_demand_ratio"), 48 | } 49 | } 50 | 51 | //go:embed vrops_project_noisiness.sql 52 | var vropsProjectNoisinessSQL string 53 | 54 | func (e *VROpsProjectNoisinessExtractor) Extract() ([]plugins.Feature, error) { 55 | return e.ExtractSQL(vropsProjectNoisinessSQL) 56 | } 57 | -------------------------------------------------------------------------------- /internal/features/plugins/vmware/vrops_project_noisiness.sql: -------------------------------------------------------------------------------- 1 | -- Extract the noisiness for each project in OpenStack with the following steps: 2 | -- 1. Get the average cpu usage of each project through the vROps metrics. 3 | -- 2. Find on which hosts the projects are currently running through the 4 | -- OpenStack servers and hypervisors. 5 | -- 3. Store the avg cpu usage together with the current hosts in the database. 6 | -- This feature can then be used to draw new VMs away from VMs of the same 7 | -- project in case this project is known to cause high cpu usage. 8 | WITH projects_avg_cpu AS ( 9 | SELECT 10 | m.project AS tenant_id, 11 | AVG(m.value) AS avg_cpu 12 | FROM vrops_vm_metrics m 13 | WHERE m.name = 'vrops_virtualmachine_cpu_demand_ratio' 14 | GROUP BY m.project 15 | ORDER BY avg_cpu DESC 16 | ), 17 | host_cpu_usage AS ( 18 | SELECT 19 | s.tenant_id, 20 | h.service_host, 21 | AVG(p.avg_cpu) AS avg_cpu_of_project 22 | FROM openstack_servers s 23 | JOIN vrops_vm_metrics m ON s.id = m.instance_uuid 24 | JOIN projects_avg_cpu p ON s.tenant_id = p.tenant_id 25 | JOIN openstack_hypervisors h ON s.os_ext_srv_attr_hypervisor_hostname = h.hostname 26 | GROUP BY s.tenant_id, h.service_host 27 | ORDER BY avg_cpu_of_project DESC 28 | ) 29 | SELECT 30 | tenant_id AS project, 31 | service_host AS compute_host, 32 | avg_cpu_of_project 33 | FROM host_cpu_usage; -------------------------------------------------------------------------------- /internal/kpis/pipeline.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kpis 5 | 6 | import ( 7 | "fmt" 8 | "log/slog" 9 | 10 | "github.com/trickyteache/cortex/internal/conf" 11 | "github.com/trickyteache/cortex/internal/db" 12 | "github.com/trickyteache/cortex/internal/kpis/plugins" 13 | "github.com/trickyteache/cortex/internal/kpis/plugins/shared" 14 | "github.com/trickyteache/cortex/internal/kpis/plugins/vmware" 15 | "github.com/trickyteache/cortex/internal/monitoring" 16 | ) 17 | 18 | // Configuration of supported kpis. 19 | var SupportedKPIs = []plugins.KPI{ 20 | // VMware kpis. 21 | &vmware.VMwareHostContentionKPI{}, 22 | &vmware.VMwareProjectNoisinessKPI{}, 23 | // Shared kpis. 24 | &shared.HostUtilizationKPI{}, 25 | &shared.VMMigrationStatisticsKPI{}, 26 | &shared.VMLifeSpanKPI{}, 27 | } 28 | 29 | // Pipeline that extracts kpis from the database. 30 | type KPIPipeline struct { 31 | // Config to use for the kpis. 32 | config conf.KPIsConfig 33 | } 34 | 35 | // Create a new kpi pipeline with kpis contained in the configuration. 36 | func NewPipeline(config conf.KPIsConfig) KPIPipeline { 37 | return KPIPipeline{config: config} 38 | } 39 | 40 | // Initialize the kpi pipeline with the given database and registry. 41 | func (p *KPIPipeline) Init(kpis []plugins.KPI, db db.DB, registry *monitoring.Registry) error { 42 | supportedKPIsByName := make(map[string]plugins.KPI) 43 | for _, kpi := range kpis { 44 | supportedKPIsByName[kpi.GetName()] = kpi 45 | } 46 | // Load all kpis from the configuration. 47 | for _, kpiConf := range p.config.Plugins { 48 | kpi, ok := supportedKPIsByName[kpiConf.Name] 49 | if !ok { 50 | return fmt.Errorf("kpi %s not supported", kpiConf.Name) 51 | } 52 | if err := kpi.Init(db, kpiConf.Options); err != nil { 53 | return fmt.Errorf("failed to initialize kpi %s: %w", kpiConf.Name, err) 54 | } 55 | registry.MustRegister(kpi) 56 | slog.Info( 57 | "kpi: added kpi", 58 | "name", kpiConf.Name, 59 | "options", kpiConf.Options, 60 | ) 61 | } 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /internal/kpis/pipeline_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kpis 5 | 6 | import ( 7 | "errors" 8 | "testing" 9 | 10 | "github.com/trickyteache/cortex/internal/conf" 11 | "github.com/trickyteache/cortex/internal/db" 12 | "github.com/trickyteache/cortex/internal/kpis/plugins" 13 | "github.com/trickyteache/cortex/internal/monitoring" 14 | testlibDB "github.com/trickyteache/cortex/testlib/db" 15 | "github.com/prometheus/client_golang/prometheus" 16 | ) 17 | 18 | type mockKPI struct { 19 | name string 20 | initErr error 21 | } 22 | 23 | func (m *mockKPI) GetName() string { 24 | return m.name 25 | } 26 | 27 | func (m *mockKPI) Init(db db.DB, opts conf.RawOpts) error { 28 | return m.initErr 29 | } 30 | 31 | func (m *mockKPI) Collect(ch chan<- prometheus.Metric) { 32 | // Mock implementation 33 | } 34 | func (m *mockKPI) Describe(ch chan<- *prometheus.Desc) { 35 | // Mock implementation 36 | } 37 | 38 | func TestKPIPipeline_Init(t *testing.T) { 39 | dbEnv := testlibDB.SetupDBEnv(t) 40 | testDB := db.DB{DbMap: dbEnv.DbMap} 41 | defer testDB.Close() 42 | defer dbEnv.Close() 43 | registry := monitoring.NewRegistry(conf.MonitoringConfig{ 44 | Labels: map[string]string{"env": "test"}, 45 | }) 46 | 47 | mockKPI1 := &mockKPI{name: "mock_kpi_1"} 48 | mockKPI2 := &mockKPI{name: "mock_kpi_2", initErr: errors.New("init error")} 49 | 50 | config := conf.KPIsConfig{ 51 | Plugins: []conf.KPIPluginConfig{ 52 | {Name: "mock_kpi_1", Options: conf.RawOpts{}}, 53 | {Name: "mock_kpi_2", Options: conf.RawOpts{}}, 54 | }, 55 | } 56 | 57 | pipeline := NewPipeline(config) 58 | 59 | err := pipeline.Init([]plugins.KPI{mockKPI1, mockKPI2}, testDB, registry) 60 | if err == nil { 61 | t.Fatalf("expected error, got nil") 62 | } 63 | 64 | expectedError := "failed to initialize kpi mock_kpi_2: init error" 65 | if err.Error() != expectedError { 66 | t.Errorf("expected error %q, got %q", expectedError, err.Error()) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /internal/kpis/plugins/base.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/trickyteache/cortex/internal/db" 9 | ) 10 | 11 | // Common base for all KPIs that provides some functionality 12 | // that would otherwise be duplicated across all KPIs. 13 | type BaseKPI[Opts any] struct { 14 | // Options to pass via json to this step. 15 | conf.JsonOpts[Opts] 16 | // Database connection. 17 | DB db.DB 18 | } 19 | 20 | // Init the KPI with the database, options, and the registry to publish metrics on. 21 | func (k *BaseKPI[Opts]) Init(db db.DB, opts conf.RawOpts) error { 22 | if err := k.Load(opts); err != nil { 23 | return err 24 | } 25 | k.DB = db 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /internal/kpis/plugins/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | testlibDB "github.com/trickyteache/cortex/testlib/db" 12 | ) 13 | 14 | type MockOptions struct { 15 | Option1 string `yaml:"option1"` 16 | Option2 int `yaml:"option2"` 17 | } 18 | 19 | type MockKPI struct { 20 | BaseKPI[MockOptions] 21 | ID int `db:"id,primarykey"` 22 | Name string `db:"name"` 23 | } 24 | 25 | func (MockKPI) TableName() string { 26 | return "mock_kpi" 27 | } 28 | 29 | func TestBaseKPI_Init(t *testing.T) { 30 | dbEnv := testlibDB.SetupDBEnv(t) 31 | testDB := db.DB{DbMap: dbEnv.DbMap} 32 | defer testDB.Close() 33 | defer dbEnv.Close() 34 | 35 | opts := conf.NewRawOpts(`{ 36 | "option1": "value1", 37 | "option2": 2 38 | }`) 39 | baseKPI := MockKPI{} 40 | err := baseKPI.Init(testDB, opts) 41 | if err != nil { 42 | t.Errorf("Init() failed: %v", err) 43 | } 44 | 45 | if baseKPI.Options.Option1 != "value1" { 46 | t.Errorf("expected Option1 to be 'value1', got %s", baseKPI.Options.Option1) 47 | } 48 | if baseKPI.Options.Option2 != 2 { 49 | t.Errorf("expected Option2 to be 2, got %d", baseKPI.Options.Option2) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /internal/kpis/plugins/histogram.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import "github.com/trickyteache/cortex/internal/features/plugins" 7 | 8 | // Create a histogram from features. 9 | func Histogram[O plugins.Feature]( 10 | features []O, 11 | buckets []float64, 12 | keysFunc func(O) []string, 13 | valueFunc func(O) float64, 14 | ) ( 15 | hists map[string]map[float64]uint64, // By key 16 | counts map[string]uint64, // By key 17 | sums map[string]float64, // By key 18 | ) { 19 | 20 | hists = map[string]map[float64]uint64{} 21 | counts = map[string]uint64{} 22 | sums = map[string]float64{} 23 | for _, feature := range features { 24 | keys := keysFunc(feature) 25 | val := valueFunc(feature) 26 | for _, key := range keys { 27 | if _, ok := hists[key]; !ok { 28 | hists[key] = make(map[float64]uint64, len(buckets)) 29 | } 30 | for _, bucket := range buckets { 31 | if val <= bucket { 32 | hists[key][bucket]++ 33 | } 34 | } 35 | counts[key]++ 36 | sums[key] += val 37 | } 38 | } 39 | // Fill up empty buckets 40 | for key, hist := range hists { 41 | for _, bucket := range buckets { 42 | if _, ok := hist[bucket]; !ok { 43 | hists[key][bucket] = 0 44 | } 45 | } 46 | } 47 | return hists, counts, sums 48 | } 49 | -------------------------------------------------------------------------------- /internal/kpis/plugins/histogram_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | type mockFeature struct { 12 | keys []string 13 | value float64 14 | } 15 | 16 | func TestHistogram(t *testing.T) { 17 | // Mock data 18 | features := []mockFeature{ 19 | {keys: []string{"key1"}, value: 1.0}, 20 | {keys: []string{"key1", "key2"}, value: 2.5}, 21 | {keys: []string{"key2"}, value: 3.0}, 22 | } 23 | buckets := []float64{1.0, 2.0, 3.0} 24 | 25 | // Mock key and value functions 26 | keysFunc := func(f mockFeature) []string { 27 | return f.keys 28 | } 29 | valueFunc := func(f mockFeature) float64 { 30 | return f.value 31 | } 32 | 33 | // Call the Histogram function 34 | hists, counts, sums := Histogram(features, buckets, keysFunc, valueFunc) 35 | 36 | // Expected results 37 | expectedHists := map[string]map[float64]uint64{ 38 | "key1": {1.0: 1, 2.0: 1, 3.0: 2}, 39 | "key2": {1.0: 0, 2.0: 0, 3.0: 2}, 40 | } 41 | expectedCounts := map[string]uint64{ 42 | "key1": 2, 43 | "key2": 2, 44 | } 45 | expectedSums := map[string]float64{ 46 | "key1": 3.5, 47 | "key2": 5.5, 48 | } 49 | 50 | // Validate results 51 | if !reflect.DeepEqual(hists, expectedHists) { 52 | t.Errorf("hists = %v, want %v", hists, expectedHists) 53 | } 54 | if !reflect.DeepEqual(counts, expectedCounts) { 55 | t.Errorf("counts = %v, want %v", counts, expectedCounts) 56 | } 57 | if !reflect.DeepEqual(sums, expectedSums) { 58 | t.Errorf("sums = %v, want %v", sums, expectedSums) 59 | } 60 | } 61 | 62 | func TestHistogram_EmptyFeatures(t *testing.T) { 63 | // Test with no features 64 | features := []mockFeature{} 65 | buckets := []float64{1.0, 2.0, 3.0} 66 | 67 | keysFunc := func(f mockFeature) []string { 68 | return f.keys 69 | } 70 | valueFunc := func(f mockFeature) float64 { 71 | return f.value 72 | } 73 | 74 | hists, counts, sums := Histogram(features, buckets, keysFunc, valueFunc) 75 | 76 | if len(hists) != 0 { 77 | t.Errorf("expected no histograms, got %v", hists) 78 | } 79 | if len(counts) != 0 { 80 | t.Errorf("expected no counts, got %v", counts) 81 | } 82 | if len(sums) != 0 { 83 | t.Errorf("expected no sums, got %v", sums) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /internal/kpis/plugins/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/trickyteache/cortex/internal/db" 9 | "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | // Each kpi plugin must conform to this interface. 13 | type KPI interface { 14 | // Configure the kpi with a database, options, and the 15 | // registry to publish metrics on. 16 | Init(db db.DB, opts conf.RawOpts) error 17 | // Collect the kpi from the given data. 18 | Collect(ch chan<- prometheus.Metric) 19 | // Describe this metric. 20 | Describe(ch chan<- *prometheus.Desc) 21 | // Get the name of this kpi. 22 | // This name is used to identify the kpi in metrics, config, logs, etc. 23 | // Should be something like: "my_cool_kpi". 24 | GetName() string 25 | } 26 | -------------------------------------------------------------------------------- /internal/kpis/plugins/shared/host_utilization_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/sync/openstack" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | func TestHostUtilizationKPI_Init(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | kpi := &HostUtilizationKPI{} 23 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | } 27 | 28 | func TestHostUtilizationKPI_Collect(t *testing.T) { 29 | dbEnv := testlibDB.SetupDBEnv(t) 30 | testDB := db.DB{DbMap: dbEnv.DbMap} 31 | defer testDB.Close() 32 | defer dbEnv.Close() 33 | 34 | // Create dependency tables 35 | if err := testDB.CreateTable( 36 | testDB.AddTable(openstack.Hypervisor{}), 37 | ); err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | 41 | // Insert mock data into the hypervisors table 42 | _, err := testDB.Exec(` 43 | INSERT INTO openstack_hypervisors ( 44 | id, service_host, hostname, state, status, hypervisor_type, hypervisor_version, host_ip, service_id, service_disabled_reason, vcpus, memory_mb, local_gb, vcpus_used, memory_mb_used, local_gb_used, free_ram_mb, free_disk_gb, current_workload, running_vms, disk_available_least, cpu_info 45 | ) 46 | VALUES 47 | (1, 'host1', 'hypervisor1', 'active', 'enabled', 'QEMU', 1000, '192.168.1.1', 1, 'none', 16, 32000, 1000, 8, 16000, 500, 16000, 500, 0, 10, 100, 'Intel'), 48 | (2, 'host2', 'hypervisor2', 'active', 'enabled', 'QEMU', 1000, '192.168.1.2', 2, 'none', 32, 64000, 2000, 16, 32000, 1000, 32000, 1000, 0, 20, 200, 'AMD') 49 | `) 50 | if err != nil { 51 | t.Fatalf("expected no error, got %v", err) 52 | } 53 | 54 | kpi := &HostUtilizationKPI{} 55 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 56 | t.Fatalf("expected no error, got %v", err) 57 | } 58 | 59 | ch := make(chan prometheus.Metric, 10) 60 | kpi.Collect(ch) 61 | close(ch) 62 | 63 | metricsCount := 0 64 | for range ch { 65 | metricsCount++ 66 | } 67 | 68 | if metricsCount == 0 { 69 | t.Errorf("expected metrics to be collected, got %d", metricsCount) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/kpis/plugins/shared/vm_life_span.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | "log/slog" 8 | "strings" 9 | 10 | "github.com/trickyteache/cortex/internal/conf" 11 | "github.com/trickyteache/cortex/internal/db" 12 | "github.com/trickyteache/cortex/internal/features/plugins/shared" 13 | "github.com/trickyteache/cortex/internal/kpis/plugins" 14 | "github.com/prometheus/client_golang/prometheus" 15 | ) 16 | 17 | // Advanced statistics about vm life spans. 18 | type VMLifeSpanKPI struct { 19 | // Common base for all KPIs that provides standard functionality. 20 | plugins.BaseKPI[struct{}] // No options passed through yaml config 21 | 22 | // Time a vm was alive before it was deleted. 23 | lifeSpanDesc *prometheus.Desc 24 | } 25 | 26 | func (VMLifeSpanKPI) GetName() string { 27 | return "vm_life_span_kpi" 28 | } 29 | 30 | func (k *VMLifeSpanKPI) Init(db db.DB, opts conf.RawOpts) error { 31 | if err := k.BaseKPI.Init(db, opts); err != nil { 32 | return err 33 | } 34 | k.lifeSpanDesc = prometheus.NewDesc( 35 | "cortex_vm_life_span", 36 | "Time a VM was alive before it was deleted", 37 | []string{"flavor_name", "flavor_id"}, 38 | nil, 39 | ) 40 | return nil 41 | } 42 | 43 | func (k *VMLifeSpanKPI) Describe(ch chan<- *prometheus.Desc) { 44 | ch <- k.lifeSpanDesc 45 | } 46 | 47 | func (k *VMLifeSpanKPI) Collect(ch chan<- prometheus.Metric) { 48 | var vmLifeSpans []shared.VMLifeSpan 49 | tableName := shared.VMLifeSpan{}.TableName() 50 | if _, err := k.DB.Select(&vmLifeSpans, "SELECT * FROM "+tableName); err != nil { 51 | slog.Error("failed to select vm life spans", "err", err) 52 | return 53 | } 54 | buckets := prometheus.ExponentialBucketsRange(5, 365*24*60*60, 30) 55 | keysFunc := func(lifeSpan shared.VMLifeSpan) []string { 56 | return []string{lifeSpan.FlavorName + "," + lifeSpan.FlavorID, "all,all"} 57 | } 58 | valueFunc := func(lifeSpan shared.VMLifeSpan) float64 { 59 | return float64(lifeSpan.Duration) 60 | } 61 | hists, counts, sums := plugins.Histogram(vmLifeSpans, buckets, keysFunc, valueFunc) 62 | for key, hist := range hists { 63 | labels := strings.Split(key, ",") 64 | if len(labels) != 2 { 65 | slog.Warn("vm_life_span: unexpected comma in flavor name or id") 66 | continue 67 | } 68 | ch <- prometheus.MustNewConstHistogram(k.lifeSpanDesc, counts[key], sums[key], hist, labels...) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /internal/kpis/plugins/shared/vm_life_span_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/shared" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | func TestVMLifeSpanKPI_Init(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | kpi := &VMLifeSpanKPI{} 23 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | } 27 | 28 | func TestVMLifeSpanKPI_Collect(t *testing.T) { 29 | dbEnv := testlibDB.SetupDBEnv(t) 30 | testDB := db.DB{DbMap: dbEnv.DbMap} 31 | defer testDB.Close() 32 | defer dbEnv.Close() 33 | 34 | // Create dependency tables 35 | if err := testDB.CreateTable( 36 | testDB.AddTable(shared.VMLifeSpan{}), 37 | ); err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | 41 | // Insert mock data into the vm_life_span table 42 | _, err := testDB.Exec(` 43 | INSERT INTO feature_vm_life_span ( 44 | duration, flavor_id, flavor_name, instance_uuid 45 | ) 46 | VALUES 47 | (3600, 'id1', 'flavor1', 'uuid1'), 48 | (7200, 'id2', 'flavor2', 'uuid2') 49 | `) 50 | if err != nil { 51 | t.Fatalf("expected no error, got %v", err) 52 | } 53 | 54 | kpi := &VMLifeSpanKPI{} 55 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 56 | t.Fatalf("expected no error, got %v", err) 57 | } 58 | 59 | ch := make(chan prometheus.Metric, 10) 60 | kpi.Collect(ch) 61 | close(ch) 62 | 63 | metricsCount := 0 64 | for range ch { 65 | metricsCount++ 66 | } 67 | 68 | if metricsCount == 0 { 69 | t.Errorf("expected metrics to be collected, got %d", metricsCount) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/kpis/plugins/shared/vm_migration_statistics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | "fmt" 8 | "log/slog" 9 | "strings" 10 | 11 | "github.com/trickyteache/cortex/internal/conf" 12 | "github.com/trickyteache/cortex/internal/db" 13 | "github.com/trickyteache/cortex/internal/features/plugins/shared" 14 | "github.com/trickyteache/cortex/internal/kpis/plugins" 15 | "github.com/prometheus/client_golang/prometheus" 16 | ) 17 | 18 | // Advanced statistics about openstack migrations. 19 | type VMMigrationStatisticsKPI struct { 20 | // Common base for all KPIs that provides standard functionality. 21 | plugins.BaseKPI[struct{}] // No options passed through yaml config 22 | 23 | // Time a VM has been on a host before migration. 24 | timeUntilMigrationDesc *prometheus.Desc 25 | // Number of migrations. 26 | nMigrations *prometheus.GaugeVec 27 | } 28 | 29 | func (VMMigrationStatisticsKPI) GetName() string { 30 | return "vm_migration_statistics_kpi" 31 | } 32 | 33 | func (k *VMMigrationStatisticsKPI) Init(db db.DB, opts conf.RawOpts) error { 34 | if err := k.BaseKPI.Init(db, opts); err != nil { 35 | return err 36 | } 37 | k.timeUntilMigrationDesc = prometheus.NewDesc( 38 | "cortex_vm_time_until_migration", 39 | "Time a VM has been on a host before migration", 40 | []string{"type", "flavor_name", "flavor_id"}, 41 | nil, 42 | ) 43 | k.nMigrations = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 44 | Name: "cortex_migrations_total", 45 | Help: "Number of migrations", 46 | }, []string{"type", "source_host", "target_host", "source_node", "target_node"}) 47 | return nil 48 | } 49 | 50 | func (k *VMMigrationStatisticsKPI) Describe(ch chan<- *prometheus.Desc) { 51 | ch <- k.timeUntilMigrationDesc 52 | k.nMigrations.Describe(ch) 53 | } 54 | 55 | func (k *VMMigrationStatisticsKPI) Collect(ch chan<- prometheus.Metric) { 56 | slog.Info("collecting vm migration statistics") 57 | defer slog.Info("finished collecting vm migration statistics") 58 | 59 | var hostResidencies []shared.VMHostResidency 60 | tableName := shared.VMHostResidency{}.TableName() 61 | if _, err := k.DB.Select(&hostResidencies, "SELECT * FROM "+tableName); err != nil { 62 | slog.Error("failed to select vm host residencies", "err", err) 63 | return 64 | } 65 | buckets := prometheus.ExponentialBucketsRange(5, 365*24*60*60, 30) 66 | keysFunc := func(residency shared.VMHostResidency) []string { 67 | return []string{ 68 | residency.Type + "," + residency.FlavorName + "," + residency.FlavorID, 69 | "all,all,all", 70 | } 71 | } 72 | valueFunc := func(residency shared.VMHostResidency) float64 { 73 | return float64(residency.Duration) 74 | } 75 | hists, counts, sums := plugins.Histogram(hostResidencies, buckets, keysFunc, valueFunc) 76 | for key, hist := range hists { 77 | labels := strings.Split(key, ",") 78 | if len(labels) != 3 { 79 | slog.Warn("vm_migration_statistics: unexpected comma in migration type, flavor name or id") 80 | continue 81 | } 82 | ch <- prometheus.MustNewConstHistogram(k.timeUntilMigrationDesc, counts[key], sums[key], hist, labels...) 83 | } 84 | 85 | nMigrations := make(map[string]int) 86 | for _, r := range hostResidencies { 87 | key := fmt.Sprintf( 88 | "%s,%s,%s,%s,%s", 89 | r.Type, r.SourceHost, r.TargetHost, r.SourceNode, r.TargetNode, 90 | ) 91 | nMigrations[key]++ 92 | } 93 | for key, n := range nMigrations { 94 | parts := strings.Split(key, ",") 95 | k.nMigrations.WithLabelValues( 96 | parts[0], parts[1], parts[2], parts[3], parts[4], 97 | ).Set(float64(n)) 98 | } 99 | k.nMigrations.Collect(ch) 100 | } 101 | -------------------------------------------------------------------------------- /internal/kpis/plugins/shared/vm_migration_statistics_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package shared 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/shared" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | func TestVMMigrationStatisticsKPI_Init(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | kpi := &VMMigrationStatisticsKPI{} 23 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | } 27 | 28 | func TestVMMigrationStatisticsKPI_Collect(t *testing.T) { 29 | dbEnv := testlibDB.SetupDBEnv(t) 30 | testDB := db.DB{DbMap: dbEnv.DbMap} 31 | defer testDB.Close() 32 | defer dbEnv.Close() 33 | 34 | // Create dependency tables 35 | if err := testDB.CreateTable( 36 | testDB.AddTable(shared.VMHostResidency{}), 37 | ); err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | _, err := testDB.Exec(` 41 | INSERT INTO feature_vm_host_residency ( 42 | duration, flavor_id, flavor_name, instance_uuid, migration_uuid, source_host, target_host, source_node, target_node, user_id, project_id, type, time 43 | ) 44 | VALUES 45 | (120, 'flavor1', 'small', 'uuid1', 'migration1', 'host1', 'host2', 'node1', 'node2', 'user1', 'project1', 'live-migration', 1620000000), 46 | (300, 'flavor2', 'medium', 'uuid2', 'migration2', 'host3', 'host4', 'node3', 'node4', 'user2', 'project2', 'resize', 1620000300) 47 | `) 48 | if err != nil { 49 | t.Fatalf("expected no error, got %v", err) 50 | } 51 | 52 | kpi := &VMMigrationStatisticsKPI{} 53 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 54 | t.Fatalf("expected no error, got %v", err) 55 | } 56 | 57 | ch := make(chan prometheus.Metric, 10) 58 | kpi.Collect(ch) 59 | close(ch) 60 | 61 | metricsCount := 0 62 | for range ch { 63 | metricsCount++ 64 | } 65 | 66 | if metricsCount == 0 { 67 | t.Errorf("expected metrics to be collected, got %d", metricsCount) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /internal/kpis/plugins/vmware/host_contention.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | "log/slog" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/vmware" 12 | "github.com/trickyteache/cortex/internal/kpis/plugins" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | type VMwareHostContentionKPI struct { 17 | // Common base for all KPIs that provides standard functionality. 18 | plugins.BaseKPI[struct{}] // No options passed through yaml config 19 | 20 | hostCPUContentionMax *prometheus.Desc 21 | hostCPUContentionAvg *prometheus.Desc 22 | } 23 | 24 | func (VMwareHostContentionKPI) GetName() string { 25 | return "vmware_host_contention_kpi" 26 | } 27 | 28 | func (k *VMwareHostContentionKPI) Init(db db.DB, opts conf.RawOpts) error { 29 | if err := k.BaseKPI.Init(db, opts); err != nil { 30 | return err 31 | } 32 | k.hostCPUContentionMax = prometheus.NewDesc( 33 | "cortex_vmware_host_cpu_contention_max", 34 | "Max CPU contention of vROps hostsystems over the configured prometheus sync period.", 35 | nil, nil, 36 | ) 37 | k.hostCPUContentionAvg = prometheus.NewDesc( 38 | "cortex_vmware_host_cpu_contention_avg", 39 | "Avg CPU contention of vROps hostsystems over the configured prometheus sync period.", 40 | nil, nil, 41 | ) 42 | return nil 43 | } 44 | 45 | func (k *VMwareHostContentionKPI) Describe(ch chan<- *prometheus.Desc) { 46 | ch <- k.hostCPUContentionMax 47 | ch <- k.hostCPUContentionAvg 48 | } 49 | 50 | func (k *VMwareHostContentionKPI) Collect(ch chan<- prometheus.Metric) { 51 | var contentions []vmware.VROpsHostsystemContention 52 | tableName := vmware.VROpsHostsystemContention{}.TableName() 53 | if _, err := k.DB.Select(&contentions, "SELECT * FROM "+tableName); err != nil { 54 | slog.Error("failed to select hostsystem contention", "err", err) 55 | return 56 | } 57 | buckets := prometheus.LinearBuckets(0, 5, 20) 58 | keysFunc := func(contention vmware.VROpsHostsystemContention) []string { 59 | return []string{"cpu_contention_max"} 60 | } 61 | valueFunc := func(contention vmware.VROpsHostsystemContention) float64 { 62 | return contention.MaxCPUContention 63 | } 64 | hists, counts, sums := plugins.Histogram(contentions, buckets, keysFunc, valueFunc) 65 | for key, hist := range hists { 66 | ch <- prometheus.MustNewConstHistogram(k.hostCPUContentionMax, counts[key], sums[key], hist) 67 | } 68 | keysFunc = func(contention vmware.VROpsHostsystemContention) []string { 69 | return []string{"cpu_contention_avg"} 70 | } 71 | valueFunc = func(contention vmware.VROpsHostsystemContention) float64 { 72 | return float64(contention.AvgCPUContention) 73 | } 74 | hists, counts, sums = plugins.Histogram(contentions, buckets, keysFunc, valueFunc) 75 | for key, hist := range hists { 76 | ch <- prometheus.MustNewConstHistogram(k.hostCPUContentionAvg, counts[key], sums[key], hist) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /internal/kpis/plugins/vmware/host_contention_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/vmware" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | func TestVMwareHostContentionKPI_Init(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | kpi := &VMwareHostContentionKPI{} 23 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | } 27 | 28 | func TestVMwareHostContentionKPI_Collect(t *testing.T) { 29 | dbEnv := testlibDB.SetupDBEnv(t) 30 | testDB := db.DB{DbMap: dbEnv.DbMap} 31 | defer testDB.Close() 32 | defer dbEnv.Close() 33 | 34 | // Create dependency tables 35 | if err := testDB.CreateTable( 36 | testDB.AddTable(vmware.VROpsHostsystemContention{}), 37 | ); err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | 41 | // Insert mock data into the feature_vrops_hostsystem_contention table 42 | _, err := testDB.Exec(` 43 | INSERT INTO feature_vrops_hostsystem_contention ( 44 | compute_host, avg_cpu_contention, max_cpu_contention 45 | ) 46 | VALUES 47 | ('host1', 10.5, 20.0), 48 | ('host2', 15.0, 25.0) 49 | `) 50 | if err != nil { 51 | t.Fatalf("expected no error, got %v", err) 52 | } 53 | 54 | kpi := &VMwareHostContentionKPI{} 55 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 56 | t.Fatalf("expected no error, got %v", err) 57 | } 58 | 59 | ch := make(chan prometheus.Metric, 10) 60 | kpi.Collect(ch) 61 | close(ch) 62 | 63 | metricsCount := 0 64 | for range ch { 65 | metricsCount++ 66 | } 67 | 68 | if metricsCount == 0 { 69 | t.Errorf("expected metrics to be collected, got %d", metricsCount) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/kpis/plugins/vmware/project_noisiness.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | "log/slog" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/vmware" 12 | "github.com/trickyteache/cortex/internal/kpis/plugins" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | type VMwareProjectNoisinessKPI struct { 17 | // Common base for all KPIs that provides standard functionality. 18 | plugins.BaseKPI[struct{}] // No options passed through yaml config 19 | 20 | projectNoisinessDesc *prometheus.Desc 21 | } 22 | 23 | func (VMwareProjectNoisinessKPI) GetName() string { 24 | return "vmware_project_noisiness_kpi" 25 | } 26 | 27 | func (k *VMwareProjectNoisinessKPI) Init(db db.DB, opts conf.RawOpts) error { 28 | if err := k.BaseKPI.Init(db, opts); err != nil { 29 | return err 30 | } 31 | k.projectNoisinessDesc = prometheus.NewDesc( 32 | "cortex_vmware_project_noisiness", 33 | "Project noisiness of vROps projects over the configured prometheus sync period.", 34 | nil, nil, 35 | ) 36 | return nil 37 | } 38 | 39 | func (k *VMwareProjectNoisinessKPI) Describe(ch chan<- *prometheus.Desc) { 40 | ch <- k.projectNoisinessDesc 41 | } 42 | 43 | func (k *VMwareProjectNoisinessKPI) Collect(ch chan<- prometheus.Metric) { 44 | var features []vmware.VROpsProjectNoisiness 45 | tableName := vmware.VROpsProjectNoisiness{}.TableName() 46 | if _, err := k.DB.Select(&features, "SELECT * FROM "+tableName); err != nil { 47 | slog.Error("failed to select project noisiness", "err", err) 48 | return 49 | } 50 | buckets := prometheus.LinearBuckets(0, 5, 20) 51 | keysFunc := func(noisiness vmware.VROpsProjectNoisiness) []string { 52 | return []string{"project_noisiness"} 53 | } 54 | valueFunc := func(noisiness vmware.VROpsProjectNoisiness) float64 { 55 | return float64(noisiness.AvgCPUOfProject) 56 | } 57 | hists, counts, sums := plugins.Histogram(features, buckets, keysFunc, valueFunc) 58 | for key, hist := range hists { 59 | ch <- prometheus.MustNewConstHistogram(k.projectNoisinessDesc, counts[key], sums[key], hist) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /internal/kpis/plugins/vmware/project_noisiness_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/vmware" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | func TestVMwareProjectNoisinessKPI_Init(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | kpi := &VMwareProjectNoisinessKPI{} 23 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | } 27 | 28 | func TestVMwareProjectNoisinessKPI_Collect(t *testing.T) { 29 | dbEnv := testlibDB.SetupDBEnv(t) 30 | testDB := db.DB{DbMap: dbEnv.DbMap} 31 | defer testDB.Close() 32 | defer dbEnv.Close() 33 | 34 | // Create dependency tables 35 | if err := testDB.CreateTable( 36 | testDB.AddTable(vmware.VROpsProjectNoisiness{}), 37 | ); err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | 41 | // Insert mock data into the feature_vrops_project_noisiness table 42 | _, err := testDB.Exec(` 43 | INSERT INTO feature_vrops_project_noisiness ( 44 | project, compute_host, avg_cpu_of_project 45 | ) 46 | VALUES 47 | ('project1', 'host1', 10.5), 48 | ('project2', 'host2', 15.0) 49 | `) 50 | if err != nil { 51 | t.Fatalf("expected no error, got %v", err) 52 | } 53 | 54 | kpi := &VMwareProjectNoisinessKPI{} 55 | if err := kpi.Init(testDB, conf.NewRawOpts("{}")); err != nil { 56 | t.Fatalf("expected no error, got %v", err) 57 | } 58 | 59 | ch := make(chan prometheus.Metric, 10) 60 | kpi.Collect(ch) 61 | close(ch) 62 | 63 | metricsCount := 0 64 | for range ch { 65 | metricsCount++ 66 | } 67 | 68 | if metricsCount == 0 { 69 | t.Errorf("expected metrics to be collected, got %d", metricsCount) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/monitoring/monitoring.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package monitoring 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/collectors" 10 | dto "github.com/prometheus/client_model/go" 11 | ) 12 | 13 | // Custom prometheus registry that adds functionality to the default registry. 14 | type Registry struct { 15 | // Inherited prometheus registry. 16 | *prometheus.Registry 17 | // Custom configuration for the monitoring. 18 | config conf.MonitoringConfig 19 | } 20 | 21 | // Create a new registry with the given configuration. 22 | // This registry will include the default go collector and process collector. 23 | func NewRegistry(config conf.MonitoringConfig) *Registry { 24 | registry := &Registry{ 25 | Registry: prometheus.NewRegistry(), 26 | config: config, 27 | } 28 | // Add go execution stats and process metrics to the registry. 29 | registry.MustRegister(collectors.NewGoCollector()) 30 | registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) 31 | return registry 32 | } 33 | 34 | // Custom gather method that adds custom labels to all metrics. 35 | func (r *Registry) Gather() ([]*dto.MetricFamily, error) { 36 | families, err := r.Registry.Gather() 37 | if err != nil { 38 | return nil, err 39 | } 40 | // Add a custom label to all metrics. This is useful for distinguishing 41 | // the metrics from other golang services that also use the default 42 | // go collector metrics. 43 | for name, value := range r.config.Labels { 44 | for _, family := range families { 45 | for _, metric := range family.Metric { 46 | metric.Label = append(metric.Label, &dto.LabelPair{ 47 | Name: &name, 48 | Value: &value, 49 | }) 50 | } 51 | } 52 | } 53 | return families, nil 54 | } 55 | -------------------------------------------------------------------------------- /internal/monitoring/monitoring_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package monitoring 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/prometheus/client_golang/prometheus" 11 | ) 12 | 13 | func TestNewRegistry(t *testing.T) { 14 | config := conf.MonitoringConfig{ 15 | Labels: map[string]string{ 16 | "env": "test", 17 | }, 18 | } 19 | registry := NewRegistry(config) 20 | 21 | if registry == nil { 22 | t.Fatalf("expected registry to be non-nil") 23 | return 24 | } 25 | if registry.config.Labels["env"] != "test" { 26 | t.Fatalf("expected registry config label 'env' to be 'test', got %v", registry.config.Labels["env"]) 27 | } 28 | } 29 | 30 | func TestRegistry_Gather(t *testing.T) { 31 | config := conf.MonitoringConfig{ 32 | Labels: map[string]string{ 33 | "env": "test", 34 | }, 35 | } 36 | registry := NewRegistry(config) 37 | 38 | // Register a custom metric 39 | counter := prometheus.NewCounter(prometheus.CounterOpts{ 40 | Name: "test_counter", 41 | Help: "A test counter", 42 | }) 43 | registry.MustRegister(counter) 44 | counter.Inc() 45 | 46 | // Gather metrics 47 | families, err := registry.Gather() 48 | if err != nil { 49 | t.Fatalf("expected no error, got %v", err) 50 | } 51 | 52 | // Check that the custom label is added to all metrics 53 | for _, family := range families { 54 | for _, metric := range family.Metric { 55 | found := false 56 | for _, label := range metric.Label { 57 | if *label.Name == "env" && *label.Value == "test" { 58 | found = true 59 | break 60 | } 61 | } 62 | if !found { 63 | t.Fatalf("expected custom label 'env' with value 'test' in metric, but not found") 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/mqtt/mqtt_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package mqtt 5 | 6 | import ( 7 | "os" 8 | "sync" 9 | "testing" 10 | 11 | "github.com/trickyteache/cortex/internal/conf" 12 | "github.com/trickyteache/cortex/testlib/mqtt/containers" 13 | mqtt "github.com/eclipse/paho.mqtt.golang" 14 | ) 15 | 16 | func TestConnect(t *testing.T) { 17 | if os.Getenv("RABBITMQ_CONTAINER") != "1" { 18 | t.Skip("skipping test; set RABBITMQ_CONTAINER=1 to run") 19 | } 20 | 21 | container := containers.RabbitMQContainer{} 22 | container.Init(t) 23 | defer container.Close() 24 | conf := conf.MQTTConfig{URL: "tcp://localhost:" + container.GetPort()} 25 | c := client{conf: conf, lock: &sync.Mutex{}} 26 | 27 | err := c.Connect() 28 | if err != nil { 29 | t.Fatalf("expected no error, got %v", err) 30 | } 31 | c.Disconnect() 32 | } 33 | 34 | func TestPublish(t *testing.T) { 35 | if os.Getenv("RABBITMQ_CONTAINER") != "1" { 36 | t.Skip("skipping test; set RABBITMQ_CONTAINER=1 to run") 37 | } 38 | // FIXME: It seems like GitHub Actions kills the container on the publish call. 39 | if os.Getenv("GITHUB_ACTIONS") == "1" { 40 | t.Skip("skipping test; GITHUB_ACTIONS=1") 41 | } 42 | 43 | container := containers.RabbitMQContainer{} 44 | container.Init(t) 45 | defer container.Close() 46 | conf := conf.MQTTConfig{URL: "tcp://localhost:" + container.GetPort()} 47 | c := client{conf: conf, lock: &sync.Mutex{}} 48 | err := c.publish("test/topic", map[string]string{"key": "value"}) 49 | if err != nil { 50 | t.Fatalf("expected no error, got %v", err) 51 | } 52 | t.Log("published message to test/topic") 53 | c.Disconnect() 54 | } 55 | 56 | func TestSubscribe(t *testing.T) { 57 | if os.Getenv("RABBITMQ_CONTAINER") != "1" { 58 | t.Skip("skipping test; set RABBITMQ_CONTAINER=1 to run") 59 | } 60 | 61 | container := containers.RabbitMQContainer{} 62 | container.Init(t) 63 | defer container.Close() 64 | conf := conf.MQTTConfig{URL: "tcp://localhost:" + container.GetPort()} 65 | c := client{conf: conf, lock: &sync.Mutex{}} 66 | 67 | err := c.Subscribe("test/topic", func(client mqtt.Client, msg mqtt.Message) {}) 68 | if err != nil { 69 | t.Fatalf("expected no error, got %v", err) 70 | } 71 | c.Disconnect() 72 | } 73 | 74 | func TestDisconnect(t *testing.T) { 75 | if os.Getenv("RABBITMQ_CONTAINER") != "1" { 76 | t.Skip("skipping test; set RABBITMQ_CONTAINER=1 to run") 77 | } 78 | 79 | container := containers.RabbitMQContainer{} 80 | container.Init(t) 81 | defer container.Close() 82 | conf := conf.MQTTConfig{URL: "tcp://localhost:" + container.GetPort()} 83 | c := client{conf: conf, lock: &sync.Mutex{}} 84 | err := c.Connect() 85 | if err != nil { 86 | t.Fatalf("expected no error, got %v", err) 87 | } 88 | c.Disconnect() 89 | c.Disconnect() // Should do nothing (already disconnected) 90 | } 91 | -------------------------------------------------------------------------------- /internal/scheduler/api/http/messages.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package http 5 | 6 | import "github.com/trickyteache/cortex/internal/scheduler/api" 7 | 8 | // Host object from the Nova scheduler pipeline. 9 | // See: https://github.com/sapcc/nova/blob/stable/xena-m3/nova/scheduler/host_manager.py class HostState 10 | type ExternalSchedulerHost struct { 11 | // Name of the Nova compute host, e.g. nova-compute-bb123. 12 | ComputeHost string `json:"host"` 13 | // Name of the hypervisor hostname, e.g. domain-c123. 14 | HypervisorHostname string `json:"hypervisor_hostname"` 15 | } 16 | 17 | // Request generated by the Nova scheduler when calling cortex. 18 | // The request contains a spec of the VM to be scheduled, a list of hosts and 19 | // their status, and a map of weights that were calculated by the Nova weigher 20 | // pipeline. Some additional flags are also included. 21 | type ExternalSchedulerRequest struct { 22 | Spec api.NovaObject[api.NovaSpec] `json:"spec"` 23 | 24 | // Request context from Nova that contains additional meta information. 25 | Context api.NovaRequestContext `json:"context"` 26 | 27 | // Whether the Nova scheduling request is a rebuild request. 28 | Rebuild bool `json:"rebuild"` 29 | // Whether the Nova scheduling request is a resize request. 30 | Resize bool `json:"resize"` 31 | // Whether the Nova scheduling request is a live migration. 32 | Live bool `json:"live"` 33 | // Whether the affected VM is a VMware VM. 34 | VMware bool `json:"vmware"` 35 | 36 | Hosts []ExternalSchedulerHost `json:"hosts"` 37 | Weights map[string]float64 `json:"weights"` 38 | } 39 | 40 | // Conform to the Request interface. 41 | 42 | func (r *ExternalSchedulerRequest) GetSpec() api.NovaObject[api.NovaSpec] { 43 | return r.Spec 44 | } 45 | func (r *ExternalSchedulerRequest) GetContext() api.NovaRequestContext { 46 | return r.Context 47 | } 48 | func (r *ExternalSchedulerRequest) GetRebuild() bool { 49 | return r.Rebuild 50 | } 51 | func (r *ExternalSchedulerRequest) GetResize() bool { 52 | return r.Resize 53 | } 54 | func (r *ExternalSchedulerRequest) GetLive() bool { 55 | return r.Live 56 | } 57 | func (r *ExternalSchedulerRequest) GetVMware() bool { 58 | return r.VMware 59 | } 60 | func (r *ExternalSchedulerRequest) GetHosts() []string { 61 | hosts := make([]string, len(r.Hosts)) 62 | for i, host := range r.Hosts { 63 | hosts[i] = host.ComputeHost 64 | } 65 | return hosts 66 | } 67 | func (r *ExternalSchedulerRequest) GetWeights() map[string]float64 { 68 | return r.Weights 69 | } 70 | 71 | // Response generated by cortex for the Nova scheduler. 72 | // Cortex returns an ordered list of hosts that the VM should be scheduled on. 73 | type ExternalSchedulerResponse struct { 74 | Hosts []string `json:"hosts"` 75 | } 76 | -------------------------------------------------------------------------------- /internal/scheduler/api/http/monitor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package http 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/monitoring" 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | // Collection of Prometheus metrics to monitor scheduler pipeline 12 | type Monitor struct { 13 | // A histogram to measure how long the API requests take to run. 14 | apiRequestsTimer *prometheus.HistogramVec 15 | } 16 | 17 | // Create a new scheduler monitor and register the necessary Prometheus metrics. 18 | func NewSchedulerMonitor(registry *monitoring.Registry) Monitor { 19 | apiRequestsTimer := prometheus.NewHistogramVec(prometheus.HistogramOpts{ 20 | Name: "cortex_scheduler_api_request_duration_seconds", 21 | Help: "Duration of API requests", 22 | Buckets: prometheus.DefBuckets, 23 | }, []string{"method", "path", "status", "error"}) 24 | registry.MustRegister( 25 | apiRequestsTimer, 26 | ) 27 | return Monitor{ 28 | apiRequestsTimer: apiRequestsTimer, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /internal/scheduler/api/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package api 5 | 6 | type Pipeline interface { 7 | // Run the scheduling pipeline with the given request. 8 | Run(request Request) ([]string, error) 9 | } 10 | 11 | // Request to the Cortex scheduling pipeline. 12 | type Request interface { 13 | // Specification of the scheduling request. 14 | GetSpec() NovaObject[NovaSpec] 15 | // Request context from Nova that contains additional meta information. 16 | GetContext() NovaRequestContext 17 | // Whether the Nova scheduling request is a rebuild request. 18 | GetRebuild() bool 19 | // Whether the Nova scheduling request is a resize request. 20 | GetResize() bool 21 | // Whether the Nova scheduling request is a live migration. 22 | GetLive() bool 23 | // Whether the affected VM is a VMware VM. 24 | GetVMware() bool 25 | // List of hosts to be considered for scheduling. 26 | // If the list is nil, all hosts are considered. 27 | GetHosts() []string 28 | // Map of weights to start with. 29 | // If the map is nil, all hosts will have the default weight starting. 30 | GetWeights() map[string]float64 31 | } 32 | -------------------------------------------------------------------------------- /internal/scheduler/api/messages.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package api 5 | 6 | // Wrapped Nova object. Nova returns objects in this format. 7 | type NovaObject[V any] struct { 8 | Name string `json:"nova_object.name"` 9 | Namespace string `json:"nova_object.namespace"` 10 | Version string `json:"nova_object.version"` 11 | Data V `json:"nova_object.data"` 12 | Changes []string `json:"nova_object.changes"` 13 | } 14 | 15 | // Spec object from the Nova scheduler pipeline. 16 | // See: https://github.com/sapcc/nova/blob/stable/xena-m3/nova/objects/request_spec.py 17 | type NovaSpec struct { 18 | ProjectID string `json:"project_id"` 19 | UserID string `json:"user_id"` 20 | AvailabilityZone string `json:"availability_zone"` 21 | NInstances int `json:"num_instances"` 22 | Image NovaObject[NovaImageMeta] `json:"image"` 23 | Flavor NovaObject[NovaFlavor] `json:"flavor"` 24 | } 25 | 26 | // Nova image metadata for the specified VM. 27 | type NovaImageMeta struct { 28 | Name string `json:"name"` 29 | Size int `json:"size"` 30 | MinRAM int `json:"min_ram"` 31 | MinDisk int `json:"min_disk"` 32 | } 33 | 34 | // Nova flavor metadata for the specified VM. 35 | type NovaFlavor struct { 36 | Name string `json:"name"` 37 | MemoryMB int `json:"memory_mb"` 38 | VCPUs int `json:"vcpus"` 39 | RootDiskGB int `json:"root_gb"` 40 | EphemeralDiskGB int `json:"ephemeral_gb"` 41 | FlavorID string `json:"flavorid"` 42 | Swap int `json:"swap"` 43 | RXTXFactor float64 `json:"rxtx_factor"` 44 | VCPUsWeight float64 `json:"vcpus_weight"` 45 | ExtraSpecs map[string]string `json:"extra_specs"` 46 | } 47 | 48 | // Nova request context object. For the spec of this object, see: 49 | // 50 | // - This: https://github.com/sapcc/nova/blob/a56409/nova/context.py#L166 51 | // - And: https://github.com/openstack/oslo.context/blob/db20dd/oslo_context/context.py#L329 52 | // 53 | // Some fields are omitted: "service_catalog", "read_deleted" (same as "show_deleted") 54 | type NovaRequestContext struct { 55 | // Fields added by oslo.context 56 | 57 | UserID string `json:"user"` 58 | ProjectID string `json:"project_id"` 59 | SystemScope string `json:"system_scope"` 60 | DomainID string `json:"domain"` 61 | UserDomainID string `json:"user_domain"` 62 | ProjectDomainID string `json:"project_domain"` 63 | IsAdmin bool `json:"is_admin"` 64 | ReadOnly bool `json:"read_only"` 65 | ShowDeleted bool `json:"show_deleted"` 66 | AuthToken string `json:"auth_token"` 67 | RequestID string `json:"request_id"` 68 | GlobalRequestID string `json:"global_request_id"` 69 | ResourceUUID string `json:"resource_uuid"` 70 | Roles []string `json:"roles"` 71 | UserIdentity string `json:"user_identity"` 72 | IsAdminProject bool `json:"is_admin_project"` 73 | 74 | // Fields added by the Nova scheduler 75 | 76 | RemoteAddress string `json:"remote_address"` 77 | Timestamp string `json:"timestamp"` 78 | QuotaClass string `json:"quota_class"` 79 | UserName string `json:"user_name"` 80 | ProjectName string `json:"project_name"` 81 | } 82 | -------------------------------------------------------------------------------- /internal/scheduler/api/messages_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package api 5 | 6 | import ( 7 | "encoding/json" 8 | "testing" 9 | ) 10 | 11 | func TestNovaSpecUnmarshal(t *testing.T) { 12 | var jsonData = `{ 13 | "spec": { 14 | "nova_object.name": "RequestSpec", 15 | "nova_object.namespace": "nova", 16 | "nova_object.version": "1.14", 17 | "nova_object.data": { 18 | "image": { 19 | "nova_object.name": "ImageMeta", 20 | "nova_object.namespace": "nova", 21 | "nova_object.version": "1.8", 22 | "nova_object.data": { 23 | "name": "example-name", 24 | "size": 123456789, 25 | "min_ram": 2048, 26 | "min_disk": 20 27 | }, 28 | "nova_object.changes": ["id", "name", "size", "min_ram", "min_disk"] 29 | }, 30 | "project_id": "example-project-id", 31 | "user_id": "example-user-id", 32 | "availability_zone": "example-az", 33 | "flavor": { 34 | "nova_object.name": "Flavor", 35 | "nova_object.namespace": "nova", 36 | "nova_object.version": "1.2", 37 | "nova_object.data": { 38 | "name": "example-flavor-name", 39 | "memory_mb": 4096, 40 | "vcpus": 2, 41 | "root_gb": 40, 42 | "ephemeral_gb": 0, 43 | "flavorid": "example-flavorid", 44 | "swap": 0, 45 | "rxtx_factor": 1.0, 46 | "vcpu_weight": 0, 47 | "extra_specs": { 48 | "example-key": "example-value" 49 | } 50 | }, 51 | "nova_object.changes": ["id", "name", "memory_mb", "vcpus", "root_gb", "ephemeral_gb", "flavorid", "swap", "rxtx_factor", "vcpu_weight", "extra_specs"] 52 | }, 53 | "num_instances": 1 54 | }, 55 | "nova_object.changes": ["image", "project_id", "user_id", "availability_zone", "flavor", "num_instances"] 56 | } 57 | }` 58 | 59 | var spec struct { 60 | Spec NovaObject[NovaSpec] `json:"spec"` 61 | } 62 | err := json.Unmarshal([]byte(jsonData), &spec) 63 | if err != nil { 64 | t.Fatalf("Failed to unmarshal JSON: %v", err) 65 | } 66 | 67 | if spec.Spec.Data.ProjectID != "example-project-id" { 68 | t.Errorf("Expected ProjectID to be 'example-project-id', got '%s'", spec.Spec.Data.ProjectID) 69 | } 70 | if spec.Spec.Data.UserID != "example-user-id" { 71 | t.Errorf("Expected UserID to be 'example-user-id', got '%s'", spec.Spec.Data.UserID) 72 | } 73 | if spec.Spec.Data.AvailabilityZone != "example-az" { 74 | t.Errorf("Expected AvailabilityZone to be 'example-az', got '%s'", spec.Spec.Data.AvailabilityZone) 75 | } 76 | if spec.Spec.Data.NInstances != 1 { 77 | t.Errorf("Expected NInstances to be 1, got %d", spec.Spec.Data.NInstances) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/activation.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import "math" 7 | 8 | type Weight = float64 9 | type Weights = map[string]float64 10 | 11 | // Mixin that can be embedded in a step to provide some activation function tooling. 12 | type ActivationFunction struct{} 13 | 14 | // Get activations that will have no effect on the host. 15 | func (m *ActivationFunction) NoEffect() Weight { return 0 } 16 | 17 | // Apply the activation function to the weights map. 18 | // All hosts that are not in the activations map are removed. 19 | func (m *ActivationFunction) Apply(in, activations Weights) Weights { 20 | for host, prevWeight := range in { 21 | // Remove hosts that are not in the weights map. 22 | if _, ok := activations[host]; !ok { 23 | delete(in, host) 24 | } else { 25 | // Apply the activation from the step. 26 | (in)[host] = prevWeight + math.Tanh(activations[host]) 27 | } 28 | } 29 | return in 30 | } 31 | 32 | // Clamp a value between lower and upper bounds. 33 | func clamp(value, lowerBound, upperBound float64) float64 { 34 | if lowerBound > upperBound { 35 | lowerBound, upperBound = upperBound, lowerBound 36 | } 37 | if value < lowerBound { 38 | return lowerBound 39 | } 40 | if value > upperBound { 41 | return upperBound 42 | } 43 | return value 44 | } 45 | 46 | // Min-max scale a value between lower and upper bounds and apply the given activation. 47 | // Note: the resulting value is clamped between the activation bounds. 48 | func MinMaxScale(value, lowerBound, upperBound, activationLowerBound, activationUpperBound float64) float64 { 49 | // Avoid zero-division during min-max scaling. 50 | if lowerBound == upperBound { 51 | return 0 52 | } 53 | if activationLowerBound == activationUpperBound { 54 | return 0 55 | } 56 | normalized := (value - lowerBound) / (upperBound - lowerBound) 57 | activation := activationLowerBound + normalized*(activationUpperBound-activationLowerBound) 58 | return clamp(activation, activationLowerBound, activationUpperBound) 59 | } 60 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/activation_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "math" 8 | "testing" 9 | ) 10 | 11 | func TestActivationFunction_NoEffect(t *testing.T) { 12 | af := ActivationFunction{} 13 | expected := Weight(0) 14 | if af.NoEffect() != expected { 15 | t.Errorf("expected %v, got %v", expected, af.NoEffect()) 16 | } 17 | } 18 | 19 | func TestActivationFunction_Apply(t *testing.T) { 20 | af := ActivationFunction{} 21 | 22 | tests := []struct { 23 | name string 24 | in Weights 25 | activations Weights 26 | expected Weights 27 | }{ 28 | { 29 | name: "all hosts in activations", 30 | in: Weights{ 31 | "host1": 1.0, 32 | "host2": 2.0, 33 | }, 34 | activations: Weights{ 35 | "host1": 0.5, 36 | "host2": -0.5, 37 | }, 38 | expected: Weights{ 39 | "host1": 1.0 + math.Tanh(0.5), 40 | "host2": 2.0 + math.Tanh(-0.5), 41 | }, 42 | }, 43 | { 44 | name: "some hosts not in activations", 45 | in: Weights{ 46 | "host1": 1.0, 47 | "host2": 2.0, 48 | "host3": 3.0, 49 | }, 50 | activations: Weights{ 51 | "host1": 0.5, 52 | }, 53 | expected: Weights{ 54 | "host1": 1.0 + math.Tanh(0.5), 55 | }, 56 | }, 57 | { 58 | name: "no hosts in activations", 59 | in: Weights{ 60 | "host1": 1.0, 61 | "host2": 2.0, 62 | }, 63 | activations: Weights{}, 64 | expected: Weights{}, 65 | }, 66 | } 67 | 68 | for _, tt := range tests { 69 | t.Run(tt.name, func(t *testing.T) { 70 | result := af.Apply(tt.in, tt.activations) 71 | if len(result) != len(tt.expected) { 72 | t.Fatalf("expected %d hosts, got %d", len(tt.expected), len(result)) 73 | } 74 | for host, weight := range tt.expected { 75 | if result[host] != weight { 76 | t.Errorf("expected weight for host %s to be %v, got %v", host, weight, result[host]) 77 | } 78 | } 79 | }) 80 | } 81 | } 82 | 83 | func TestClamp(t *testing.T) { 84 | tests := []struct { 85 | value, lowerBound, upperBound, expected float64 86 | }{ 87 | {5, 0, 10, 5}, 88 | {15, 0, 10, 10}, 89 | {-5, 0, 10, 0}, 90 | {5, 10, 0, 5}, // bounds are swapped 91 | {15, 10, 0, 10}, // bounds are swapped 92 | {-5, 10, 0, 0}, // bounds are swapped 93 | } 94 | 95 | for _, test := range tests { 96 | result := clamp(test.value, test.lowerBound, test.upperBound) 97 | if result != test.expected { 98 | t.Errorf("clamp(%v, %v, %v) = %v; want %v", test.value, test.lowerBound, test.upperBound, result, test.expected) 99 | } 100 | } 101 | } 102 | 103 | func TestMinMaxScale(t *testing.T) { 104 | tests := []struct { 105 | value, lowerBound, upperBound, activationLowerBound, activationUpperBound, expected float64 106 | }{ 107 | {5, 0, 10, 0, 1, 0.5}, 108 | {15, 0, 10, 0, 1, 1}, 109 | {-5, 0, 10, 0, 1, 0}, 110 | {5, 0, 10, 1, 2, 1.5}, 111 | {5, 0, 0, 0, 1, 0}, // avoid zero-division 112 | {5, 0, 10, 1, 1, 0}, // avoid zero-division 113 | } 114 | 115 | for _, test := range tests { 116 | result := MinMaxScale(test.value, test.lowerBound, test.upperBound, test.activationLowerBound, test.activationUpperBound) 117 | if result != test.expected { 118 | t.Errorf("MinMaxScale(%v, %v, %v, %v, %v) = %v; want %v", test.value, test.lowerBound, test.upperBound, test.activationLowerBound, test.activationUpperBound, result, test.expected) 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/base.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/trickyteache/cortex/internal/db" 9 | "github.com/trickyteache/cortex/internal/scheduler/api" 10 | ) 11 | 12 | // Interface to which step options must conform. 13 | type StepOpts interface { 14 | // Validate the options for this step. 15 | Validate() error 16 | } 17 | 18 | // Empty options for steps that don't need any. 19 | type EmptyStepOpts struct{} 20 | 21 | func (o EmptyStepOpts) Validate() error { return nil } 22 | 23 | // Common base for all steps that provides some functionality 24 | // that would otherwise be duplicated across all steps. 25 | type BaseStep[Opts StepOpts] struct { 26 | // Options to pass via yaml to this step. 27 | conf.JsonOpts[Opts] 28 | // The activation function to use. 29 | ActivationFunction 30 | // Database connection. 31 | DB db.DB 32 | } 33 | 34 | // Init the step with the database and options. 35 | func (s *BaseStep[Opts]) Init(db db.DB, opts conf.RawOpts) error { 36 | if err := s.Load(opts); err != nil { 37 | return err 38 | } 39 | s.DB = db 40 | return s.Options.Validate() 41 | } 42 | 43 | // Get zero activations for all hosts. 44 | func (s *BaseStep[Opts]) BaseActivations(request api.Request) Weights { 45 | weights := make(Weights) 46 | for _, host := range request.GetHosts() { 47 | weights[host] = s.NoEffect() 48 | } 49 | return weights 50 | } 51 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | testlibDB "github.com/trickyteache/cortex/testlib/db" 12 | ) 13 | 14 | type MockOptions struct { 15 | Option1 string `json:"option1"` 16 | Option2 int `json:"option2"` 17 | } 18 | 19 | func (o MockOptions) Validate() error { 20 | return nil 21 | } 22 | 23 | func TestBaseStep_Init(t *testing.T) { 24 | dbEnv := testlibDB.SetupDBEnv(t) 25 | testDB := db.DB{DbMap: dbEnv.DbMap} 26 | defer testDB.Close() 27 | defer dbEnv.Close() 28 | 29 | opts := conf.NewRawOpts(`{ 30 | "option1": "value1", 31 | "option2": 2 32 | }`) 33 | 34 | step := BaseStep[MockOptions]{} 35 | err := step.Init(testDB, opts) 36 | if err != nil { 37 | t.Fatalf("expected no error, got %v", err) 38 | } 39 | 40 | if step.Options.Option1 != "value1" { 41 | t.Errorf("expected Option1 to be 'value1', got %s", step.Options.Option1) 42 | } 43 | 44 | if step.Options.Option2 != 2 { 45 | t.Errorf("expected Option2 to be 2, got %d", step.Options.Option2) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/trickyteache/cortex/internal/db" 9 | "github.com/trickyteache/cortex/internal/scheduler/api" 10 | ) 11 | 12 | // Interface for a scheduler step. 13 | type Step interface { 14 | // Configure the step with a database and options. 15 | Init(db db.DB, opts conf.RawOpts) error 16 | // Run this step of the scheduling pipeline. 17 | // Return a map of hostnames to activation values. Important: hosts that are 18 | // not in the map are considered as filtered out. 19 | Run(request api.Request) (map[string]float64, error) 20 | // Get the name of this step. 21 | // The name is used to identify the step in metrics, config, logs, and more. 22 | // Should be something like: "my_cool_scheduler_step". 23 | GetName() string 24 | } 25 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/kvm/avoid_overloaded_hosts_cpu.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | "errors" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins/kvm" 10 | "github.com/trickyteache/cortex/internal/scheduler/api" 11 | "github.com/trickyteache/cortex/internal/scheduler/plugins" 12 | ) 13 | 14 | // Options for the scheduling step, given through the step config in the service yaml file. 15 | // Use the options contained in this struct to configure the bounds for min-max scaling. 16 | type AvoidOverloadedHostsCPUStepOpts struct { 17 | AvgCPUUsageLowerBound float64 `json:"avgCPUUsageLowerBound"` // -> mapped to ActivationLowerBound 18 | AvgCPUUsageUpperBound float64 `json:"avgCPUUsageUpperBound"` // -> mapped to ActivationUpperBound 19 | 20 | AvgCPUUsageActivationLowerBound float64 `json:"avgCPUUsageActivationLowerBound"` 21 | AvgCPUUsageActivationUpperBound float64 `json:"avgCPUUsageActivationUpperBound"` 22 | 23 | MaxCPUUsageLowerBound float64 `json:"maxCPUUsageLowerBound"` // -> mapped to ActivationLowerBound 24 | MaxCPUUsageUpperBound float64 `json:"maxCPUUsageUpperBound"` // -> mapped to ActivationUpperBound 25 | 26 | MaxCPUUsageActivationLowerBound float64 `json:"maxCPUUsageActivationLowerBound"` 27 | MaxCPUUsageActivationUpperBound float64 `json:"maxCPUUsageActivationUpperBound"` 28 | } 29 | 30 | func (o AvoidOverloadedHostsCPUStepOpts) Validate() error { 31 | // Avoid zero-division during min-max scaling. 32 | if o.AvgCPUUsageLowerBound == o.AvgCPUUsageUpperBound { 33 | return errors.New("avgCPUUsageLowerBound and avgCPUUsageUpperBound must not be equal") 34 | } 35 | if o.MaxCPUUsageLowerBound == o.MaxCPUUsageUpperBound { 36 | return errors.New("maxCPUUsageLowerBound and maxCPUUsageUpperBound must not be equal") 37 | } 38 | return nil 39 | } 40 | 41 | // Step to avoid high cpu hosts by downvoting them. 42 | type AvoidOverloadedHostsCPUStep struct { 43 | // BaseStep is a helper struct that provides common functionality for all steps. 44 | plugins.BaseStep[AvoidOverloadedHostsCPUStepOpts] 45 | } 46 | 47 | // Get the name of this step, used for identification in config, logs, metrics, etc. 48 | func (s *AvoidOverloadedHostsCPUStep) GetName() string { 49 | return "kvm_avoid_overloaded_hosts_cpu" 50 | } 51 | 52 | // Downvote hosts that have high cpu load. 53 | func (s *AvoidOverloadedHostsCPUStep) Run(request api.Request) (map[string]float64, error) { 54 | activations := s.BaseActivations(request) 55 | if request.GetVMware() { 56 | // Don't run this step for VMware VMs. 57 | return activations, nil 58 | } 59 | 60 | var hostCPUUsages []kvm.NodeExporterHostCPUUsage 61 | if _, err := s.DB.Select(&hostCPUUsages, ` 62 | SELECT * FROM feature_host_cpu_usage 63 | `); err != nil { 64 | return nil, err 65 | } 66 | 67 | for _, host := range hostCPUUsages { 68 | // Only modify the weight if the host is in the scenario. 69 | if _, ok := activations[host.ComputeHost]; !ok { 70 | continue 71 | } 72 | activationAvg := plugins.MinMaxScale( 73 | host.AvgCPUUsage, 74 | s.Options.AvgCPUUsageLowerBound, 75 | s.Options.AvgCPUUsageUpperBound, 76 | s.Options.AvgCPUUsageActivationLowerBound, 77 | s.Options.AvgCPUUsageActivationUpperBound, 78 | ) 79 | activationMax := plugins.MinMaxScale( 80 | host.MaxCPUUsage, 81 | s.Options.MaxCPUUsageLowerBound, 82 | s.Options.MaxCPUUsageUpperBound, 83 | s.Options.MaxCPUUsageActivationLowerBound, 84 | s.Options.MaxCPUUsageActivationUpperBound, 85 | ) 86 | activations[host.ComputeHost] = activationAvg + activationMax 87 | } 88 | return activations, nil 89 | } 90 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/kvm/avoid_overloaded_hosts_cpu_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/kvm" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/trickyteache/cortex/testlib/scheduler/api" 14 | ) 15 | 16 | func TestAvoidOverloadedHostsCPUStep_Run(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | // Create dependency tables 23 | err := testDB.CreateTable(testDB.AddTable(kvm.NodeExporterHostCPUUsage{})) 24 | if err != nil { 25 | t.Fatalf("expected no error, got %v", err) 26 | } 27 | 28 | // Insert mock data into the feature_host_cpu_usage table 29 | _, err = testDB.Exec(` 30 | INSERT INTO feature_host_cpu_usage (compute_host, avg_cpu_usage, max_cpu_usage) 31 | VALUES 32 | ('host1', 15.0, 25.0), 33 | ('host2', 5.0, 10.0), 34 | ('host3', 20.0, 30.0) 35 | `) 36 | if err != nil { 37 | t.Fatalf("expected no error, got %v", err) 38 | } 39 | 40 | // Create an instance of the step 41 | opts := conf.NewRawOpts(`{ 42 | "avgCPUUsageLowerBound": 10, 43 | "avgCPUUsageUpperBound": 100, 44 | "avgCPUUsageActivationLowerBound": 0.0, 45 | "avgCPUUsageActivationUpperBound": -0.5, 46 | "maxCPUUsageLowerBound": 20, 47 | "maxCPUUsageUpperBound": 100, 48 | "maxCPUUsageActivationLowerBound": 0.0, 49 | "maxCPUUsageActivationUpperBound": -0.5 50 | }`) 51 | step := &AvoidOverloadedHostsCPUStep{} 52 | if err := step.Init(testDB, opts); err != nil { 53 | t.Fatalf("expected no error, got %v", err) 54 | } 55 | 56 | tests := []struct { 57 | name string 58 | request api.MockRequest 59 | downvotedHosts map[string]struct{} 60 | }{ 61 | { 62 | name: "Non-vmware vm", 63 | request: api.MockRequest{ 64 | VMware: false, 65 | Hosts: []string{"host1", "host2", "host3"}, 66 | }, 67 | // Should downvote hosts with high CPU usage 68 | downvotedHosts: map[string]struct{}{ 69 | "host1": {}, 70 | "host3": {}, 71 | }, 72 | }, 73 | { 74 | name: "VMware vm", 75 | request: api.MockRequest{ 76 | VMware: true, 77 | Hosts: []string{"host1", "host2", "host3"}, 78 | }, 79 | // Should not do anything for VMware VMs 80 | downvotedHosts: map[string]struct{}{}, 81 | }, 82 | { 83 | name: "No overloaded hosts", 84 | request: api.MockRequest{ 85 | VMware: false, 86 | Hosts: []string{"host4", "host5"}, 87 | }, 88 | // Should not downvote any hosts 89 | downvotedHosts: map[string]struct{}{}, 90 | }, 91 | } 92 | 93 | for _, tt := range tests { 94 | t.Run(tt.name, func(t *testing.T) { 95 | weights, err := step.Run(&tt.request) 96 | if err != nil { 97 | t.Fatalf("expected no error, got %v", err) 98 | } 99 | // Check that the weights have decreased 100 | for host, weight := range weights { 101 | if _, ok := tt.downvotedHosts[host]; ok { 102 | if weight >= 0 { 103 | t.Errorf("expected weight for host %s to be less than 0, got %f", host, weight) 104 | } 105 | } else { 106 | if weight != 0 { 107 | t.Errorf("expected weight for host %s to be 0, got %f", host, weight) 108 | } 109 | } 110 | } 111 | }) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/kvm/avoid_overloaded_hosts_memory_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package kvm 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/trickyteache/cortex/internal/conf" 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/features/plugins/kvm" 12 | testlibDB "github.com/trickyteache/cortex/testlib/db" 13 | "github.com/trickyteache/cortex/testlib/scheduler/api" 14 | ) 15 | 16 | func TestAvoidOverloadedHostsMemoryStep_Run(t *testing.T) { 17 | dbEnv := testlibDB.SetupDBEnv(t) 18 | testDB := db.DB{DbMap: dbEnv.DbMap} 19 | defer testDB.Close() 20 | defer dbEnv.Close() 21 | 22 | // Create dependency tables 23 | err := testDB.CreateTable(testDB.AddTable(kvm.NodeExporterHostMemoryActive{})) 24 | if err != nil { 25 | t.Fatalf("expected no error, got %v", err) 26 | } 27 | 28 | // Insert mock data into the feature_host_memory_active table 29 | _, err = testDB.Exec(` 30 | INSERT INTO feature_host_memory_active 31 | (compute_host, avg_memory_active, max_memory_active) 32 | VALUES 33 | ('host1', 15.0, 25.0), 34 | ('host2', 5.0, 10.0), 35 | ('host3', 20.0, 30.0) 36 | `) 37 | if err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | 41 | // Create an instance of the step 42 | opts := conf.NewRawOpts(`{ 43 | "avgMemoryUsageLowerBound": 10, 44 | "avgMemoryUsageUpperBound": 100, 45 | "avgMemoryUsageActivationLowerBound": 0.0, 46 | "avgMemoryUsageActivationUpperBound": -0.5, 47 | "maxMemoryUsageLowerBound": 20, 48 | "maxMemoryUsageUpperBound": 100, 49 | "maxMemoryUsageActivationLowerBound": 0.0, 50 | "maxMemoryUsageActivationUpperBound": -0.5 51 | }`) 52 | step := &AvoidOverloadedHostsMemoryStep{} 53 | if err := step.Init(testDB, opts); err != nil { 54 | t.Fatalf("expected no error, got %v", err) 55 | } 56 | 57 | tests := []struct { 58 | name string 59 | request api.MockRequest 60 | downvotedHosts map[string]struct{} 61 | }{ 62 | { 63 | name: "Non-vmware vm", 64 | request: api.MockRequest{ 65 | VMware: false, 66 | Hosts: []string{"host1", "host2", "host3"}, 67 | }, 68 | // Should downvote hosts with high CPU usage 69 | downvotedHosts: map[string]struct{}{ 70 | "host1": {}, 71 | "host3": {}, 72 | }, 73 | }, 74 | { 75 | name: "VMware vm", 76 | request: api.MockRequest{ 77 | VMware: true, 78 | Hosts: []string{"host1", "host2", "host3"}, 79 | }, 80 | // Should not do anything for VMware VMs 81 | downvotedHosts: map[string]struct{}{}, 82 | }, 83 | { 84 | name: "No overloaded hosts", 85 | request: api.MockRequest{ 86 | VMware: false, 87 | Hosts: []string{"host4", "host5"}, 88 | }, 89 | // Should not downvote any hosts 90 | downvotedHosts: map[string]struct{}{}, 91 | }, 92 | } 93 | 94 | for _, tt := range tests { 95 | t.Run(tt.name, func(t *testing.T) { 96 | weights, err := step.Run(&tt.request) 97 | if err != nil { 98 | t.Fatalf("expected no error, got %v", err) 99 | } 100 | // Check that the weights have decreased 101 | for host, weight := range weights { 102 | if _, ok := tt.downvotedHosts[host]; ok { 103 | if weight >= 0 { 104 | t.Errorf("expected weight for host %s to be less than 0, got %f", host, weight) 105 | } 106 | } else { 107 | if weight != 0 { 108 | t.Errorf("expected weight for host %s to be 0, got %f", host, weight) 109 | } 110 | } 111 | } 112 | }) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /internal/scheduler/plugins/vmware/anti_affinity_noisy_projects.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package vmware 5 | 6 | import ( 7 | "errors" 8 | 9 | "github.com/trickyteache/cortex/internal/features/plugins/vmware" 10 | "github.com/trickyteache/cortex/internal/scheduler/api" 11 | "github.com/trickyteache/cortex/internal/scheduler/plugins" 12 | ) 13 | 14 | // Options for the scheduling step, given through the step config in the service yaml file. 15 | // Use the options contained in this struct to configure the bounds for min-max scaling. 16 | type AntiAffinityNoisyProjectsStepOpts struct { 17 | AvgCPUUsageLowerBound float64 `json:"avgCPUUsageLowerBound"` // -> mapped to ActivationLowerBound 18 | AvgCPUUsageUpperBound float64 `json:"avgCPUUsageUpperBound"` // -> mapped to ActivationUpperBound 19 | 20 | AvgCPUUsageActivationLowerBound float64 `json:"avgCPUUsageActivationLowerBound"` 21 | AvgCPUUsageActivationUpperBound float64 `json:"avgCPUUsageActivationUpperBound"` 22 | } 23 | 24 | func (o AntiAffinityNoisyProjectsStepOpts) Validate() error { 25 | // Avoid zero-division during min-max scaling. 26 | if o.AvgCPUUsageLowerBound == o.AvgCPUUsageUpperBound { 27 | return errors.New("avgCPUUsageLowerBound and avgCPUUsageUpperBound must not be equal") 28 | } 29 | return nil 30 | } 31 | 32 | // Step to avoid noisy projects by downvoting the hosts they are running on. 33 | type AntiAffinityNoisyProjectsStep struct { 34 | // BaseStep is a helper struct that provides common functionality for all steps. 35 | plugins.BaseStep[AntiAffinityNoisyProjectsStepOpts] 36 | } 37 | 38 | // Get the name of this step, used for identification in config, logs, metrics, etc. 39 | func (s *AntiAffinityNoisyProjectsStep) GetName() string { 40 | return "vmware_anti_affinity_noisy_projects" 41 | } 42 | 43 | // Downvote the hosts a project is currently running on if it's noisy. 44 | func (s *AntiAffinityNoisyProjectsStep) Run(request api.Request) (map[string]float64, error) { 45 | activations := s.BaseActivations(request) 46 | if !request.GetVMware() { 47 | // Only run this step for VMware VMs. 48 | return activations, nil 49 | } 50 | 51 | // Check how noisy the project is on the compute hosts. 52 | var projectNoisinessOnHosts []vmware.VROpsProjectNoisiness 53 | if _, err := s.DB.Select(&projectNoisinessOnHosts, ` 54 | SELECT * FROM feature_vrops_project_noisiness 55 | WHERE project = :project_id 56 | `, map[string]any{ 57 | "project_id": request.GetSpec().Data.ProjectID, 58 | }); err != nil { 59 | return nil, err 60 | } 61 | 62 | for _, p := range projectNoisinessOnHosts { 63 | // Only modify the weight if the host is in the scenario. 64 | if _, ok := activations[p.ComputeHost]; !ok { 65 | continue 66 | } 67 | activations[p.ComputeHost] = plugins.MinMaxScale( 68 | p.AvgCPUOfProject, 69 | s.Options.AvgCPUUsageLowerBound, 70 | s.Options.AvgCPUUsageUpperBound, 71 | s.Options.AvgCPUUsageActivationLowerBound, 72 | s.Options.AvgCPUUsageActivationUpperBound, 73 | ) 74 | } 75 | return activations, nil 76 | } 77 | -------------------------------------------------------------------------------- /internal/scheduler/validation.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package scheduler 5 | 6 | import ( 7 | "errors" 8 | "log/slog" 9 | 10 | "github.com/trickyteache/cortex/internal/conf" 11 | "github.com/trickyteache/cortex/internal/db" 12 | "github.com/trickyteache/cortex/internal/scheduler/api" 13 | "github.com/trickyteache/cortex/internal/scheduler/plugins" 14 | ) 15 | 16 | // The config type has a long name, so we use a shorter alias here. 17 | // The name is intentionally long to make it explicit that we disable 18 | // validations for the scheduler step instead of enabling them. 19 | type disabledValidations = conf.SchedulerStepDisabledValidationsConfig 20 | 21 | // Wrapper for scheduler steps that validates them before/after execution. 22 | type StepValidator struct { 23 | // The wrapped step to validate. 24 | Step plugins.Step 25 | // By default, we execute all validations. However, through the config, 26 | // we can also disable some validations if necessary. 27 | DisabledValidations disabledValidations 28 | } 29 | 30 | // Get the name of the wrapped step. 31 | func (s *StepValidator) GetName() string { 32 | return s.Step.GetName() 33 | } 34 | 35 | // Initialize the wrapped step with the database and options. 36 | func (s *StepValidator) Init(db db.DB, opts conf.RawOpts) error { 37 | slog.Info( 38 | "scheduler: init validation for step", "name", s.GetName(), 39 | "disabled", s.DisabledValidations, 40 | ) 41 | return s.Step.Init(db, opts) 42 | } 43 | 44 | // Validate the wrapped step with the database and options. 45 | func validateStep[S plugins.Step](step S, disabledValidations disabledValidations) *StepValidator { 46 | return &StepValidator{ 47 | Step: step, 48 | DisabledValidations: disabledValidations, 49 | } 50 | } 51 | 52 | // Run the step and validate what happens. 53 | func (s *StepValidator) Run(request api.Request) (map[string]float64, error) { 54 | weights, err := s.Step.Run(request) 55 | if err != nil { 56 | return nil, err 57 | } 58 | // If not disabled, validate that the number of hosts stayed the same. 59 | if !s.DisabledValidations.SameHostNumberInOut { 60 | if len(weights) != len(request.GetHosts()) { 61 | return nil, errors.New("number of hosts changed during step execution") 62 | } 63 | } 64 | return weights, nil 65 | } 66 | -------------------------------------------------------------------------------- /internal/sync/datasource.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package sync 5 | 6 | import "context" 7 | 8 | // Common interface for data sources. 9 | type Datasource interface { 10 | // Initialize the data source, e.g. create database tables. 11 | Init(context.Context) 12 | // Download data from the data source. 13 | Sync(context.Context) 14 | } 15 | -------------------------------------------------------------------------------- /internal/sync/monitor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package sync 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/monitoring" 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | // Monitor is a collection of Prometheus metrics for the sync package. 12 | type Monitor struct { 13 | // A histogram to measure how long each sync run takes. 14 | PipelineRunTimer *prometheus.HistogramVec 15 | // A gauge to observe the number of objects synced. 16 | PipelineObjectsGauge *prometheus.GaugeVec 17 | // A histogram to measure how long each sync request takes. 18 | PipelineRequestTimer *prometheus.HistogramVec 19 | // A counter to observe the number of processed sync requests. 20 | PipelineRequestProcessedCounter *prometheus.CounterVec 21 | } 22 | 23 | // NewSyncMonitor creates a new sync monitor and registers the necessary Prometheus metrics. 24 | func NewSyncMonitor(registry *monitoring.Registry) Monitor { 25 | pipelineRunTimer := prometheus.NewHistogramVec(prometheus.HistogramOpts{ 26 | Name: "cortex_sync_run_duration_seconds", 27 | Help: "Duration of sync run", 28 | Buckets: prometheus.DefBuckets, 29 | }, []string{"datasource"}) 30 | pipelineObjectsGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ 31 | Name: "cortex_sync_objects", 32 | Help: "Number of objects synced", 33 | }, []string{"datasource"}) 34 | pipelineRequestTimer := prometheus.NewHistogramVec(prometheus.HistogramOpts{ 35 | Name: "cortex_sync_request_duration_seconds", 36 | Help: "Duration of sync request", 37 | Buckets: prometheus.DefBuckets, 38 | }, []string{"datasource"}) 39 | pipelineRequestProcessedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ 40 | Name: "cortex_sync_request_processed_total", 41 | Help: "Number of processed sync requests", 42 | }, []string{"datasource"}) 43 | registry.MustRegister( 44 | pipelineRunTimer, 45 | pipelineObjectsGauge, 46 | pipelineRequestTimer, 47 | pipelineRequestProcessedCounter, 48 | ) 49 | return Monitor{ 50 | PipelineRunTimer: pipelineRunTimer, 51 | PipelineObjectsGauge: pipelineObjectsGauge, 52 | PipelineRequestTimer: pipelineRequestTimer, 53 | PipelineRequestProcessedCounter: pipelineRequestProcessedCounter, 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /internal/sync/openstack/keystone.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "context" 8 | "log/slog" 9 | 10 | "github.com/trickyteache/cortex/internal/conf" 11 | "github.com/trickyteache/cortex/internal/sync" 12 | "github.com/gophercloud/gophercloud/v2" 13 | "github.com/gophercloud/gophercloud/v2/openstack" 14 | ) 15 | 16 | // Type alias for the OpenStack keystone configuration. 17 | type KeystoneConf = conf.SyncOpenStackKeystoneConfig 18 | 19 | // KeystoneAPI for OpenStack. 20 | type KeystoneAPI interface { 21 | // Authenticate against the OpenStack keystone. 22 | Authenticate(context.Context) error 23 | // Get the OpenStack provider client. 24 | Client() *gophercloud.ProviderClient 25 | // Find the endpoint for the given service type and availability. 26 | FindEndpoint(availability, serviceType string) (string, error) 27 | } 28 | 29 | // KeystoneAPI implementation. 30 | type keystoneAPI struct { 31 | // OpenStack provider client. 32 | client *gophercloud.ProviderClient 33 | // OpenStack keystone configuration. 34 | keystoneConf KeystoneConf 35 | } 36 | 37 | // Create a new OpenStack keystone API. 38 | func newKeystoneAPI(keystoneConf KeystoneConf) KeystoneAPI { 39 | return &keystoneAPI{keystoneConf: keystoneConf} 40 | } 41 | 42 | // Authenticate against OpenStack keystone. 43 | func (api *keystoneAPI) Authenticate(ctx context.Context) error { 44 | if api.client != nil { 45 | // Already authenticated. 46 | return nil 47 | } 48 | slog.Info("authenticating against openstack", "url", api.keystoneConf.URL) 49 | authOptions := gophercloud.AuthOptions{ 50 | IdentityEndpoint: api.keystoneConf.URL, 51 | Username: api.keystoneConf.OSUsername, 52 | DomainName: api.keystoneConf.OSUserDomainName, 53 | Password: api.keystoneConf.OSPassword, 54 | AllowReauth: true, 55 | Scope: &gophercloud.AuthScope{ 56 | ProjectName: api.keystoneConf.OSProjectName, 57 | DomainName: api.keystoneConf.OSProjectDomainName, 58 | }, 59 | } 60 | httpClient, err := sync.NewHTTPClient(api.keystoneConf.SSO) 61 | if err != nil { 62 | panic(err) 63 | } 64 | provider, err := openstack.NewClient(authOptions.IdentityEndpoint) 65 | if err != nil { 66 | panic(err) 67 | } 68 | provider.HTTPClient = *httpClient 69 | if err = openstack.Authenticate(ctx, provider, authOptions); err != nil { 70 | panic(err) 71 | } 72 | api.client = provider 73 | slog.Info("authenticated against openstack") 74 | return nil 75 | } 76 | 77 | // Find the endpoint for the given service type and availability. 78 | func (api *keystoneAPI) FindEndpoint(availability, serviceType string) (string, error) { 79 | return api.client.EndpointLocator(gophercloud.EndpointOpts{ 80 | Type: serviceType, 81 | Availability: gophercloud.Availability(availability), 82 | }) 83 | } 84 | 85 | // Get the OpenStack provider client. 86 | func (api *keystoneAPI) Client() *gophercloud.ProviderClient { 87 | return api.client 88 | } 89 | -------------------------------------------------------------------------------- /internal/sync/openstack/keystone_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "context" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | 12 | "github.com/gophercloud/gophercloud/v2" 13 | ) 14 | 15 | type mockKeystoneAPI struct { 16 | url string 17 | } 18 | 19 | func (m *mockKeystoneAPI) Authenticate(ctx context.Context) error { 20 | return nil 21 | } 22 | 23 | func (m *mockKeystoneAPI) Client() *gophercloud.ProviderClient { 24 | return &gophercloud.ProviderClient{} 25 | } 26 | 27 | func (m *mockKeystoneAPI) FindEndpoint(availability, serviceType string) (string, error) { 28 | return m.url, nil 29 | } 30 | 31 | //nolint:gocritic 32 | func setupKeystoneMockServer(handler http.HandlerFunc) (*httptest.Server, KeystoneConf) { 33 | server := httptest.NewServer(handler) 34 | conf := KeystoneConf{ 35 | URL: server.URL + "/v3", 36 | OSUsername: "testuser", 37 | OSUserDomainName: "default", 38 | OSPassword: "password", 39 | OSProjectName: "testproject", 40 | OSProjectDomainName: "default", 41 | } 42 | return server, conf 43 | } 44 | 45 | func TestNewKeystoneAPI(t *testing.T) { 46 | keystoneConf := KeystoneConf{ 47 | URL: "http://example.com", 48 | OSUsername: "testuser", 49 | OSUserDomainName: "default", 50 | OSPassword: "password", 51 | OSProjectName: "testproject", 52 | OSProjectDomainName: "default", 53 | } 54 | 55 | api := newKeystoneAPI(keystoneConf) 56 | if api == nil { 57 | t.Fatal("expected non-nil api") 58 | } 59 | } 60 | 61 | func TestKeystoneAPI_Authenticate(t *testing.T) { 62 | handler := func(w http.ResponseWriter, r *http.Request) { 63 | w.Header().Add("Content-Type", "application/json") 64 | w.WriteHeader(http.StatusAccepted) 65 | if _, err := w.Write([]byte(`{"token": {"catalog": []}}`)); err != nil { 66 | t.Fatalf("error writing response: %v", err) 67 | } 68 | } 69 | server, keystoneConf := setupKeystoneMockServer(handler) 70 | defer server.Close() 71 | 72 | api := newKeystoneAPI(keystoneConf).(*keystoneAPI) 73 | 74 | err := api.Authenticate(t.Context()) 75 | if err != nil { 76 | t.Fatalf("expected no error, got %v", err) 77 | } 78 | if api.client == nil { 79 | t.Fatal("expected non-nil client after authentication") 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /internal/sync/openstack/placement_api_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | 11 | "github.com/trickyteache/cortex/internal/sync" 12 | ) 13 | 14 | func setupPlacementMockServer(handler http.HandlerFunc) (*httptest.Server, KeystoneAPI) { 15 | server := httptest.NewServer(handler) 16 | return server, &mockKeystoneAPI{url: server.URL + "/"} 17 | } 18 | 19 | func TestNewPlacementAPI(t *testing.T) { 20 | mon := sync.Monitor{} 21 | k := &mockKeystoneAPI{} 22 | conf := PlacementConf{} 23 | 24 | api := newPlacementAPI(mon, k, conf) 25 | if api == nil { 26 | t.Fatal("expected non-nil api") 27 | } 28 | } 29 | 30 | func TestPlacementAPI_GetAllResourceProviders(t *testing.T) { 31 | handler := func(w http.ResponseWriter, r *http.Request) { 32 | w.Header().Add("Content-Type", "application/json") 33 | w.WriteHeader(http.StatusOK) 34 | if _, err := w.Write([]byte(`{"resource_providers": [{"uuid": "1", "name": "rp1", "parent_provider_uuid": "pp1", "root_provider_uuid": "rootp1", "resource_provider_generation": 1}]}`)); err != nil { 35 | t.Fatalf("failed to write response: %v", err) 36 | } 37 | } 38 | server, k := setupPlacementMockServer(handler) 39 | defer server.Close() 40 | 41 | mon := sync.Monitor{} 42 | conf := PlacementConf{} 43 | 44 | api := newPlacementAPI(mon, k, conf).(*placementAPI) 45 | api.Init(t.Context()) 46 | 47 | ctx := t.Context() 48 | rps, err := api.GetAllResourceProviders(ctx) 49 | if err != nil { 50 | t.Fatalf("expected no error, got %v", err) 51 | } 52 | if len(rps) != 1 { 53 | t.Fatalf("expected 1 resource provider, got %d", len(rps)) 54 | } 55 | } 56 | 57 | func TestPlacementAPI_GetAllTraits(t *testing.T) { 58 | handler := func(w http.ResponseWriter, r *http.Request) { 59 | if r.URL.Path == "/resource_providers/1/traits" { 60 | w.WriteHeader(http.StatusOK) 61 | if _, err := w.Write([]byte(`{"traits": ["trait1"]}`)); err != nil { 62 | t.Fatalf("failed to write response: %v", err) 63 | } 64 | } else { 65 | w.WriteHeader(http.StatusNotFound) 66 | } 67 | } 68 | server, pc := setupPlacementMockServer(handler) 69 | defer server.Close() 70 | 71 | mon := sync.Monitor{} 72 | conf := PlacementConf{} 73 | 74 | api := newPlacementAPI(mon, pc, conf).(*placementAPI) 75 | api.Init(t.Context()) 76 | 77 | ctx := t.Context() 78 | providers := []ResourceProvider{{UUID: "1", Name: "rp1"}} 79 | traits, err := api.GetAllTraits(ctx, providers) 80 | if err != nil { 81 | t.Fatalf("expected no error, got %v", err) 82 | } 83 | if len(traits) != 1 { 84 | t.Fatalf("expected 1 trait, got %d", len(traits)) 85 | } 86 | } 87 | 88 | func TestPlacementAPI_GetAllTraits_Error(t *testing.T) { 89 | handler := func(w http.ResponseWriter, r *http.Request) { 90 | if r.URL.Path == "/resource_providers/error/traits" { 91 | w.WriteHeader(http.StatusInternalServerError) 92 | if _, err := w.Write([]byte(`{"error": "error fetching traits"}`)); err != nil { 93 | t.Fatalf("failed to write response: %v", err) 94 | } 95 | } else { 96 | w.WriteHeader(http.StatusNotFound) 97 | } 98 | } 99 | server, pc := setupPlacementMockServer(handler) 100 | defer server.Close() 101 | 102 | mon := sync.Monitor{} 103 | conf := PlacementConf{} 104 | 105 | api := newPlacementAPI(mon, pc, conf).(*placementAPI) 106 | api.Init(t.Context()) 107 | 108 | ctx := t.Context() 109 | providers := []ResourceProvider{{UUID: "error", Name: "rp1"}} 110 | _, err := api.GetAllTraits(ctx, providers) 111 | if err == nil { 112 | t.Fatal("expected error, got nil") 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /internal/sync/openstack/placement_sync.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "context" 8 | "slices" 9 | 10 | "github.com/trickyteache/cortex/internal/db" 11 | "github.com/trickyteache/cortex/internal/mqtt" 12 | "github.com/trickyteache/cortex/internal/sync" 13 | "github.com/go-gorp/gorp" 14 | ) 15 | 16 | // Syncer for OpenStack placement. 17 | type placementSyncer struct { 18 | // Database to store the placement objects in. 19 | db db.DB 20 | // Monitor to track the syncer. 21 | mon sync.Monitor 22 | // Configuration for the placement syncer. 23 | conf PlacementConf 24 | // Placement API client to fetch the data. 25 | api PlacementAPI 26 | // MQTT client to publish mqtt data. 27 | mqttClient mqtt.Client 28 | } 29 | 30 | // Init the OpenStack resource provider and trait syncer. 31 | func (s *placementSyncer) Init(ctx context.Context) { 32 | s.api.Init(ctx) 33 | var tables = []*gorp.TableMap{} 34 | // Only add the tables that are configured in the yaml conf. 35 | if slices.Contains(s.conf.Types, "resource_providers") { 36 | tables = append(tables, s.db.AddTable(ResourceProvider{})) 37 | // Depends on the resource providers. (Checked during conf validation) 38 | if slices.Contains(s.conf.Types, "traits") { 39 | tables = append(tables, s.db.AddTable(Trait{})) 40 | } 41 | } 42 | if err := s.db.CreateTable(tables...); err != nil { 43 | panic(err) 44 | } 45 | } 46 | 47 | // Sync the OpenStack placement objects. 48 | func (s *placementSyncer) Sync(ctx context.Context) error { 49 | // Only sync the objects that are configured in the yaml conf. 50 | if slices.Contains(s.conf.Types, "resource_providers") { 51 | rps, err := s.SyncResourceProviders(ctx) 52 | if err != nil { 53 | return err 54 | } 55 | go s.mqttClient.Publish(TriggerPlacementResourceProvidersSynced, "") 56 | // Dependencies of the resource providers. 57 | if slices.Contains(s.conf.Types, "traits") { 58 | if _, err := s.SyncTraits(ctx, rps); err != nil { 59 | return err 60 | } 61 | go s.mqttClient.Publish(TriggerPlacementTraitsSynced, "") 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | // Sync the OpenStack resource providers into the database. 68 | func (s *placementSyncer) SyncResourceProviders(ctx context.Context) ([]ResourceProvider, error) { 69 | label := ResourceProvider{}.TableName() 70 | rps, err := s.api.GetAllResourceProviders(ctx) 71 | if err != nil { 72 | return nil, err 73 | } 74 | if err := db.ReplaceAll(s.db, rps...); err != nil { 75 | return nil, err 76 | } 77 | if s.mon.PipelineObjectsGauge != nil { 78 | gauge := s.mon.PipelineObjectsGauge.WithLabelValues(label) 79 | gauge.Set(float64(len(rps))) 80 | } 81 | if s.mon.PipelineRequestProcessedCounter != nil { 82 | counter := s.mon.PipelineRequestProcessedCounter.WithLabelValues(label) 83 | counter.Inc() 84 | } 85 | return rps, nil 86 | } 87 | 88 | // Sync the OpenStack traits into the database. 89 | func (s *placementSyncer) SyncTraits(ctx context.Context, rps []ResourceProvider) ([]Trait, error) { 90 | label := Trait{}.TableName() 91 | traits, err := s.api.GetAllTraits(ctx, rps) 92 | if err != nil { 93 | return nil, err 94 | } 95 | if err := db.ReplaceAll(s.db, traits...); err != nil { 96 | return nil, err 97 | } 98 | if s.mon.PipelineObjectsGauge != nil { 99 | gauge := s.mon.PipelineObjectsGauge.WithLabelValues(label) 100 | gauge.Set(float64(len(traits))) 101 | } 102 | if s.mon.PipelineRequestProcessedCounter != nil { 103 | counter := s.mon.PipelineRequestProcessedCounter.WithLabelValues(label) 104 | counter.Inc() 105 | } 106 | return traits, err 107 | } 108 | -------------------------------------------------------------------------------- /internal/sync/openstack/placement_types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import "github.com/trickyteache/cortex/internal/conf" 7 | 8 | // Type alias for the OpenStack placement configuration. 9 | type PlacementConf = conf.SyncOpenStackPlacementConfig 10 | 11 | // Resource provider model from the OpenStack placement API. 12 | // This model is returned when listing resource providers. 13 | type ResourceProvider struct { 14 | UUID string `json:"uuid" db:"uuid,primarykey"` 15 | Name string `json:"name" db:"name"` 16 | ParentProviderUUID string `json:"parent_provider_uuid" db:"parent_provider_uuid"` 17 | RootProviderUUID string `json:"root_provider_uuid" db:"root_provider_uuid"` 18 | ResourceProviderGeneration int `json:"resource_provider_generation" db:"resource_provider_generation"` 19 | } 20 | 21 | // Table in which the openstack model is stored. 22 | func (r ResourceProvider) TableName() string { return "openstack_resource_providers" } 23 | 24 | // Resource provider trait model from the OpenStack placement API. 25 | type Trait struct { 26 | ResourceProviderUUID string `db:"resource_provider_uuid,primarykey"` 27 | Name string `db:"name,primarykey"` 28 | ResourceProviderGeneration int `db:"resource_provider_generation"` 29 | } 30 | 31 | // Table in which the openstack trait model is stored. 32 | func (r Trait) TableName() string { return "openstack_resource_provider_traits" } 33 | -------------------------------------------------------------------------------- /internal/sync/openstack/placement_types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | func TestResourceProvider_TableName(t *testing.T) { 11 | rp := ResourceProvider{} 12 | expected := "openstack_resource_providers" 13 | if rp.TableName() != expected { 14 | t.Errorf("expected %s, got %s", expected, rp.TableName()) 15 | } 16 | } 17 | 18 | func TestTrait_TableName(t *testing.T) { 19 | trait := Trait{} 20 | expected := "openstack_resource_provider_traits" 21 | if trait.TableName() != expected { 22 | t.Errorf("expected %s, got %s", expected, trait.TableName()) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /internal/sync/openstack/sync.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "context" 8 | "log/slog" 9 | gosync "sync" 10 | 11 | "github.com/trickyteache/cortex/internal/conf" 12 | "github.com/trickyteache/cortex/internal/db" 13 | "github.com/trickyteache/cortex/internal/mqtt" 14 | "github.com/trickyteache/cortex/internal/sync" 15 | "github.com/prometheus/client_golang/prometheus" 16 | ) 17 | 18 | type Syncer interface { 19 | Init(context.Context) 20 | Sync(context.Context) error 21 | } 22 | 23 | // Combined syncer that combines multiple syncers. 24 | type CombinedSyncer struct { 25 | monitor sync.Monitor 26 | // List of syncers to run in parallel. 27 | syncers []Syncer 28 | } 29 | 30 | // Create a new combined syncer that runs multiple syncers in parallel. 31 | func NewCombinedSyncer( 32 | ctx context.Context, 33 | config conf.SyncOpenStackConfig, 34 | monitor sync.Monitor, 35 | db db.DB, 36 | mqttClient mqtt.Client, 37 | ) sync.Datasource { 38 | 39 | keystoneAPI := newKeystoneAPI(config.Keystone) 40 | slog.Info("loading openstack sub-syncers") 41 | syncers := []Syncer{ 42 | &novaSyncer{ 43 | db: db, 44 | mon: monitor, 45 | conf: config.Nova, 46 | api: newNovaAPI(monitor, keystoneAPI, config.Nova), 47 | mqttClient: mqttClient, 48 | }, 49 | &placementSyncer{ 50 | db: db, 51 | mon: monitor, 52 | conf: config.Placement, 53 | api: newPlacementAPI(monitor, keystoneAPI, config.Placement), 54 | mqttClient: mqttClient, 55 | }, 56 | } 57 | return CombinedSyncer{monitor: monitor, syncers: syncers} 58 | } 59 | 60 | // Create all needed database tables if they do not exist. 61 | func (s CombinedSyncer) Init(ctx context.Context) { 62 | for _, syncer := range s.syncers { 63 | syncer.Init(ctx) 64 | } 65 | } 66 | 67 | // Sync all objects from OpenStack to the database. 68 | func (s CombinedSyncer) Sync(context context.Context) { 69 | if s.monitor.PipelineRunTimer != nil { 70 | hist := s.monitor.PipelineRunTimer.WithLabelValues("openstack") 71 | timer := prometheus.NewTimer(hist) 72 | defer timer.ObserveDuration() 73 | } 74 | 75 | // Sync all objects in parallel. 76 | var wg gosync.WaitGroup 77 | for _, syncer := range s.syncers { 78 | wg.Add(1) 79 | go func(syncer Syncer) { 80 | defer wg.Done() 81 | if err := syncer.Sync(context); err != nil { 82 | slog.Error("failed to sync objects", "error", err) 83 | } 84 | }(syncer) 85 | } 86 | wg.Wait() 87 | } 88 | -------------------------------------------------------------------------------- /internal/sync/openstack/sync_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | 10 | "github.com/trickyteache/cortex/internal/conf" 11 | "github.com/trickyteache/cortex/internal/db" 12 | "github.com/trickyteache/cortex/internal/sync" 13 | testlibDB "github.com/trickyteache/cortex/testlib/db" 14 | "github.com/trickyteache/cortex/testlib/mqtt" 15 | ) 16 | 17 | type mockSyncer struct { 18 | initCalled bool 19 | syncCalled bool 20 | } 21 | 22 | func (m *mockSyncer) Init(ctx context.Context) { 23 | m.initCalled = true 24 | } 25 | 26 | func (m *mockSyncer) Sync(ctx context.Context) error { 27 | m.syncCalled = true 28 | return nil 29 | } 30 | 31 | func TestCombinedSyncer_Init(t *testing.T) { 32 | dbEnv := testlibDB.SetupDBEnv(t) 33 | testDB := db.DB{DbMap: dbEnv.DbMap} 34 | defer testDB.Close() 35 | defer dbEnv.Close() 36 | 37 | monitor := sync.Monitor{} 38 | 39 | mockSyncer1 := &mockSyncer{} 40 | mockSyncer2 := &mockSyncer{} 41 | syncers := []Syncer{mockSyncer1, mockSyncer2} 42 | 43 | combinedSyncer := CombinedSyncer{monitor: monitor, syncers: syncers} 44 | combinedSyncer.Init(t.Context()) 45 | 46 | if !mockSyncer1.initCalled { 47 | t.Fatal("expected mockSyncer1.Init to be called") 48 | } 49 | if !mockSyncer2.initCalled { 50 | t.Fatal("expected mockSyncer2.Init to be called") 51 | } 52 | } 53 | 54 | func TestCombinedSyncer_Sync(t *testing.T) { 55 | dbEnv := testlibDB.SetupDBEnv(t) 56 | testDB := db.DB{DbMap: dbEnv.DbMap} 57 | defer testDB.Close() 58 | defer dbEnv.Close() 59 | 60 | monitor := sync.Monitor{} 61 | 62 | mockSyncer1 := &mockSyncer{} 63 | mockSyncer2 := &mockSyncer{} 64 | syncers := []Syncer{mockSyncer1, mockSyncer2} 65 | 66 | combinedSyncer := CombinedSyncer{monitor: monitor, syncers: syncers} 67 | combinedSyncer.Sync(t.Context()) 68 | 69 | if !mockSyncer1.syncCalled { 70 | t.Fatal("expected mockSyncer1.Sync to be called") 71 | } 72 | if !mockSyncer2.syncCalled { 73 | t.Fatal("expected mockSyncer2.Sync to be called") 74 | } 75 | } 76 | 77 | func TestNewCombinedSyncer(t *testing.T) { 78 | dbEnv := testlibDB.SetupDBEnv(t) 79 | testDB := db.DB{DbMap: dbEnv.DbMap} 80 | defer testDB.Close() 81 | defer dbEnv.Close() 82 | 83 | monitor := sync.Monitor{} 84 | mqttClient := &mqtt.MockClient{} // Mock or initialize as needed 85 | config := conf.SyncOpenStackConfig{} // Populate with test configuration 86 | 87 | combinedSyncer := NewCombinedSyncer(t.Context(), config, monitor, testDB, mqttClient) 88 | 89 | if combinedSyncer == nil { 90 | t.Fatal("expected NewCombinedSyncer to return a non-nil CombinedSyncer") 91 | } 92 | 93 | // Additional assertions can be added here to verify the state of the returned CombinedSyncer 94 | } 95 | -------------------------------------------------------------------------------- /internal/sync/openstack/triggers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package openstack 5 | 6 | // Trigger executed when new servers are available. 7 | const TriggerNovaServersSynced = "triggers/sync/openstack/nova/types/servers" 8 | 9 | // Trigger executed when new hypervisors are available. 10 | const TriggerNovaHypervisorsSynced = "triggers/sync/openstack/nova/types/hypervisors" 11 | 12 | // Trigger executed when new flavors are available. 13 | const TriggerNovaFlavorsSynced = "triggers/sync/openstack/nova/types/flavors" 14 | 15 | // Trigger executed when new migrations are available. 16 | const TriggerNovaMigrationsSynced = "triggers/sync/openstack/nova/types/migrations" 17 | 18 | // Trigger executed when new resource providers are available. 19 | const TriggerPlacementResourceProvidersSynced = "triggers/sync/openstack/placement/types/resource_providers" 20 | 21 | // Trigger executed when new traits are available. 22 | const TriggerPlacementTraitsSynced = "triggers/sync/openstack/placement/types/traits" 23 | -------------------------------------------------------------------------------- /internal/sync/pipeline.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package sync 5 | 6 | import ( 7 | "context" 8 | "log/slog" 9 | "sync" 10 | "time" 11 | 12 | "github.com/sapcc/go-bits/jobloop" 13 | ) 14 | 15 | // Pipeline wrapper for all datasources. 16 | type Pipeline struct { 17 | Syncers []Datasource 18 | } 19 | 20 | // Initialize all datasources. 21 | func (p *Pipeline) Init(ctx context.Context) { 22 | for _, syncer := range p.Syncers { 23 | syncer.Init(ctx) 24 | } 25 | } 26 | 27 | // Sync all datasources in parallel. 28 | func (p *Pipeline) SyncPeriodic(ctx context.Context) { 29 | for { 30 | select { 31 | case <-ctx.Done(): 32 | slog.Info("syncer shutting down") 33 | return 34 | default: 35 | var wg sync.WaitGroup 36 | for _, syncer := range p.Syncers { 37 | wg.Add(1) 38 | go func(syncer Datasource) { 39 | defer wg.Done() 40 | syncer.Sync(ctx) 41 | }(syncer) 42 | } 43 | wg.Wait() 44 | time.Sleep(jobloop.DefaultJitter(time.Minute)) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/sync/pipeline_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package sync 5 | 6 | import ( 7 | "context" 8 | "sync" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | // MockDatasource implements the Datasource interface for testing purposes. 14 | type MockDatasource struct { 15 | InitCalled bool 16 | SyncCalled bool 17 | mu sync.Mutex 18 | } 19 | 20 | func (m *MockDatasource) Init(ctx context.Context) { 21 | m.mu.Lock() 22 | defer m.mu.Unlock() 23 | m.InitCalled = true 24 | } 25 | 26 | func (m *MockDatasource) Sync(ctx context.Context) { 27 | m.mu.Lock() 28 | defer m.mu.Unlock() 29 | m.SyncCalled = true 30 | } 31 | 32 | func TestPipeline(t *testing.T) { 33 | // Create mock datasources 34 | ds1 := &MockDatasource{} 35 | ds2 := &MockDatasource{} 36 | 37 | // Create a pipeline with the mock datasources 38 | pipeline := &Pipeline{ 39 | Syncers: []Datasource{ds1, ds2}, 40 | } 41 | 42 | // Test Init method 43 | ctx := t.Context() 44 | pipeline.Init(ctx) 45 | 46 | if !ds1.InitCalled || !ds2.InitCalled { 47 | t.Errorf("Init was not called on all datasources") 48 | } 49 | 50 | // Test SyncPeriodic method 51 | ctx, cancel := context.WithCancel(t.Context()) 52 | defer cancel() 53 | 54 | // Run SyncPeriodic in a separate goroutine 55 | go func() { 56 | pipeline.SyncPeriodic(ctx) 57 | }() 58 | 59 | // Allow some time for SyncPeriodic to run 60 | time.Sleep(100 * time.Millisecond) 61 | cancel() // Stop the SyncPeriodic loop 62 | 63 | // Check if Sync was called on all datasources 64 | if !ds1.SyncCalled || !ds2.SyncCalled { 65 | t.Errorf("Sync was not called on all datasources") 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/sync/prometheus/prometheus_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package prometheus 5 | 6 | import ( 7 | "encoding/json" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | "time" 12 | 13 | "github.com/trickyteache/cortex/internal/conf" 14 | ) 15 | 16 | func TestFetchMetrics(t *testing.T) { 17 | // Mock the Prometheus API response 18 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | if r.URL.Path == "/api/v1/query_range" && r.Method == http.MethodGet { 20 | w.WriteHeader(http.StatusOK) 21 | //nolint:errcheck 22 | json.NewEncoder(w).Encode(map[string]interface{}{ 23 | "status": "success", 24 | "data": map[string]interface{}{ 25 | "resultType": "matrix", 26 | "result": []map[string]interface{}{ 27 | { 28 | "metric": map[string]interface{}{ 29 | "cluster": "test_cluster", 30 | "cluster_type": "test_cluster_type", 31 | "collector": "test_collector", 32 | "datacenter": "test_datacenter", 33 | "hostsystem": "test_hostsystem", 34 | "instance_uuid": "test_instance_uuid", 35 | "internal_name": "test_internal_name", 36 | "job": "test_job", 37 | "project": "test_project", 38 | "prometheus": "test_prometheus", 39 | "region": "test_region", 40 | "vccluster": "test_vccluster", 41 | "vcenter": "test_vcenter", 42 | "virtualmachine": "test_virtualmachine", 43 | }, 44 | "values": [][]interface{}{ 45 | {float64(time.Now().Unix()), "123.45"}, 46 | }, 47 | }, 48 | }, 49 | }, 50 | }) 51 | } else { 52 | w.WriteHeader(http.StatusNotFound) 53 | } 54 | }) 55 | 56 | server := httptest.NewServer(handler) 57 | defer server.Close() 58 | 59 | start := time.Now().Add(-time.Hour) 60 | end := time.Now() 61 | resolutionSeconds := 60 62 | 63 | api := &prometheusAPI[VROpsVMMetric]{ 64 | hostConf: conf.SyncPrometheusHostConfig{ 65 | URL: server.URL, 66 | }, 67 | metricConf: conf.SyncPrometheusMetricConfig{ 68 | Alias: "test_metric", 69 | }, 70 | } 71 | data, err := api.FetchMetrics("test_query", start, end, resolutionSeconds) 72 | if err != nil { 73 | t.Fatalf("expected no error, got %v", err) 74 | } 75 | 76 | // Verify the results 77 | if len(data.Metrics) != 1 { 78 | t.Errorf("expected 1 metric, got %d", len(data.Metrics)) 79 | } 80 | metric := data.Metrics[0] 81 | if metric.Name != "test_metric" { 82 | t.Errorf("expected metric name to be %s, got %s", "test_metric", metric.Name) 83 | } 84 | if metric.Value != 123.45 { 85 | t.Errorf("expected value to be %f, got %f", 123.45, metric.Value) 86 | } 87 | } 88 | 89 | func TestFetchMetricsFailure(t *testing.T) { 90 | // Mock the Prometheus API response 91 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 92 | w.WriteHeader(http.StatusInternalServerError) 93 | }) 94 | 95 | server := httptest.NewServer(handler) 96 | defer server.Close() 97 | 98 | start := time.Now().Add(-time.Hour) 99 | end := time.Now() 100 | resolutionSeconds := 60 101 | 102 | api := &prometheusAPI[*VROpsVMMetric]{ 103 | hostConf: conf.SyncPrometheusHostConfig{ 104 | URL: server.URL, 105 | }, 106 | } 107 | _, err := api.FetchMetrics("test_query", start, end, resolutionSeconds) 108 | if err == nil { 109 | t.Fatalf("expected error, got none") 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /internal/sync/prometheus/triggers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package prometheus 5 | 6 | // Trigger executed when the prometheus metric with this alias has finished syncing. 7 | func TriggerMetricAliasSynced(metricAlias string) string { 8 | return "triggers/sync/prometheus/alias/" + metricAlias 9 | } 10 | 11 | // Trigger executed when the prometheus metric with this type has finished syncing. 12 | func TriggerMetricTypeSynced(metricType string) string { 13 | return "triggers/sync/prometheus/type/" + metricType 14 | } 15 | -------------------------------------------------------------------------------- /internal/sync/prometheus/types_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package prometheus 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestVROpsHostMetric(t *testing.T) { 12 | metric := VROpsHostMetric{ 13 | Name: "cpu_usage", 14 | Cluster: "cluster1", 15 | ClusterType: "type1", 16 | Collector: "collector1", 17 | Datacenter: "datacenter1", 18 | HostSystem: "host1", 19 | InternalName: "internal1", 20 | Job: "job1", 21 | Prometheus: "prometheus1", 22 | Region: "region1", 23 | VCCluster: "vccluster1", 24 | VCenter: "vcenter1", 25 | Timestamp: time.Now(), 26 | Value: 0.5, 27 | } 28 | 29 | if metric.GetName() != "cpu_usage" { 30 | t.Errorf("expected name to be 'cpu_usage', got %s", metric.GetName()) 31 | } 32 | 33 | if metric.GetTimestamp().IsZero() { 34 | t.Error("expected timestamp to be set") 35 | } 36 | 37 | newMetric := metric.With("whatever", time.Unix(0, 0), 1.0) 38 | if newMetric.GetName() != "whatever" { 39 | t.Errorf("expected name to be 'whatever', got %s", newMetric.GetName()) 40 | } 41 | if !newMetric.GetTimestamp().Equal(time.Unix(0, 0)) { 42 | t.Errorf("expected timestamp to be '1970-01-01 00:00:00 +0000 UTC', got %s", newMetric.GetTimestamp()) 43 | } 44 | if newMetric.GetValue() != 1.0 { 45 | t.Errorf("expected value to be 1.0, got %f", newMetric.GetValue()) 46 | } 47 | } 48 | 49 | func TestVROpsVMMetric(t *testing.T) { 50 | metric := VROpsVMMetric{ 51 | Name: "ram_usage", 52 | Cluster: "cluster1", 53 | ClusterType: "type1", 54 | Collector: "collector1", 55 | Datacenter: "datacenter1", 56 | HostSystem: "host1", 57 | InternalName: "internal1", 58 | Job: "job1", 59 | Project: "project1", 60 | Prometheus: "prometheus1", 61 | Region: "region1", 62 | VCCluster: "vccluster1", 63 | VCenter: "vcenter1", 64 | VirtualMachine: "vm1", 65 | InstanceUUID: "uuid1", 66 | Timestamp: time.Now(), 67 | Value: 0.5, 68 | } 69 | 70 | if metric.GetName() != "ram_usage" { 71 | t.Errorf("expected name to be 'ram_usage', got %s", metric.GetName()) 72 | } 73 | 74 | if metric.GetTimestamp().IsZero() { 75 | t.Error("expected timestamp to be set") 76 | } 77 | 78 | newMetric := metric.With("cpu_usage", time.Unix(0, 0), 1.0) 79 | if newMetric.GetName() != "cpu_usage" { 80 | t.Errorf("expected name to be 'cpu_usage', got %s", newMetric.GetName()) 81 | } 82 | if !newMetric.GetTimestamp().Equal(time.Unix(0, 0)) { 83 | t.Errorf("expected timestamp to be '1970-01-01 00:00:00 +0000 UTC', got %s", newMetric.GetTimestamp()) 84 | } 85 | if newMetric.GetValue() != 1.0 { 86 | t.Errorf("expected value to be 1.0, got %f", newMetric.GetValue()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /internal/sync/sso.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package sync 5 | 6 | import ( 7 | "crypto/tls" 8 | "crypto/x509" 9 | "errors" 10 | "fmt" 11 | "log/slog" 12 | "net/http" 13 | 14 | "github.com/trickyteache/cortex/internal/conf" 15 | ) 16 | 17 | // Custom HTTP round tripper that logs each request. 18 | type requestLogger struct { 19 | T http.RoundTripper 20 | } 21 | 22 | // RoundTrip logs the request URL before making the request. 23 | func (lrt *requestLogger) RoundTrip(req *http.Request) (*http.Response, error) { 24 | slog.Info("making http request", "url", req.URL.String()) 25 | return lrt.T.RoundTrip(req) 26 | } 27 | 28 | // Create a new HTTP client with the given SSO configuration 29 | // and logging for each request. 30 | func NewHTTPClient(conf conf.SSOConfig) (*http.Client, error) { 31 | if conf.Cert == "" { 32 | // Disable SSO if no certificate is provided. 33 | slog.Debug("making http requests without SSO") 34 | return &http.Client{Transport: &requestLogger{T: &http.Transport{}}}, nil 35 | } 36 | // If we have a public key, we also need a private key. 37 | if conf.CertKey == "" { 38 | return nil, errors.New("missing cert key for SSO") 39 | } 40 | cert, err := tls.X509KeyPair( 41 | []byte(conf.Cert), 42 | []byte(conf.CertKey), 43 | ) 44 | if err != nil { 45 | return nil, fmt.Errorf("failed to load client certificate: %w", err) 46 | } 47 | caCertPool := x509.NewCertPool() 48 | caCertPool.AddCert(cert.Leaf) 49 | return &http.Client{Transport: &requestLogger{T: &http.Transport{ 50 | TLSClientConfig: &tls.Config{ 51 | Certificates: []tls.Certificate{cert}, 52 | RootCAs: caCertPool, 53 | // If the cert is self signed, skip verification. 54 | //nolint:gosec 55 | InsecureSkipVerify: conf.SelfSigned, 56 | }, 57 | }}}, nil 58 | } 59 | -------------------------------------------------------------------------------- /plutono/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Plutono is a fork of Grafana 7.5.17 under the Apache 2.0 License. 5 | FROM ghcr.io/credativ/plutono:v7.5.37 6 | COPY provisioning /etc/plutono/provisioning 7 | COPY plutono.ini /etc/plutono/plutono.ini -------------------------------------------------------------------------------- /plutono/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: cortex-plutono 8 | labels: 9 | app: cortex-plutono 10 | spec: 11 | replicas: 1 12 | selector: 13 | matchLabels: 14 | app: cortex-plutono 15 | template: 16 | metadata: 17 | labels: 18 | app: cortex-plutono 19 | spec: 20 | containers: 21 | - name: cortex-plutono # from local tilt build 22 | image: cortex-plutono 23 | ports: 24 | - containerPort: 3000 -------------------------------------------------------------------------------- /plutono/plutono.ini: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # When we use plutono in the Tilt setup, we don't want to login all the time. 5 | [auth.anonymous] 6 | enabled = true 7 | org_role = Admin 8 | [auth] 9 | disable_login_form = true 10 | -------------------------------------------------------------------------------- /plutono/provisioning/dashboards/cortex.json.license: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 -------------------------------------------------------------------------------- /plutono/provisioning/dashboards/dashboards.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: 1 5 | 6 | providers: 7 | - name: 'default' 8 | orgId: 1 9 | folder: '' 10 | type: file 11 | disableDeletion: false 12 | editable: true 13 | allowUiUpdates: true 14 | options: 15 | path: /etc/plutono/provisioning/dashboards -------------------------------------------------------------------------------- /plutono/provisioning/datasources/datasources.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: 1 5 | 6 | datasources: 7 | - name: prometheus-openstack 8 | type: prometheus 9 | access: proxy 10 | # Created by the prometheus-operator. 11 | url: prometheus-operated:9090 12 | jsonData: 13 | timeInterval: "5s" 14 | editable: true 15 | - name: Postgres 16 | type: postgres 17 | access: proxy 18 | # Point at the postgres proxy that is running locally. 19 | url: cortex-postgresql:5432 20 | user: postgres 21 | uid: postgres 22 | secureJsonData: 23 | password: 'secret' 24 | jsonData: 25 | database: postgres 26 | sslmode: 'disable' # disable/require/verify-ca/verify-full 27 | maxOpenConns: 100 # Grafana v5.4+ 28 | maxIdleConns: 100 # Grafana v5.4+ 29 | maxIdleConnsAuto: true # Grafana v9.5.1+ 30 | connMaxLifetime: 14400 # Grafana v5.4+ 31 | postgresVersion: 1700 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 32 | timescaledb: false 33 | editable: true -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright 2024 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | { pkgs ? import { } }: 5 | 6 | with pkgs; 7 | 8 | mkShell { 9 | nativeBuildInputs = [ 10 | addlicense 11 | go-licence-detector 12 | go_1_24 13 | gotools # goimports 14 | postgresql_17 15 | 16 | # keep this line if you use bash 17 | bashInteractive 18 | ]; 19 | } 20 | -------------------------------------------------------------------------------- /testlib/db/containers/postgres.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package containers 5 | 6 | import ( 7 | "database/sql" 8 | "fmt" 9 | "log" 10 | "testing" 11 | 12 | "github.com/ory/dockertest" 13 | "github.com/ory/dockertest/docker" 14 | ) 15 | 16 | type PostgresContainer struct { 17 | pool *dockertest.Pool 18 | resource *dockertest.Resource 19 | } 20 | 21 | func (c PostgresContainer) GetPort() string { 22 | return c.resource.GetPort("5432/tcp") 23 | } 24 | 25 | func (c *PostgresContainer) Init(t *testing.T) { 26 | pool, err := dockertest.NewPool("") 27 | if err != nil { 28 | log.Fatalf("could not construct pool: %s", err) 29 | } 30 | c.pool = pool 31 | if err = pool.Client.Ping(); err != nil { 32 | log.Fatalf("could not connect to Docker: %s", err) 33 | } 34 | resource, err := pool.RunWithOptions(&dockertest.RunOptions{ 35 | Repository: "postgres", 36 | Tag: "17", 37 | Env: []string{ 38 | "POSTGRES_USER=postgres", 39 | "POSTGRES_PASSWORD=secret", 40 | "listen_addresses = '*'", 41 | }, 42 | }, func(config *docker.HostConfig) { 43 | // set AutoRemove to true so that stopped container goes away by itself 44 | config.AutoRemove = true 45 | config.RestartPolicy = docker.RestartPolicy{ 46 | Name: "no", 47 | } 48 | }) 49 | if err != nil { 50 | log.Fatalf("could not start resource: %s", err) 51 | } 52 | c.resource = resource 53 | if err := c.resource.Expire(10); err != nil { 54 | log.Fatalf("could not set expiration: %s", err) 55 | } 56 | psqlInfo := fmt.Sprintf( 57 | "host=localhost port=%s user=postgres password=secret dbname=postgres sslmode=disable", 58 | resource.GetPort("5432/tcp"), 59 | ) 60 | sqlDB, err := sql.Open("postgres", psqlInfo) 61 | if err != nil { 62 | log.Fatalf("could not connect to sql db: %s", err) 63 | } 64 | if err = pool.Retry(sqlDB.Ping); err != nil { 65 | log.Fatalf("postgres db is not ready in time: %s", err) 66 | } 67 | } 68 | 69 | func (c *PostgresContainer) Close() { 70 | if err := c.pool.Purge(c.resource); err != nil { 71 | log.Fatalf("could not purge resource: %s", err) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /testlib/db/containers/postgres_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package containers 5 | 6 | import ( 7 | "database/sql" 8 | "fmt" 9 | "os" 10 | "testing" 11 | 12 | _ "github.com/lib/pq" 13 | ) 14 | 15 | func TestPostgresContainer_Init(t *testing.T) { 16 | if os.Getenv("POSTGRES_CONTAINER") != "1" { 17 | t.Skip("skipping test; set POSTGRES_CONTAINER=1 to run") 18 | } 19 | 20 | container := PostgresContainer{} 21 | container.Init(t) 22 | defer container.Close() 23 | 24 | psqlInfo := fmt.Sprintf( 25 | "host=localhost port=%s user=postgres password=secret dbname=postgres sslmode=disable", 26 | container.GetPort(), 27 | ) 28 | db, err := sql.Open("postgres", psqlInfo) 29 | if err != nil { 30 | t.Fatalf("expected no error, got %v", err) 31 | } 32 | defer db.Close() 33 | 34 | if err := db.Ping(); err != nil { 35 | t.Fatalf("expected no error, got %v", err) 36 | } 37 | } 38 | 39 | func TestPostgresContainer_Close(t *testing.T) { 40 | if os.Getenv("POSTGRES_CONTAINER") != "1" { 41 | t.Skip("skipping test; set POSTGRES_CONTAINER=1 to run") 42 | } 43 | 44 | container := PostgresContainer{} 45 | container.Init(t) 46 | 47 | psqlInfo := fmt.Sprintf( 48 | "host=localhost port=%s user=postgres password=secret dbname=postgres sslmode=disable", 49 | container.GetPort(), 50 | ) 51 | db, err := sql.Open("postgres", psqlInfo) 52 | if err != nil { 53 | t.Fatalf("expected no error, got %v", err) 54 | } 55 | defer db.Close() 56 | 57 | container.Close() 58 | 59 | if err := db.Ping(); err == nil { 60 | t.Fatal("expected error, got nil") 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /testlib/db/env.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package db 5 | 6 | import ( 7 | "database/sql" 8 | "log" 9 | "log/slog" 10 | "os" 11 | "testing" 12 | 13 | "github.com/trickyteache/cortex/testlib/db/containers" 14 | "github.com/go-gorp/gorp" 15 | _ "github.com/mattn/go-sqlite3" 16 | "github.com/sapcc/go-bits/easypg" 17 | ) 18 | 19 | type DBEnv struct { 20 | *gorp.DbMap 21 | Close func() 22 | } 23 | 24 | func SetupDBEnv(t *testing.T) DBEnv { 25 | var env DBEnv 26 | // To run tests faster, the default is running with sqlite. 27 | if os.Getenv("POSTGRES_CONTAINER") == "1" { 28 | slog.Info("Using real postgres container") 29 | container := containers.PostgresContainer{} 30 | container.Init(t) 31 | dbURL, err := easypg.URLFrom(easypg.URLParts{ 32 | HostName: "localhost", 33 | Port: container.GetPort(), 34 | UserName: "postgres", 35 | Password: "secret", 36 | ConnectionOptions: "sslmode=disable", 37 | DatabaseName: "postgres", 38 | }) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | db, err := sql.Open("postgres", dbURL.String()) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | env.DbMap = &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}} 47 | env.Close = container.Close 48 | } else { 49 | slog.Info("Using sqlite") 50 | tmpDir := t.TempDir() 51 | sqlDB, err := sql.Open("sqlite3", tmpDir+"/test.db") 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | env.DbMap = &gorp.DbMap{Db: sqlDB, Dialect: gorp.SqliteDialect{}} 56 | env.Close = func() {} 57 | } 58 | env.TraceOn("[gorp]", log.New(os.Stdout, "cortex:", log.Lmicroseconds)) 59 | return env 60 | } 61 | -------------------------------------------------------------------------------- /testlib/monitoring/mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package monitoring 5 | 6 | type MockObserver struct { 7 | // Observations recorded by the mock observer. 8 | Observations []float64 9 | } 10 | 11 | func (m *MockObserver) Observe(value float64) { 12 | m.Observations = append(m.Observations, value) 13 | } 14 | -------------------------------------------------------------------------------- /testlib/mqtt/containers/rabbitmq-entrypoint.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | rabbitmq-plugins enable rabbitmq_mqtt 5 | rabbitmq-server -------------------------------------------------------------------------------- /testlib/mqtt/containers/rabbitmq.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | mqtt.listeners.tcp.default = 1883 5 | mqtt.allow_anonymous = true 6 | default_user = guest 7 | default_pass = guest -------------------------------------------------------------------------------- /testlib/mqtt/containers/rabbitmq_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package containers 5 | 6 | import ( 7 | "os" 8 | "testing" 9 | 10 | _ "github.com/lib/pq" 11 | ) 12 | 13 | func TestRabbitMQContainer_Init(t *testing.T) { 14 | if os.Getenv("RABBITMQ_CONTAINER") != "1" { 15 | t.Skip("skipping test; set RABBITMQ_CONTAINER=1 to run") 16 | } 17 | 18 | container := RabbitMQContainer{} 19 | container.Init(t) 20 | 21 | // Should not panic. 22 | 23 | container.Close() 24 | } 25 | -------------------------------------------------------------------------------- /testlib/mqtt/mock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package mqtt 5 | 6 | import ( 7 | pahomqtt "github.com/eclipse/paho.mqtt.golang" 8 | ) 9 | 10 | // Mock mqtt client that does nothing and can be used for testing. 11 | type MockClient struct{} 12 | 13 | func (m *MockClient) Publish(topic string, payload any) { 14 | // Do nothing 15 | } 16 | 17 | func (m *MockClient) Connect() error { 18 | return nil 19 | } 20 | 21 | func (m *MockClient) Disconnect() { 22 | // Do nothing 23 | } 24 | 25 | func (m *MockClient) Subscribe(topic string, callback pahomqtt.MessageHandler) error { 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /testlib/scheduler/api/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package api 5 | 6 | import "github.com/trickyteache/cortex/internal/scheduler/api" 7 | 8 | type MockRequest struct { 9 | Spec api.NovaObject[api.NovaSpec] 10 | Context api.NovaRequestContext 11 | Rebuild bool 12 | Resize bool 13 | Live bool 14 | VMware bool 15 | Hosts []string 16 | Weights map[string]float64 17 | } 18 | 19 | func (r *MockRequest) GetSpec() api.NovaObject[api.NovaSpec] { return r.Spec } 20 | func (r *MockRequest) GetContext() api.NovaRequestContext { return r.Context } 21 | func (r *MockRequest) GetRebuild() bool { return r.Rebuild } 22 | func (r *MockRequest) GetResize() bool { return r.Resize } 23 | func (r *MockRequest) GetLive() bool { return r.Live } 24 | func (r *MockRequest) GetVMware() bool { return r.VMware } 25 | func (r *MockRequest) GetHosts() []string { return r.Hosts } 26 | func (r *MockRequest) GetWeights() map[string]float64 { return r.Weights } 27 | -------------------------------------------------------------------------------- /testlib/scheduler/plugins/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 SAP SE 2 | // SPDX-License-Identifier: Apache-2.0 3 | 4 | package plugins 5 | 6 | import ( 7 | "github.com/trickyteache/cortex/internal/conf" 8 | "github.com/trickyteache/cortex/internal/db" 9 | "github.com/trickyteache/cortex/internal/scheduler/api" 10 | ) 11 | 12 | // MockStep is a manual mock implementation of the plugins.Step interface. 13 | type MockStep struct { 14 | Name string 15 | InitFunc func(db db.DB, opts conf.RawOpts) error 16 | RunFunc func(request api.Request) (map[string]float64, error) 17 | } 18 | 19 | func (m *MockStep) GetName() string { 20 | return m.Name 21 | } 22 | 23 | func (m *MockStep) Init(db db.DB, opts conf.RawOpts) error { 24 | return m.InitFunc(db, opts) 25 | } 26 | 27 | func (m *MockStep) Run(request api.Request) (map[string]float64, error) { 28 | return m.RunFunc(request) 29 | } 30 | -------------------------------------------------------------------------------- /visualizer/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | FROM nginx 5 | COPY vendor/mqtt.min.js /usr/share/nginx/html/mqtt.min.js 6 | COPY index.html /usr/share/nginx/html/index.html -------------------------------------------------------------------------------- /visualizer/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 SAP SE 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: cortex-visualizer 8 | labels: 9 | app: cortex-visualizer 10 | spec: 11 | replicas: 1 12 | selector: 13 | matchLabels: 14 | app: cortex-visualizer 15 | template: 16 | metadata: 17 | labels: 18 | app: cortex-visualizer 19 | spec: 20 | containers: 21 | - name: cortex-visualizer # from local tilt build 22 | image: cortex-visualizer 23 | ports: 24 | - containerPort: 3000 -------------------------------------------------------------------------------- /visualizer/vendor/mqtt.min.js.license: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | ===================== 3 | 4 | Copyright (c) 2015-2016 MQTT.js contributors 5 | --------------------------------------- 6 | 7 | *MQTT.js contributors listed at * 8 | 9 | Copyright 2011-2014 by Adam Rudd 10 | 11 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------