├── .dockerignore ├── .editorconfig ├── .github ├── dependabot.yml └── workflows │ ├── build-debug.yml │ ├── build.yml │ ├── lint.yml │ ├── release.yml │ ├── scorecard.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .releaserc ├── CONTRIBUTING.md ├── Makefile ├── PROJECT ├── README.md ├── SECURITY.md ├── analyzers └── patches.go ├── api ├── cache.go ├── context.go ├── global.go └── v1 │ ├── aws.go │ ├── aws_test.go │ ├── azure.go │ ├── clickhouse.go │ ├── common.go │ ├── config.go │ ├── const.go │ ├── data.go │ ├── file.go │ ├── gcp.go │ ├── github.go │ ├── groupversion_info.go │ ├── http.go │ ├── interface.go │ ├── interface_test.go │ ├── json_types.go │ ├── kubernetes.go │ ├── kubernetes_test.go │ ├── scrapeconfig_types.go │ ├── scrapeplugin_types.go │ ├── slack.go │ ├── sql.go │ ├── terraform.go │ ├── trivy.go │ ├── types.go │ └── zz_generated.deepcopy.go ├── build ├── Dockerfile └── Dockerfile.debug ├── changes ├── fingerprint.go ├── fingerprint_test.go └── testdata │ ├── change_1.json │ ├── change_2.json │ ├── change_3.json │ ├── change_4.json │ ├── health_passed_1.json │ ├── health_passed_2.json │ ├── helm_upgrade_failed.json │ └── helm_upgrade_failed_2.json ├── chart ├── .helmignore ├── Chart.yaml ├── README.md ├── crds │ ├── configs.flanksource.com_scrapeconfigs.yaml │ └── configs.flanksource.com_scrapeplugins.yaml ├── templates │ ├── _helpers.tpl │ ├── clickhouse.yaml │ ├── configmap.yaml │ ├── deployment.yaml │ ├── ingress.yaml │ ├── postgres.yaml │ ├── service.yaml │ ├── serviceaccount.yaml │ └── servicemonitor.yaml ├── values.schema.json └── values.yaml ├── cmd ├── analyze.go ├── offline.go ├── operator.go ├── root.go ├── run.go └── server.go ├── config └── schemas │ ├── config_aws.schema.json │ ├── config_azure.schema.json │ ├── config_azuredevops.schema.json │ ├── config_file.schema.json │ ├── config_gcp.schema.json │ ├── config_githubactions.schema.json │ ├── config_http.schema.json │ ├── config_kubernetes.schema.json │ ├── config_kubernetesfile.schema.json │ ├── config_slack.schema.json │ ├── config_sql.schema.json │ ├── config_terraform.schema.json │ ├── config_trivy.schema.json │ └── scrape_config.schema.json ├── db ├── analysis.go ├── changes.go ├── config.go ├── config_scraper.go ├── diff.go ├── diff_test.go ├── models │ ├── config_change.go │ ├── config_item.go │ └── config_relationship.go ├── people.go ├── scrape_plugin.go ├── testdata │ ├── person-new.json │ ├── person-old.json │ ├── person.diff │ ├── simple-new.json │ ├── simple-old.json │ └── simple.diff ├── ulid │ └── ulid.go └── update.go ├── debug.go ├── deploy ├── kustomization.yaml ├── manager.yaml ├── namespace.yaml └── rbac.yaml ├── external └── diffgen │ ├── Cargo.lock │ ├── Cargo.toml │ ├── libdiffgen.h │ └── src │ └── lib.rs ├── fixtures ├── access_logs.yaml ├── aws.yaml ├── azure-devops.yaml ├── azure.yaml ├── clickhouse.yaml ├── crds │ └── scrape-config-kubernetes.yaml ├── data │ ├── car.json │ ├── car_changes.json │ ├── echo-playbook.yaml │ ├── multiple-configs.json │ ├── single-config.json │ └── test.yaml ├── expected │ ├── file-exclusion.json │ ├── file-git.json │ ├── file-mask.json │ ├── file-postgres-properties.json │ ├── file-script-gotemplate.json │ └── file-script.json ├── file-car-change.yaml ├── file-car.yaml ├── file-crd-sync.yaml ├── file-exclusion.yaml ├── file-git.yaml ├── file-local-creation-date.yaml ├── file-local.yaml ├── file-mask.yaml ├── file-postgres-properties.conf ├── file-postgres-properties.yaml ├── file-script-gotemplate.yaml ├── file-script.yaml ├── file.yaml ├── github-actions.yaml ├── http-lastfm.yaml ├── http-scraper.yaml ├── kubernetes.yaml ├── kubernetes_file.yaml ├── load │ ├── .gitignore │ ├── Makefile │ └── load.ts ├── plugin-change-exclusion.yaml ├── plugin-change-mapping.yaml ├── scrapper.yaml ├── slack.yaml ├── sql.yaml ├── terraform.yaml └── trivy.yaml ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt └── generate-schemas │ ├── .gitignore │ └── main.go ├── jobs ├── cleanup.go ├── jobs.go ├── jobs_test.go ├── retention.go ├── suite_test.go └── sync_upstream.go ├── main.go ├── rustdiffgen.go ├── scrapers ├── analysis │ ├── rules.go │ └── rules.yaml ├── aws │ ├── analyzer.go │ ├── aws.go │ ├── backup.go │ ├── cloudtrail.go │ ├── cloudtrail_test.go │ ├── config.go │ ├── cost.go │ ├── scratch.go │ ├── trusted_advisor.go │ └── types.go ├── azure │ ├── active_directory.go │ ├── advisors.go │ ├── azure.go │ ├── azure_test.go │ └── devops │ │ ├── client.go │ │ └── pipelines.go ├── changes │ ├── changes_suite_test.go │ ├── extraction.go │ ├── extraction_test.go │ ├── rules.go │ ├── rules.yaml │ └── rules_test.go ├── clickhouse │ └── clickhouse.go ├── common.go ├── cron.go ├── event.go ├── file │ ├── file.go │ └── file_test.go ├── gcp │ ├── cloud_logging.go │ └── gcp.go ├── github │ ├── client.go │ ├── client_test.go │ └── workflows.go ├── http │ └── http.go ├── incremental.go ├── kubernetes │ ├── context.go │ ├── events.go │ ├── events_test.go │ ├── exclusions.go │ ├── hook_argo.go │ ├── hook_aws.go │ ├── hook_azure.go │ ├── hook_flux.go │ ├── hooks.go │ ├── informers.go │ ├── informers_test.go │ ├── ketall.go │ ├── kubernetes.go │ ├── kubernetes_file.go │ ├── kubernetes_test.go │ └── resource_map.go ├── processors │ ├── json.go │ └── script.go ├── retention.go ├── run.go ├── run_now.go ├── run_test.go ├── runscrapers_suite_test.go ├── slack │ ├── api.go │ └── slack.go ├── sql │ └── sql.go ├── stale.go ├── terraform │ ├── mask.go │ ├── mask_test.go │ ├── models.go │ ├── path.go │ ├── terraform.go │ └── testdata │ │ ├── cloudflare.tfstate │ │ └── cloudflare.tfstate.expected └── trivy │ ├── models.go │ └── trivy.go ├── telemetry ├── pyroscope.go └── tracer.go ├── testdata └── .gitignore ├── tests └── load_test.go └── utils ├── concurrency.go ├── debug.go ├── files.go ├── hash.go ├── hash_test.go ├── json.go ├── json_test.go ├── kube ├── fetcher.go └── name.go └── struct.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .bin/ 2 | .config-db/ 3 | .DS_Store 4 | .env 5 | .github/ 6 | .idea/ 7 | .releaserc 8 | .vscode/ 9 | build/ 10 | chart/ 11 | CONTRIBUTING.md 12 | cover.out 13 | Dockerfile 14 | PROJECT 15 | README.md 16 | SECURITY.md 17 | test.test 18 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.ts] 2 | indent_style = space 3 | indent_size = 2 4 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: 'gomod' 4 | directory: '/' 5 | schedule: 6 | interval: 'daily' 7 | 8 | - package-ecosystem: github-actions 9 | directory: / 10 | schedule: 11 | interval: daily 12 | -------------------------------------------------------------------------------- /.github/workflows/build-debug.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | inputs: 4 | image_tag: 5 | description: image tag 6 | required: true 7 | 8 | name: Build Debug 9 | jobs: 10 | docker: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Harden Runner 14 | uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 15 | with: 16 | egress-policy: audit 17 | 18 | - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 19 | 20 | - name: Publish to DockerHub Registry 21 | uses: elgohr/Publish-Docker-Github-Action@ec61b713af46c32efaa27ac2626c2acb82ce6435 # v5 22 | with: 23 | name: flanksource/config-db 24 | username: ${{ secrets.DOCKER_USERNAME }} 25 | password: ${{ secrets.DOCKER_PASSWORD }} 26 | snapshot: true 27 | dockerfile: build/Dockerfile.debug 28 | tags: "v${{inputs.image_tag}}" 29 | 30 | - name: Configure AWS Credentials 31 | uses: aws-actions/configure-aws-credentials@v4 32 | with: 33 | aws-access-key-id: ${{ secrets.ECR_AWS_ACCESS_KEY }} 34 | aws-secret-access-key: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} 35 | aws-region: us-east-1 36 | 37 | - name: Login to Amazon ECR Public 38 | id: login-ecr-public 39 | uses: aws-actions/amazon-ecr-login@v2 40 | with: 41 | registry-type: public 42 | 43 | - name: Publish to ECR Public 44 | env: 45 | REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} 46 | REGISTRY_ALIAS: k4y9r6y5 47 | REPOSITORY: config-db 48 | IMAGE_TAG: "v${{inputs.image_tag}}" 49 | run: | 50 | docker build -t $REGISTRY/$REGISTRY_ALIAS/$REPOSITORY:$IMAGE_TAG -f build/Dockerfile.debug . 51 | docker push $REGISTRY/$REGISTRY_ALIAS/$REPOSITORY:$IMAGE_TAG 52 | 53 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | on: pull_request 2 | name: Build 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | timeout-minutes: 10 7 | steps: 8 | - name: Harden Runner 9 | uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 10 | with: 11 | egress-policy: audit 12 | 13 | - name: Checkout code 14 | uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 15 | - name: Build Container 16 | run: make docker 17 | env: 18 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 19 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | on: 3 | pull_request: 4 | 5 | permissions: 6 | contents: read 7 | 8 | jobs: 9 | golangci: 10 | permissions: 11 | contents: read # for actions/checkout to fetch code 12 | pull-requests: read # for golangci/golangci-lint-action to fetch pull requests 13 | name: lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 17 | - name: Install Go 18 | uses: buildjet/setup-go@v5 19 | with: 20 | go-version: 1.24.x 21 | 22 | - run: make resources 23 | - name: golangci-lint 24 | uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 25 | -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. They are provided 2 | # by a third-party and are governed by separate terms of service, privacy 3 | # policy, and support documentation. 4 | 5 | name: Scorecard supply-chain security 6 | on: 7 | # For Branch-Protection check. Only the default branch is supported. See 8 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection 9 | branch_protection_rule: 10 | # To guarantee Maintained check is occasionally updated. See 11 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained 12 | schedule: 13 | - cron: '18 5 * * 0' 14 | push: 15 | branches: [ "main" ] 16 | 17 | # Declare default permissions as read only. 18 | permissions: read-all 19 | 20 | jobs: 21 | analysis: 22 | name: Scorecard analysis 23 | runs-on: ubuntu-latest 24 | permissions: 25 | # Needed to upload the results to code-scanning dashboard. 26 | security-events: write 27 | # Needed to publish results and get a badge (see publish_results below). 28 | id-token: write 29 | # Uncomment the permissions below if installing in a private repository. 30 | # contents: read 31 | # actions: read 32 | 33 | steps: 34 | - name: "Checkout code" 35 | uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 36 | with: 37 | persist-credentials: false 38 | 39 | - name: "Run analysis" 40 | uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2 41 | with: 42 | results_file: results.sarif 43 | results_format: sarif 44 | # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: 45 | # - you want to enable the Branch-Protection check on a *public* repository, or 46 | # - you are installing Scorecard on a *private* repository 47 | # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. 48 | # repo_token: ${{ secrets.SCORECARD_TOKEN }} 49 | 50 | # Public repositories: 51 | # - Publish results to OpenSSF REST API for easy access by consumers 52 | # - Allows the repository to include the Scorecard badge. 53 | # - See https://github.com/ossf/scorecard-action#publishing-results. 54 | # For private repositories: 55 | # - `publish_results` will always be set to `false`, regardless 56 | # of the value entered here. 57 | publish_results: true 58 | 59 | # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF 60 | # format to the repository Actions tab. 61 | - name: "Upload artifact" 62 | uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 63 | with: 64 | name: SARIF file 65 | path: results.sarif 66 | retention-days: 5 67 | 68 | # Upload the results to GitHub's code scanning dashboard. 69 | - name: "Upload to code-scanning" 70 | uses: github/codeql-action/upload-sarif@c36620d31ac7c881962c3d9dd939c40ec9434f2b # v3.26.12 71 | with: 72 | sarif_file: results.sarif 73 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | _DS_Store 3 | .bin/ 4 | vendor 5 | config-db 6 | scraped/ 7 | 8 | tests/log.txt 9 | 10 | .config-db 11 | ginkgo.report 12 | 13 | config-db.properties 14 | 15 | *.gob 16 | *.out 17 | *.test 18 | 19 | output-*/ 20 | configs/ 21 | *.pprof 22 | 23 | 24 | # Rust 25 | external/diffgen/target 26 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | exclusions: 4 | generated: lax 5 | presets: 6 | - comments 7 | - common-false-positives 8 | - legacy 9 | #- std-error-handling 10 | paths: 11 | - third_party$ 12 | - builtin$ 13 | - examples$ 14 | rules: 15 | - linters: 16 | - staticcheck 17 | text: 'QF1008:' 18 | formatters: 19 | exclusions: 20 | generated: lax 21 | paths: 22 | - third_party$ 23 | - builtin$ 24 | - examples$ 25 | -------------------------------------------------------------------------------- /.releaserc: -------------------------------------------------------------------------------- 1 | branches: 2 | - name: main 3 | plugins: 4 | - - "@semantic-release/commit-analyzer" 5 | - releaseRules: 6 | - { type: doc, scope: README, release: patch } 7 | - { type: fix, release: patch } 8 | - { type: chore, release: patch } 9 | - { type: refactor, release: patch } 10 | - { type: feat, release: patch } 11 | - { type: ci, release: false } 12 | - { type: style, release: false } 13 | parserOpts: 14 | noteKeywords: 15 | - MAJOR RELEASE 16 | - "@semantic-release/release-notes-generator" 17 | - - "@semantic-release/github" 18 | - assets: 19 | - path: ./.bin/config-db-amd64 20 | name: config-db-amd64 21 | - path: ./.bin/config-db.exe 22 | name: config-db.exe 23 | - path: ./.bin/config-db_osx-amd64 24 | name: config-db_osx-amd64 25 | - path: ./.bin/config-db_osx-arm64 26 | name: config-db_osx-arm64 27 | 28 | # From: https://github.com/semantic-release/github/pull/487#issuecomment-1486298997 29 | successComment: false 30 | failTitle: false 31 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Adding a new scraper 2 | 3 | 1. Create a new file in `scrapers/` which implements the `api/v1/Scraper` interface 4 | 2. Add a reference to the scraper in `scrapers/common` 5 | 3. Create a configuration struct in `api/v1` and add it into the `api/v1/types/ConfigScraper` struct 6 | 4. Add a fixture in `fixtures/` 7 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: flanksource.com 2 | layout: 3 | - go.kubebuilder.io/v3 4 | projectName: config-db 5 | repo: github.com/flanksource/config-db 6 | resources: 7 | - api: 8 | crdVersion: v1 9 | namespaced: true 10 | controller: true 11 | domain: flanksource.com 12 | group: configs 13 | kind: ScrapeConfig 14 | path: github.com/flanksource/config-db/api/v1 15 | version: v1 16 | version: "3" 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # config-db 2 | 3 | **config-db** is developer first, JSON based configuration management database (CMDB) 4 | 5 | ## Setup 6 | 7 | ```bash 8 | make build 9 | ``` 10 | 11 | ### Run as server 12 | 13 | Starting the server will run the migrations and start scraping in background (The `default-schedule` configuration will run scraping every 60 minutes if configuration is not explicitly specified). 14 | 15 | ```bash 16 | DB_URL=postgres://:@localhost:5432/ ./.bin/config-db serve --db-migrations 17 | ``` 18 | 19 | ### Scape config 20 | 21 | To explicitly run scraping with a particular configuration: 22 | 23 | ```bash 24 | ./.bin/config-db run -vvv 25 | ``` 26 | 27 | See `fixtures/` for example scraping configurations. 28 | 29 | ## Principles 30 | 31 | * **JSON Based** - Configuration is stored in JSON, with changes recorded as JSON patches that enables highly structured search. 32 | * **SPAM Free** - Not all configuration data is useful, and overly verbose change histories are difficult to navigate. 33 | * **GitOps Ready** - Configuration should be stored in Git, config-db enables the extraction of configuration out of Git repositories with branch/environment awareness. 34 | * **Topology Aware** - Configuration can often have an inheritance or override hierarchy. 35 | 36 | ## Capabilities 37 | 38 | * View and search change history in any dimension (node, zone, environment, application, technology) 39 | * Compare and diff configuration across environments. 40 | 41 | ## Configuration Sources 42 | 43 | * AWS 44 | * [x] EC2 (including trusted advisor, compliance and patch reporting) 45 | * [x] VPC 46 | * [x] IAM 47 | * Azure 48 | * Kubernetes 49 | * [x] Pods 50 | * [x] Secrets / ConfigMaps 51 | * [x] LoadBalancers / Ingress 52 | * [x] Nodes 53 | * Configuration Files 54 | * [ ] YAML/JSON 55 | * [ ] Properties files 56 | * Dependency Graphs 57 | * [ ] pom.xml 58 | * [ ] package.json 59 | * [ ] go.mod 60 | * Infrastructure as Code 61 | * [ ] Terraform 62 | * [ ] CloudFormation 63 | * [ ] Ansible 64 | 65 | ## Contributing 66 | 67 | See [CONTRIBUTING.md](./CONTRIBUTING.md) 68 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | If you discover any security vulnerabilities within this project, please report them to our team immediately. We appreciate your help in making this project more secure for everyone. 6 | 7 | To report a vulnerability, please follow these steps: 8 | 9 | 1. **Email**: Send an email to our security team at [security@flanksource.com](mailto:security@flanksource.com) with a detailed description of the vulnerability. 10 | 2. **Subject Line**: Use the subject line "Security Vulnerability Report" to ensure prompt attention. 11 | 3. **Information**: Provide as much information as possible about the vulnerability, including steps to reproduce it and any supporting documentation or code snippets. 12 | 4. **Confidentiality**: We prioritize the confidentiality of vulnerability reports. Please avoid publicly disclosing the issue until we have had an opportunity to address it. 13 | 14 | Our team will respond to your report as soon as possible and work towards a solution. We appreciate your responsible disclosure and cooperation in maintaining the security of this project. 15 | 16 | Thank you for your contribution to the security of this project! 17 | 18 | **Note:** This project follows responsible disclosure practices. 19 | -------------------------------------------------------------------------------- /analyzers/patches.go: -------------------------------------------------------------------------------- 1 | package analyzers 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/flanksource/commons/logger" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | ) 11 | 12 | // PatchAnalyzer ... 13 | func PatchAnalyzer(configs []v1.ScrapeResult) v1.AnalysisResult { 14 | 15 | result := v1.AnalysisResult{ 16 | Analyzer: "patch", 17 | } 18 | 19 | var platforms = make(map[string][]v1.Host) 20 | var hostIndex = make(map[string]v1.Host) 21 | for _, config := range configs { 22 | switch config.Config.(type) { 23 | case v1.Host: 24 | host := config.Config.(v1.Host) 25 | hostIndex[host.GetId()] = host 26 | platforms[host.GetPlatform()] = append(platforms[host.GetPlatform()], host) 27 | } 28 | } 29 | 30 | for platform, hosts := range platforms { 31 | if len(hosts) == 1 { 32 | logger.Infof("[%s] skipping analysis on a single host", platform) 33 | continue 34 | } 35 | logger.Infof("[%s] %d hosts", platform, len(hosts)) 36 | hostPatches := make(map[string]map[string]string) 37 | allPatches := make(map[string]bool) 38 | for _, host := range hosts { 39 | for _, patch := range host.GetPatches() { 40 | if _, ok := hostPatches[host.GetId()]; !ok { 41 | hostPatches[host.GetId()] = make(map[string]string) 42 | } 43 | hostPatches[host.GetId()][patch.GetTitle()] = patch.GetVersion() 44 | allPatches[patch.GetTitle()] = true 45 | } 46 | } 47 | 48 | logger.Infof("Unique Patches: %d, Hosts with patches: %d, Hosts without: %d", len(allPatches), len(hostPatches), len(hosts)-len(hostPatches)) 49 | for _, patches := range hostPatches { 50 | for patch := range allPatches { 51 | if _, found := patches[patch]; !found { 52 | allPatches[patch] = false 53 | } 54 | } 55 | } 56 | 57 | appliedByHost := map[string][]string{} 58 | notAppliedByHost := map[string][]string{} 59 | for patch, common := range allPatches { 60 | if common { 61 | continue 62 | } 63 | appliedHosts := []string{} 64 | for host, patches := range hostPatches { 65 | if _, ok := patches[patch]; ok { 66 | appliedHosts = append(appliedHosts, hostIndex[host].GetHostname()) 67 | } 68 | } 69 | 70 | notApplied := []string{} 71 | for _, host := range hosts { 72 | if !inSlice(host.GetHostname(), appliedHosts) { 73 | notApplied = append(notApplied, host.GetHostname()) 74 | } 75 | } 76 | 77 | if len(notApplied) == 1 { 78 | notAppliedByHost[notApplied[0]] = append(notAppliedByHost[notApplied[0]], patch) 79 | } else if len(appliedHosts) == 1 { 80 | appliedByHost[appliedHosts[0]] = append(appliedByHost[appliedHosts[0]], patch) 81 | } else if len(notApplied) > len(appliedHosts) { 82 | result.Messages = append(result.Messages, fmt.Sprintf("%s is only applied to %s", patch, strings.Join(appliedHosts, ","))) 83 | } else { 84 | result.Messages = append(result.Messages, fmt.Sprintf("%s is not applied to %s", patch, strings.Join(notApplied, ","))) 85 | } 86 | } 87 | for host, patches := range notAppliedByHost { 88 | sort.Strings(patches) 89 | result.Messages = append(result.Messages, fmt.Sprintf("%s has not applied\n\t%s", host, strings.Join(patches, "\n\t"))) 90 | } 91 | for host, patches := range appliedByHost { 92 | sort.Strings(patches) 93 | result.Messages = append(result.Messages, fmt.Sprintf("%s has only applied \n\t%s", host, strings.Join(patches, "\n\t"))) 94 | } 95 | } 96 | return result 97 | } 98 | 99 | func inSlice(v string, in []string) bool { 100 | for _, i := range in { 101 | if i == v { 102 | return true 103 | } 104 | } 105 | return false 106 | } 107 | -------------------------------------------------------------------------------- /api/global.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | v1 "github.com/flanksource/config-db/api/v1" 5 | "github.com/flanksource/duty/upstream" 6 | "k8s.io/client-go/kubernetes" 7 | "k8s.io/client-go/rest" 8 | ) 9 | 10 | var ( 11 | KubernetesClient kubernetes.Interface 12 | KubernetesRestConfig *rest.Config 13 | Namespace string 14 | 15 | UpstreamConfig upstream.UpstreamConfig 16 | ) 17 | 18 | const MissionControlConfigTypePrefix = "MissionControl::" 19 | 20 | type Scraper interface { 21 | Scrape(ctx ScrapeContext) v1.ScrapeResults 22 | CanScrape(config v1.ScraperSpec) bool 23 | } 24 | -------------------------------------------------------------------------------- /api/v1/aws_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestAWS_Includes(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | config AWS 13 | resource string 14 | want bool 15 | }{ 16 | { 17 | name: "empty include list, not in default exclusions", 18 | config: AWS{}, 19 | resource: "ec2", 20 | want: true, 21 | }, 22 | { 23 | name: "empty include list, in default exclusions", 24 | config: AWS{}, 25 | resource: "ECSTASKDEFINITION", 26 | want: false, 27 | }, 28 | { 29 | name: "explicit inclusion of default exclusion", 30 | config: AWS{Include: []string{"EcsTaskDefinition"}}, 31 | resource: "ECSTASKDEFINITION", 32 | want: true, 33 | }, 34 | { 35 | name: "non-empty include list, resource included", 36 | config: AWS{ 37 | Include: []string{"s3", "ec2", "rds"}, 38 | }, 39 | resource: "ec2", 40 | want: true, 41 | }, 42 | { 43 | name: "non-empty include list, resource not included", 44 | config: AWS{ 45 | Include: []string{"s3", "ec2", "rds"}, 46 | }, 47 | resource: "lambda", 48 | want: false, 49 | }, 50 | { 51 | name: "case-insensitive include", 52 | config: AWS{ 53 | Include: []string{"S3", "EC2", "RDS"}, 54 | }, 55 | resource: "ec2", 56 | want: true, 57 | }, 58 | } 59 | 60 | for _, tt := range tests { 61 | t.Run(tt.name, func(t *testing.T) { 62 | got := tt.config.Includes(tt.resource) 63 | assert.Equal(t, tt.want, got) 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /api/v1/azure.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/flanksource/commons/collections" 5 | "github.com/flanksource/duty/types" 6 | ) 7 | 8 | type AzureDevops struct { 9 | BaseScraper `json:",inline"` 10 | ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` 11 | Organization string `yaml:"organization,omitempty" json:"organization,omitempty"` 12 | PersonalAccessToken types.EnvVar `yaml:"personalAccessToken,omitempty" json:"personalAccessToken,omitempty"` 13 | Projects []string `yaml:"projects" json:"projects"` 14 | Pipelines []string `yaml:"pipelines" json:"pipelines"` 15 | } 16 | 17 | type Entra struct { 18 | Users []types.ResourceSelector `yaml:"users,omitempty" json:"users,omitempty"` 19 | Groups []types.ResourceSelector `yaml:"groups,omitempty" json:"groups,omitempty"` 20 | AppRegistrations []types.ResourceSelector `yaml:"appRegistrations,omitempty" json:"appRegistrations,omitempty"` 21 | EnterpriseApps []types.ResourceSelector `yaml:"enterpriseApps,omitempty" json:"enterpriseApps,omitempty"` 22 | AppRoleAssignments []types.ResourceSelector `yaml:"appRoleAssignments,omitempty" json:"appRoleAssignments,omitempty"` 23 | } 24 | 25 | type Azure struct { 26 | BaseScraper `json:",inline"` 27 | ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` 28 | SubscriptionID string `yaml:"subscriptionID" json:"subscriptionID"` 29 | ClientID types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` 30 | ClientSecret types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"` 31 | TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"` 32 | Include []string `yaml:"include,omitempty" json:"include,omitempty"` 33 | Exclusions *AzureExclusions `yaml:"exclusions,omitempty" json:"exclusions,omitempty"` 34 | Entra *Entra `yaml:"entra,omitempty" json:"entra,omitempty"` 35 | } 36 | 37 | func (azure Azure) Includes(resource string) bool { 38 | if len(azure.Include) == 0 { 39 | return true 40 | } 41 | return collections.MatchItems(resource, azure.Include...) 42 | } 43 | 44 | type AzureExclusions struct { 45 | // ActivityLogs is a list of operations to exclude from activity logs. 46 | // Example: 47 | // "Microsoft.ContainerService/managedClusters/listClusterAdminCredential/action" 48 | // "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action" 49 | ActivityLogs []string `yaml:"activityLogs,omitempty" json:"activityLogs,omitempty"` 50 | } 51 | -------------------------------------------------------------------------------- /api/v1/clickhouse.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/flanksource/duty/connection" 7 | ) 8 | 9 | type Clickhouse struct { 10 | BaseScraper `yaml:",inline" json:",inline"` 11 | AWSS3 *AWSS3 `json:"awsS3,omitempty"` 12 | AzureBlobStorage *AzureBlobStorage `json:"azureBlobStorage,omitempty"` 13 | 14 | // clickhouse://:@:/?param1=value1¶m2=value2 15 | ClickhouseURL string `json:"clickhouseURL,omitempty"` 16 | Query string `json:"query"` 17 | } 18 | 19 | type AzureBlobStorage struct { 20 | *connection.AzureConnection `yaml:",inline" json:",inline"` 21 | 22 | Account string `json:"account,omitempty"` 23 | Container string `json:"container,omitempty"` 24 | Path string `json:"path,omitempty"` 25 | EndpointSuffix string `json:"endpoint,omitempty"` 26 | CollectionName string `json:"collection"` 27 | } 28 | 29 | type AWSS3 struct { 30 | *AWSConnection `yaml:",inline" json:",inline"` 31 | 32 | Bucket string `json:"bucket,omitempty"` 33 | Path string `json:"path,omitempty"` 34 | Endpoint string `json:"endpoint,omitempty"` 35 | } 36 | 37 | func (az AzureBlobStorage) GetAccountKeyCommand() string { 38 | return fmt.Sprintf(`az storage account keys list -n %s | jq -r '.[0].value'`, az.Account) 39 | } 40 | 41 | func (az AzureBlobStorage) GetConnectionString(accKey string) string { 42 | ep := "core.windows.net" 43 | if az.EndpointSuffix != "" { 44 | ep = az.EndpointSuffix 45 | } 46 | return fmt.Sprintf("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=%s", az.Account, accKey, ep) 47 | } 48 | -------------------------------------------------------------------------------- /api/v1/config.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "regexp" 8 | "strings" 9 | 10 | yamlutil "k8s.io/apimachinery/pkg/util/yaml" 11 | ) 12 | 13 | var yamlDividerRegexp = regexp.MustCompile(`(?m)^---\n`) 14 | 15 | func readFile(path string) (string, error) { 16 | var data []byte 17 | var err error 18 | if path == "-" { 19 | if data, err = io.ReadAll(os.Stdin); err != nil { 20 | return "", err 21 | } 22 | } else { 23 | if data, err = os.ReadFile(path); err != nil { 24 | return "", err 25 | } 26 | } 27 | return string(data), nil 28 | } 29 | 30 | func ParseConfigs(files ...string) ([]ScrapeConfig, error) { 31 | scrapers := make([]ScrapeConfig, 0, len(files)) 32 | 33 | for _, f := range files { 34 | _scrapers, err := parseConfig(f) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | scrapers = append(scrapers, _scrapers...) 40 | } 41 | 42 | return scrapers, nil 43 | } 44 | 45 | // ParseConfig : Read config file 46 | func parseConfig(configfile string) ([]ScrapeConfig, error) { 47 | configs, err := readFile(configfile) 48 | if err != nil { 49 | return nil, fmt.Errorf("error reading config file=%s: %w", configfile, err) 50 | } 51 | 52 | var scrapers []ScrapeConfig 53 | for _, chunk := range yamlDividerRegexp.Split(configs, -1) { 54 | if strings.TrimSpace(chunk) == "" { 55 | continue 56 | } 57 | 58 | var config ScrapeConfig 59 | decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(chunk), 1024) 60 | if err := decoder.Decode(&config); err != nil { 61 | return nil, fmt.Errorf("error decoding yaml. file=%s: %w", configfile, err) 62 | } 63 | 64 | scrapers = append(scrapers, config) 65 | } 66 | 67 | return scrapers, nil 68 | } 69 | -------------------------------------------------------------------------------- /api/v1/const.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | type ChangeAction string 4 | 5 | var ( 6 | Delete ChangeAction = "delete" 7 | Ignore ChangeAction = "ignore" 8 | ) 9 | 10 | const ( 11 | ChangeTypeDiff = "diff" 12 | ) 13 | -------------------------------------------------------------------------------- /api/v1/data.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | // Host ... 4 | // +kubebuilder:object:generate=false 5 | type Host interface { 6 | GetHostname() string 7 | GetPlatform() string 8 | GetId() string 9 | GetIP() string 10 | GetPatches() []Patch 11 | } 12 | 13 | // Patch ... 14 | // +kubebuilder:object:generate=false 15 | type Patch interface { 16 | GetName() string 17 | GetVersion() string 18 | GetTitle() string 19 | IsInstalled() bool 20 | IsMissing() bool 21 | IsPendingReboot() bool 22 | IsFailed() bool 23 | } 24 | 25 | // Properties ... 26 | type Properties []Property 27 | 28 | // Property ... 29 | type Property struct { 30 | Name string `json:"name"` 31 | // Line comments or description associated with this property 32 | Description string `json:"description,omitempty"` 33 | Value string `json:"value,omitempty"` 34 | Type string `json:"type,omitempty"` 35 | GitLocation *GitLocation `json:"location,omitempty"` 36 | FileLocation *FileLocation `json:"fileLocation,omitempty"` 37 | // A path to an OpenAPI spec and fieldRef that describes the field 38 | OpenAPI *OpenAPIFieldRef `json:"openapiRef,omitempty"` 39 | } 40 | 41 | // FileLocation ... 42 | type FileLocation struct { 43 | Host string `json:"host,omitempty"` 44 | FilePath string `json:"filePath"` 45 | LineNumber int `json:"lineNumber"` 46 | } 47 | 48 | // GitLocation ... 49 | type GitLocation struct { 50 | Repository string `json:"repository"` 51 | FilePath string `json:"filePath"` 52 | LineNumber int `json:"lineNumber"` 53 | GitRef string `json:"gitRef"` 54 | } 55 | 56 | // OpenAPIFieldRef ... 57 | type OpenAPIFieldRef struct { 58 | // Location of the OpenAPI spec 59 | Location string `json:"location,omitempty"` 60 | // Reference to the field 61 | FieldRef string `json:"fieldRef,omitempty"` 62 | } 63 | -------------------------------------------------------------------------------- /api/v1/file.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/flanksource/duty/models" 7 | ) 8 | 9 | // File ... 10 | type File struct { 11 | BaseScraper `json:",inline"` 12 | URL string `json:"url,omitempty" yaml:"url,omitempty"` 13 | Paths []string `json:"paths,omitempty" yaml:"paths,omitempty"` 14 | Ignore []string `json:"ignore,omitempty" yaml:"ignore,omitempty"` 15 | Format string `json:"format,omitempty" yaml:"format,omitempty"` 16 | Icon string `json:"icon,omitempty" yaml:"icon,omitempty"` 17 | 18 | // ConnectionName is used to populate the URL 19 | ConnectionName string `json:"connection,omitempty" yaml:"connection,omitempty"` 20 | } 21 | 22 | func (f File) RedactedString() string { 23 | if f.URL == "" { 24 | return f.URL 25 | } 26 | 27 | url, err := url.Parse(f.URL) 28 | if err != nil { 29 | return f.URL 30 | } 31 | 32 | return url.Redacted() 33 | } 34 | 35 | func (t File) GetConnection() *models.Connection { 36 | return &models.Connection{ 37 | URL: t.URL, 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /api/v1/gcp.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/flanksource/duty/connection" 7 | ) 8 | 9 | const ( 10 | GCSBucket = "GCP::Bucket" 11 | GKECluster = "GCP::GKECluster" 12 | RedisInstance = "GCP::Redis" 13 | MemcacheInstance = "GCP::MemCache" 14 | PubSubTopic = "GCP::PubSub" 15 | CloudSQLInstance = "GCP::CloudSQL" 16 | IAMRole = "GCP::IAMRole" 17 | IAMServiceAccount = "GCP::ServiceAccount" 18 | ) 19 | 20 | type GCP struct { 21 | BaseScraper `json:",inline"` 22 | connection.GCPConnection `json:",inline"` 23 | Project string `json:"project"` 24 | Include []string `json:"include,omitempty"` 25 | Exclude []string `json:"exclude,omitempty"` 26 | } 27 | 28 | func (gcp GCP) Includes(resource string) bool { 29 | if len(gcp.Include) == 0 { 30 | return true 31 | } 32 | for _, include := range gcp.Include { 33 | if strings.EqualFold(include, resource) { 34 | return true 35 | } 36 | } 37 | return false 38 | } 39 | 40 | func (gcp GCP) Excludes(resource string) bool { 41 | if len(gcp.Exclude) == 0 { 42 | return false 43 | } 44 | for _, exclude := range gcp.Exclude { 45 | if strings.EqualFold(exclude, resource) { 46 | return true 47 | } 48 | } 49 | return false 50 | } 51 | -------------------------------------------------------------------------------- /api/v1/github.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "github.com/flanksource/duty/types" 4 | 5 | type GitHubActions struct { 6 | BaseScraper `json:",inline"` 7 | Owner string `yaml:"owner" json:"owner"` 8 | Repository string `yaml:"repository" json:"repository"` 9 | PersonalAccessToken types.EnvVar `yaml:"personalAccessToken" json:"personalAccessToken"` 10 | // ConnectionName, if provided, will be used to populate personalAccessToken 11 | ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"` 12 | Workflows []string `yaml:"workflows" json:"workflows"` 13 | } 14 | -------------------------------------------------------------------------------- /api/v1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package v1 contains API Schema definitions for the configs v1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=configs.flanksource.com 20 | package v1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "configs.flanksource.com", Version: "v1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /api/v1/http.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/flanksource/duty/connection" 5 | "github.com/flanksource/duty/types" 6 | ) 7 | 8 | type HTTP struct { 9 | BaseScraper `json:",inline"` 10 | connection.HTTPConnection `json:",inline"` 11 | // Environment variables to be used in the templating. 12 | Env []types.EnvVar `json:"env,omitempty"` 13 | Method *string `json:"method,omitempty"` 14 | Body *string `json:"body,omitempty"` 15 | Headers map[string]types.EnvVar `json:"headers,omitempty"` 16 | } 17 | -------------------------------------------------------------------------------- /api/v1/interface_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestChangeSummary_Merge(t *testing.T) { 8 | tests := []struct { 9 | name string 10 | summary1 ChangeSummary 11 | summary2 ChangeSummary 12 | expected ChangeSummary 13 | }{ 14 | { 15 | name: "merge empty summaries", 16 | summary1: ChangeSummary{}, 17 | summary2: ChangeSummary{}, 18 | expected: ChangeSummary{}, 19 | }, 20 | { 21 | name: "merge summaries with orphaned changes", 22 | summary1: ChangeSummary{ 23 | Orphaned: map[string]int{ 24 | "foo": 1, 25 | "bar": 2, 26 | }, 27 | }, 28 | summary2: ChangeSummary{ 29 | Orphaned: map[string]int{ 30 | "foo": 3, 31 | "baz": 4, 32 | }, 33 | }, 34 | expected: ChangeSummary{ 35 | Orphaned: map[string]int{ 36 | "foo": 4, 37 | "bar": 2, 38 | "baz": 4, 39 | }, 40 | }, 41 | }, 42 | { 43 | name: "merge summaries with ignored changes", 44 | summary1: ChangeSummary{ 45 | Ignored: map[string]int{ 46 | "foo": 1, 47 | "bar": 2, 48 | }, 49 | }, 50 | summary2: ChangeSummary{ 51 | Ignored: map[string]int{ 52 | "foo": 3, 53 | "baz": 4, 54 | }, 55 | }, 56 | expected: ChangeSummary{ 57 | Ignored: map[string]int{ 58 | "foo": 4, 59 | "bar": 2, 60 | "baz": 4, 61 | }, 62 | }, 63 | }, 64 | { 65 | name: "merge summaries with both orphaned and ignored changes", 66 | summary1: ChangeSummary{ 67 | Orphaned: map[string]int{ 68 | "foo": 1, 69 | "bar": 2, 70 | }, 71 | Ignored: map[string]int{ 72 | "baz": 3, 73 | "qux": 4, 74 | }, 75 | }, 76 | summary2: ChangeSummary{ 77 | Orphaned: map[string]int{ 78 | "foo": 5, 79 | "quux": 6, 80 | }, 81 | Ignored: map[string]int{ 82 | "baz": 7, 83 | "corge": 8, 84 | }, 85 | }, 86 | expected: ChangeSummary{ 87 | Orphaned: map[string]int{ 88 | "foo": 6, 89 | "bar": 2, 90 | "quux": 6, 91 | }, 92 | Ignored: map[string]int{ 93 | "baz": 10, 94 | "qux": 4, 95 | "corge": 8, 96 | }, 97 | }, 98 | }, 99 | } 100 | 101 | for _, tt := range tests { 102 | t.Run(tt.name, func(t *testing.T) { 103 | tt.summary1.Merge(tt.summary2) 104 | if len(tt.summary1.Orphaned) != len(tt.expected.Orphaned) { 105 | t.Errorf("Expected %d orphaned changes, got %d", len(tt.expected.Orphaned), len(tt.summary1.Orphaned)) 106 | } 107 | for k, v := range tt.expected.Orphaned { 108 | if tt.summary1.Orphaned[k] != v { 109 | t.Errorf("Expected %d orphaned changes for %s, got %d", v, k, tt.summary1.Orphaned[k]) 110 | } 111 | } 112 | if len(tt.summary1.Ignored) != len(tt.expected.Ignored) { 113 | t.Errorf("Expected %d ignored changes, got %d", len(tt.expected.Ignored), len(tt.summary1.Ignored)) 114 | } 115 | for k, v := range tt.expected.Ignored { 116 | if tt.summary1.Ignored[k] != v { 117 | t.Errorf("Expected %d ignored changes for %s, got %d", v, k, tt.summary1.Ignored[k]) 118 | } 119 | } 120 | }) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /api/v1/kubernetes_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestKubernetesConfigExclusions_Filter(t *testing.T) { 8 | type args struct { 9 | name string 10 | namespace string 11 | kind string 12 | labels map[string]string 13 | } 14 | tests := []struct { 15 | name string 16 | config KubernetesExclusionConfig 17 | args args 18 | shouldExclude bool 19 | }{ 20 | { 21 | name: "exclusion by name", 22 | config: KubernetesExclusionConfig{ 23 | Names: []string{"junit-*"}, 24 | }, 25 | args: args{ 26 | name: "junit-123", 27 | }, 28 | shouldExclude: true, 29 | }, 30 | { 31 | name: "exclusion by namespace", 32 | config: KubernetesExclusionConfig{ 33 | Namespaces: []string{"*-canaries"}, 34 | }, 35 | args: args{ 36 | namespace: "customer-canaries", 37 | }, 38 | shouldExclude: true, 39 | }, 40 | { 41 | name: "exclusion by kind", 42 | config: KubernetesExclusionConfig{ 43 | Kinds: []string{"*Chart"}, 44 | }, 45 | args: args{ 46 | kind: "HelmChart", 47 | }, 48 | shouldExclude: true, 49 | }, 50 | { 51 | name: "exclusion by labels | exact match", 52 | config: KubernetesExclusionConfig{ 53 | Labels: map[string]string{ 54 | "prod": "env", 55 | }, 56 | }, 57 | args: args{ 58 | labels: map[string]string{ 59 | "prod": "env", 60 | }, 61 | }, 62 | shouldExclude: true, 63 | }, 64 | { 65 | name: "exclusion by labels | one matches", 66 | config: KubernetesExclusionConfig{ 67 | Labels: map[string]string{ 68 | "prod": "env", 69 | "is-billed": "true", 70 | "trace-enabled": "true", 71 | }, 72 | }, 73 | args: args{ 74 | labels: map[string]string{ 75 | "prod": "env", 76 | "trace-enabled": "false", 77 | }, 78 | }, 79 | shouldExclude: true, 80 | }, 81 | { 82 | name: "no exclusions", 83 | config: KubernetesExclusionConfig{}, 84 | args: args{ 85 | namespace: "default", 86 | name: "test-foo", 87 | }, 88 | shouldExclude: false, 89 | }, 90 | } 91 | for _, tt := range tests { 92 | t.Run(tt.name, func(t *testing.T) { 93 | if got := tt.config.Filter(tt.args.name, tt.args.namespace, tt.args.kind, tt.args.labels); got != tt.shouldExclude { 94 | t.Errorf("KubernetesConfigExclusions.Filter() = %v, want %v", got, tt.shouldExclude) 95 | } 96 | }) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /api/v1/scrapeplugin_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/flanksource/duty/models" 8 | "github.com/google/uuid" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | ) 11 | 12 | // ScrapePluginStatus defines the observed state of Plugin 13 | type ScrapePluginStatus struct { 14 | ObservedGeneration int64 `json:"observedGeneration,omitempty" protobuf:"varint,3,opt,name=observedGeneration"` 15 | } 16 | 17 | //+kubebuilder:object:root=true 18 | //+kubebuilder:subresource:status 19 | 20 | // ScrapePlugin is the Schema for the scraper plugins 21 | type ScrapePlugin struct { 22 | metav1.TypeMeta `json:",inline"` 23 | metav1.ObjectMeta `json:"metadata,omitempty"` 24 | 25 | Spec ScrapePluginSpec `json:"spec,omitempty"` 26 | Status ScrapePluginStatus `json:"status,omitempty"` 27 | } 28 | 29 | type ScrapePluginSpec struct { 30 | Change TransformChange `json:"changes,omitempty"` 31 | 32 | // Relationship allows you to form relationships between config items using selectors. 33 | Relationship []RelationshipConfig `json:"relationship,omitempty"` 34 | 35 | // Properties are custom templatable properties for the scraped config items 36 | // grouped by the config type. 37 | Properties []ConfigProperties `json:"properties,omitempty" template:"true"` 38 | } 39 | 40 | func (t ScrapePlugin) ToModel() (*models.ScrapePlugin, error) { 41 | var id uuid.UUID 42 | if v, err := uuid.Parse(string(t.GetUID())); err == nil { 43 | id = v 44 | } 45 | 46 | specJSON, err := json.Marshal(t.Spec) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | return &models.ScrapePlugin{ 52 | ID: id, 53 | Name: t.Name, 54 | Namespace: t.Namespace, 55 | Spec: specJSON, 56 | CreatedAt: t.CreationTimestamp.Time, 57 | }, nil 58 | } 59 | 60 | func (t ScrapePlugin) LoggerName() string { 61 | return fmt.Sprintf("plugin.%s.%s", t.Namespace, t.Name) 62 | } 63 | 64 | func (t ScrapePlugin) GetContext() map[string]any { 65 | return map[string]any{ 66 | "namespace": t.Namespace, 67 | "name": t.Name, 68 | "scraper_id": t.GetPersistedID(), 69 | } 70 | } 71 | 72 | func (t *ScrapePlugin) GetPersistedID() *uuid.UUID { 73 | if t.GetUID() == "" { 74 | return nil 75 | } 76 | 77 | u, _ := uuid.Parse(string(t.GetUID())) 78 | return &u 79 | } 80 | 81 | //+kubebuilder:object:root=true 82 | 83 | // ScrapePluginList contains a list of Plugin 84 | type ScrapePluginList struct { 85 | metav1.TypeMeta `json:",inline"` 86 | metav1.ListMeta `json:"metadata,omitempty"` 87 | Items []ScrapePlugin `json:"items"` 88 | } 89 | 90 | func init() { 91 | SchemeBuilder.Register(&ScrapePlugin{}, &ScrapePluginList{}) 92 | } 93 | -------------------------------------------------------------------------------- /api/v1/slack.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import "github.com/flanksource/duty/types" 4 | 5 | type Slack struct { 6 | BaseScraper `json:",inline"` 7 | 8 | // Slack token 9 | Token types.EnvVar `yaml:"token" json:"token"` 10 | 11 | // Fetch the messages since this period. 12 | // Default: 7d 13 | // 14 | // Specify the duration string. 15 | // eg: 1h, 7d, ... 16 | Since string `yaml:"since,omitempty" json:"since,omitempty"` 17 | 18 | // Process messages from these channels and discard others. 19 | // If empty, all channels are matched. 20 | Channels types.MatchExpressions `yaml:"channels,omitempty" json:"channels,omitempty"` 21 | 22 | // Rules define the change extraction rules. 23 | // +kubebuilder:validation:MinItems=1 24 | Rules []SlackChangeExtractionRule `yaml:"rules" json:"rules"` 25 | } 26 | 27 | type SlackChangeExtractionRule struct { 28 | ChangeExtractionRule `json:",inline" yaml:",inline"` 29 | 30 | // Only those messages matching this filter will be processed. 31 | Filter *SlackChangeAcceptanceFilter `yaml:"filter,omitempty" json:"filter,omitempty"` 32 | } 33 | 34 | type SlackChangeAcceptanceFilter struct { 35 | // Bot name to match 36 | Bot types.MatchExpression `yaml:"bot,omitempty" json:"bot,omitempty"` 37 | 38 | // Slack User to match 39 | User SlackUserFilter `yaml:"user,omitempty" json:"user,omitempty"` 40 | 41 | // Must match the given expression 42 | Expr types.CelExpression `yaml:"expr,omitempty" json:"expr,omitempty"` 43 | } 44 | 45 | type SlackUserFilter struct { 46 | Name types.MatchExpression `yaml:"name,omitempty" json:"name,omitempty"` 47 | DisplayName types.MatchExpression `yaml:"displayName,omitempty" json:"displayName,omitempty"` 48 | } 49 | -------------------------------------------------------------------------------- /api/v1/sql.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | type SQL struct { 4 | BaseScraper `json:",inline"` 5 | Connection `json:",inline"` 6 | Driver string `json:"driver,omitempty"` 7 | Query string `json:"query"` 8 | } 9 | -------------------------------------------------------------------------------- /api/v1/terraform.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/flanksource/duty/connection" 8 | "github.com/flanksource/duty/context" 9 | "github.com/flanksource/duty/models" 10 | "github.com/flanksource/duty/types" 11 | ) 12 | 13 | type TerraformStateSource struct { 14 | S3 *connection.S3Connection `json:"s3,omitempty"` 15 | GCS *connection.GCSConnection `json:"gcs,omitempty"` 16 | Local string `json:"local,omitempty"` 17 | } 18 | 19 | func (t *TerraformStateSource) Path() string { 20 | if t.Local != "" { 21 | return t.Local 22 | } 23 | 24 | if t.S3 != nil { 25 | return t.S3.ObjectPath 26 | } 27 | 28 | if t.GCS != nil { 29 | // TODO: 30 | return "" 31 | } 32 | 33 | return "" 34 | } 35 | 36 | func (t *TerraformStateSource) Connection(ctx context.Context) (*models.Connection, error) { 37 | if t.Local != "" { 38 | return &models.Connection{Type: models.ConnectionTypeFolder}, nil 39 | } 40 | 41 | if t.S3 != nil { 42 | if err := t.S3.Populate(ctx); err != nil { 43 | return nil, fmt.Errorf("failed to populate S3 connection: %v", err) 44 | } 45 | 46 | connection := &models.Connection{Type: models.ConnectionTypeS3} 47 | connection, err := connection.Merge(ctx, t.S3) 48 | if err != nil { 49 | return nil, fmt.Errorf("failed to merge S3 connection: %v", err) 50 | } 51 | 52 | return connection, nil 53 | } 54 | 55 | if t.GCS != nil { 56 | if err := t.GCS.HydrateConnection(ctx); err != nil { 57 | return nil, fmt.Errorf("failed to populate GCP connection: %v", err) 58 | } 59 | 60 | connection := &models.Connection{Type: models.ConnectionTypeGCP} 61 | connection, err := connection.Merge(ctx, t.GCS) 62 | if err != nil { 63 | return nil, fmt.Errorf("failed to merge GCP connection: %v", err) 64 | } 65 | 66 | return connection, nil 67 | } 68 | 69 | return nil, errors.New("state source is empty") 70 | } 71 | 72 | type Terraform struct { 73 | BaseScraper `json:",inline"` 74 | Name types.GoTemplate `json:"name"` 75 | State TerraformStateSource `json:"state"` 76 | } 77 | -------------------------------------------------------------------------------- /api/v1/trivy.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type Trivy struct { 8 | BaseScraper `json:",inline"` 9 | 10 | // Common Trivy Flags ... 11 | Version string `json:"version,omitempty" yaml:"version,omitempty"` // Specify the version of Trivy to use 12 | Compliance []string `json:"compliance,omitempty" yaml:"compliance,omitempty"` 13 | IgnoredLicenses []string `json:"ignoredLicenses,omitempty" yaml:"ignoredLicenses,omitempty"` 14 | IgnoreUnfixed bool `json:"ignoreUnfixed,omitempty" yaml:"ignoreUnfixed,omitempty"` 15 | LicenseFull bool `json:"licenseFull,omitempty" yaml:"licenseFull,omitempty"` 16 | Severity []string `json:"severity,omitempty" yaml:"severity,omitempty"` 17 | VulnType []string `json:"vulnType,omitempty" yaml:"vulnType,omitempty"` 18 | Scanners []string `json:"scanners,omitempty" yaml:"scanners,omitempty"` 19 | Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` 20 | 21 | Kubernetes *TrivyK8sOptions `json:"kubernetes,omitempty"` 22 | } 23 | 24 | func (t Trivy) IsEmpty() bool { 25 | return t.Kubernetes == nil 26 | } 27 | 28 | // GetK8sArgs returns a slice of arguments that Trivy uses to scan Kubernetes objects. 29 | func (t Trivy) GetK8sArgs() []string { 30 | var args []string 31 | args = append(args, "k8s") 32 | args = append(args, "--format", "json") // hardcoded here. don't allow users this option. 33 | args = append(args, t.getCommonArgs()...) 34 | args = append(args, t.Kubernetes.getArgs()...) 35 | args = append(args, "all") 36 | return args 37 | } 38 | 39 | func (t Trivy) getCommonArgs() []string { 40 | var args []string 41 | if len(t.Compliance) > 0 { 42 | args = append(args, "--compliance", strings.Join(t.Compliance, ",")) 43 | } 44 | if len(t.IgnoredLicenses) > 0 { 45 | args = append(args, "--ignored-licenses", strings.Join(t.IgnoredLicenses, ",")) 46 | } 47 | if t.IgnoreUnfixed { 48 | args = append(args, "--ignore-unfixed") 49 | } 50 | if t.LicenseFull { 51 | args = append(args, "--license-full") 52 | } 53 | if len(t.Severity) > 0 { 54 | args = append(args, "--severity", strings.Join(t.Severity, ",")) 55 | } 56 | if len(t.VulnType) > 0 { 57 | args = append(args, "--vuln-type", strings.Join(t.VulnType, ",")) 58 | } 59 | if len(t.Scanners) > 0 { 60 | args = append(args, "--scanners", strings.Join(t.Scanners, ",")) 61 | } 62 | if t.Timeout != "" { 63 | args = append(args, "--timeout", t.Timeout) 64 | } 65 | 66 | return args 67 | } 68 | 69 | // TrivyK8sOptions holds in Trivy flags that are Kubernetes specific. 70 | type TrivyK8sOptions struct { 71 | Components []string `json:"components,omitempty" yaml:"components,omitempty"` 72 | Context string `json:"context,omitempty" yaml:"context,omitempty"` 73 | Kubeconfig string `json:"kubeconfig,omitempty" yaml:"kubeconfig,omitempty"` 74 | Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` 75 | } 76 | 77 | func (t TrivyK8sOptions) getArgs() []string { 78 | var args []string 79 | if len(t.Components) > 0 { 80 | args = append(args, "--components", strings.Join(t.Components, ",")) 81 | } 82 | if t.Kubeconfig != "" { 83 | args = append(args, "--kubeconfig", t.Kubeconfig) 84 | } 85 | if t.Namespace != "" { 86 | args = append(args, "--namespace", t.Namespace) 87 | } 88 | if t.Context != "" { 89 | args = append(args, "--context", t.Context) 90 | } 91 | return args 92 | } 93 | -------------------------------------------------------------------------------- /build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:bookworm@sha256:29fe4376919e25b7587a1063d7b521d9db735fc137d3cf30ae41eb326d209471 AS rust-builder 2 | 3 | WORKDIR /app 4 | COPY Makefile /app 5 | COPY external/diffgen /app/external/diffgen 6 | RUN make rust-diffgen 7 | 8 | FROM golang:1.24.3-bookworm@sha256:89a04cc2e2fbafef82d4a45523d4d4ae4ecaf11a197689036df35fef3bde444a AS builder 9 | WORKDIR /app 10 | 11 | ARG VERSION 12 | 13 | COPY go.mod /app/go.mod 14 | COPY go.sum /app/go.sum 15 | RUN go mod download 16 | 17 | COPY ./ ./ 18 | 19 | COPY --from=rust-builder /app/external/diffgen/target ./external/diffgen/target 20 | 21 | RUN make build-prod 22 | 23 | FROM flanksource/base-image:v0.0.7@sha256:c3cda640ca7033a89e52c7f27776edfc95f825ece4b49de3b9c5af981d34a44e 24 | WORKDIR /app 25 | 26 | COPY --from=builder /app/.bin/config-db /app 27 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 28 | 29 | RUN mkdir /opt/database && groupadd --gid 1000 catalog && \ 30 | useradd catalog --uid 1000 -g catalog -m -d /var/lib/catalog && \ 31 | chown -R 1000:1000 /opt/database && chown -R 1000:1000 /app 32 | 33 | USER catalog:catalog 34 | 35 | RUN /app/config-db go-offline 36 | ENTRYPOINT ["/app/config-db"] 37 | -------------------------------------------------------------------------------- /build/Dockerfile.debug: -------------------------------------------------------------------------------- 1 | FROM rust:bookworm@sha256:29fe4376919e25b7587a1063d7b521d9db735fc137d3cf30ae41eb326d209471 AS rust-builder 2 | 3 | WORKDIR /app 4 | COPY Makefile /app 5 | COPY external/diffgen /app/external/diffgen 6 | RUN make rust-diffgen 7 | 8 | FROM golang:1.23.4@sha256:574185e5c6b9d09873f455a7c205ea0514bfd99738c5dc7750196403a44ed4b7 AS builder 9 | WORKDIR /app 10 | 11 | ARG VERSION 12 | 13 | COPY go.mod /app/go.mod 14 | COPY go.sum /app/go.sum 15 | RUN go mod download 16 | 17 | COPY ./ ./ 18 | 19 | COPY --from=rust-builder /app/external/diffgen/target ./external/diffgen/target 20 | 21 | RUN make build-debug 22 | 23 | FROM flanksource/base-image:v0.0.7@sha256:c3cda640ca7033a89e52c7f27776edfc95f825ece4b49de3b9c5af981d34a44e 24 | WORKDIR /app 25 | 26 | COPY --from=builder /app/.bin/config-db /app 27 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 28 | 29 | RUN mkdir /opt/database 30 | RUN /app/config-db go-offline 31 | ENTRYPOINT ["/app/config-db"] 32 | -------------------------------------------------------------------------------- /changes/fingerprint.go: -------------------------------------------------------------------------------- 1 | package changes 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "regexp" 9 | "sort" 10 | "time" 11 | 12 | "github.com/Jeffail/gabs/v2" 13 | "github.com/flanksource/config-db/db/models" 14 | "github.com/samber/lo" 15 | ) 16 | 17 | type Replacement struct { 18 | Value string 19 | Regex *regexp.Regexp 20 | } 21 | 22 | type Replacements []Replacement 23 | 24 | var tokenizer Replacements 25 | 26 | func init() { 27 | tokenizer = NewReplacements( 28 | "UUID", `\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b`, 29 | "TIMESTAMP", `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})`, 30 | "DURATION", `\s+\d+(.\d+){0,1}(ms|s|h|d|m)`, 31 | "SHA256", `[a-z0-9]{64}`, 32 | "NUMBER", `^\d+$`, 33 | "HEX16", `[0-9a-f]{16}`, // matches a 16 character long hex string 34 | ) 35 | } 36 | 37 | func NewReplacements(pairs ...string) Replacements { 38 | var r Replacements 39 | for i := 0; i < len(pairs)-1; i = i + 2 { 40 | r = append(r, Replacement{ 41 | Value: pairs[i], 42 | Regex: regexp.MustCompile(pairs[i+1]), 43 | }) 44 | } 45 | return r 46 | } 47 | 48 | func Fingerprint(change *models.ConfigChange) (string, error) { 49 | if change == nil { 50 | return "", nil 51 | } 52 | 53 | if change.Patches == "" && change.Details == nil { 54 | return "", nil 55 | } 56 | 57 | input := []byte(change.Patches) 58 | if len(input) == 0 { 59 | detailsJSON, err := json.Marshal(change.Details) 60 | if err != nil { 61 | return "", err 62 | } 63 | input = detailsJSON 64 | } 65 | 66 | container, err := gabs.ParseJSON(input) 67 | if err != nil { 68 | return "", err 69 | } 70 | 71 | flat, err := container.Flatten() 72 | if err != nil { 73 | return "", err 74 | } 75 | 76 | out := map[string]any{"__change_type": change.ChangeType} 77 | for k, v := range flat { 78 | out[k] = tokenizer.Tokenize(v) 79 | } 80 | 81 | // logger.GetLogger("fingerprint").Infof("in-->\n%s\nout-->\n%s\n", logger.Pretty(flat), logger.Pretty(out)) 82 | 83 | hash := Hash(out) 84 | return hash, nil 85 | } 86 | 87 | func Hash(data map[string]interface{}) string { 88 | keys := lo.Keys(data) 89 | sort.Strings(keys) 90 | h := md5.New() 91 | for _, k := range keys { 92 | h.Write([]byte(k)) 93 | h.Write([]byte(data[k].(string))) 94 | } 95 | 96 | return hex.EncodeToString(h.Sum(nil)[:]) 97 | } 98 | 99 | func (replacements Replacements) Tokenize(data interface{}) string { 100 | switch v := data.(type) { 101 | 102 | case int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64: 103 | return "0" 104 | case time.Duration: 105 | return "DURATION" 106 | case time.Time: 107 | return "TIMESTAMP" 108 | case string: 109 | out := v 110 | for _, r := range replacements { 111 | out = r.Regex.ReplaceAllString(out, r.Value) 112 | if out == r.Value { 113 | break 114 | } 115 | } 116 | return out 117 | } 118 | 119 | return fmt.Sprintf("%v", data) 120 | } 121 | -------------------------------------------------------------------------------- /changes/fingerprint_test.go: -------------------------------------------------------------------------------- 1 | package changes_test 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | 8 | "github.com/flanksource/config-db/changes" 9 | "github.com/flanksource/config-db/db/models" 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | func readChange(name string) *models.ConfigChange { 15 | data, err := os.ReadFile("testdata/" + name) 16 | if err != nil { 17 | Fail("failed to read test data file: " + err.Error()) 18 | } 19 | var change models.ConfigChange 20 | if err := json.Unmarshal(data, &change); err != nil { 21 | Fail("failed to unmarshal change from testdata: " + err.Error()) 22 | } 23 | return &change 24 | } 25 | 26 | var _ = Describe("Change Fingerprints", func() { 27 | It("Should calculate the same fingerprints for pod stop", func() { 28 | fp1, err1 := changes.Fingerprint(readChange("change_1.json")) 29 | fp3, err3 := changes.Fingerprint(readChange("change_3.json")) 30 | Expect(err1).ToNot(HaveOccurred()) 31 | Expect(err3).ToNot(HaveOccurred()) 32 | Expect(fp1).To(Equal(fp3)) 33 | }) 34 | 35 | It("Should calculate the same fingerprints for pod start", func() { 36 | fp2, err2 := changes.Fingerprint(readChange("change_2.json")) 37 | fp4, err4 := changes.Fingerprint(readChange("change_4.json")) 38 | Expect(err2).ToNot(HaveOccurred()) 39 | Expect(err4).ToNot(HaveOccurred()) 40 | Expect(fp2).To(Equal(fp4)) 41 | }) 42 | 43 | It("Should calculate the diff fingerprints for pod start", func() { 44 | fp1, err1 := changes.Fingerprint(readChange("change_1.json")) 45 | fp2, err2 := changes.Fingerprint(readChange("change_2.json")) 46 | Expect(err1).ToNot(HaveOccurred()) 47 | Expect(err2).ToNot(HaveOccurred()) 48 | Expect(fp1).ToNot(Equal(fp2)) 49 | }) 50 | 51 | It("durations should be ignored", func() { 52 | fp1, err1 := changes.Fingerprint(readChange("health_passed_1.json")) 53 | fp2, err2 := changes.Fingerprint(readChange("health_passed_2.json")) 54 | Expect(err1).ToNot(HaveOccurred()) 55 | Expect(err2).ToNot(HaveOccurred()) 56 | Expect(fp1).To(Equal(fp2)) 57 | }) 58 | 59 | It("16 character long hex should be ignored", func() { 60 | fp1, err1 := changes.Fingerprint(readChange("helm_upgrade_failed.json")) 61 | fp2, err2 := changes.Fingerprint(readChange("helm_upgrade_failed_2.json")) 62 | Expect(err1).ToNot(HaveOccurred()) 63 | Expect(err2).ToNot(HaveOccurred()) 64 | Expect(fp1).To(Equal(fp2)) 65 | }) 66 | }) 67 | 68 | func TestChangeFingerprints(t *testing.T) { 69 | RegisterFailHandler(Fail) 70 | RunSpecs(t, "Change Fingerprints Suite") 71 | } 72 | -------------------------------------------------------------------------------- /changes/testdata/health_passed_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "abf65d53-d011-429d-b659-ea20991acd42", 3 | "config_id": "3397183d-cb66-40bd-9257-3e2a86550b2b", 4 | "change_type": "HealthCheckPassed", 5 | "created_at": "2024-09-17T10:58:03.925589+00:00", 6 | "external_created_by": null, 7 | "source": "", 8 | "diff": "--- before\n+++ after\n@@ -74,7 +74,7 @@\n \"type\": \"Ready\"\n },\n {\n- \"message\": \"Health check passed in 50.309434ms\",\n+ \"message\": \"Health check passed in 83.594302ms\",\n \"reason\": \"Succeeded\",\n \"status\": \"True\",\n \"type\": \"Healthy\"\n", 9 | "details": null, 10 | "patches": "{\"status\":{\"conditions\":[{\"type\":\"Ready\",\"reason\":\"ReconciliationSucceeded\",\"status\":\"True\",\"message\":\"Applied revision: v1.53.1@sha1:8b257f050d3748497382e41bc22f95c9022e6f0d\"},{\"type\":\"Healthy\",\"reason\":\"Succeeded\",\"status\":\"True\",\"message\":\"Health check passed in 50.309434ms\"}]}}", 11 | "created_by": null, 12 | "config": { 13 | "id": "3397183d-cb66-40bd-9257-3e2a86550b2b", 14 | "name": "mission-control-agent", 15 | "type": "Kubernetes::Kustomization", 16 | "config_class": "Kustomization" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /changes/testdata/health_passed_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "abf65d53-d011-429d-b659-ea20991acd42", 3 | "config_id": "3397183d-cb66-40bd-9257-3e2a86550b2b", 4 | "change_type": "HealthCheckPassed", 5 | "created_at": "2024-09-17T10:05:03.925589+00:00", 6 | "external_created_by": null, 7 | "source": "", 8 | "diff": "--- before\n+++ after\n@@ -74,7 +74,7 @@\n \"type\": \"Ready\"\n },\n {\n- \"message\": \"Health check passed in 50.309434ms\",\n+ \"message\": \"Health check passed in 83.594302ms\",\n \"reason\": \"Succeeded\",\n \"status\": \"True\",\n \"type\": \"Healthy\"\n", 9 | "details": null, 10 | "patches": "{\"status\":{\"conditions\":[{\"type\":\"Ready\",\"reason\":\"ReconciliationSucceeded\",\"status\":\"True\",\"message\":\"Applied revision: v1.53.1@sha1:8b257f050d3748497382e41bc22f95c9022e6f0d\"},{\"type\":\"Healthy\",\"reason\":\"Succeeded\",\"status\":\"True\",\"message\":\"Health check passed in 52.309434ms\"}]}}", 11 | "created_by": null, 12 | "config": { 13 | "id": "3397183d-cb66-40bd-9257-3e2a86550b2b", 14 | "name": "mission-control-agent", 15 | "type": "Kubernetes::Kustomization", 16 | "config_class": "Kustomization" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /changes/testdata/helm_upgrade_failed.json: -------------------------------------------------------------------------------- 1 | { 2 | "details": { 3 | "reason": "UpgradeFailed", 4 | "source": { 5 | "component": "helm-controller" 6 | }, 7 | "message": "Helm upgrade failed for release mission-control-agent/mission-control-kubernetes-bundle with chart mission-control-kubernetes@0.1.69: values don't meet the specifications of the schema(s) in the following chart(s):\nmission-control-kubernetes:\n- (root): Additional property interval is not allowed\n\nLast Helm logs:\n\n2024-10-09T04:46:13.116068801Z: preparing upgrade for mission-control-kubernetes-bundle\n2024-10-09T04:46:13.156697628Z: resetting values to the chart's original version", 8 | "metadata": { 9 | "uid": "852af766-4928-49d5-a2a6-a5668e3715e3", 10 | "name": "mission-control-kubernetes-bundle.17fcaf5d944b5886", 11 | "namespace": "mission-control-agent", 12 | "annotations": { 13 | "helm.toolkit.fluxcd.io/token": "sha256:eb7505628756567105a575c2dcaf473756212793a2b43537b976db9e18c7b233", 14 | "helm.toolkit.fluxcd.io/revision": "0.1.69", 15 | "helm.toolkit.fluxcd.io/app-version": "1.0.0" 16 | }, 17 | "resourceVersion": "43136301", 18 | "creationTimestamp": "2024-10-09T04:46:13Z" 19 | }, 20 | "involvedObject": { 21 | "uid": "e280b462-01b2-41a4-a53a-50541240a3e2", 22 | "kind": "HelmRelease", 23 | "name": "mission-control-kubernetes-bundle", 24 | "namespace": "mission-control-agent", 25 | "apiVersion": "helm.toolkit.fluxcd.io/v2", 26 | "resourceVersion": "43130040" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /changes/testdata/helm_upgrade_failed_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "details": { 3 | "reason": "UpgradeFailed", 4 | "source": { 5 | "component": "helm-controller" 6 | }, 7 | "message": "Helm upgrade failed for release mission-control-agent/mission-control-kubernetes-bundle with chart mission-control-kubernetes@0.1.69: values don't meet the specifications of the schema(s) in the following chart(s):\nmission-control-kubernetes:\n- (root): Additional property interval is not allowed\n\nLast Helm logs:\n\n2024-10-09T03:31:10.99090145Z: preparing upgrade for mission-control-kubernetes-bundle\n2024-10-09T03:31:11.022444234Z: resetting values to the chart's original version", 8 | "metadata": { 9 | "uid": "552e28a9-54a4-4ba7-8dc2-a68924cbba60", 10 | "name": "mission-control-kubernetes-bundle.17fcab45546704af", 11 | "namespace": "mission-control-agent", 12 | "annotations": { 13 | "helm.toolkit.fluxcd.io/token": "sha256:eb7505628756567105a575c2dcaf473756212793a2b43537b976db9e18c7b233", 14 | "helm.toolkit.fluxcd.io/revision": "0.1.69", 15 | "helm.toolkit.fluxcd.io/app-version": "1.0.0" 16 | }, 17 | "resourceVersion": "43130433", 18 | "creationTimestamp": "2024-10-09T03:31:11Z" 19 | }, 20 | "involvedObject": { 21 | "uid": "e280b462-01b2-41a4-a53a-50541240a3e2", 22 | "kind": "HelmRelease", 23 | "name": "mission-control-kubernetes-bundle", 24 | "namespace": "mission-control-agent", 25 | "apiVersion": "helm.toolkit.fluxcd.io/v2", 26 | "resourceVersion": "43098665" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: config-db 3 | description: A Helm chart for config-db 4 | 5 | type: application 6 | 7 | version: 0.3.0 8 | 9 | appVersion: "0.0.5" 10 | -------------------------------------------------------------------------------- /chart/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "config-db.name" . }} 5 | labels: 6 | {{- include "config-db.labels" . | nindent 4 }} 7 | data: 8 | config-db.properties: | 9 | {{- range $k, $v := .Values.properties }} 10 | {{ $k }}={{ $v }} 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /chart/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "config-db.name" . -}} 3 | apiVersion: networking.k8s.io/v1 4 | kind: Ingress 5 | metadata: 6 | name: {{ $fullName }} 7 | labels: 8 | {{- include "config-db.labels" . | nindent 4 }} 9 | {{- with .Values.ingress.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | spec: 14 | {{- if .Values.ingress.tls }} 15 | tls: 16 | {{- range .Values.ingress.tls }} 17 | - hosts: 18 | {{- range .hosts }} 19 | - {{ . | quote }} 20 | {{- end }} 21 | secretName: {{ .secretName }} 22 | {{- end }} 23 | {{- end }} 24 | rules: 25 | - host: {{ .Values.ingress.host | quote }} 26 | http: 27 | paths: 28 | - path: / 29 | pathType: ImplementationSpecific 30 | backend: 31 | service: 32 | name: {{ $fullName }} 33 | port: 34 | number: 8080 35 | {{- end }} 36 | -------------------------------------------------------------------------------- /chart/templates/postgres.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.db.external.create }} 2 | --- 3 | # PostgreSQL StatefulSet 4 | apiVersion: apps/v1 5 | kind: StatefulSet 6 | metadata: 7 | name: {{ template "config-db.name" . }}-postgresql 8 | spec: 9 | serviceName: postgresql 10 | selector: 11 | matchLabels: 12 | app: postgresql 13 | replicas: 1 14 | template: 15 | metadata: 16 | labels: 17 | app: postgresql 18 | spec: 19 | containers: 20 | - name: postgresql 21 | image: {{ tpl .Values.global.imageRegistry . }}/supabase/postgres:14.1.0.89 22 | volumeMounts: 23 | - name: postgresql 24 | mountPath: /data 25 | envFrom: 26 | - secretRef: 27 | name: {{ .Values.db.external.secretKeyRef.name }} 28 | volumeClaimTemplates: 29 | - metadata: 30 | name: postgresql 31 | spec: 32 | accessModes: ["ReadWriteOnce"] 33 | {{- if ne .Values.db.external.storageClass "" }} 34 | storageClassName: {{ .Values.db.external.storageClass }} 35 | {{- end }} 36 | resources: 37 | requests: 38 | storage: {{ .Values.db.external.storage }} 39 | --- 40 | # PostgreSQL StatefulSet Service 41 | apiVersion: v1 42 | kind: Service 43 | metadata: 44 | name: {{ template "config-db.name" . }}-postgresql 45 | spec: 46 | selector: 47 | app: postgresql 48 | ports: 49 | - port: 5432 50 | targetPort: 5432 51 | --- 52 | apiVersion: v1 53 | kind: Secret 54 | metadata: 55 | name: {{ .Values.db.external.secretKeyRef.name }} 56 | annotations: 57 | "helm.sh/resource-policy": "keep" 58 | type: Opaque 59 | stringData: 60 | {{- $secretObj := ( lookup "v1" "Secret" .Release.Namespace "postgres-connection" ) | default dict }} 61 | {{- $secretData := ( get $secretObj "data" | default dict ) }} 62 | {{- $user := (( get $secretData "POSTGRES_USER" ) | b64dec ) | default "postgres" }} 63 | {{- $password := (( get $secretData "POSTGRES_PASSWORD" ) | b64dec ) | default (randAlphaNum 32) }} 64 | {{- $dbname := (( get $secretData "POSTGRES_DB" ) | b64dec ) | default "config_db" }} 65 | {{- $host := print (include "config-db.name" .) "-postgresql." .Release.Namespace ".svc.cluster.local:5432" }} 66 | {{- $url := print "postgresql://" $user ":" $password "@" $host }} 67 | {{- $configDbUrl := ( get $secretData .Values.db.external.secretKeyRef.key ) | default ( print $url "/config_db?sslmode=disable" ) }} 68 | POSTGRES_USER: {{ $user | quote }} 69 | POSTGRES_PASSWORD: {{ $password | quote }} 70 | POSTGRES_HOST: {{ $host | quote }} 71 | POSTGRES_DB: {{ $dbname | quote }} 72 | {{ .Values.db.external.secretKeyRef.key }}: {{ $configDbUrl | quote }} 73 | --- 74 | 75 | {{- end }} 76 | -------------------------------------------------------------------------------- /chart/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "config-db.name" . }} 5 | labels: 6 | {{- include "config-db.labels" . | nindent 4 }} 7 | spec: 8 | ports: 9 | - port: 8080 10 | targetPort: 8080 11 | protocol: TCP 12 | name: http 13 | selector: 14 | {{- include "config-db.selectorLabels" . | nindent 4 }} 15 | -------------------------------------------------------------------------------- /chart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "serviceAccountName" . }} 6 | labels: {{- include "config-db.labels" . | nindent 4 }} 7 | {{- with merge .Values.serviceAccount.annotations .Values.global.serviceAccount.annotations }} 8 | annotations: {{ toYaml . | nindent 4 }} 9 | {{- end }} 10 | {{- end }} 11 | --- 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: "{{if .Values.serviceAccount.rbac.clusterRole}}Cluster{{end}}RoleBinding" 14 | metadata: 15 | name: {{ template "serviceAccountName" . }}-rolebinding 16 | labels: {{- include "config-db.labels" . | nindent 4 }} 17 | roleRef: 18 | apiGroup: rbac.authorization.k8s.io 19 | kind: "{{if .Values.serviceAccount.rbac.clusterRole}}Cluster{{end}}Role" 20 | name: {{ template "serviceAccountName" . }}-role 21 | subjects: 22 | - kind: ServiceAccount 23 | name: {{ template "serviceAccountName" . }} 24 | namespace: {{ .Release.Namespace }} 25 | --- 26 | apiVersion: rbac.authorization.k8s.io/v1 27 | kind: "{{if .Values.serviceAccount.rbac.clusterRole}}Cluster{{end}}Role" 28 | metadata: 29 | name: {{ template "serviceAccountName" . }}-role 30 | labels: {{- include "config-db.labels" . | nindent 4 }} 31 | rules: 32 | {{- if .Values.serviceAccount.rbac.secrets}} 33 | - apiGroups: 34 | - v1 35 | resources: 36 | - secrets 37 | verbs: 38 | - get 39 | - list 40 | {{- end}} 41 | {{- if .Values.serviceAccount.rbac.configmaps}} 42 | - apiGroups: 43 | - v1 44 | resources: 45 | - configmaps 46 | verbs: 47 | - get 48 | - list 49 | {{- end}} 50 | {{- if .Values.serviceAccount.rbac.exec}} 51 | - apiGroups: [""] 52 | resources: 53 | - pods/attach 54 | - pods/exec 55 | - pods/log 56 | verbs: 57 | - '*' 58 | {{- end}} 59 | {{- if .Values.serviceAccount.rbac.tokenRequest}} 60 | - apiGroups: ['authentication.k8s.io/v1'] 61 | resources: ['serviceaccounts/token'] 62 | verbs: ['create'] 63 | {{- end}} 64 | {{- if .Values.serviceAccount.rbac.readAll}} 65 | - apiGroups: 66 | - '*' 67 | resources: 68 | - '*' 69 | verbs: 70 | - "list" 71 | - "get" 72 | - "watch" 73 | {{- end}} 74 | - apiGroups: 75 | - configs.flanksource.com 76 | resources: 77 | - scrapeconfigs 78 | verbs: 79 | - create 80 | - delete 81 | - get 82 | - list 83 | - patch 84 | - update 85 | - watch 86 | - apiGroups: 87 | - configs.flanksource.com 88 | resources: 89 | - scrapeconfigs/finalizers 90 | verbs: 91 | - update 92 | - apiGroups: 93 | - configs.flanksource.com 94 | resources: 95 | - scrapeconfigs/status 96 | verbs: 97 | - get 98 | - patch 99 | - update 100 | # Leader election 101 | - apiGroups: 102 | - "" 103 | resources: 104 | - configmaps 105 | verbs: 106 | - get 107 | - list 108 | - watch 109 | - create 110 | - update 111 | - patch 112 | - delete 113 | - apiGroups: 114 | - coordination.k8s.io 115 | resources: 116 | - leases 117 | verbs: 118 | - get 119 | - list 120 | - watch 121 | - create 122 | - update 123 | - patch 124 | - delete 125 | - apiGroups: 126 | - "" 127 | resources: 128 | - events 129 | verbs: 130 | - create 131 | - patch 132 | - apiGroups: 133 | - "" 134 | resources: 135 | - pods/exec 136 | verbs: 137 | - create 138 | -------------------------------------------------------------------------------- /chart/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if eq "true" (include "truthy" ( list .Values.serviceMonitor.enabled .Values.global.serviceMonitor.enabled )) }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "config-db.name" . }}-monitor 6 | labels: 7 | {{- include "config-db.labels" . | nindent 4 }} 8 | {{- range $k, $v := (merge .Values.serviceMonitor.labels .Values.global.serviceMonitor.labels )}} 9 | {{$k}}: {{$v | quote}} 10 | {{- end }} 11 | spec: 12 | jobLabel: {{ include "config-db.name" . }} 13 | endpoints: 14 | - port: http 15 | interval: 30s 16 | selector: 17 | matchLabels: 18 | {{- include "config-db.labels" . | nindent 6 }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /cmd/analyze.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | 7 | "github.com/flanksource/commons/logger" 8 | "github.com/flanksource/config-db/analyzers" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | "github.com/flanksource/config-db/scrapers/aws" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | var outputFile, outputFormat string 15 | 16 | // Analyzers ... 17 | var Analyzers = []v1.Analyzer{ 18 | analyzers.PatchAnalyzer, 19 | aws.EC2InstanceAnalyzer, 20 | } 21 | 22 | // Analyze ... 23 | var Analyze = &cobra.Command{ 24 | Use: "analyze ", 25 | Short: "Analyze configuration items and report discrepencies/issues.", 26 | Run: func(cmd *cobra.Command, configs []string) { 27 | 28 | objects := []v1.ScrapeResult{} 29 | for _, path := range configs { 30 | obj := v1.ScrapeResult{} 31 | data, err := os.ReadFile(path) 32 | if err != nil { 33 | logger.Fatalf("could not read %s: %v", path, err) 34 | } 35 | if err := json.Unmarshal(data, &obj); err != nil { 36 | logger.Fatalf("Could not unmarshall %s: %v", path, err) 37 | } 38 | 39 | if obj.ConfigClass == "EC2Instance" { 40 | nested, _ := json.Marshal(obj.Config) 41 | instance := aws.Instance{} 42 | if err := json.Unmarshal(nested, &instance); err != nil { 43 | logger.Fatalf("Failed to unmarshal object into ec2 instance %s", obj.ID) 44 | } 45 | obj.Config = instance 46 | } 47 | objects = append(objects, obj) 48 | } 49 | results := []v1.AnalysisResult{} 50 | for _, analyzer := range Analyzers { 51 | results = append(results, analyzer(objects)) 52 | } 53 | if outputFormat == "json" { 54 | data, _ := json.Marshal(results) 55 | if err := os.WriteFile(outputFile, data, 0644); err != nil { 56 | logger.Fatalf("Failed to write to %s: %v", outputFile, err) 57 | } 58 | } 59 | }, 60 | } 61 | 62 | func init() { 63 | Analyze.Flags().StringVarP(&outputFile, "output", "o", "analysis.json", "Output file") 64 | Analyze.Flags().StringVarP(&outputFormat, "format", "f", "json", "Output format") 65 | 66 | } 67 | -------------------------------------------------------------------------------- /cmd/offline.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/flanksource/commons/logger" 5 | "github.com/flanksource/duty/postgrest" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | // GoOffline ... 10 | var GoOffline = &cobra.Command{ 11 | Use: "go-offline", 12 | Long: "Download all dependencies so that config-db can work without an internet connection", 13 | Run: func(cmd *cobra.Command, args []string) { 14 | if err := postgrest.GoOffline(); err != nil { 15 | logger.Fatalf("Failed to go offline: %+v", err) 16 | } 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /db/analysis.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/flanksource/config-db/api" 7 | "github.com/flanksource/duty/models" 8 | "gorm.io/gorm" 9 | ) 10 | 11 | func getAnalysis(ctx api.ScrapeContext, analysis models.ConfigAnalysis) (*models.ConfigAnalysis, error) { 12 | existing := models.ConfigAnalysis{} 13 | err := ctx.DB().First(&existing, "config_id = ? AND analyzer = ?", analysis.ConfigID, analysis.Analyzer).Error 14 | if errors.Is(err, gorm.ErrRecordNotFound) { 15 | return nil, nil 16 | } 17 | 18 | return &existing, err 19 | } 20 | 21 | func CreateAnalysis(ctx api.ScrapeContext, analysis models.ConfigAnalysis) error { 22 | // get analysis by config_id, and summary 23 | existingAnalysis, err := getAnalysis(ctx, analysis) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | if existingAnalysis != nil { 29 | analysis.ID = existingAnalysis.ID 30 | return ctx.DB().Model(&analysis).Updates(map[string]interface{}{ 31 | "last_observed": gorm.Expr("now()"), 32 | "message": analysis.Message, 33 | "status": analysis.Status, 34 | }).Error 35 | } 36 | 37 | return ctx.DB().Create(&analysis).Error 38 | } 39 | -------------------------------------------------------------------------------- /db/changes.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/flanksource/config-db/api" 8 | "github.com/flanksource/config-db/db/models" 9 | "github.com/flanksource/duty/context" 10 | "github.com/patrickmn/go-cache" 11 | ) 12 | 13 | var ChangeCacheByFingerprint = cache.New(time.Hour, time.Hour) 14 | 15 | func changeFingeprintCacheKey(configID, fingerprint string) string { 16 | return fmt.Sprintf("%s:%s", configID, fingerprint) 17 | } 18 | 19 | func InitChangeFingerprintCache(ctx context.Context, window time.Duration) error { 20 | var changes []*models.ConfigChange 21 | if err := ctx.DB().Where("fingerprint IS NOT NULL").Where(fmt.Sprintf("created_at >= NOW() - INTERVAL '%d SECOND'", int(window.Seconds()))).Find(&changes).Error; err != nil { 22 | return err 23 | } 24 | 25 | ctx.Logger.Debugf("initializing changes cache with %d changes", len(changes)) 26 | 27 | for _, c := range changes { 28 | key := changeFingeprintCacheKey(c.ConfigID, *c.Fingerprint) 29 | ChangeCacheByFingerprint.Set(key, c.ID, time.Until(c.CreatedAt.Add(window))) 30 | } 31 | 32 | return nil 33 | } 34 | 35 | func dedupChanges(window time.Duration, changes []*models.ConfigChange) ([]*models.ConfigChange, []models.ConfigChangeUpdate) { 36 | if len(changes) == 0 { 37 | return nil, nil 38 | } 39 | 40 | var nonDuped []*models.ConfigChange 41 | var fingerprinted = map[string]models.ConfigChangeUpdate{} 42 | 43 | for _, change := range changes { 44 | if change.Fingerprint == nil { 45 | nonDuped = append(nonDuped, change) 46 | continue 47 | } 48 | 49 | key := changeFingeprintCacheKey(change.ConfigID, *change.Fingerprint) 50 | if existingChangeID, ok := ChangeCacheByFingerprint.Get(key); !ok { 51 | ChangeCacheByFingerprint.Set(key, change.ID, window) 52 | fingerprinted[change.ID] = models.ConfigChangeUpdate{Change: change, CountIncrement: 0} 53 | } else { 54 | change.ID = existingChangeID.(string) 55 | ChangeCacheByFingerprint.Set(key, change.ID, window) // Refresh the cache expiry 56 | 57 | if existing, ok := fingerprinted[change.ID]; ok { 58 | fingerprinted[change.ID] = models.ConfigChangeUpdate{Change: change, CountIncrement: existing.CountIncrement + 1} 59 | } else { 60 | fingerprinted[change.ID] = models.ConfigChangeUpdate{Change: change, CountIncrement: 1} 61 | } 62 | } 63 | } 64 | 65 | var deduped []models.ConfigChangeUpdate 66 | for _, v := range fingerprinted { 67 | if v.CountIncrement == 0 { 68 | nonDuped = append(nonDuped, v.Change) 69 | } else { 70 | deduped = append(deduped, v) 71 | } 72 | } 73 | 74 | return nonDuped, deduped 75 | } 76 | 77 | func GetWorkflowRunCount(ctx api.ScrapeContext, workflowID string) (int64, error) { 78 | var count int64 79 | err := ctx.DB().Table("config_changes"). 80 | Where("config_id = (?)", ctx.DB().Table("config_items").Select("id").Where("? = ANY(external_id)", workflowID)). 81 | Count(&count). 82 | Error 83 | return count, err 84 | } 85 | -------------------------------------------------------------------------------- /db/diff.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/flanksource/commons/properties" 9 | dutyContext "github.com/flanksource/duty/context" 10 | "github.com/hexops/gotextdiff" 11 | "github.com/hexops/gotextdiff/myers" 12 | "github.com/ohler55/ojg" 13 | "github.com/ohler55/ojg/oj" 14 | ) 15 | 16 | // We expose this function to replace it with a rust function called via FFI 17 | // when the build tag rustdiffgen is provided 18 | var DiffFunc func(string, string) string = TextDiff 19 | 20 | // NormalizeJSON returns an indented json string. 21 | // The keys are sorted lexicographically. 22 | func NormalizeJSONOj(object any) (string, error) { 23 | data := object 24 | switch v := object.(type) { 25 | case string: 26 | var err error 27 | var jsonStrMap map[string]any 28 | err = oj.Unmarshal([]byte(v), &jsonStrMap) 29 | 30 | if err != nil { 31 | return "", err 32 | } 33 | data = jsonStrMap 34 | } 35 | 36 | out, err := oj.Marshal(data, &ojg.Options{ 37 | Indent: 2, 38 | Sort: true, 39 | OmitNil: true, 40 | UseTags: true, 41 | FloatFormat: "%0.0f", 42 | }) 43 | if err != nil { 44 | return "", err 45 | } 46 | return string(out), nil 47 | } 48 | 49 | // normalizeJSON returns an indented json string. 50 | // The keys are sorted lexicographically. 51 | func NormalizeJSON(object any) (string, error) { 52 | data := object 53 | switch v := object.(type) { 54 | case string: 55 | var jsonStrMap map[string]any 56 | if err := json.Unmarshal([]byte(v), &jsonStrMap); err != nil { 57 | return "", err 58 | } 59 | data = jsonStrMap 60 | } 61 | 62 | jsonStrIndented, err := json.MarshalIndent(data, "", "\t") 63 | if err != nil { 64 | return "", err 65 | } 66 | 67 | return string(jsonStrIndented), nil 68 | } 69 | 70 | // generateDiff calculates the diff (git style) between the given 2 configs. 71 | func GenerateDiff(ctx dutyContext.Context, newConf, prevConfig string) (string, error) { 72 | if ctx.Properties().On(false, "scraper.diff.disable") { 73 | return "", nil 74 | } 75 | 76 | return generateDiff(newConf, prevConfig) 77 | } 78 | 79 | func generateDiff(newConf, prevConfig string) (string, error) { 80 | if newConf == prevConfig { 81 | return "", nil 82 | } 83 | 84 | normalizer := NormalizeJSONOj 85 | 86 | // We want a nicely indented json config with each key-vals in new line 87 | // because that gives us a better diff. A one-line json string config produces diff 88 | // that's not very helpful. 89 | before, err := normalizer(prevConfig) 90 | if err != nil { 91 | return "", fmt.Errorf("failed to normalize json for previous config: %w", err) 92 | } 93 | 94 | after, err := normalizer(newConf) 95 | if err != nil { 96 | return "", fmt.Errorf("failed to normalize json for new config: %w", err) 97 | } 98 | 99 | if before == after { 100 | return "", nil 101 | } 102 | 103 | // If we compile the code with rustdiffgen tag, we still might 104 | // want to disable rust invokation 105 | var once sync.Once 106 | once.Do(func() { 107 | if properties.On(false, "diff.rust-gen") { 108 | DiffFunc = TextDiff 109 | } 110 | }) 111 | 112 | return DiffFunc(before, after), nil 113 | } 114 | 115 | func TextDiff(before, after string) string { 116 | edits := myers.ComputeEdits("", before, after) 117 | if len(edits) == 0 { 118 | return "" 119 | } 120 | return fmt.Sprint(gotextdiff.ToUnified("before", "after", before, edits)) 121 | } 122 | -------------------------------------------------------------------------------- /db/models/config_change.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | v1 "github.com/flanksource/config-db/api/v1" 8 | "github.com/google/uuid" 9 | "github.com/samber/lo" 10 | "gorm.io/gorm" 11 | "gorm.io/gorm/clause" 12 | ) 13 | 14 | type ConfigChangeUpdate struct { 15 | Change *ConfigChange 16 | CountIncrement int 17 | } 18 | 19 | // ConfigChange represents the config change database table 20 | type ConfigChange struct { 21 | ExternalID string `gorm:"-"` 22 | ConfigType string `gorm:"-"` 23 | Fingerprint *string `gorm:"column:fingerprint" json:"fingerprint"` 24 | ExternalChangeID *string `gorm:"column:external_change_id;default:null" json:"external_change_id"` 25 | ID string `gorm:"primaryKey;unique_index;not null;column:id" json:"id"` 26 | ConfigID string `gorm:"column:config_id;default:''" json:"config_id"` 27 | ChangeType string `gorm:"column:change_type" json:"change_type"` 28 | Diff *string `gorm:"column:diff" json:"diff,omitempty"` 29 | Severity string `gorm:"column:severity" json:"severity"` 30 | Source string `gorm:"column:source" json:"source"` 31 | Summary string `gorm:"column:summary" json:"summary,omitempty"` 32 | Patches string `gorm:"column:patches;default:null" json:"patches,omitempty"` 33 | Details v1.JSON `gorm:"column:details" json:"details,omitempty"` 34 | Count int `gorm:"column:count;<-" json:"count"` 35 | FirstObserved *time.Time `gorm:"column:first_observed;default:NOW()" json:"first_observed"` 36 | CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` 37 | CreatedBy *string `json:"created_by"` 38 | ExternalCreatedBy *string `json:"external_created_by"` 39 | } 40 | 41 | func (c ConfigChange) GetExternalID() v1.ExternalID { 42 | return v1.ExternalID{ 43 | ExternalID: c.ExternalID, 44 | ConfigType: c.ConfigType, 45 | } 46 | } 47 | 48 | func (c ConfigChange) String() string { 49 | return fmt.Sprintf("[%s/%s] %s", c.ConfigType, c.ExternalID, c.ChangeType) 50 | } 51 | 52 | func NewConfigChangeFromV1(result v1.ScrapeResult, change v1.ChangeResult) *ConfigChange { 53 | _change := ConfigChange{ 54 | ID: uuid.NewString(), 55 | ExternalID: change.ExternalID, 56 | ConfigType: change.ConfigType, 57 | ChangeType: change.ChangeType, 58 | Source: change.Source, 59 | Diff: change.Diff, 60 | Severity: change.Severity, 61 | Details: v1.JSON(change.Details), 62 | Summary: change.Summary, 63 | Patches: change.Patches, 64 | CreatedBy: change.CreatedBy, 65 | Count: 1, 66 | ConfigID: change.ConfigID, 67 | } 68 | if change.CreatedAt != nil && !change.CreatedAt.IsZero() { 69 | _change.CreatedAt = lo.FromPtr(change.CreatedAt) 70 | } 71 | 72 | if change.ExternalChangeID != "" { 73 | _change.ExternalChangeID = &change.ExternalChangeID 74 | } 75 | 76 | return &_change 77 | } 78 | 79 | func (c *ConfigChange) BeforeCreate(tx *gorm.DB) (err error) { 80 | if c.ID == "" { 81 | c.ID = uuid.New().String() 82 | } 83 | 84 | tx.Statement.AddClause(clause.OnConflict{ 85 | Columns: []clause.Column{ 86 | {Name: "config_id"}, 87 | {Name: "external_change_id"}, 88 | }, 89 | UpdateAll: true, 90 | }) 91 | 92 | return 93 | } 94 | -------------------------------------------------------------------------------- /db/models/config_relationship.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type ConfigRelationship struct { 4 | ConfigID string `gorm:"column:config_id" json:"config_id"` 5 | RelatedID string `gorm:"column:related_id" json:"related_id"` 6 | Relation string `gorm:"column:relation" json:"relation"` 7 | SelectorID string `gorm:"selector_id" json:"selector_id"` 8 | } 9 | -------------------------------------------------------------------------------- /db/people.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/flanksource/config-db/api" 7 | "github.com/flanksource/duty/models" 8 | "gorm.io/gorm" 9 | ) 10 | 11 | func FindPersonByEmail(ctx api.ScrapeContext, email string) (*models.Person, error) { 12 | var person models.Person 13 | err := ctx.DB().Where("email = ?", email).First(&person).Error 14 | if err != nil { 15 | if errors.Is(err, gorm.ErrRecordNotFound) { 16 | return nil, nil 17 | } 18 | 19 | return nil, err 20 | } 21 | 22 | return &person, err 23 | } 24 | -------------------------------------------------------------------------------- /db/scrape_plugin.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | v1 "github.com/flanksource/config-db/api/v1" 9 | "github.com/flanksource/duty" 10 | "github.com/flanksource/duty/context" 11 | "github.com/flanksource/duty/models" 12 | gocache "github.com/patrickmn/go-cache" 13 | ) 14 | 15 | func PersistScrapePluginFromCRD(ctx context.Context, plugin *v1.ScrapePlugin) error { 16 | m, err := plugin.ToModel() 17 | if err != nil { 18 | return err 19 | } 20 | m.Source = models.SourceCRD 21 | 22 | return ctx.DB().Save(m).Error 23 | } 24 | 25 | func DeleteScrapePlugin(ctx context.Context, id string) error { 26 | return ctx.DB().Model(&models.ScrapePlugin{}).Where("id = ?", id).Update("deleted_at", duty.Now()).Error 27 | } 28 | 29 | var cachedPlugin = gocache.New(time.Hour, time.Hour) 30 | 31 | func LoadAllPlugins(ctx context.Context) ([]v1.ScrapePluginSpec, error) { 32 | if v, found := cachedPlugin.Get("only"); found { 33 | return v.([]v1.ScrapePluginSpec), nil 34 | } 35 | 36 | return ReloadAllScrapePlugins(ctx) 37 | } 38 | 39 | func ReloadAllScrapePlugins(ctx context.Context) ([]v1.ScrapePluginSpec, error) { 40 | var plugins []models.ScrapePlugin 41 | if err := ctx.DB().Where("deleted_at IS NULL").Find(&plugins).Error; err != nil { 42 | return nil, err 43 | } 44 | 45 | specs := make([]v1.ScrapePluginSpec, 0, len(plugins)) 46 | for _, p := range plugins { 47 | var spec v1.ScrapePluginSpec 48 | if err := json.Unmarshal(p.Spec, &spec); err != nil { 49 | return nil, fmt.Errorf("failed to unmarshal scrape plugin spec(%s): %w", p.ID, err) 50 | } 51 | 52 | specs = append(specs, spec) 53 | } 54 | 55 | cachedPlugin.SetDefault("only", specs) 56 | 57 | return specs, nil 58 | } 59 | -------------------------------------------------------------------------------- /db/testdata/person-new.json: -------------------------------------------------------------------------------- 1 | { 2 | "address": { 3 | "city": "Kathmandu", 4 | "street": "Baneshwor", 5 | "state": "Bagmati", 6 | "zip": "44600" 7 | }, 8 | "age": 31, 9 | "email": "john.smith@example.com", 10 | "isVerified": true, 11 | "name": "John Smith" 12 | } 13 | -------------------------------------------------------------------------------- /db/testdata/person-old.json: -------------------------------------------------------------------------------- 1 | { 2 | "address": { 3 | "city": "Anytown", 4 | "state": "CA", 5 | "street": "123 Main St", 6 | "zip": "12345" 7 | }, 8 | "age": 30, 9 | "email": "john.smith@example.com", 10 | "isVerified": true, 11 | "name": "John Smith" 12 | } 13 | -------------------------------------------------------------------------------- /db/testdata/person.diff: -------------------------------------------------------------------------------- 1 | --- before 2 | +++ after 3 | @@ -1,11 +1,11 @@ 4 | { 5 | "address": { 6 | - "city": "Anytown", 7 | - "state": "CA", 8 | - "street": "123 Main St", 9 | - "zip": "12345" 10 | + "city": "Kathmandu", 11 | + "state": "Bagmati", 12 | + "street": "Baneshwor", 13 | + "zip": "44600" 14 | }, 15 | - "age": 30, 16 | + "age": 31, 17 | "email": "john.smith@example.com", 18 | "isVerified": true, 19 | "name": "John Smith" 20 | -------------------------------------------------------------------------------- /db/testdata/simple-new.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Mission Control", 3 | "stack": "Go", 4 | "number": 1, 5 | "date": "20223-01-01", 6 | "bool": true, 7 | "float": 1.1, 8 | "bigint": 32493274893274 9 | 10 | } 11 | -------------------------------------------------------------------------------- /db/testdata/simple-old.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Mission Control", 3 | "stack": "Golang", 4 | "number": 2, 5 | "date": "20223-01-01", 6 | "bool": true, 7 | "float": 1.1, 8 | "bigint": 32493293274 9 | } 10 | -------------------------------------------------------------------------------- /db/testdata/simple.diff: -------------------------------------------------------------------------------- 1 | --- before 2 | +++ after 3 | @@ -1,9 +1,9 @@ 4 | { 5 | - "bigint": 32493293274, 6 | + "bigint": 32493274893274, 7 | "bool": true, 8 | "date": "20223-01-01", 9 | "float": 1, 10 | "name": "Mission Control", 11 | - "number": 2, 12 | - "stack": "Golang" 13 | + "number": 1, 14 | + "stack": "Go" 15 | } 16 | \ No newline at end of file 17 | -------------------------------------------------------------------------------- /db/ulid/ulid.go: -------------------------------------------------------------------------------- 1 | package ulid 2 | 3 | import ( 4 | "io" 5 | "math/rand" 6 | "sync" 7 | "time" 8 | 9 | "github.com/google/uuid" 10 | v2 "github.com/oklog/ulid/v2" 11 | ) 12 | 13 | // ULID ... 14 | type ULID [16]byte 15 | 16 | // AsUUID ... 17 | func (u ULID) AsUUID() string { 18 | return uuid.UUID(u).String() 19 | } 20 | 21 | var pool = sync.Pool{ 22 | New: func() interface{} { 23 | return v2.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0) 24 | }, 25 | } 26 | 27 | // New ... 28 | func New() (ULID, error) { 29 | entropy := pool.Get() 30 | result, err := v2.New(v2.Timestamp(time.Now()), entropy.(io.Reader)) 31 | pool.Put(entropy) 32 | return ULID(result), err 33 | } 34 | 35 | // MustNew ... 36 | func MustNew() ULID { 37 | entropy := pool.Get() 38 | defer pool.Put(entropy) 39 | return ULID(v2.MustNew(v2.Timestamp(time.Now()), entropy.(io.Reader))) 40 | } 41 | -------------------------------------------------------------------------------- /debug.go: -------------------------------------------------------------------------------- 1 | //go:build debug 2 | 3 | package main 4 | 5 | import ( 6 | "time" 7 | 8 | "github.com/fjl/memsize" 9 | "github.com/fjl/memsize/memsizeui" 10 | "github.com/flanksource/config-db/api" 11 | "github.com/flanksource/config-db/db" 12 | "github.com/flanksource/config-db/scrapers" 13 | "github.com/flanksource/config-db/scrapers/kubernetes" 14 | "github.com/flanksource/config-db/utils" 15 | "github.com/labstack/echo/v4" 16 | ) 17 | 18 | func init() { 19 | var memsizeHandler memsizeui.Handler 20 | utils.TrackObject = func(name string, obj any) { 21 | if obj == nil { 22 | go func() { 23 | time.Sleep(5 * time.Minute) 24 | utils.TrackObject(name, obj) 25 | }() 26 | } else { 27 | memsizeHandler.Add(name, obj) 28 | } 29 | } 30 | 31 | utils.MemsizeScan = func(obj any) uintptr { 32 | sizes := memsize.Scan(obj) 33 | return sizes.Total 34 | } 35 | 36 | utils.MemsizeEchoHandler = func(c echo.Context) error { 37 | memsizeHandler.ServeHTTP(c.Response(), c.Request()) 38 | return nil 39 | } 40 | 41 | utils.TrackObject("TempCacheStore", &scrapers.TempCacheStore) 42 | utils.TrackObject("ScraperTempCache", &api.ScraperTempCache) 43 | utils.TrackObject("IgnoreCache", &kubernetes.IgnoreCache) 44 | utils.TrackObject("OrphanCache", &db.OrphanCache) 45 | utils.TrackObject("ChangeCacheByFingerprint", &db.ChangeCacheByFingerprint) 46 | utils.TrackObject("ParentCache", &db.ParentCache) 47 | utils.TrackObject("ResourceIDMapPerCluster", &kubernetes.ResourceIDMapPerCluster) 48 | } 49 | -------------------------------------------------------------------------------- /deploy/kustomization.yaml: -------------------------------------------------------------------------------- 1 | namespace: config-db 2 | resources: 3 | - "namespace.yaml" 4 | - "rbac.yaml" 5 | - "manager.yaml" -------------------------------------------------------------------------------- /deploy/manager.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: config-db 6 | labels: 7 | control-plane: config-db 8 | spec: 9 | selector: 10 | matchLabels: 11 | control-plane: config-db 12 | replicas: 1 13 | template: 14 | metadata: 15 | labels: 16 | control-plane: config-db 17 | spec: 18 | serviceAccountName: config-db-sa 19 | containers: 20 | - name: config-db 21 | image: docker.io/flanksource/config-db:latest 22 | command: 23 | - /app/config-db 24 | args: 25 | - serve 26 | - -vvv 27 | env: 28 | - name: DB_URL 29 | valueFrom: 30 | secretKeyRef: 31 | name: postgres-connection-string 32 | key: connection-string 33 | resources: 34 | requests: 35 | cpu: 200m 36 | memory: 200Mi 37 | limits: 38 | memory: 512Mi 39 | cpu: 500m 40 | --- 41 | apiVersion: v1 42 | kind: Service 43 | metadata: 44 | labels: 45 | control-plane: config-db 46 | name: config-db 47 | namespace: config-db 48 | spec: 49 | ports: 50 | - port: 8080 51 | protocol: TCP 52 | targetPort: 8080 53 | selector: 54 | control-plane: config-db 55 | -------------------------------------------------------------------------------- /deploy/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: config-db 5 | labels: 6 | control-plane: config-db -------------------------------------------------------------------------------- /deploy/rbac.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: config-db-sa 6 | labels: 7 | control-plane: config-db 8 | --- 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | kind: ClusterRoleBinding 11 | metadata: 12 | name: config-db-rolebinding 13 | roleRef: 14 | apiGroup: rbac.authorization.k8s.io 15 | kind: ClusterRole 16 | name: config-db-role 17 | subjects: 18 | - kind: ServiceAccount 19 | name: config-db-sa 20 | --- 21 | apiVersion: rbac.authorization.k8s.io/v1 22 | kind: ClusterRole 23 | metadata: 24 | name: config-db-role 25 | rules: 26 | - apiGroups: 27 | - '*' 28 | resources: 29 | - '*' 30 | verbs: 31 | - "list" 32 | - "get" 33 | - "watch" 34 | -------------------------------------------------------------------------------- /external/diffgen/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "diffgen" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "libc", 10 | "similar", 11 | ] 12 | 13 | [[package]] 14 | name = "libc" 15 | version = "0.2.158" 16 | source = "registry+https://github.com/rust-lang/crates.io-index" 17 | checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" 18 | 19 | [[package]] 20 | name = "similar" 21 | version = "2.6.0" 22 | source = "git+https://github.com/mitsuhiko/similar#7e15c44de11a1cd61e1149189929e189ef977fd8" 23 | -------------------------------------------------------------------------------- /external/diffgen/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "diffgen" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | libc = "0.2.158" 8 | similar = { git = "https://github.com/mitsuhiko/similar", version = "2.6.0" } 9 | 10 | 11 | [lib] 12 | crate-type = ["cdylib", "staticlib"] 13 | -------------------------------------------------------------------------------- /external/diffgen/libdiffgen.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | char *diff(const char *before, const char *after); 7 | -------------------------------------------------------------------------------- /external/diffgen/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate similar; 2 | 3 | use similar::TextDiff; 4 | use std::ffi::{CStr, CString}; 5 | 6 | #[no_mangle] 7 | pub extern "C" fn diff( 8 | before: *const libc::c_char, 9 | after: *const libc::c_char, 10 | ) -> *mut libc::c_char { 11 | let before_cstr = unsafe { CStr::from_ptr(before) }; 12 | let before_str = before_cstr.to_str().unwrap(); 13 | 14 | let after_cstr = unsafe { CStr::from_ptr(after) }; 15 | let after_str = after_cstr.to_str().unwrap(); 16 | 17 | let diff = TextDiff::from_lines(before_str, after_str); 18 | 19 | CString::new(diff.unified_diff().to_string()) 20 | .unwrap() 21 | .into_raw() 22 | } 23 | 24 | #[cfg(test)] 25 | pub mod test { 26 | use super::*; 27 | use std::ffi::CString; 28 | 29 | #[test] 30 | fn test_diff() { 31 | let diff_result = diff( 32 | CString::new("hello\nworld\n").unwrap().into_raw(), 33 | CString::new("bye\nworld\n").unwrap().into_raw(), 34 | ); 35 | 36 | assert_eq!( 37 | unsafe { CStr::from_ptr(diff_result) }.to_str().unwrap(), 38 | "@@ -1,2 +1,2 @@\n-hello\n+bye\n world\n" 39 | ) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /fixtures/access_logs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: azure-access-logs 5 | namespace: mc 6 | spec: 7 | full: true 8 | http: 9 | - type: 'None' 10 | name: 'none' 11 | id: 'none' 12 | url: 'http://localhost:8000/azure_enterprise_app_access_logs.json' 13 | transform: 14 | expr: | 15 | dyn(config). 16 | filter(item, len(catalog.traverse(item.app_id, "MissionControl::Application")) > 0). 17 | map(item, { 18 | "access_logs": [{ 19 | "config_id": item.app_id, 20 | "external_user_id": item.user_id, 21 | "created_at": item.created_at 22 | }] 23 | }).toJSON() 24 | 25 | -------------------------------------------------------------------------------- /fixtures/aws.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: aws-scraper 5 | spec: 6 | aws: 7 | - region: 8 | - eu-west-2 9 | - us-east-1 10 | - af-south-1 11 | - ap-south-1 12 | - eu-central-1 13 | properties: 14 | - name: AWS Link 15 | filter: 'config_type == AWS::IAM::Role' 16 | icon: aws-iam 17 | links: 18 | - text: AWS Link 19 | url: https://us-east-1.console.aws.amazon.com/iamv2/home#/roles/details/{{.name}}?section=permissions 20 | compliance: true 21 | patch_states: false 22 | trusted_advisor_check: false 23 | patch_details: false 24 | costReporting: 25 | s3BucketPath: s3://flanksource-cost-reports/query-results 26 | database: athenacurcfn_flanksource_report 27 | table: flanksource_report 28 | region: af-south-1 29 | inventory: true 30 | exclude: 31 | - Amazon EC2 Reserved Instances Optimization 32 | - Savings Plan 33 | # - trusted_advisor 34 | # - cloudtrail 35 | # include: 36 | # - vpc 37 | # # - subnet 38 | # - vpc 39 | # - SecurityGroup 40 | transform: 41 | relationship: 42 | # EKS Cluster to Kubernetes Cluster & Kubernetes Node 43 | - filter: config_type == 'AWS::EKS::Cluster' 44 | expr: | 45 | [ 46 | {"type": "Kubernetes::Cluster","tags": {"account": tags['account'],"cluster": labels["alpha.eksctl.io/cluster-name"]}}, 47 | {"type": "Kubernetes::Node","tags": {"account": tags['account'],"cluster": labels["alpha.eksctl.io/cluster-name"]}} 48 | ].toJSON() 49 | # EC2 Instance to kubernetes node 50 | - filter: config_type == 'AWS::EC2:Instance' 51 | expr: | 52 | [{"type": "Kubernetes::Node", "labels": {"alpha.eksctl.io/instance-id": config["instance_id"]}}].toJSON() 53 | # IAM Role to Kubernetes Node 54 | - filter: config_type == 'AWS::IAM::Role' 55 | expr: | 56 | [{"type": "Kubernetes::Node", "labels": {"aws/iam-role": config["Arn"]}}].toJSON() 57 | # AvailabilityZone to Zone ID & Kubernetes Node 58 | - filter: config_type == 'AWS::AvailabilityZone' 59 | expr: | 60 | [ 61 | {"type": "Kubernetes::Node", "tags": {"account": labels['account'], "topology.kubernetes.io/zone": name}} 62 | ].toJSON() 63 | # Region to ZoneID 64 | - filter: config_type == 'AWS::Region' 65 | expr: | 66 | [{"type": "AWS::AvailabilityZoneID", "tags": {"region": name}}].toJSON() 67 | exclude: 68 | - jsonpath: $.tags 69 | - jsonpath: $.privateDnsNameOptionsOnLaunch 70 | # - jsonpath: availableIpAddressCount 71 | - jsonpath: outpostArn 72 | - jsonpath: mapCustomerOwnedIpOnLaunch 73 | - jsonpath: subnetArn 74 | # - jsonpath: usageOperationUpdateTime 75 | # - jsonpath: $..privateIPAddresses 76 | -------------------------------------------------------------------------------- /fixtures/azure-devops.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: azure-devops-scraper 5 | spec: 6 | azureDevops: 7 | - connection: connection://Azure Devops/Flanksource 8 | projects: 9 | - Demo1 10 | pipelines: 11 | - "adhoc-release" 12 | - "git automation" -------------------------------------------------------------------------------- /fixtures/azure.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: azure-scraper 5 | spec: 6 | azure: 7 | - connection: connection://azure/flanksource 8 | subscriptionID: e3911016-5810-415f-b075-682db169988f 9 | transform: 10 | relationship: 11 | # Link AKS Cluster to Kubernetes Cluster 12 | - filter: config_class == 'KubernetesCluster' 13 | expr: | 14 | [{ 15 | "type": "Kubernetes::Cluster", 16 | "labels": { 17 | "aks-nodeResourceGroup": config["properties"]["nodeResourceGroup"], 18 | "subscriptionID": tags["subscriptionID"] 19 | } 20 | }].toJSON() 21 | # Link Azure Virtual Machine Scale Sets to the Kubernetes Nodes 22 | - filter: config_class == 'Node' 23 | expr: | 24 | [{ 25 | "type": "Kubernetes::Node", 26 | "labels": { 27 | "azure/vm-scale-set": name, 28 | "subscriptionID": tags["subscriptionID"] 29 | } 30 | }].toJSON() 31 | -------------------------------------------------------------------------------- /fixtures/clickhouse.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: clickhouse-scraper 5 | spec: 6 | clickhouse: 7 | - query: | 8 | SELECT 9 | concat('ORD-', toString(10000 + number)) as order_id, 10 | ['Electronics', 'Clothing', 'Books', 'Home', 'Sports'][rand() % 5 + 1] as category, 11 | ['New York', 'London', 'Tokyo', 'Paris', 'Sydney'][rand() % 5 + 1] as city, 12 | round((rand() % 50000) / 100, 2) as amount, 13 | ['completed', 'pending', 'cancelled'][rand() % 3 + 1] as status, 14 | toDateTime('2024-01-01 00:00:00') + toIntervalSecond(rand() % 31536000) as order_date 15 | FROM numbers(1000) 16 | type: Order 17 | id: $.order_id 18 | transform: 19 | #full: true 20 | expr: "[config].toJSON()" 21 | -------------------------------------------------------------------------------- /fixtures/crds/scrape-config-kubernetes.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: scrapeconfig-kubernetes 5 | spec: 6 | kubernetes: 7 | - clusterName: local-kind-cluster 8 | exclusions: 9 | - Secret 10 | - ReplicaSet 11 | - APIService 12 | - events 13 | - endpoints.discovery.k8s.io 14 | - endpointslices.discovery.k8s.io 15 | - leases.coordination.k8s.io 16 | - podmetrics.metrics.k8s.io 17 | - nodemetrics.metrics.k8s.io 18 | - customresourcedefinition 19 | - controllerrevision 20 | - certificaterequest 21 | - orders.acme.cert-manager.io 22 | -------------------------------------------------------------------------------- /fixtures/data/car.json: -------------------------------------------------------------------------------- 1 | { 2 | "reg_no": "A123", 3 | "color": "red", 4 | "top_speed_mile": 120, 5 | "cylinders": 8 6 | } 7 | -------------------------------------------------------------------------------- /fixtures/data/car_changes.json: -------------------------------------------------------------------------------- 1 | { 2 | "reg_no": "A123", 3 | "changes": [ 4 | { 5 | "action": "drive", 6 | "summary": "car color changed to blue", 7 | "unrelated_stuff": 123 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /fixtures/data/echo-playbook.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: mission-control.flanksource.com/v1 3 | kind: Playbook 4 | metadata: 5 | name: echo-input-name 6 | namespace: mc 7 | spec: 8 | category: Echoer 9 | description: Echos the input 10 | parameters: 11 | - name: name 12 | label: Name 13 | actions: 14 | - name: Echo 15 | exec: 16 | script: echo "{{.params.name}}" -------------------------------------------------------------------------------- /fixtures/data/multiple-configs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Config1", 4 | "id": 1, 5 | "password": "p1", 6 | "secret": "secret_1" 7 | }, 8 | { 9 | "name": "Config2", 10 | "id": 2, 11 | "password": "p2", 12 | "secret": "secret_2" 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /fixtures/data/single-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Config1", 3 | "id": 1, 4 | "password": "p1", 5 | "secret": "secret_1" 6 | } 7 | -------------------------------------------------------------------------------- /fixtures/data/test.yaml: -------------------------------------------------------------------------------- 1 | aws: 2 | - region: eu-west-1 3 | compliance: true 4 | patch_states: true 5 | patch_details: true 6 | inventory: true 7 | made_at: "2017-03-06T21:04:11Z" 8 | deleted_at: "2017-04-04T15:04:05Z" 9 | -------------------------------------------------------------------------------- /fixtures/expected/file-exclusion.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "last_modified": "0001-01-01T00:00:00Z", 4 | "config_type": "Config1", 5 | "config_class": "MySecrets", 6 | "name": "Config1", 7 | "id": "1", 8 | "config": { 9 | "id": 1, 10 | "name": "Config1", 11 | "secret": "secret_1" 12 | } 13 | }, 14 | { 15 | "last_modified": "0001-01-01T00:00:00Z", 16 | "config_type": "Config2", 17 | "config_class": "MySecrets", 18 | "name": "Config2", 19 | "id": "2", 20 | "config": { 21 | "id": 2, 22 | "name": "Config2" 23 | } 24 | } 25 | ] 26 | -------------------------------------------------------------------------------- /fixtures/expected/file-git.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "last_modified": "0001-01-01T00:00:00Z", 4 | "source": "github.com/flanksource/canary-checker/fixtures/minimal/http_pass_single.yaml", 5 | "id": "http-pass", 6 | "config_type": "Canary", 7 | "config_class": "Canary", 8 | "config": { 9 | "apiVersion": "canaries.flanksource.com/v1", 10 | "kind": "Canary", 11 | "metadata": { 12 | "name": "http-pass", 13 | "labels": { 14 | "canary": "http" 15 | } 16 | }, 17 | "spec": { 18 | "schedule": "@every 5m", 19 | "http": [ 20 | { 21 | "url": "https://httpbin.demo.aws.flanksource.com/status/200", 22 | "name": "http-deprecated-endpoint" 23 | }, 24 | { 25 | "name": "http-minimal-check", 26 | "url": "https://httpbin.demo.aws.flanksource.com/status/200", 27 | "metrics": [ 28 | { 29 | "name": "httpbin_2xx_count", 30 | "type": "counter", 31 | "value": "code == 200 ? 1 : 0", 32 | "labels": [ 33 | { 34 | "name": "name", 35 | "value": "httpbin_2xx_count" 36 | }, 37 | { 38 | "name": "check_name", 39 | "valueExpr": "check.name" 40 | }, 41 | { 42 | "name": "status_class", 43 | "valueExpr": "string(code).charAt(0)" 44 | } 45 | ] 46 | } 47 | ] 48 | }, 49 | { 50 | "name": "http-param-tests", 51 | "url": "https://httpbin.demo.aws.flanksource.com/status/200", 52 | "responseCodes": [201, 200, 301], 53 | "responseContent": "", 54 | "maxSSLExpiry": 7 55 | }, 56 | { 57 | "name": "http-expr-tests", 58 | "url": "https://httpbin.demo.aws.flanksource.com/status/200", 59 | "test": { 60 | "expr": "code in [200,201,301] && sslAge > Duration('7d')" 61 | }, 62 | "display": { 63 | "template": "code={{.code}}, age={{.sslAge}}" 64 | } 65 | }, 66 | { 67 | "name": "http-headers", 68 | "url": "https://httpbin.demo.aws.flanksource.com/headers", 69 | "test": { 70 | "expr": "json.headers[\"User-Agent\"].startsWith(\"canary-checker/\")" 71 | } 72 | } 73 | ] 74 | } 75 | } 76 | } 77 | ] 78 | -------------------------------------------------------------------------------- /fixtures/expected/file-mask.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "last_modified": "0001-01-01T00:00:00Z", 3 | "config_class": "Config", 4 | "config_type": "Config", 5 | "name": "Config1", 6 | "id": "1", 7 | "config": { 8 | "id": 1, 9 | "name": "Config1", 10 | "password": "ec6ef230f1828039ee794566b9c58adc", 11 | "secret": "***" 12 | } 13 | }] 14 | -------------------------------------------------------------------------------- /fixtures/expected/file-postgres-properties.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "last_modified": "0001-01-01T00:00:00Z", 4 | "id": "postgresql-properties", 5 | "config_class": "PostgreSQLProperties", 6 | "config_type": "PostgreSQLProperties", 7 | "config": { 8 | "authentication_timeout": "1min", 9 | "checkpoint_completion_target": "0.5", 10 | "checkpoint_flush_after": "256kB", 11 | "cluster_name": "'main'", 12 | "cron.database_name": "'postgres'", 13 | "data_directory": "'/var/lib/postgresql/data'", 14 | "db_user_namespace": "off", 15 | "default_text_search_config": "'pg_catalog.english'", 16 | "effective_cache_size": "128MB", 17 | "extra_float_digits": "0", 18 | "hba_file": "'/etc/postgresql/pg_hba.conf'", 19 | "ident_file": "'/etc/postgresql/pg_ident.conf'", 20 | "jit_provider": "'llvmjit'", 21 | "lc_messages": "'en_US.UTF-8'", 22 | "lc_monetary": "'en_US.UTF-8'", 23 | "lc_numeric": "'en_US.UTF-8'", 24 | "lc_time": "'en_US.UTF-8'", 25 | "listen_addresses": "'*'", 26 | "log_destination": "'csvlog'", 27 | "log_directory": "'/var/log/postgresql'", 28 | "log_file_mode": "0640", 29 | "log_filename": "'postgresql.log'", 30 | "log_line_prefix": "'%h %m [%p] %q%u@%d '", 31 | "log_rotation_age": "0", 32 | "log_rotation_size": "0", 33 | "log_statement": "'all'", 34 | "log_timezone": "'UTC'", 35 | "logging_collector": "on", 36 | "max_replication_slots": "5", 37 | "max_slot_wal_keep_size": "1024", 38 | "max_wal_senders": "10", 39 | "password_encryption": "scram-sha-256", 40 | "pgsodium.getkey_script": "'/usr/lib/postgresql/14/bin/pgsodium_getkey_urandom.sh'", 41 | "pljava.libjvm_location": "'/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'", 42 | "row_security": "on", 43 | "shared_buffers": "128MB", 44 | "shared_preload_libraries": "'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium'", 45 | "ssl": "off", 46 | "ssl_ca_file": "''", 47 | "ssl_cert_file": "''", 48 | "ssl_ciphers": "'HIGH:MEDIUM:+3DES:!aNULL'", 49 | "ssl_crl_dir": "''", 50 | "ssl_crl_file": "''", 51 | "ssl_dh_params_file": "''", 52 | "ssl_ecdh_curve": "'prime256v1'", 53 | "ssl_key_file": "''", 54 | "ssl_max_protocol_version": "''", 55 | "ssl_min_protocol_version": "'TLSv1.2'", 56 | "ssl_passphrase_command": "''", 57 | "ssl_passphrase_command_supports_reload": "off", 58 | "ssl_prefer_server_ciphers": "on", 59 | "timezone": "'UTC'", 60 | "unix_socket_directories": "'/var/run/postgresql'", 61 | "wal_level": "logical" 62 | } 63 | } 64 | ] 65 | -------------------------------------------------------------------------------- /fixtures/expected/file-script-gotemplate.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "config_class": "MyConfig", 4 | "config_type": "MyConfig", 5 | "name": "scraped", 6 | "id": "1", 7 | "config": { 8 | "name-1": "hi Config1", 9 | "name-2": "hi Config2", 10 | "id": "1" 11 | } 12 | } 13 | ] 14 | -------------------------------------------------------------------------------- /fixtures/expected/file-script.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "last_modified": "0001-01-01T00:00:00Z", 4 | "config_class": "Config", 5 | "config_type": "Config", 6 | "name": "Config1", 7 | "id": "1", 8 | "config": { 9 | "added": "a", 10 | "id": 1, 11 | "name": "Config1", 12 | "password": "p1", 13 | "secret": "secret_1" 14 | } 15 | }, 16 | { 17 | "last_modified": "0001-01-01T00:00:00Z", 18 | "config_class": "Config", 19 | "config_type": "Config", 20 | "name": "Config2", 21 | "id": "2", 22 | "config": { 23 | "added": "a", 24 | "id": 2, 25 | "name": "Config2", 26 | "password": "p2", 27 | "secret": "secret_2" 28 | } 29 | } 30 | ] 31 | -------------------------------------------------------------------------------- /fixtures/file-car-change.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-car-change-scraper 5 | spec: 6 | full: true 7 | file: 8 | - type: Car 9 | id: $.reg_no 10 | paths: 11 | - fixtures/data/car_changes.json -------------------------------------------------------------------------------- /fixtures/file-car.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-car-scraper 5 | spec: 6 | file: 7 | - type: Car 8 | class: Car 9 | id: $.reg_no 10 | paths: 11 | - fixtures/data/car.json 12 | -------------------------------------------------------------------------------- /fixtures/file-crd-sync.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: playbook-syncer 5 | spec: 6 | crdSync: true 7 | file: 8 | - type: MissionControl::Playbook 9 | id: $.metadata.name 10 | name: $.metadata.name 11 | paths: 12 | - fixtures/data/echo-playbook.yaml 13 | -------------------------------------------------------------------------------- /fixtures/file-exclusion.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: exclude-file-secrets 5 | spec: 6 | file: 7 | - type: $.name 8 | id: $.id 9 | name: $.name 10 | class: MySecrets 11 | transform: 12 | javascript: |+ 13 | for (var i = 0; i < config.length; i++) { 14 | config[i].id = i + 1 15 | } 16 | JSON.stringify(config) 17 | exclude: 18 | - jsonpath: '.password' 19 | - types: 20 | - Config2 21 | jsonpath: '.secret' 22 | paths: 23 | - fixtures/data/multiple-configs.json 24 | -------------------------------------------------------------------------------- /fixtures/file-git.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-git-scraper 5 | spec: 6 | file: 7 | - type: $.kind 8 | id: $.metadata.name 9 | url: github.com/flanksource/canary-checker?ref=076cf8b888f2dbaca26a7cc98a4153c154220a22 10 | paths: 11 | - fixtures/minimal/http_pass.yaml 12 | -------------------------------------------------------------------------------- /fixtures/file-local-creation-date.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-local-creation-date.yaml 5 | spec: 6 | file: 7 | - type: $.aws[0].region 8 | class: AWS 9 | id: $.aws[0].region 10 | createFields: 11 | - $.aws[0].made_at 12 | - $.aws[0].created_at 13 | deleteFields: 14 | - "$.aws[0].removed_at" 15 | - "$.aws[0].deleted_at" 16 | paths: 17 | - fixtures/data/test.yaml 18 | -------------------------------------------------------------------------------- /fixtures/file-local.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-local-scraper 5 | spec: 6 | file: 7 | - type: $.aws[0].region 8 | class: $.aws[0].region 9 | id: $.aws[0].region 10 | paths: 11 | - fixtures/data/test.yaml 12 | -------------------------------------------------------------------------------- /fixtures/file-mask.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-mask-scraper 5 | spec: 6 | file: 7 | - type: Config 8 | id: $.id 9 | name: $.name 10 | transform: 11 | mask: 12 | - selector: config.name == 'Config1' 13 | jsonpath: $.password 14 | value: md5sum 15 | - selector: config.name == 'Config1' 16 | jsonpath: $.secret 17 | value: '***' 18 | paths: 19 | - fixtures/data/single-config.json 20 | 21 | -------------------------------------------------------------------------------- /fixtures/file-postgres-properties.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-postgres-properties-scraper 5 | spec: 6 | file: 7 | - format: properties 8 | type: PostgreSQLProperties 9 | id: postgresql-properties 10 | paths: 11 | - ./fixtures/file-postgres-properties.conf 12 | -------------------------------------------------------------------------------- /fixtures/file-script-gotemplate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-script-gotemplate-scraper 5 | spec: 6 | file: 7 | - type: MyConfig 8 | id: "$.id" 9 | name: "scraped" 10 | transform: 11 | gotemplate: |+ 12 | [ 13 | { 14 | {{range $i := .config}} 15 | "name-{{.id}}": "hi {{.name}}", 16 | {{end}} 17 | "id": "1" 18 | } 19 | ] 20 | paths: 21 | - fixtures/data/multiple-configs.json 22 | -------------------------------------------------------------------------------- /fixtures/file-script.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-script-scraper 5 | spec: 6 | file: 7 | - type: Config 8 | id: $.id 9 | name: $.name 10 | transform: 11 | javascript: |+ 12 | for (var i = 0; i < config.length; i++) { 13 | config[i].added ="a" 14 | } 15 | JSON.stringify(config) 16 | paths: 17 | - fixtures/data/multiple-configs.json -------------------------------------------------------------------------------- /fixtures/file.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: file-scraper 5 | spec: 6 | file: 7 | - type: $.Config.InstanceType 8 | class: $.Config.InstanceType 9 | id: $.Config.InstanceId 10 | path: 11 | - config*.json 12 | - test*.json 13 | -------------------------------------------------------------------------------- /fixtures/github-actions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: github-actions-scraper 5 | spec: 6 | githubActions: 7 | - owner: flanksource 8 | repository: config-db 9 | personalAccessToken: 10 | value: 11 | -------------------------------------------------------------------------------- /fixtures/http-lastfm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: lastfm-scraper 5 | spec: 6 | http: 7 | - type: 'LastFM::Singer' 8 | name: '$.name' 9 | id: '$.url' 10 | env: 11 | - name: api_key 12 | valueFrom: 13 | secretKeyRef: 14 | name: lastfm 15 | key: API_KEY 16 | url: 'http://ws.audioscrobbler.com/2.0/?method=chart.gettopartists&api_key={{.api_key}}&format=json' 17 | transform: 18 | expr: | 19 | dyn(config).artists.artist.map(item, item).toJSON() 20 | -------------------------------------------------------------------------------- /fixtures/http-scraper.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: todo-scraper 5 | spec: 6 | http: 7 | - type: 'Todo::Task' 8 | name: '$.title' 9 | id: '$.id' 10 | url: 'https://jsonplaceholder.typicode.com/todos' 11 | transform: 12 | expr: | 13 | dyn(config).filter(item, item.id <= 10).map(item, { 14 | "title": item.title, 15 | "id": item.id, 16 | }).toJSON() 17 | -------------------------------------------------------------------------------- /fixtures/kubernetes_file.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: kubernetes-file-scraper 5 | spec: 6 | kubernetesFile: 7 | - selector: 8 | namespace: default 9 | kind: Statefulset 10 | name: postgresql 11 | container: postgresql 12 | files: 13 | - path: 14 | - /etc/postgresql/postgresql.conf 15 | format: properties 16 | -------------------------------------------------------------------------------- /fixtures/load/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | k6 3 | -------------------------------------------------------------------------------- /fixtures/load/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: 2 | k6: 3 | go install go.k6.io/xk6/cmd/xk6@latest 4 | xk6 build v0.45.1 --with github.com/grafana/xk6-kubernetes@v0.10.0 --with github.com/avitalique/xk6-file@latest 5 | 6 | .PHONY: 7 | run: k6 8 | kubectl delete pods --all -n testns 9 | ./k6 run load.ts --insecure-skip-tls-verify 10 | -------------------------------------------------------------------------------- /fixtures/plugin-change-exclusion.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapePlugin 3 | metadata: 4 | name: exclude-info-level-changes 5 | namespace: mc 6 | spec: 7 | changes: 8 | exclude: 9 | - severity == "info" 10 | -------------------------------------------------------------------------------- /fixtures/plugin-change-mapping.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapePlugin 3 | metadata: 4 | name: change-mapping-rules 5 | namespace: mc 6 | spec: 7 | changes: 8 | mapping: 9 | - filter: > 10 | change_type == 'diff' && summary == "status.containerStatuses" && 11 | patch != null && has(patch.status) && has(patch.status.containerStatuses) && 12 | patch.status.containerStatuses.size() > 0 && 13 | has(patch.status.containerStatuses[0].restartCount) 14 | type: PodCrashLooping 15 | - filter: > 16 | change_type == 'diff' && summary == "status.images" && config.kind == "Node" 17 | type: ImageUpdated 18 | -------------------------------------------------------------------------------- /fixtures/scrapper.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: aws-general-scraper 5 | spec: 6 | aws: 7 | - region: ap-south-1 8 | compliance: true 9 | patch_states: true 10 | patch_details: true 11 | inventory: true 12 | -------------------------------------------------------------------------------- /fixtures/slack.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # The following regexp captures 3 | # name: | config_type: | severity: | summary: | type: 4 | # An example slack message that would match the regexp is: 5 | # name: terraform.tfstate | config_type: Terraform::StateFile | severity: high | summary: ishealthy | type: health_update 6 | # 7 | # NOTE: Even though the mapping is left empty, the `severity`, `type` & `summary` for the change 8 | # are defaulted from the captured groups in the regexp. 9 | apiVersion: configs.flanksource.com/v1 10 | kind: ScrapeConfig 11 | metadata: 12 | name: slack-flanksource 13 | namespace: default 14 | spec: 15 | schedule: '@every 1m' 16 | slack: 17 | - channels: 18 | - 'notification-*' 19 | since: 14d 20 | token: 21 | valueFrom: 22 | secretKeyRef: 23 | name: slack-mission-control-bot 24 | key: token 25 | rules: 26 | - regexp: name:\s*(?P[\w\s.-]+?)\s*\|\s*config_type:\s*(?P[\w:]+)\s*\|\s*severity:\s*(?P\w+)\s*\|\s*summary:\s*(?P[\w\s]+?)\s*\|\s*type:\s*(?P[\w_]+) 27 | filter: 28 | bot: 'Notifier' 29 | config: 30 | - name: 31 | expr: env.name 32 | types: 33 | - expr: env.config_type 34 | mapping: {} 35 | -------------------------------------------------------------------------------- /fixtures/sql.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: incident-commander-postgres-scraper 5 | spec: 6 | sql: 7 | - connection: postgresql://postgres:postgres@localhost:5432/incident_commander?sslmode=disable 8 | type: Postgres::Database 9 | id: "incident_commander" 10 | items: .database 11 | query: | 12 | WITH settings AS ( 13 | select json_object_agg(name, concat(setting,unit)) as setting from pg_settings where source != 'default' 14 | ), 15 | roles as ( 16 | SELECT json_object_agg(usename, 17 | CASE 18 | WHEN usesuper AND usecreatedb THEN 19 | CAST('superuser, create database' AS pg_catalog.text) 20 | WHEN usesuper THEN 21 | CAST('superuser' AS pg_catalog.text) 22 | WHEN usecreatedb THEN 23 | CAST('create database' AS pg_catalog.text) 24 | ELSE 25 | CAST('' AS pg_catalog.text) 26 | END) as role 27 | FROM pg_catalog.pg_user 28 | ) 29 | 30 | select json_build_object('version', version(), 'settings', s.setting, 'roles', r.role ) as database FROM (SELECT * from settings) as s, (Select * from roles) as r 31 | # - connection: connection://Postgres/incident-commander (Alternatively, you can use a connection) 32 | # - connection: postgresql://$(username):$(password)@localhost:5432/incident_commander?sslmode=disable (connection string can also be templatized) 33 | # auth: 34 | # username: 35 | # value: postgres 36 | # password: 37 | # value: postgres 38 | -------------------------------------------------------------------------------- /fixtures/terraform.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: configs.flanksource.com/v1 3 | kind: ScrapeConfig 4 | metadata: 5 | name: terraform 6 | spec: 7 | schedule: '@every 5m' 8 | terraform: 9 | - name: '{{ filepath.Base .path}}' 10 | state: 11 | s3: 12 | bucket: terraform 13 | connection: connection://aws 14 | objectPath: 'states/**/*.tfstate' 15 | -------------------------------------------------------------------------------- /fixtures/trivy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configs.flanksource.com/v1 2 | kind: ScrapeConfig 3 | metadata: 4 | name: trivy-scraper 5 | spec: 6 | trivy: 7 | - version: "0.40.0" 8 | ignoreUnfixed: true 9 | severity: 10 | - critical 11 | - high 12 | scanners: 13 | - config 14 | - license 15 | - rbac 16 | - secret 17 | - vuln 18 | kubernetes: {} 19 | timeout: "20m" # Increased from the default 5m timeout 20 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ -------------------------------------------------------------------------------- /hack/generate-schemas/.gitignore: -------------------------------------------------------------------------------- 1 | go.mod 2 | go.sum 3 | -------------------------------------------------------------------------------- /hack/generate-schemas/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | 8 | "github.com/flanksource/commons/logger" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | "github.com/flanksource/duty/schema/openapi" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | var schemas = map[string]any{ 15 | "scrape_config": &v1.ScrapeConfig{}, 16 | } 17 | 18 | var generateSchema = &cobra.Command{ 19 | Use: "generate-schema", 20 | RunE: func(cmd *cobra.Command, args []string) error { 21 | if err := os.Mkdir(schemaPath, 0755); err != nil { 22 | logger.Warnf(err.Error()) 23 | } 24 | 25 | for file, obj := range schemas { 26 | p := path.Join(schemaPath, file+".schema.json") 27 | if err := openapi.WriteSchemaToFile(p, obj); err != nil { 28 | logger.Fatalf("unable to save schema: %v", err) 29 | } 30 | logger.Infof("Saved OpenAPI schema to %s", p) 31 | } 32 | 33 | for name, obj := range v1.AllScraperConfigs { 34 | p := path.Join(schemaPath, fmt.Sprintf("config_%s.schema.json", name)) 35 | if err := openapi.WriteSchemaToFile(p, obj); err != nil { 36 | logger.Fatalf("unable to save schema: %v", err) 37 | } 38 | logger.Infof("Saved OpenAPI schema to %s", p) 39 | } 40 | 41 | return nil 42 | }, 43 | } 44 | 45 | var schemaPath string 46 | 47 | func main() { 48 | generateSchema.Flags().StringVar(&schemaPath, "schema-path", "../../config/schemas", "Path to save JSON schema to") 49 | if err := generateSchema.Execute(); err != nil { 50 | os.Exit(1) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /jobs/jobs.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "github.com/flanksource/commons/logger" 5 | "github.com/flanksource/config-db/api" 6 | "github.com/flanksource/duty/context" 7 | dutyEcho "github.com/flanksource/duty/echo" 8 | "github.com/flanksource/duty/job" 9 | "github.com/robfig/cron/v3" 10 | ) 11 | 12 | const JobResourceType = "configs" 13 | 14 | var FuncScheduler = cron.New() 15 | 16 | func init() { 17 | dutyEcho.RegisterCron(FuncScheduler) 18 | } 19 | 20 | func ScheduleJobs(ctx context.Context) { 21 | for _, j := range cleanupJobs { 22 | job := j 23 | job.Context = ctx 24 | if err := job.AddToScheduler(FuncScheduler); err != nil { 25 | logger.Fatalf(err.Error()) 26 | } 27 | } 28 | 29 | if err := job.NewJob(ctx, "Process Change Retention Rules", "@every 1h", ProcessChangeRetentionRules). 30 | RunOnStart().AddToScheduler(FuncScheduler); err != nil { 31 | logger.Errorf("Failed to schedule sync jobs for team component: %v", err) 32 | } 33 | 34 | if api.UpstreamConfig.Valid() { 35 | for _, j := range UpstreamJobs { 36 | job := j 37 | job.Context = ctx 38 | if err := job.AddToScheduler(FuncScheduler); err != nil { 39 | logger.Fatalf(err.Error()) 40 | } 41 | } 42 | } 43 | } 44 | 45 | func Stop() { 46 | FuncScheduler.Stop() 47 | } 48 | -------------------------------------------------------------------------------- /jobs/jobs_test.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | ) 7 | 8 | var _ = ginkgo.Describe("Job Tests", ginkgo.Ordered, func() { 9 | var totalconfigitems int 10 | 11 | ginkgo.BeforeAll(func() { 12 | CleanupConfigItems.Context = DefaultContext 13 | ConfigItemRetentionDays = 7 14 | 15 | err := DefaultContext.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&totalconfigitems).Error 16 | Expect(err).ToNot(HaveOccurred()) 17 | }) 18 | 19 | ginkgo.It("should not cleanup recently deleted config items", func() { 20 | err := DefaultContext.DB().Exec("UPDATE config_items SET deleted_at = NOW()").Error 21 | Expect(err).ToNot(HaveOccurred()) 22 | 23 | CleanupConfigItems.Run() 24 | expectJobToPass(CleanupConfigItems) 25 | 26 | var after int 27 | err = DefaultContext.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&after).Error 28 | Expect(err).ToNot(HaveOccurred()) 29 | Expect(after).To(Equal(totalconfigitems)) 30 | }) 31 | 32 | ginkgo.It("should delete all the existing relationship, changes & analyses", func() { 33 | err := DefaultContext.DB().Exec("DELETE FROM config_changes; DELETE FROM config_analysis; DELETE FROM config_relationships;").Error 34 | Expect(err).ToNot(HaveOccurred()) 35 | }) 36 | 37 | ginkgo.It("should cleanup deleted config items after reducing retentiondays", func() { 38 | ConfigItemRetentionDays = 0 39 | CleanupConfigItems.Run() 40 | expectJobToPass(CleanupConfigItems) 41 | 42 | var after int 43 | err := DefaultContext.DB().Raw("SELECT COUNT(*) FROM config_items").Scan(&after).Error 44 | Expect(err).ToNot(HaveOccurred()) 45 | Expect(after).To(Equal(0)) 46 | }) 47 | }) 48 | -------------------------------------------------------------------------------- /jobs/retention.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/flanksource/commons/logger" 7 | v1 "github.com/flanksource/config-db/api/v1" 8 | "github.com/flanksource/config-db/scrapers" 9 | "github.com/flanksource/duty/job" 10 | "github.com/flanksource/duty/models" 11 | ) 12 | 13 | func ProcessChangeRetentionRules(ctx job.JobRuntime) error { 14 | ctx.History.ResourceType = JobResourceType 15 | var allScrapers []models.ConfigScraper 16 | if err := ctx.DB().Find(&allScrapers).Error; err != nil { 17 | return err 18 | } 19 | 20 | for _, s := range allScrapers { 21 | var spec v1.ScraperSpec 22 | if err := json.Unmarshal([]byte(s.Spec), &spec); err != nil { 23 | ctx.History.AddErrorf("failed to unmarshal scraper spec (%s): %v", s.ID, err) 24 | continue 25 | } 26 | 27 | for _, changeSpec := range spec.Retention.Changes { 28 | err := scrapers.ProcessChangeRetention(ctx.Context, s.ID, changeSpec) 29 | if err != nil { 30 | logger.Errorf("Error processing change retention for scraper[%s] config analysis: %v", s.ID, err) 31 | ctx.History.AddError(err.Error()) 32 | } else { 33 | ctx.History.SuccessCount++ 34 | } 35 | } 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /jobs/suite_test.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flanksource/duty/context" 7 | "github.com/flanksource/duty/job" 8 | "github.com/flanksource/duty/models" 9 | "github.com/flanksource/duty/tests/setup" 10 | "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | func TestJobs(t *testing.T) { 15 | RegisterFailHandler(ginkgo.Fail) 16 | ginkgo.RunSpecs(t, "Jobs Test Suite") 17 | } 18 | 19 | var ( 20 | DefaultContext context.Context 21 | ) 22 | 23 | func expectJobToPass(j *job.Job) { 24 | history, err := j.FindHistory() 25 | Expect(err).To(BeNil()) 26 | Expect(len(history)).To(BeNumerically(">=", 1)) 27 | Expect(history[0].Status).To(Equal(models.StatusSuccess)) 28 | } 29 | 30 | var _ = ginkgo.BeforeSuite(func() { 31 | DefaultContext = setup.BeforeSuiteFn().WithTrace() 32 | }) 33 | 34 | var _ = ginkgo.AfterSuite(setup.AfterSuiteFn) 35 | -------------------------------------------------------------------------------- /jobs/sync_upstream.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "time" 8 | 9 | "github.com/flanksource/commons/http" 10 | "github.com/flanksource/commons/logger" 11 | "github.com/flanksource/commons/utils" 12 | "github.com/flanksource/config-db/api" 13 | "github.com/flanksource/duty/context" 14 | "github.com/flanksource/duty/job" 15 | "github.com/flanksource/duty/models" 16 | "github.com/flanksource/duty/upstream" 17 | "gorm.io/gorm/clause" 18 | ) 19 | 20 | var ReconcilePageSize int 21 | 22 | var tablesToReconcile = []string{ 23 | "config_scrapers", 24 | "config_items", 25 | "config_changes", "config_analysis", 26 | "config_relationships", 27 | } 28 | 29 | var ReconcileConfigs = &job.Job{ 30 | Name: "ReconcileConfigs", 31 | Schedule: "@every 1m", 32 | Retention: job.RetentionBalanced, 33 | Singleton: true, 34 | JobHistory: true, 35 | RunNow: true, 36 | Fn: func(ctx job.JobRuntime) error { 37 | ctx.History.ResourceType = job.ResourceTypeUpstream 38 | ctx.History.ResourceID = api.UpstreamConfig.Host 39 | summary := upstream.ReconcileSome(ctx.Context, api.UpstreamConfig, ReconcilePageSize, tablesToReconcile...) 40 | ctx.History.AddDetails("summary", summary) 41 | ctx.History.SuccessCount, ctx.History.ErrorCount = summary.GetSuccessFailure() 42 | if summary.Error() != nil { 43 | ctx.History.AddDetails("errors", summary.Error()) 44 | } 45 | 46 | return nil 47 | }, 48 | } 49 | 50 | var PullUpstreamConfigScrapers = &job.Job{ 51 | Name: "PullUpstreamConfigScrapers", 52 | JobHistory: true, 53 | Singleton: true, 54 | RunNow: true, 55 | Schedule: "@every 10m", 56 | Retention: job.RetentionFew, 57 | Fn: func(ctx job.JobRuntime) error { 58 | ctx.History.ResourceType = job.ResourceTypeUpstream 59 | ctx.History.ResourceID = api.UpstreamConfig.Host 60 | count, err := pullUpstreamConfigScrapers(ctx.Context, api.UpstreamConfig) 61 | ctx.History.SuccessCount = count 62 | return err 63 | }, 64 | } 65 | 66 | var UpstreamJobs = []*job.Job{ 67 | PullUpstreamConfigScrapers, 68 | ReconcileConfigs, 69 | } 70 | 71 | // configScrapersPullLastRuntime pulls scrape configs from the upstream server 72 | var configScrapersPullLastRuntime time.Time 73 | 74 | func pullUpstreamConfigScrapers(ctx context.Context, config upstream.UpstreamConfig) (int, error) { 75 | logger.Tracef("pulling scrape configs from upstream since: %v", configScrapersPullLastRuntime) 76 | 77 | req := http.NewClient().BaseURL(config.Host).Auth(config.Username, config.Password).R(ctx). 78 | QueryParam("since", configScrapersPullLastRuntime.Format(time.RFC3339Nano)). 79 | QueryParam(upstream.AgentNameQueryParam, config.AgentName) 80 | 81 | resp, err := req.Get("upstream/scrapeconfig/pull") 82 | if err != nil { 83 | return 0, fmt.Errorf("error making request: %w", err) 84 | } 85 | defer resp.Body.Close() // nolint:errcheck 86 | 87 | if !resp.IsOK() { 88 | body, _ := io.ReadAll(resp.Body) 89 | return 0, fmt.Errorf("server returned unexpected status:%s (%s)", resp.Status, body) 90 | } 91 | 92 | var scrapeConfigs []models.ConfigScraper 93 | if err := json.NewDecoder(resp.Body).Decode(&scrapeConfigs); err != nil { 94 | return 0, fmt.Errorf("error decoding JSON response: %w", err) 95 | } 96 | 97 | if len(scrapeConfigs) == 0 { 98 | return 0, nil 99 | } 100 | 101 | configScrapersPullLastRuntime = utils.Deref(scrapeConfigs[len(scrapeConfigs)-1].UpdatedAt) 102 | logger.Tracef("fetched %d scrape configs from upstream", len(scrapeConfigs)) 103 | 104 | return len(scrapeConfigs), ctx.DB().Omit("agent_id").Clauses(clause.OnConflict{ 105 | Columns: []clause.Column{{Name: "id"}}, 106 | UpdateAll: true, 107 | }).Create(&scrapeConfigs).Error 108 | } 109 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | 8 | "github.com/flanksource/config-db/cmd" 9 | ) 10 | 11 | func main() { 12 | if err := cmd.Root.ExecuteContext(newCancelableContext()); err != nil { 13 | os.Exit(1) 14 | } 15 | } 16 | 17 | func newCancelableContext() context.Context { 18 | doneCh := make(chan os.Signal, 1) 19 | signal.Notify(doneCh, os.Interrupt) 20 | 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | 23 | go func() { 24 | <-doneCh 25 | cancel() 26 | }() 27 | 28 | return ctx 29 | } 30 | -------------------------------------------------------------------------------- /rustdiffgen.go: -------------------------------------------------------------------------------- 1 | //go:build rustdiffgen 2 | 3 | package main 4 | 5 | /* 6 | #cgo LDFLAGS: ${SRCDIR}/external/diffgen/target/release/libdiffgen.a -ldl 7 | #include "./external/diffgen/libdiffgen.h" 8 | #include 9 | */ 10 | import "C" 11 | 12 | import ( 13 | "unsafe" 14 | 15 | "github.com/flanksource/config-db/db" 16 | ) 17 | 18 | func init() { 19 | db.DiffFunc = func(before, after string) string { 20 | beforeCString := C.CString(before) 21 | defer C.free(unsafe.Pointer(beforeCString)) 22 | 23 | afterCString := C.CString(after) 24 | defer C.free(unsafe.Pointer(afterCString)) 25 | 26 | diffChar := C.diff(beforeCString, afterCString) 27 | defer C.free(unsafe.Pointer(diffChar)) 28 | if diffChar == nil { 29 | return "" 30 | } 31 | 32 | // prefix is required for UI 33 | prefix := "--- before\n+++ after\n" 34 | diff := C.GoString(diffChar) 35 | return prefix + diff 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /scrapers/analysis/rules.go: -------------------------------------------------------------------------------- 1 | package analysis 2 | 3 | import ( 4 | _ "embed" 5 | 6 | "github.com/flanksource/commons/logger" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | //go:embed rules.yaml 11 | var configRules []byte 12 | 13 | type Category struct { 14 | Category, Severity string 15 | } 16 | 17 | var Rules map[string]Category 18 | 19 | func init() { 20 | if err := yaml.Unmarshal(configRules, &Rules); err != nil { 21 | logger.Errorf("Failed to unmarshal config rules: %s", err) 22 | } 23 | logger.Infof("Loaded %d config rules", len(Rules)) 24 | } 25 | -------------------------------------------------------------------------------- /scrapers/analysis/rules.yaml: -------------------------------------------------------------------------------- 1 | ec2-instance-no-public-ip: 2 | category: security 3 | severity: critical 4 | cloudtrail-enabled: 5 | category: security 6 | severity: critical 7 | iam-user-mfa-enabled: 8 | category: security 9 | severity: critical 10 | root-account-hardware-mfa-enabled: 11 | category: security 12 | severity: critical 13 | eks-endpoint-no-public-access: 14 | category: security 15 | severity: critical 16 | vpc-flow-logs-enabled: 17 | category: security 18 | severity: warning 19 | Exposed Access Keys: 20 | category: security 21 | severity: critical 22 | -------------------------------------------------------------------------------- /scrapers/aws/analyzer.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/flanksource/commons/console" 8 | "github.com/flanksource/commons/logger" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | ) 11 | 12 | // EC2InstanceAnalyzer ... 13 | func EC2InstanceAnalyzer(configs []v1.ScrapeResult) v1.AnalysisResult { 14 | 15 | result := v1.AnalysisResult{ 16 | Analyzer: "ec2-instance", 17 | } 18 | for _, config := range configs { 19 | switch config.Config.(type) { 20 | case Instance: 21 | host := config.Config.(Instance) 22 | state := "" 23 | if host.PatchState != nil { 24 | if host.PatchState.FailedCount > 0 { 25 | state += fmt.Sprintf(" failed=%s", console.Redf("%d", host.PatchState.FailedCount)) 26 | } 27 | if (host.PatchState.InstalledCount) > 0 { 28 | state += fmt.Sprintf(" installed=%d", host.PatchState.InstalledCount) 29 | } 30 | if host.PatchState.MissingCount > 0 { 31 | state += fmt.Sprintf(" missing=%s", console.Redf("%d", host.PatchState.MissingCount)) 32 | } 33 | if host.PatchState.OperationEndTime != nil { 34 | state += " end=" + time.Since(*host.PatchState.OperationEndTime).String() 35 | } 36 | } else { 37 | state += console.Redf(" no patch state") 38 | } 39 | logger.Infof("[%s/%s] os=%s %s", host.GetHostname(), host.GetID(), host.GetPlatform(), state) 40 | 41 | for _, compliance := range host.Compliance { 42 | if compliance.ComplianceType != "COMPLIANT" { 43 | result.Messages = append(result.Messages, fmt.Sprintf("[%s/%s] %s - %s: %s", host.GetHostname(), host.GetID(), compliance.ID, compliance.ComplianceType, compliance.Annotation)) 44 | } 45 | } 46 | } 47 | } 48 | 49 | return result 50 | } 51 | -------------------------------------------------------------------------------- /scrapers/aws/config.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go-v2/service/configservice" 5 | "github.com/aws/aws-sdk-go-v2/service/configservice/types" 6 | v1 "github.com/flanksource/config-db/api/v1" 7 | ) 8 | 9 | func (aws Scraper) config(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults) { 10 | if !config.Compliance { 11 | return 12 | } 13 | 14 | ctx.Logger.V(2).Infof("scraping Config Rules") 15 | 16 | rules, err := ctx.Config.DescribeConfigRules(ctx, nil) 17 | if err != nil { 18 | results.Errorf(err, "Failed to describe config rules") 19 | return 20 | } 21 | 22 | for _, rule := range rules.ConfigRules { 23 | details, err := ctx.Config.GetComplianceDetailsByConfigRule(ctx, &configservice.GetComplianceDetailsByConfigRuleInput{ 24 | ConfigRuleName: rule.ConfigRuleName, 25 | ComplianceTypes: []types.ComplianceType{types.ComplianceTypeNonCompliant}, 26 | }) 27 | if err != nil { 28 | results.Errorf(err, "Failed to describe config rules") 29 | return 30 | } 31 | for _, compliance := range details.EvaluationResults { 32 | if compliance.EvaluationResultIdentifier == nil { 33 | continue 34 | } 35 | if compliance.EvaluationResultIdentifier.EvaluationResultQualifier == nil { 36 | continue 37 | } 38 | obj := compliance.EvaluationResultIdentifier.EvaluationResultQualifier 39 | results.Analysis(*obj.ConfigRuleName, *obj.ResourceType, *obj.ResourceId). 40 | Message(deref(rule.Description)). 41 | Message(deref(compliance.Annotation)) 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /scrapers/aws/scratch.go: -------------------------------------------------------------------------------- 1 | package aws 2 | -------------------------------------------------------------------------------- /scrapers/azure/azure_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import "testing" 4 | 5 | func TestExtractResourceGroup(t *testing.T) { 6 | tests := []struct { 7 | input string 8 | expectedOutput string 9 | }{ 10 | // Valid input cases 11 | {"/subscriptions/0cd017bb-aa54-5121-b21f-ecf8daee0624/resourcegroups/mc_demo_demo_francecentral", "mc_demo_demo_francecentral"}, 12 | {"/subscriptions/0cd017bb-aa54-5121-b21f-ecf8daee0624/resourcegroups/crossplane", "crossplane"}, 13 | {"/subscriptions/0cd017bb-aa54-5121-b21f-ecf8daee0624/resourcegroups/crossplane/providers/microsoft.containerservice/managedclusters/workload-prod-eu-01", "crossplane"}, 14 | {"/subscriptions/0cd017bb-aa54-5121-b21f-ecf8daee0624/resourcegroups/internal-prod/providers/microsoft.storage/storageaccounts/flanksourcebackups", "internal-prod"}, 15 | {"/subscriptions/0cd017bb-aa54-5121-b21f-ecf8daee0624/resourcegroups/mc_crossplane_crossplane-master_northeurope/providers/microsoft.network/loadbalancers/kubernetes", "mc_crossplane_crossplane-master_northeurope"}, 16 | 17 | // Invalid input cases 18 | {"", ""}, 19 | {"/subscriptions/123", ""}, 20 | {"/subscriptions/456/notresourcegroup/test", ""}, 21 | } 22 | 23 | for _, test := range tests { 24 | result := extractResourceGroup(test.input) 25 | if result != test.expectedOutput { 26 | t.Errorf("Input: %s, Expected: %s, Got: %s", test.input, test.expectedOutput, result) 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /scrapers/changes/changes_suite_test.go: -------------------------------------------------------------------------------- 1 | package changes 2 | 3 | import ( 4 | "testing" 5 | 6 | dutycontext "github.com/flanksource/duty/context" 7 | "github.com/flanksource/duty/tests/setup" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | // +kubebuilder:scaffold:imports 11 | ) 12 | 13 | func TestRunScrapers(t *testing.T) { 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "Changes Suite") 16 | } 17 | 18 | var ( 19 | DefaultContext dutycontext.Context 20 | ) 21 | 22 | var _ = BeforeSuite(func() { 23 | DefaultContext = setup.BeforeSuiteFn().WithTrace() 24 | }) 25 | 26 | var _ = AfterSuite(func() { 27 | setup.AfterSuiteFn() 28 | }) 29 | -------------------------------------------------------------------------------- /scrapers/changes/rules.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /scrapers/common.go: -------------------------------------------------------------------------------- 1 | package scrapers 2 | 3 | import ( 4 | "github.com/flanksource/config-db/api" 5 | "github.com/flanksource/config-db/scrapers/azure" 6 | "github.com/flanksource/config-db/scrapers/clickhouse" 7 | "github.com/flanksource/config-db/scrapers/gcp" 8 | "github.com/flanksource/config-db/scrapers/http" 9 | "github.com/flanksource/config-db/scrapers/slack" 10 | "github.com/flanksource/config-db/scrapers/terraform" 11 | "github.com/flanksource/config-db/scrapers/trivy" 12 | "github.com/flanksource/duty/types" 13 | 14 | v1 "github.com/flanksource/config-db/api/v1" 15 | "github.com/flanksource/config-db/scrapers/aws" 16 | "github.com/flanksource/config-db/scrapers/azure/devops" 17 | "github.com/flanksource/config-db/scrapers/file" 18 | "github.com/flanksource/config-db/scrapers/github" 19 | "github.com/flanksource/config-db/scrapers/kubernetes" 20 | "github.com/flanksource/config-db/scrapers/sql" 21 | ) 22 | 23 | // All is the scrapers registry 24 | var All = []api.Scraper{ 25 | azure.Scraper{}, 26 | aws.Scraper{}, 27 | aws.CostScraper{}, 28 | file.FileScraper{}, 29 | kubernetes.KubernetesScraper{}, 30 | kubernetes.KubernetesFileScraper{}, 31 | devops.AzureDevopsScraper{}, 32 | github.GithubActionsScraper{}, 33 | clickhouse.ClickhouseScraper{}, 34 | gcp.Scraper{}, 35 | slack.Scraper{}, 36 | sql.SqlScraper{}, 37 | trivy.Scanner{}, 38 | http.Scraper{}, 39 | terraform.Scraper{}, 40 | } 41 | 42 | func GetAuthValues(ctx api.ScrapeContext, auth *v1.Authentication) (*v1.Authentication, error) { 43 | authentication := &v1.Authentication{ 44 | Username: types.EnvVar{ 45 | ValueStatic: "", 46 | }, 47 | Password: types.EnvVar{ 48 | ValueStatic: "", 49 | }, 50 | } 51 | // in case nil we are sending empty string values for username and password 52 | if auth == nil { 53 | return authentication, nil 54 | } 55 | username, err := ctx.GetEnvValueFromCache(auth.Username, ctx.GetNamespace()) 56 | if err != nil { 57 | return nil, err 58 | } 59 | authentication.Username = types.EnvVar{ 60 | ValueStatic: username, 61 | } 62 | password, err := ctx.GetEnvValueFromCache(auth.Password, ctx.GetNamespace()) 63 | if err != nil { 64 | return nil, err 65 | } 66 | authentication.Password = types.EnvVar{ 67 | ValueStatic: password, 68 | } 69 | return authentication, err 70 | } 71 | -------------------------------------------------------------------------------- /scrapers/file/file_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import "testing" 4 | 5 | // test stripPrefix 6 | func TestStripPrefix(t *testing.T) { 7 | cases := []struct { 8 | input string 9 | expected string 10 | }{ 11 | {"file://foo", "foo"}, 12 | {"git::foo", "foo"}, 13 | {"git::https://foo", "https://foo"}, 14 | {"foo", "foo"}, 15 | {"", ""}, 16 | } 17 | for _, c := range cases { 18 | actual := stripPrefix(c.input) 19 | if actual != c.expected { 20 | t.Errorf("stripPrefix(%s) == %s, expected %s", c.input, actual, c.expected) 21 | } 22 | } 23 | } 24 | 25 | func TestConvertLocalPath(t *testing.T) { 26 | cases := []struct { 27 | input string 28 | expected string 29 | }{ 30 | {"file://foo", "foo-ecf5c8ee"}, 31 | {"git::foo", "foo-b943d8a5"}, 32 | {"git::https://foo/path?query=abc", "foo-path-8f49fbdc"}, 33 | {"foo", "foo-acbd18db"}, 34 | } 35 | for _, c := range cases { 36 | actual := convertToLocalPath(c.input) 37 | if actual != c.expected { 38 | t.Errorf("convertToLocalPath(%s) == %s, expected %s", c.input, actual, c.expected) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /scrapers/gcp/cloud_logging.go: -------------------------------------------------------------------------------- 1 | package gcp 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "cloud.google.com/go/logging/logadmin" 8 | v1 "github.com/flanksource/config-db/api/v1" 9 | "google.golang.org/api/iterator" 10 | ) 11 | 12 | func (gcp Scraper) AuditLogs(ctx *GCPContext, config v1.GCP, results *v1.ScrapeResults) { 13 | if !config.Includes("AuditLog") { 14 | return 15 | } 16 | 17 | adminClient, err := logadmin.NewClient(ctx, config.Project) 18 | if err != nil { 19 | results.Errorf(err, "failed to create logging admin client") 20 | return 21 | } 22 | defer adminClient.Close() // nolint:errcheck 23 | 24 | // Define the time range for the logs 25 | startTime := time.Now().Add(-24 * time.Hour) // Last 24 hours 26 | endTime := time.Now() 27 | 28 | // Define the filter for audit logs 29 | filter := fmt.Sprintf(`logName="projects/%s/logs/cloudaudit.googleapis.com%%2Factivity" AND timestamp>="%s" AND timestamp<="%s"`, 30 | config.Project, startTime.Format(time.RFC3339), endTime.Format(time.RFC3339)) 31 | 32 | it := adminClient.Entries(ctx, logadmin.Filter(filter)) 33 | 34 | for { 35 | entry, err := it.Next() 36 | if err == iterator.Done { 37 | break 38 | } 39 | if err != nil { 40 | results.Errorf(err, "failed to list audit log entries") 41 | return 42 | } 43 | 44 | // Extract relevant information from the log entry 45 | resourceName := entry.Resource.Labels["resource_name"] 46 | if resourceName == "" { 47 | continue 48 | } 49 | 50 | change := v1.ChangeResult{ 51 | 52 | // Timestamp: entry.Timestamp, 53 | // Message: entry.TextPayload, 54 | // Details: entry, 55 | } 56 | 57 | // Find the corresponding configuration item and attach the change 58 | for i, result := range *results { 59 | if result.ID == resourceName { 60 | (*results)[i].Changes = append((*results)[i].Changes, change) 61 | break 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /scrapers/github/client_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/flanksource/config-db/api" 8 | v1 "github.com/flanksource/config-db/api/v1" 9 | "github.com/flanksource/duty/context" 10 | "github.com/flanksource/duty/types" 11 | ) 12 | 13 | var testGithubApiClient = func() (*GitHubActionsClient, error) { 14 | textCtx := api.NewScrapeContext(context.New()) 15 | ghToken := os.Getenv("GH_ACCESS_TOKEN") 16 | testGh := v1.GitHubActions{ 17 | Owner: "flanksource", 18 | Repository: "config-db", 19 | PersonalAccessToken: types.EnvVar{ValueStatic: ghToken}, 20 | } 21 | client, err := NewGitHubActionsClient(textCtx, testGh) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | return client, nil 27 | } 28 | 29 | var client *GitHubActionsClient 30 | var workflows []Workflow 31 | 32 | func init() { 33 | var err error 34 | client, err = testGithubApiClient() 35 | if err != nil { 36 | panic(err) 37 | } 38 | } 39 | 40 | func TestGetWorkFlows(t *testing.T) { 41 | var err error 42 | workflows, err = client.GetWorkflows() 43 | if err != nil { 44 | t.Fatalf("error was not expected %v", err) 45 | } 46 | // (TODO: basebandit) we could probably assert that there is something in the returned workflows slice 47 | } 48 | 49 | func TestGetWorkFlowRuns(t *testing.T) { 50 | for _, workflow := range workflows { 51 | _, err := client.GetWorkflowRuns(workflow.ID, 1) 52 | if err != nil { 53 | t.Fatalf("error was not expected %v", err) 54 | } 55 | 56 | // (TODO: basebandit) we could probably assert that there is something in the returned runs slice 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /scrapers/github/workflows.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "github.com/flanksource/commons/collections" 8 | "github.com/flanksource/config-db/api" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | "github.com/flanksource/config-db/db" 11 | ) 12 | 13 | const WorkflowRun = "GitHubActions::WorkflowRun" 14 | 15 | type GithubActionsScraper struct { 16 | } 17 | 18 | func (gh GithubActionsScraper) CanScrape(spec v1.ScraperSpec) bool { 19 | return len(spec.GithubActions) > 0 20 | } 21 | 22 | // Scrape fetches github workflows and workflow runs from github API and converts the action executions (workflow runs) to change events. 23 | func (gh GithubActionsScraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { 24 | results := v1.ScrapeResults{} 25 | for _, config := range ctx.ScrapeConfig().Spec.GithubActions { 26 | client, err := NewGitHubActionsClient(ctx, config) 27 | if err != nil { 28 | results.Errorf(err, "failed to create github actions client for owner %s with repository %v", config.Owner, config.Repository) 29 | continue 30 | } 31 | 32 | workflows, err := client.GetWorkflows() 33 | if err != nil { 34 | results.Errorf(err, "failed to get projects for %s", config.Repository) 35 | continue 36 | } 37 | 38 | for _, workflow := range workflows { 39 | if !collections.MatchItems(workflow.Name, config.Workflows...) { 40 | continue 41 | } 42 | runs, err := getNewWorkflowRuns(client, workflow) 43 | if err != nil { 44 | results.Errorf(err, "failed to get workflow runs for %s", workflow.GetID()) 45 | continue 46 | } 47 | results = append(results, v1.ScrapeResult{ 48 | ConfigClass: "Deployment", 49 | Config: workflow, 50 | Type: WorkflowRun, 51 | ID: workflow.GetID(), 52 | Name: workflow.Name, 53 | Changes: runs, 54 | Aliases: []string{fmt.Sprintf("%s/%d", workflow.Name, workflow.ID)}, 55 | }) 56 | } 57 | } 58 | return results 59 | } 60 | 61 | func getNewWorkflowRuns(client *GitHubActionsClient, workflow Workflow) ([]v1.ChangeResult, error) { 62 | runs, err := client.GetWorkflowRuns(workflow.ID, 1) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | var allRuns []v1.ChangeResult 68 | for _, run := range runs.Value { 69 | allRuns = append(allRuns, v1.ChangeResult{ 70 | ChangeType: "GithubAction", 71 | CreatedAt: &run.CreatedAt, 72 | Severity: fmt.Sprint(run.Conclusion), 73 | ExternalID: workflow.GetID(), 74 | ConfigType: WorkflowRun, 75 | Source: run.Event, 76 | Details: v1.NewJSON(run), 77 | ExternalChangeID: fmt.Sprintf("%s/%d/%d", workflow.Name, workflow.ID, run.ID), 78 | }) 79 | } 80 | 81 | // Get total runs from DB for that workflow 82 | totalRunsInDB, err := db.GetWorkflowRunCount(client.ScrapeContext, workflow.GetID()) 83 | if err != nil { 84 | return nil, err 85 | } 86 | delta := runs.Count - totalRunsInDB 87 | pagesToFetch := int(math.Ceil(float64(delta) / 100)) 88 | for page := 2; page <= pagesToFetch; page += 1 { 89 | runs, err := client.GetWorkflowRuns(workflow.ID, page) 90 | if err != nil { 91 | return nil, err 92 | } 93 | for _, run := range runs.Value { 94 | allRuns = append(allRuns, v1.ChangeResult{ 95 | ChangeType: "GithubWorkflowRun", 96 | CreatedAt: &run.CreatedAt, 97 | Severity: run.Conclusion.(string), 98 | ExternalID: workflow.GetID(), 99 | ConfigType: WorkflowRun, 100 | Source: run.Event, 101 | Details: v1.NewJSON(run), 102 | ExternalChangeID: fmt.Sprintf("%s/%d/%d", workflow.Name, workflow.ID, run.ID), 103 | }) 104 | } 105 | } 106 | return allRuns, nil 107 | } 108 | -------------------------------------------------------------------------------- /scrapers/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/flanksource/config-db/api" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | "github.com/flanksource/duty/connection" 11 | "github.com/flanksource/gomplate/v3" 12 | "github.com/samber/lo" 13 | ) 14 | 15 | type Scraper struct{} 16 | 17 | func (file Scraper) CanScrape(configs v1.ScraperSpec) bool { 18 | return len(configs.HTTP) > 0 19 | } 20 | 21 | func (file Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { 22 | var results v1.ScrapeResults 23 | 24 | for _, spec := range ctx.ScrapeConfig().Spec.HTTP { 25 | if result, err := scrape(ctx, spec); err != nil { 26 | results = append(results, v1.ScrapeResult{Error: err}) 27 | } else { 28 | results = append(results, *result) 29 | } 30 | } 31 | 32 | return results 33 | } 34 | 35 | func scrape(ctx api.ScrapeContext, spec v1.HTTP) (*v1.ScrapeResult, error) { 36 | conn, err := spec.HTTPConnection.Hydrate(ctx, ctx.Namespace()) 37 | if err != nil { 38 | return nil, fmt.Errorf("failed to populate connection: %w", err) 39 | } 40 | 41 | client, err := connection.CreateHTTPClient(ctx, *conn) 42 | if err != nil { 43 | return nil, fmt.Errorf("failed to create http client: %w", err) 44 | } 45 | 46 | for key, val := range spec.Headers { 47 | if v, err := ctx.GetEnvValueFromCache(val, ctx.Namespace()); err != nil { 48 | return nil, fmt.Errorf("failed to get header env value for %v: %w", val, err) 49 | } else { 50 | client.Header(key, v) 51 | } 52 | } 53 | 54 | templateEnv := map[string]any{} 55 | for _, env := range spec.Env { 56 | if v, err := ctx.GetEnvValueFromCache(env, ctx.Namespace()); err != nil { 57 | return nil, fmt.Errorf("failed to get env value for %v: %w", env, err) 58 | } else { 59 | templateEnv[env.Name] = v 60 | } 61 | } 62 | 63 | url, err := gomplate.RunTemplate(templateEnv, gomplate.Template{Template: conn.URL}) 64 | if err != nil { 65 | return nil, fmt.Errorf("failed to apply template: %w", err) 66 | } 67 | 68 | method := lo.CoalesceOrEmpty(lo.FromPtr(spec.Method), "GET") 69 | request := client.R(ctx) 70 | if spec.Body != nil { 71 | if err := request.Body(*spec.Body); err != nil { 72 | return nil, fmt.Errorf("failed to apply TLS config: %w", err) 73 | } 74 | } 75 | 76 | response, err := request.Do(method, url) 77 | if err != nil { 78 | return nil, fmt.Errorf("error calling URL: %w", err) 79 | } 80 | 81 | responseBody, err := response.AsString() 82 | if err != nil { 83 | return nil, fmt.Errorf("failed to get response as a string: %w", err) 84 | } 85 | 86 | result := v1.NewScrapeResult(spec.BaseScraper) 87 | if !response.IsJSON() { 88 | result.Config = responseBody 89 | } else { 90 | if strings.HasPrefix(responseBody, "[") { 91 | var jsonArr []any 92 | if err := json.Unmarshal([]byte(responseBody), &jsonArr); err != nil { 93 | return nil, fmt.Errorf("failed to unmarshal response: %w", err) 94 | } 95 | result.Config = jsonArr 96 | } else { 97 | var jsonObj map[string]any 98 | if err := json.Unmarshal([]byte(responseBody), &jsonObj); err != nil { 99 | return nil, fmt.Errorf("failed to unmarshal response: %w", err) 100 | } 101 | result.Config = jsonObj 102 | } 103 | } 104 | 105 | return result, nil 106 | } 107 | -------------------------------------------------------------------------------- /scrapers/kubernetes/events.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | "time" 8 | 9 | "github.com/flanksource/commons/collections" 10 | "github.com/flanksource/commons/logger" 11 | v1 "github.com/flanksource/config-db/api/v1" 12 | "github.com/flanksource/is-healthy/events" 13 | "github.com/google/uuid" 14 | "github.com/samber/lo" 15 | "k8s.io/apimachinery/pkg/types" 16 | ) 17 | 18 | func maxTime(t1, t2 time.Time) time.Time { 19 | if t1.Sub(t2) > 0 { 20 | return t1 21 | } 22 | return t2 23 | } 24 | 25 | func getSeverityFromReason(reason string, errKeywords, warnKeywords []string) string { 26 | if collections.MatchItems(reason, errKeywords...) { 27 | return "error" 28 | } 29 | 30 | if collections.MatchItems(reason, warnKeywords...) { 31 | return "warn" 32 | } 33 | 34 | return "" 35 | } 36 | 37 | func getSourceFromEvent(event v1.KubernetesEvent) string { 38 | if component, ok := event.Source["component"]; ok { 39 | return component 40 | } 41 | 42 | keyVals := make([]string, 0, len(event.Source)) 43 | for k, v := range event.Source { 44 | keyVals = append(keyVals, fmt.Sprintf("%s=%s", k, v)) 45 | } 46 | 47 | sort.Slice(keyVals, func(i, j int) bool { return keyVals[i] < keyVals[j] }) 48 | return fmt.Sprintf("kubernetes/%s", strings.Join(keyVals, ",")) 49 | } 50 | 51 | func getDetailsFromEvent(event v1.KubernetesEvent) map[string]any { 52 | details, err := event.AsMap() 53 | if err != nil { 54 | logger.Errorf("failed to convert event to map: %v", err) 55 | return nil 56 | } 57 | 58 | // Don't remove involved object as it can be useful when 59 | // excluding changes during transformation. 60 | // delete(details, "involvedObject") 61 | 62 | if metadata, ok := details["metadata"].(map[string]any); ok { 63 | delete(metadata, "managedFields") 64 | } 65 | 66 | return details 67 | } 68 | 69 | func getChangeFromEvent(event v1.KubernetesEvent, severityKeywords v1.SeverityKeywords) *v1.ChangeResult { 70 | _, err := uuid.Parse(string(event.InvolvedObject.UID)) 71 | if err != nil { 72 | path := fmt.Sprintf("Kubernetes/%s/%s/%s", event.InvolvedObject.Kind, event.InvolvedObject.Namespace, event.InvolvedObject.Name) 73 | logger.Warnf("failed to parse uid (%s), using default path %s: %v", event.InvolvedObject.UID, path, err) 74 | event.InvolvedObject.UID = types.UID(path) 75 | } 76 | 77 | severity := getSeverityFromReason(event.Reason, severityKeywords.Error, severityKeywords.Warn) 78 | if severity == "" { 79 | severity = events.GetSeverity(event.Reason) 80 | } 81 | 82 | createdAt := event.Metadata.CreationTimestamp.Time 83 | for _, mf := range event.Metadata.ManagedFields { 84 | createdAt = maxTime(createdAt, lo.FromPtr(mf.Time).Time) 85 | } 86 | 87 | return &v1.ChangeResult{ 88 | ChangeType: event.Reason, 89 | CreatedAt: &createdAt, 90 | Details: getDetailsFromEvent(event), 91 | ExternalChangeID: event.GetUID(), 92 | ExternalID: string(event.InvolvedObject.UID), 93 | ConfigType: ConfigTypePrefix + event.InvolvedObject.Kind, 94 | Severity: severity, 95 | Source: getSourceFromEvent(event), 96 | Summary: event.Message, 97 | ConfigID: string(event.InvolvedObject.UID), 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /scrapers/kubernetes/events_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | v1 "github.com/flanksource/config-db/api/v1" 8 | corev1 "k8s.io/api/core/v1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | ) 11 | 12 | func Test_getSourceFromEvent(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | args v1.KubernetesEvent 16 | want string 17 | }{ 18 | { 19 | name: "simple", args: v1.KubernetesEvent{ 20 | Source: map[string]string{ 21 | "component": "kubelet", 22 | "host": "minikube", 23 | }, 24 | }, 25 | want: "kubelet", 26 | }, 27 | { 28 | name: "empty", args: v1.KubernetesEvent{ 29 | Source: map[string]string{}, 30 | }, 31 | want: "kubernetes/", 32 | }, 33 | } 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | if got := getSourceFromEvent(tt.args); got != tt.want { 37 | t.Errorf("getSourceFromEvent() = %v, want %v", got, tt.want) 38 | } 39 | }) 40 | } 41 | } 42 | 43 | func TestEvent_FromObjMap(t *testing.T) { 44 | t.Run("from object", func(t *testing.T) { 45 | eventV1 := corev1.Event{ 46 | ObjectMeta: metav1.ObjectMeta{ 47 | Name: "HI", 48 | }, 49 | } 50 | var eventFromV1 v1.KubernetesEvent 51 | if err := eventFromV1.FromObjMap(eventV1); err != nil { 52 | t.Fatalf("error was not expected %v", err) 53 | } 54 | 55 | if eventFromV1.Metadata == nil { 56 | t.Fail() 57 | } 58 | }) 59 | 60 | t.Run("from map", func(t *testing.T) { 61 | eventMap := map[string]any{ 62 | "metadata": map[string]any{ 63 | "name": "HI", 64 | "namespace": "default", 65 | "uid": "1028a8ac-b028-456c-b3ea-869b9a9fba5f", 66 | "creationTimestamp": "2020-01-01T00:00:00Z", 67 | }, 68 | } 69 | var eventFromMap v1.KubernetesEvent 70 | if err := eventFromMap.FromObjMap(eventMap); err != nil { 71 | t.Fatalf("error was not expected %v", err) 72 | } 73 | 74 | if eventFromMap.Metadata == nil { 75 | t.Fail() 76 | } 77 | }) 78 | 79 | t.Run("from map II", func(t *testing.T) { 80 | eventMap := corev1.Event{ 81 | ObjectMeta: metav1.ObjectMeta{ 82 | CreationTimestamp: metav1.Time{ 83 | Time: time.Date(1995, 8, 1, 0, 0, 0, 0, time.UTC), 84 | }, 85 | }, 86 | } 87 | 88 | var expected v1.KubernetesEvent 89 | if err := expected.FromObjMap(eventMap); err != nil { 90 | t.Fatalf("error was not expected %v", err) 91 | } 92 | 93 | if !expected.Metadata.CreationTimestamp.Time.Equal(eventMap.ObjectMeta.CreationTimestamp.Time) { 94 | t.Fatalf("creation timestamps do not match, expected %v, got %v", eventMap.ObjectMeta.CreationTimestamp.Time, expected.Metadata.CreationTimestamp.Time) 95 | } 96 | }) 97 | } 98 | -------------------------------------------------------------------------------- /scrapers/kubernetes/hook_argo.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | 7 | "github.com/Jeffail/gabs/v2" 8 | "github.com/flanksource/commons/logger" 9 | v1 "github.com/flanksource/config-db/api/v1" 10 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 11 | ) 12 | 13 | type argo struct{} 14 | 15 | func init() { 16 | childlookupHooks = append(childlookupHooks, argo{}) 17 | } 18 | 19 | func (argo argo) ChildLookupHook(ctx *KubernetesContext, obj *unstructured.Unstructured) []v1.ConfigExternalKey { 20 | children := []v1.ConfigExternalKey{} 21 | // Argo Applications have children references 22 | if strings.HasPrefix(obj.GetAPIVersion(), "argoproj.io") && obj.GetKind() == "Application" { 23 | o := gabs.Wrap(obj.Object) 24 | 25 | type argoResourceRef struct { 26 | Kind string `json:"kind"` 27 | Name string `json:"name"` 28 | Namespace string `json:"namespace"` 29 | } 30 | var ars []argoResourceRef 31 | if err := json.Unmarshal(o.S("status", "resources").Bytes(), &ars); err != nil { 32 | logger.Tracef("error marshaling status.resources for argo app[%s/%s]: %v", obj.GetNamespace(), obj.GetName(), err) 33 | } else { 34 | for _, resource := range ars { 35 | children = append([]v1.ConfigExternalKey{{ 36 | Type: ConfigTypePrefix + resource.Kind, 37 | ExternalID: alias(resource.Kind, resource.Namespace, resource.Name), 38 | }}, children...) 39 | } 40 | } 41 | } 42 | 43 | return children 44 | } 45 | -------------------------------------------------------------------------------- /scrapers/kubernetes/hook_aws.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "regexp" 5 | 6 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 7 | ) 8 | 9 | var arnRegexp = regexp.MustCompile(`arn:aws:iam::(\d+):role/`) 10 | 11 | func extractAccountIDFromARN(input string) string { 12 | matches := arnRegexp.FindStringSubmatch(input) 13 | if len(matches) >= 2 { 14 | return matches[1] 15 | } 16 | 17 | return "" 18 | } 19 | 20 | type AWS struct{} 21 | 22 | func init() { 23 | onObjectHooks = append(onObjectHooks, AWS{}) 24 | } 25 | 26 | func (aws AWS) OnObject(ctx *KubernetesContext, obj *unstructured.Unstructured) (bool, map[string]string, error) { 27 | if obj.GetKind() == "ConfigMap" && obj.GetName() == "aws-auth" { 28 | cm, ok := obj.Object["data"].(map[string]any) 29 | if ok { 30 | var accountID string 31 | if mapRolesYAML, ok := cm["mapRoles"].(string); ok { 32 | accountID = extractAccountIDFromARN(mapRolesYAML) 33 | } 34 | 35 | ctx.cluster.Tags.Append("account", accountID) 36 | 37 | if clusterScrapeResult, ok := ctx.cluster.Config.(map[string]any); ok { 38 | clusterScrapeResult["aws-auth"] = cm 39 | clusterScrapeResult["account-id"] = accountID 40 | } 41 | } 42 | } 43 | if obj.GetKind() == "Pod" && obj.GetLabels()["app.kubernetes.io/name"] == "aws-node" { 44 | nodeName := getString(obj, "spec", "nodeName") 45 | spec := obj.Object["spec"].(map[string]interface{}) 46 | for _, ownerRef := range obj.GetOwnerReferences() { 47 | if ownerRef.Kind == "DaemonSet" && ownerRef.Name == "aws-node" { 48 | var ( 49 | awsRoleARN = getContainerEnv(spec, "AWS_ROLE_ARN") 50 | vpcID = getContainerEnv(spec, "VPC_ID") 51 | awsClusterName = getContainerEnv(spec, "CLUSTER_NAME") 52 | ) 53 | 54 | ctx.labelsPerNode[nodeName] = make(map[string]string) 55 | 56 | if awsRoleARN != "" { 57 | ctx.labelsPerNode[nodeName]["aws/iam-role"] = awsRoleARN 58 | } 59 | 60 | if vpcID != "" { 61 | ctx.labelsForAllNode["aws/vpc-id"] = vpcID 62 | 63 | if clusterScrapeResult, ok := ctx.cluster.Config.(map[string]any); ok { 64 | clusterScrapeResult["vpc-id"] = vpcID 65 | } 66 | } 67 | 68 | if awsClusterName != "" { 69 | if clusterScrapeResult, ok := ctx.cluster.Config.(map[string]any); ok { 70 | clusterScrapeResult["cluster-name"] = awsClusterName 71 | } 72 | } 73 | } 74 | } 75 | } 76 | return false, nil, nil 77 | } 78 | -------------------------------------------------------------------------------- /scrapers/kubernetes/hook_azure.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "strings" 5 | 6 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 7 | ) 8 | 9 | func parseAzureURI(uri string) (string, string) { 10 | if !strings.HasPrefix(uri, "azure:///subscriptions/") { 11 | return "", "" 12 | } 13 | 14 | parts := strings.Split(uri, "/") 15 | var subscriptionID, vmScaleSetID string 16 | for i := 0; i < len(parts); i++ { 17 | if parts[i] == "subscriptions" && i+1 < len(parts) { 18 | subscriptionID = parts[i+1] 19 | } 20 | 21 | if parts[i] == "virtualMachineScaleSets" && i+1 < len(parts) { 22 | vmScaleSetID = parts[i+1] 23 | break 24 | } 25 | } 26 | 27 | return subscriptionID, vmScaleSetID 28 | } 29 | 30 | type Azure struct{} 31 | 32 | func init() { 33 | onObjectHooks = append(onObjectHooks, Azure{}) 34 | } 35 | 36 | func (azure Azure) OnObject(ctx *KubernetesContext, obj *unstructured.Unstructured) (bool, map[string]string, error) { 37 | labels := make(map[string]string) 38 | if obj.GetKind() == "Node" { 39 | if clusterName, ok := obj.GetLabels()["kubernetes.azure.com/cluster"]; ok { 40 | // kubernetes.azure.com/cluster doesn't actually contain the 41 | // AKS cluster name - it contains the node resource group. 42 | // The cluster name isn't available in the node. 43 | ctx.cluster.Labels["aks-nodeResourceGroup"] = clusterName 44 | } 45 | 46 | if spec, ok := obj.Object["spec"].(map[string]interface{}); ok { 47 | if providerID, ok := spec["providerID"].(string); ok { 48 | subID, vmScaleSetID := parseAzureURI(providerID) 49 | if subID != "" { 50 | ctx.globalLabels["azure/subscription-id"] = subID 51 | } 52 | if vmScaleSetID != "" { 53 | labels["azure/vm-scale-set"] = vmScaleSetID 54 | } 55 | } 56 | } 57 | } 58 | return false, labels, nil 59 | } 60 | -------------------------------------------------------------------------------- /scrapers/kubernetes/hook_flux.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "strings" 5 | 6 | v1 "github.com/flanksource/config-db/api/v1" 7 | "github.com/samber/lo" 8 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 9 | ) 10 | 11 | type flux struct{} 12 | 13 | func isKustomizationObject(obj *unstructured.Unstructured) bool { 14 | if obj.GetKind() == "Kustomization" && strings.HasPrefix(obj.GetAPIVersion(), "kustomize.toolkit.fluxcd.io") { 15 | return true 16 | } 17 | return false 18 | } 19 | 20 | func init() { 21 | parentlookupHooks = append(parentlookupHooks, flux{}) 22 | } 23 | 24 | func (flux flux) ParentLookupHook(ctx *KubernetesContext, obj *unstructured.Unstructured) []v1.ConfigExternalKey { 25 | helmName := obj.GetLabels()["helm.toolkit.fluxcd.io/name"] 26 | helmNamespace := obj.GetLabels()["helm.toolkit.fluxcd.io/namespace"] 27 | if helmName != "" && helmNamespace != "" { 28 | return []v1.ConfigExternalKey{{ 29 | Type: ConfigTypePrefix + "HelmRelease", 30 | ExternalID: lo.CoalesceOrEmpty( 31 | ctx.GetID(helmNamespace, "HelmRelease", helmName), 32 | alias("HelmRelease", helmNamespace, helmName)), 33 | }} 34 | } 35 | 36 | kustomizeName := obj.GetLabels()["kustomize.toolkit.fluxcd.io/name"] 37 | kustomizeNamespace := obj.GetLabels()["kustomize.toolkit.fluxcd.io/namespace"] 38 | // Kustomization objects should not have Kustomization parents 39 | if kustomizeName != "" && kustomizeNamespace != "" && !isKustomizationObject(obj) { 40 | return []v1.ConfigExternalKey{{ 41 | Type: ConfigTypePrefix + "Kustomization", 42 | ExternalID: lo.CoalesceOrEmpty( 43 | ctx.GetID(kustomizeNamespace, "Kustomization", kustomizeName), 44 | alias("Kustomization", kustomizeNamespace, kustomizeName)), 45 | }} 46 | } 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /scrapers/kubernetes/hooks.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | v1 "github.com/flanksource/config-db/api/v1" 5 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 6 | ) 7 | 8 | type OnObject interface { 9 | // OnObject is called when a new object is observed, return true to skip the object 10 | OnObject(ctx *KubernetesContext, obj *unstructured.Unstructured) (bool, map[string]string, error) 11 | } 12 | 13 | type ParentLookupHook interface { 14 | ParentLookupHook(ctx *KubernetesContext, obj *unstructured.Unstructured) []v1.ConfigExternalKey 15 | } 16 | 17 | type ChildLookupHook interface { 18 | ChildLookupHook(ctx *KubernetesContext, obj *unstructured.Unstructured) []v1.ConfigExternalKey 19 | } 20 | 21 | var childlookupHooks []ChildLookupHook 22 | var parentlookupHooks []ParentLookupHook 23 | var onObjectHooks []OnObject 24 | 25 | func OnObjectHooks(ctx *KubernetesContext, obj *unstructured.Unstructured) (bool, map[string]string, error) { 26 | labels := make(map[string]string) 27 | for _, hook := range onObjectHooks { 28 | skip, _labels, err := hook.OnObject(ctx, obj) 29 | for k, v := range _labels { 30 | labels[k] = v 31 | } 32 | if err != nil { 33 | return false, labels, err 34 | } 35 | if skip { 36 | return true, labels, nil 37 | } 38 | } 39 | return false, labels, nil 40 | } 41 | 42 | func ParentLookupHooks(ctx *KubernetesContext, obj *unstructured.Unstructured) []v1.ConfigExternalKey { 43 | parents := []v1.ConfigExternalKey{} 44 | for _, hook := range parentlookupHooks { 45 | 46 | parents = append(hook.ParentLookupHook(ctx, obj), parents...) 47 | 48 | } 49 | return parents 50 | } 51 | 52 | func ChildLookupHooks(ctx *KubernetesContext, obj *unstructured.Unstructured) []v1.ConfigExternalKey { 53 | children := []v1.ConfigExternalKey{} 54 | for _, hook := range childlookupHooks { 55 | 56 | children = append(hook.ChildLookupHook(ctx, obj), children...) 57 | 58 | } 59 | return children 60 | } 61 | -------------------------------------------------------------------------------- /scrapers/kubernetes/ketall.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | gocontext "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/flanksource/config-db/api" 10 | v1 "github.com/flanksource/config-db/api/v1" 11 | "github.com/flanksource/duty/context" 12 | "github.com/flanksource/ketall" 13 | ketallClient "github.com/flanksource/ketall/client" 14 | "github.com/flanksource/ketall/options" 15 | "github.com/sethvargo/go-retry" 16 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 17 | ) 18 | 19 | func scrape(ctx api.ScrapeContext, config v1.Kubernetes) ([]*unstructured.Unstructured, error) { 20 | ctx.Context = ctx.WithKubernetes(config.KubernetesConnection) 21 | 22 | opts := options.NewDefaultCmdOptions() 23 | opts, err := updateOptions(ctx.DutyContext(), opts, config) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | var objs []*unstructured.Unstructured 29 | 30 | backoff := retry.WithMaxRetries(3, retry.NewExponential(time.Second)) 31 | err = retry.Do(ctx, backoff, func(goctx gocontext.Context) error { 32 | objs, err = ketall.KetAll(ctx, opts) 33 | if err != nil { 34 | if errors.Is(err, ketallClient.ErrEmpty) { 35 | return fmt.Errorf("no resources returned due to insufficient access") 36 | } 37 | return err 38 | } 39 | 40 | if len(objs) == 0 { 41 | // This scenario happens when new CRDs are introduced but we have a cached 42 | // restmapper who's discovery information is outdated 43 | // We reset the internal discovery cache 44 | k8s, err := ctx.Kubernetes() 45 | if err != nil { 46 | return fmt.Errorf("error getting k8s client: %w", err) 47 | } 48 | k8s.ResetRestMapper() 49 | return retry.RetryableError(fmt.Errorf("no resources or error returned")) 50 | } 51 | 52 | return nil 53 | }) 54 | 55 | return objs, nil 56 | } 57 | 58 | func updateOptions(ctx context.Context, opts *options.KetallOptions, config v1.Kubernetes) (*options.KetallOptions, error) { 59 | opts.AllowIncomplete = config.AllowIncomplete 60 | opts.Namespace = config.Namespace 61 | opts.Scope = config.Scope 62 | opts.Selector = config.Selector 63 | opts.FieldSelector = config.FieldSelector 64 | opts.UseCache = config.UseCache 65 | opts.MaxInflight = config.MaxInflight 66 | opts.Exclusions = append(config.Exclusions.List(), "componentstatuses", "Event") 67 | opts.Since = config.Since 68 | 69 | k8s, err := ctx.Kubernetes() 70 | if err != nil { 71 | return opts, fmt.Errorf("error creating k8s client: %w", err) 72 | } 73 | opts.Flags.KubeConfig = k8s.RestConfig() 74 | 75 | rm, err := k8s.GetRestMapper() 76 | if err != nil { 77 | return opts, fmt.Errorf("error getting k8s rest mapper: %w", err) 78 | } 79 | opts.Flags.RestMapper = rm 80 | 81 | return opts, nil 82 | } 83 | -------------------------------------------------------------------------------- /scrapers/kubernetes/kubernetes_test.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_extractAccountIDFromARN(t *testing.T) { 8 | type args struct { 9 | input string 10 | } 11 | tests := []struct { 12 | name string 13 | args args 14 | want string 15 | }{ 16 | { 17 | name: "xx", 18 | args: args{input: `- groups:\n - system:masters\n rolearn: arn:aws:iam::123456789:role/kubernetes-admin\n username: admin\n- groups:\n - system:bootstrappers\n - system:nodes\n rolearn: arn:aws:iam::123456789:role/eksctl-mission-control-demo-clust-NodeInstanceRole-VRLF7VBIVK3M\n username: system:node:{{EC2PrivateDNSName}}\n`}, 19 | want: "123456789", 20 | }, 21 | } 22 | for _, tt := range tests { 23 | t.Run(tt.name, func(t *testing.T) { 24 | if got := extractAccountIDFromARN(tt.args.input); got != tt.want { 25 | t.Errorf("extractAccountIDFromARN() = %v, want %v", got, tt.want) 26 | } 27 | }) 28 | } 29 | } 30 | 31 | func Test_extractAzureSubscriptionIDFromProvider(t *testing.T) { 32 | tests := []struct { 33 | name string 34 | provider string 35 | subID string 36 | scaleSetID string 37 | }{ 38 | { 39 | name: "valid", 40 | provider: "azure:///subscriptions/3da0f5ee-405a-4dd4-a408-a635799995ea/resourceGroups/mc_demo_demo_francecentral/providers/Microsoft.Compute/virtualMachineScaleSets/aks-pool1-37358073-vmss/virtualMachines/9", 41 | subID: "3da0f5ee-405a-4dd4-a408-a635799995ea", 42 | scaleSetID: "aks-pool1-37358073-vmss", 43 | }, 44 | { 45 | name: "invalid", 46 | provider: "aws:///subscriptions/3da0f5ee-405a-4dd4-a408-a635799995ea/resourceGroups/mc_demo_demo_francecentral/providers/Microsoft.Compute/virtualMachineScaleSets/aks-pool1-37358073-vmss/virtualMachines/9", 47 | subID: "", 48 | scaleSetID: "", 49 | }, 50 | { 51 | name: "absent scale set", 52 | provider: "azure:///subscriptions/3da0f5ee-405a-4dd4-a408-a635799995ea/resourceGroups/mc_demo_demo_francecentral/providers/", 53 | subID: "3da0f5ee-405a-4dd4-a408-a635799995ea", 54 | scaleSetID: "", 55 | }, 56 | } 57 | for _, tt := range tests { 58 | t.Run(tt.name, func(t *testing.T) { 59 | subID, scaleSetID := parseAzureURI(tt.provider) 60 | if subID != tt.subID { 61 | t.Errorf("got = %v, want %v", subID, tt.subID) 62 | } 63 | 64 | if scaleSetID != tt.scaleSetID { 65 | t.Errorf("got = %v, want %v", scaleSetID, tt.scaleSetID) 66 | } 67 | }) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /scrapers/kubernetes/resource_map.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "sync" 5 | 6 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 7 | ) 8 | 9 | type ResourceIDMap map[string]string 10 | 11 | type ResourceIDMapContainer struct { 12 | mu sync.RWMutex 13 | data ResourceIDMap 14 | } 15 | 16 | func ResourceIDMapKey(namespace, kind, name string) string { 17 | return namespace + "|" + kind + "|" + name 18 | } 19 | 20 | func (t *ResourceIDMapContainer) Set(namespace, kind, name, id string) { 21 | t.mu.Lock() 22 | defer t.mu.Unlock() 23 | 24 | t.data[ResourceIDMapKey(namespace, kind, name)] = id 25 | } 26 | 27 | func (t *ResourceIDMapContainer) Get(namespace, kind, name string) string { 28 | t.mu.RLock() 29 | defer t.mu.RUnlock() 30 | return t.data[ResourceIDMapKey(namespace, kind, name)] 31 | } 32 | 33 | type PerClusterResourceIDMap struct { 34 | mu sync.Mutex 35 | data map[string]ResourceIDMap 36 | } 37 | 38 | func (t *PerClusterResourceIDMap) Swap(clusterID string, resourceIDMap ResourceIDMap) { 39 | t.mu.Lock() 40 | defer t.mu.Unlock() 41 | 42 | if t.data == nil { 43 | t.data = make(map[string]ResourceIDMap) 44 | } 45 | 46 | t.data[clusterID] = resourceIDMap 47 | } 48 | 49 | func (t *PerClusterResourceIDMap) MergeAndUpdate(clusterID string, resourceIDMap ResourceIDMap) ResourceIDMap { 50 | t.mu.Lock() 51 | defer t.mu.Unlock() 52 | 53 | cached, ok := t.data[clusterID] 54 | if ok { 55 | resourceIDMap = mergeResourceIDMap(resourceIDMap, cached) 56 | } 57 | 58 | if t.data == nil { 59 | t.data = make(map[string]ResourceIDMap) 60 | } 61 | 62 | t.data[clusterID] = resourceIDMap 63 | return resourceIDMap 64 | } 65 | 66 | func NewResourceIDMap(objs []*unstructured.Unstructured) *ResourceIDMapContainer { 67 | resourceIDMap := make(ResourceIDMap) 68 | for _, obj := range objs { 69 | resourceIDMap[ResourceIDMapKey(obj.GetNamespace(), obj.GetKind(), obj.GetName())] = string(obj.GetUID()) 70 | } 71 | 72 | return &ResourceIDMapContainer{ 73 | data: resourceIDMap, 74 | mu: sync.RWMutex{}, 75 | } 76 | } 77 | 78 | func mergeResourceIDMap(latest, cached ResourceIDMap) ResourceIDMap { 79 | if len(latest) == 0 { 80 | return cached 81 | } 82 | 83 | if len(cached) == 0 { 84 | return latest 85 | } 86 | 87 | output := make(ResourceIDMap) 88 | 89 | // First, copy all data from cached 90 | for k, v := range cached { 91 | output[k] = v 92 | } 93 | 94 | // Then, update or add data from latest 95 | for k, v := range latest { 96 | output[k] = v 97 | } 98 | 99 | return output 100 | } 101 | -------------------------------------------------------------------------------- /scrapers/processors/script.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/flanksource/config-db/api" 7 | v1 "github.com/flanksource/config-db/api/v1" 8 | ) 9 | 10 | func RunScript(ctx api.ScrapeContext, result v1.ScrapeResult, script v1.Script) ([]v1.ScrapeResult, error) { 11 | env := map[string]interface{}{ 12 | "config": result.Config, 13 | "result": result, 14 | } 15 | 16 | out, err := ctx.RunTemplate(script.ToGomplate(), env) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | configs, err := unmarshalConfigsFromString(out, result) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | return configs, nil 27 | } 28 | 29 | func unmarshalConfigsFromString(s string, parent v1.ScrapeResult) ([]v1.ScrapeResult, error) { 30 | var configs []v1.ScrapeResult 31 | var results = []map[string]interface{}{} 32 | if err := json.Unmarshal([]byte(s), &results); err != nil { 33 | return nil, err 34 | } 35 | 36 | for _, result := range results { 37 | configs = append(configs, v1.ScrapeResult{ 38 | BaseScraper: parent.BaseScraper.WithoutTransform(), 39 | Config: result, 40 | }) 41 | } 42 | 43 | return configs, nil 44 | } 45 | -------------------------------------------------------------------------------- /scrapers/retention.go: -------------------------------------------------------------------------------- 1 | package scrapers 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/flanksource/commons/duration" 9 | "github.com/flanksource/commons/logger" 10 | v1 "github.com/flanksource/config-db/api/v1" 11 | "github.com/flanksource/duty/context" 12 | "github.com/google/uuid" 13 | ) 14 | 15 | func ProcessChangeRetention(ctx context.Context, scraperID uuid.UUID, spec v1.ChangeRetentionSpec) error { 16 | var whereClauses []string 17 | 18 | var ageMinutes int 19 | if spec.Age != "" { 20 | age, err := duration.ParseDuration(spec.Age) 21 | if err != nil { 22 | return fmt.Errorf("error parsing age %s as duration: %w", spec.Age, err) 23 | } 24 | ageMinutes = int(age.Minutes()) 25 | 26 | whereClauses = append(whereClauses, `((now()- created_at) > interval '1 minute' * @ageMinutes)`) 27 | } 28 | 29 | if spec.Count > 0 { 30 | whereClauses = append(whereClauses, `seq > @count`) 31 | } 32 | 33 | if len(whereClauses) == 0 { 34 | return fmt.Errorf("both age and count cannot be empty") 35 | } 36 | 37 | query := fmt.Sprintf(` 38 | WITH latest_config_changes AS ( 39 | SELECT id, change_type, created_at, ROW_NUMBER() OVER(PARTITION BY config_id ORDER BY created_at DESC) AS seq 40 | FROM config_changes 41 | WHERE 42 | change_type = @changeType AND 43 | config_id IN (SELECT id FROM config_items WHERE scraper_id = @scraperID) 44 | ) 45 | DELETE FROM config_changes 46 | WHERE id IN ( 47 | SELECT id from latest_config_changes 48 | WHERE %s 49 | ) 50 | `, strings.Join(whereClauses, " OR ")) 51 | 52 | result := ctx.DB().Exec(query, 53 | sql.Named("changeType", spec.Name), 54 | sql.Named("scraperID", scraperID), 55 | sql.Named("ageMinutes", ageMinutes), 56 | sql.Named("count", spec.Count), 57 | ) 58 | if err := result.Error; err != nil { 59 | return fmt.Errorf("error retaining config changes: %w", err) 60 | } 61 | 62 | if result.RowsAffected > 0 { 63 | logger.Infof("Deleted %d config_changes as per ChangeRetentionSpec[%s]", result.RowsAffected, spec.Name) 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /scrapers/run_now.go: -------------------------------------------------------------------------------- 1 | package scrapers 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/flanksource/config-db/api" 8 | v1 "github.com/flanksource/config-db/api/v1" 9 | "github.com/flanksource/config-db/db" 10 | "github.com/flanksource/duty/context" 11 | "github.com/labstack/echo/v4" 12 | ) 13 | 14 | func RunNowHandler(c echo.Context) error { 15 | id := c.Param("id") 16 | 17 | baseCtx := c.Request().Context() 18 | ctx := baseCtx.(context.Context) 19 | 20 | scraper, err := db.FindScraper(ctx, id) 21 | if err != nil { 22 | return echo.NewHTTPError(http.StatusBadRequest, err.Error()) // could mean server errors as well, but there's no trivial way to find out... 23 | } 24 | 25 | if scraper == nil { 26 | return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("scraper with id=%s was not found", id)) 27 | } 28 | 29 | configScraper, err := v1.ScrapeConfigFromModel(*scraper) 30 | if err != nil { 31 | return echo.NewHTTPError(http.StatusInternalServerError, "failed to transform config scraper model", err) 32 | } 33 | 34 | scrapeCtx := api.NewScrapeContext(ctx).WithScrapeConfig(&configScraper) 35 | j := newScraperJob(scrapeCtx) 36 | j.JitterDisable = true 37 | j.Run() 38 | 39 | return c.JSON(http.StatusOK, j.LastJob.Details) 40 | } 41 | -------------------------------------------------------------------------------- /scrapers/runscrapers_suite_test.go: -------------------------------------------------------------------------------- 1 | package scrapers 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/flanksource/commons/logger" 9 | "github.com/flanksource/config-db/api" 10 | v1 "github.com/flanksource/config-db/api/v1" 11 | dutycontext "github.com/flanksource/duty/context" 12 | "github.com/flanksource/duty/tests/setup" 13 | . "github.com/onsi/ginkgo/v2" 14 | . "github.com/onsi/gomega" 15 | "k8s.io/client-go/kubernetes/scheme" 16 | "k8s.io/client-go/rest" 17 | "k8s.io/client-go/tools/clientcmd" 18 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 19 | "sigs.k8s.io/controller-runtime/pkg/client" 20 | "sigs.k8s.io/controller-runtime/pkg/envtest" 21 | // +kubebuilder:scaffold:imports 22 | ) 23 | 24 | func TestRunScrapers(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Scrapers Suite") 27 | } 28 | 29 | var ( 30 | DefaultContext dutycontext.Context 31 | ctx api.ScrapeContext 32 | ) 33 | 34 | var _ = BeforeSuite(func() { 35 | 36 | DefaultContext = setup.BeforeSuiteFn().WithDBLogLevel("trace") 37 | ctx = api.NewScrapeContext(DefaultContext) 38 | 39 | if err := os.Chdir(".."); err != nil { 40 | Fail(err.Error()) 41 | } 42 | 43 | setupTestK8s() 44 | }) 45 | 46 | var _ = AfterSuite(func() { 47 | setup.AfterSuiteFn() 48 | if err := testEnv.Stop(); err != nil { 49 | logger.Errorf("Error stopping test environment: %v", err) 50 | } 51 | }) 52 | 53 | var ( 54 | cfg *rest.Config 55 | k8sClient client.Client 56 | testEnv *envtest.Environment 57 | kubeConfigPath string 58 | ) 59 | 60 | func setupTestK8s() { 61 | By("bootstrapping test environment") 62 | testEnv = &envtest.Environment{ 63 | CRDDirectoryPaths: []string{filepath.Join("chart", "crds")}, 64 | } 65 | 66 | var err error 67 | cfg, err = testEnv.Start() 68 | Expect(err).ToNot(HaveOccurred()) 69 | Expect(cfg).ToNot(BeNil()) 70 | 71 | err = v1.AddToScheme(scheme.Scheme) 72 | Expect(err).NotTo(HaveOccurred()) 73 | 74 | // +kubebuilder:scaffold:scheme 75 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) 76 | Expect(err).ToNot(HaveOccurred()) 77 | Expect(k8sClient).ToNot(BeNil()) 78 | 79 | kubeConfigPath, err = createKubeconfigFileForRestConfig(*cfg) 80 | Expect(err).ToNot(HaveOccurred()) 81 | } 82 | 83 | func createKubeconfigFileForRestConfig(restConfig rest.Config) (string, error) { 84 | clusters := make(map[string]*clientcmdapi.Cluster) 85 | clusters["default-cluster"] = &clientcmdapi.Cluster{ 86 | Server: restConfig.Host, 87 | CertificateAuthorityData: restConfig.CAData, 88 | } 89 | contexts := make(map[string]*clientcmdapi.Context) 90 | contexts["default-context"] = &clientcmdapi.Context{ 91 | Cluster: "default-cluster", 92 | AuthInfo: "default-user", 93 | } 94 | authinfos := make(map[string]*clientcmdapi.AuthInfo) 95 | authinfos["default-user"] = &clientcmdapi.AuthInfo{ 96 | ClientCertificateData: restConfig.CertData, 97 | ClientKeyData: restConfig.KeyData, 98 | } 99 | clientConfig := clientcmdapi.Config{ 100 | Kind: "Config", 101 | APIVersion: "v1", 102 | Clusters: clusters, 103 | Contexts: contexts, 104 | CurrentContext: "default-context", 105 | AuthInfos: authinfos, 106 | } 107 | kubeConfigFile, err := os.CreateTemp("", "kubeconfig-*") 108 | if err != nil { 109 | return "", err 110 | } 111 | _ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name()) 112 | return kubeConfigFile.Name(), nil 113 | } 114 | -------------------------------------------------------------------------------- /scrapers/stale.go: -------------------------------------------------------------------------------- 1 | package scrapers 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/flanksource/commons/duration" 8 | v1 "github.com/flanksource/config-db/api/v1" 9 | "github.com/flanksource/config-db/db/models" 10 | "github.com/flanksource/duty/context" 11 | "github.com/google/uuid" 12 | ) 13 | 14 | var ( 15 | DefaultStaleTimeout = "24h" 16 | ) 17 | 18 | func DeleteStaleConfigItems(ctx context.Context, staleTimeout string, scraperID uuid.UUID) (int64, error) { 19 | var staleDuration time.Duration 20 | if val := ctx.Value(contextKeyScrapeStart); val != nil { 21 | if start, ok := val.(time.Time); ok { 22 | staleDuration = time.Since(start) 23 | } 24 | } 25 | 26 | switch staleTimeout { 27 | case "keep": 28 | return 0, nil 29 | case "": 30 | if defaultVal, exists := ctx.Properties()["config.retention.stale_item_age"]; exists { 31 | staleTimeout = defaultVal 32 | } else { 33 | staleTimeout = DefaultStaleTimeout 34 | } 35 | } 36 | 37 | if parsed, err := duration.ParseDuration(staleTimeout); err != nil { 38 | return 0, fmt.Errorf("failed to parse stale timeout %s: %w", staleTimeout, err) 39 | } else if time.Duration(parsed) > staleDuration { 40 | // Use which ever is greater 41 | staleDuration = time.Duration(parsed) 42 | } 43 | 44 | deleteQuery := ` 45 | UPDATE config_items 46 | SET 47 | deleted_at = NOW(), 48 | delete_reason = ? 49 | WHERE 50 | ((NOW() - last_scraped_time) > INTERVAL '1 SECOND' * ?) AND 51 | deleted_at IS NULL AND 52 | scraper_id = ? 53 | RETURNING type` 54 | 55 | var deletedConfigs []models.ConfigItem 56 | result := ctx.DB().Raw(deleteQuery, v1.DeletedReasonStale, staleDuration.Seconds(), scraperID).Scan(&deletedConfigs) 57 | if err := result.Error; err != nil { 58 | return 0, fmt.Errorf("failed to delete stale config items: %w", err) 59 | } 60 | 61 | if len(deletedConfigs) > 0 { 62 | ctx.Logger.V(3).Infof("deleted %d stale config items for scraper: %s", len(deletedConfigs), scraperID) 63 | } 64 | 65 | for _, c := range deletedConfigs { 66 | ctx.Counter("scraper_deleted", "scraper_id", scraperID.String(), "kind", c.Type, "reason", string(v1.DeletedReasonStale)).Add(1) 67 | } 68 | 69 | return result.RowsAffected, nil 70 | } 71 | -------------------------------------------------------------------------------- /scrapers/terraform/mask.go: -------------------------------------------------------------------------------- 1 | package terraform 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | 9 | "github.com/zclconf/go-cty/cty" 10 | gocty "github.com/zclconf/go-cty/cty/gocty" 11 | ctyjson "github.com/zclconf/go-cty/cty/json" 12 | ) 13 | 14 | func attributeToCtyValue(attributes map[string]any) (cty.Value, error) { 15 | attributeJSON, err := json.Marshal(attributes) 16 | if err != nil { 17 | return cty.Value{}, err 18 | } 19 | 20 | impliedType, err := ctyjson.ImpliedType(attributeJSON) 21 | if err != nil { 22 | return cty.Value{}, fmt.Errorf("error unmarshaling state to gocty value: %w", err) 23 | } 24 | 25 | return gocty.ToCtyValue(attributes, impliedType) 26 | } 27 | 28 | func maskSensitiveAttributes(state State, data []byte) (map[string]any, error) { 29 | var stateFileRaw map[string]any 30 | if err := json.Unmarshal(data, &stateFileRaw); err != nil { 31 | return nil, err 32 | } 33 | 34 | resources := stateFileRaw["resources"].([]any) 35 | for i, resource := range state.Resources { 36 | rawResource := resources[i] 37 | rawInstances := rawResource.(map[string]any)["instances"].([]any) 38 | for j, instance := range resource.Instances { 39 | ctyValue, err := attributeToCtyValue(instance.Attributes) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | sensitivePaths, err := unmarshalPaths(instance.SensitiveAttributes) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | // Transform the ctyValue, masking sensitive attributes 50 | maskedValue, err := cty.Transform(ctyValue, func(path cty.Path, v cty.Value) (cty.Value, error) { 51 | for _, sensitivePath := range sensitivePaths { 52 | if path.Equals(sensitivePath) { 53 | var strToHash string 54 | switch v.Type() { 55 | case cty.Number: 56 | bf := v.AsBigFloat() 57 | strToHash = bf.Text('f', -1) 58 | case cty.String: 59 | strToHash = v.AsString() 60 | default: 61 | // For complex types or unsupported types, use cty's string representation 62 | strToHash = v.GoString() 63 | } 64 | 65 | hash := sha256.Sum256([]byte(strToHash)) 66 | hashString := hex.EncodeToString(hash[:]) 67 | return cty.StringVal(fmt.Sprintf("sha256(%s)", hashString)), nil 68 | } 69 | } 70 | return v, nil 71 | }) 72 | if err != nil { 73 | return nil, fmt.Errorf("error masking values: %w", err) 74 | } 75 | 76 | maskedJSON, err := ctyjson.Marshal(maskedValue, maskedValue.Type()) 77 | if err != nil { 78 | return nil, fmt.Errorf("error marshaling masked value to JSON: %w", err) 79 | } 80 | 81 | var maskedAttribute map[string]any 82 | if err := json.Unmarshal(maskedJSON, &maskedAttribute); err != nil { 83 | return nil, fmt.Errorf("error unmarshaling masked JSON to instance: %w", err) 84 | } 85 | 86 | rawInstances[j].(map[string]any)["attributes"] = maskedAttribute 87 | } 88 | } 89 | 90 | return stateFileRaw, nil 91 | } 92 | -------------------------------------------------------------------------------- /scrapers/terraform/mask_test.go: -------------------------------------------------------------------------------- 1 | package terraform 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | func Test_maskSensitiveAttributes(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | wantErr bool 15 | }{ 16 | { 17 | name: "cloudflare.tfstate", 18 | wantErr: false, 19 | }, 20 | } 21 | for _, tt := range tests { 22 | t.Run(tt.name, func(t *testing.T) { 23 | content, err := os.ReadFile(fmt.Sprintf("testdata/%s", tt.name)) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | 28 | var state State 29 | if err := json.Unmarshal(content, &state); err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | got, err := maskSensitiveAttributes(state, content) 34 | if (err != nil) != tt.wantErr { 35 | t.Errorf("maskSensitiveAttributes() error = %v, wantErr %v", err, tt.wantErr) 36 | return 37 | } 38 | 39 | expected, err := os.ReadFile(fmt.Sprintf("testdata/%s.expected", tt.name)) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | var expectedMap map[string]any 45 | if err := json.Unmarshal(expected, &expectedMap); err != nil { 46 | t.Fatal(err) 47 | } 48 | 49 | if !reflect.DeepEqual(got, expectedMap) { 50 | t.Errorf("maskSensitiveAttributes() = %v, want %v", got, expectedMap) 51 | } 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /scrapers/terraform/models.go: -------------------------------------------------------------------------------- 1 | package terraform 2 | 3 | import "encoding/json" 4 | 5 | type ResourceInstance struct { 6 | SchemaVersion int `json:"schema_version"` 7 | Attributes map[string]any `json:"attributes"` 8 | SensitiveAttributes json.RawMessage `json:"sensitive_attributes"` 9 | Private string `json:"private"` 10 | } 11 | 12 | type Resource struct { 13 | Module string `json:"module"` 14 | Mode string `json:"mode"` 15 | Type string `json:"type"` 16 | Name string `json:"name"` 17 | Provider string `json:"provider"` 18 | Instances []ResourceInstance `json:"instances"` 19 | } 20 | 21 | type State struct { 22 | Version int `json:"version"` 23 | TerraformVersion string `json:"terraform_version"` 24 | Serial int `json:"serial"` 25 | Lineage string `json:"lineage"` 26 | Outputs map[string]interface{} `json:"outputs"` 27 | Resources []Resource `json:"resources"` 28 | } 29 | -------------------------------------------------------------------------------- /scrapers/terraform/path.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package terraform 5 | 6 | import ( 7 | "encoding/json" 8 | "fmt" 9 | 10 | "github.com/zclconf/go-cty/cty" 11 | ctyjson "github.com/zclconf/go-cty/cty/json" 12 | ) 13 | 14 | // pathStep is an intermediate representation of a cty.pathStep to facilitate 15 | // consistent JSON serialization. The Value field can either be a cty.Value of 16 | // dynamic type (for index steps), or a string (for get attr steps). 17 | type pathStep struct { 18 | Type string `json:"type"` 19 | Value json.RawMessage `json:"value"` 20 | } 21 | 22 | const ( 23 | indexPathStepType = "index" 24 | getAttrPathStepType = "get_attr" 25 | ) 26 | 27 | func unmarshalPaths(buf []byte) ([]cty.Path, error) { 28 | var jsonPaths [][]pathStep 29 | 30 | err := json.Unmarshal(buf, &jsonPaths) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | if len(jsonPaths) == 0 { 36 | return nil, nil 37 | } 38 | paths := make([]cty.Path, 0, len(jsonPaths)) 39 | 40 | for _, jsonPath := range jsonPaths { 41 | var path cty.Path 42 | for _, jsonStep := range jsonPath { 43 | switch jsonStep.Type { 44 | case indexPathStepType: 45 | key, err := ctyjson.Unmarshal(jsonStep.Value, cty.DynamicPseudoType) 46 | if err != nil { 47 | return nil, fmt.Errorf("failed to unmarshal index step key: %w", err) 48 | } 49 | path = append(path, cty.IndexStep{Key: key}) 50 | case getAttrPathStepType: 51 | var name string 52 | if err := json.Unmarshal(jsonStep.Value, &name); err != nil { 53 | return nil, fmt.Errorf("failed to unmarshal get attr step name: %w", err) 54 | } 55 | path = append(path, cty.GetAttrStep{Name: name}) 56 | default: 57 | return nil, fmt.Errorf("unsupported path step type %q", jsonStep.Type) 58 | } 59 | } 60 | paths = append(paths, path) 61 | } 62 | 63 | return paths, nil 64 | } 65 | -------------------------------------------------------------------------------- /scrapers/terraform/testdata/cloudflare.tfstate: -------------------------------------------------------------------------------- 1 | { 2 | "version": 4, 3 | "terraform_version": "1.9.4", 4 | "serial": 118, 5 | "lineage": "bef3a10e-9947-edde-9ccd-79f630c9ef13", 6 | "outputs": {}, 7 | "resources": [ 8 | { 9 | "mode": "data", 10 | "type": "sops_file", 11 | "name": "cloudflare_secrets", 12 | "provider": "provider[\"registry.terraform.io/carlpett/sops\"]", 13 | "instances": [ 14 | { 15 | "schema_version": 0, 16 | "attributes": { 17 | "data": "flanksource.com", 18 | "id": "-", 19 | "input_type": null, 20 | "source_file": "secret.sops.yaml" 21 | }, 22 | "sensitive_attributes": [ 23 | [ 24 | { 25 | "type": "get_attr", 26 | "value": "data" 27 | } 28 | ] 29 | ] 30 | } 31 | ] 32 | } 33 | ], 34 | "check_results": null 35 | } 36 | -------------------------------------------------------------------------------- /scrapers/terraform/testdata/cloudflare.tfstate.expected: -------------------------------------------------------------------------------- 1 | { 2 | "version": 4, 3 | "terraform_version": "1.9.4", 4 | "serial": 118, 5 | "lineage": "bef3a10e-9947-edde-9ccd-79f630c9ef13", 6 | "outputs": {}, 7 | "resources": [ 8 | { 9 | "mode": "data", 10 | "type": "sops_file", 11 | "name": "cloudflare_secrets", 12 | "provider": "provider[\"registry.terraform.io/carlpett/sops\"]", 13 | "instances": [ 14 | { 15 | "schema_version": 0, 16 | "attributes": { 17 | "data": "sha256(8e2e88380fc2685aa448d2ad4725e9bc15d232d89c3e027e3504be135d853bfb)", 18 | "id": "-", 19 | "input_type": null, 20 | "source_file": "secret.sops.yaml" 21 | }, 22 | "sensitive_attributes": [ 23 | [ 24 | { 25 | "type": "get_attr", 26 | "value": "data" 27 | } 28 | ] 29 | ] 30 | } 31 | ] 32 | } 33 | ], 34 | "check_results": null 35 | } 36 | -------------------------------------------------------------------------------- /telemetry/pyroscope.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/flanksource/commons/logger" 7 | "github.com/grafana/pyroscope-go" 8 | ) 9 | 10 | func StartPyroscope(serviceName, address string) error { 11 | _, err := pyroscope.Start(pyroscope.Config{ 12 | ApplicationName: serviceName, 13 | 14 | // address of pyroscope server 15 | ServerAddress: address, 16 | 17 | BasicAuthUser: os.Getenv("PYROSCOPE_USER"), 18 | BasicAuthPassword: os.Getenv("PYROSCOPE_PASSWORD"), 19 | 20 | // disable logging by setting this to nil 21 | Logger: logger.GetLogger("pyroscope"), 22 | 23 | Tags: map[string]string{ 24 | "hostname": os.Getenv("HOSTNAME"), 25 | "env": os.Getenv("PYROSCOPE_ENV"), 26 | }, 27 | 28 | ProfileTypes: []pyroscope.ProfileType{ 29 | // these profile types are enabled by default: 30 | pyroscope.ProfileCPU, 31 | pyroscope.ProfileAllocObjects, 32 | pyroscope.ProfileAllocSpace, 33 | pyroscope.ProfileInuseObjects, 34 | pyroscope.ProfileInuseSpace, 35 | 36 | // these profile types are optional: 37 | //pyroscope.ProfileGoroutines, 38 | //pyroscope.ProfileMutexCount, 39 | //pyroscope.ProfileMutexDuration, 40 | //pyroscope.ProfileBlockCount, 41 | //pyroscope.ProfileBlockDuration, 42 | }, 43 | }) 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /telemetry/tracer.go: -------------------------------------------------------------------------------- 1 | package telemetry 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strings" 7 | 8 | "go.opentelemetry.io/otel" 9 | "go.opentelemetry.io/otel/attribute" 10 | "go.opentelemetry.io/otel/exporters/otlp/otlptrace" 11 | "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" 12 | "go.opentelemetry.io/otel/propagation" 13 | "google.golang.org/grpc/credentials" 14 | 15 | "github.com/flanksource/commons/collections" 16 | "github.com/flanksource/commons/logger" 17 | "go.opentelemetry.io/otel/sdk/resource" 18 | sdktrace "go.opentelemetry.io/otel/sdk/trace" 19 | ) 20 | 21 | func InitTracer(serviceName, collectorURL string, insecure bool) func(context.Context) error { 22 | var secureOption otlptracegrpc.Option 23 | if !insecure { 24 | secureOption = otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, "")) 25 | } else { 26 | secureOption = otlptracegrpc.WithInsecure() 27 | } 28 | 29 | exporter, err := otlptrace.New( 30 | context.Background(), 31 | otlptracegrpc.NewClient( 32 | secureOption, 33 | otlptracegrpc.WithEndpoint(collectorURL), 34 | ), 35 | ) 36 | 37 | if err != nil { 38 | logger.Errorf("failed to create opentelemetry exporter: %v", err) 39 | return func(_ context.Context) error { return nil } 40 | } 41 | 42 | attributes := []attribute.KeyValue{attribute.String("service.name", serviceName)} 43 | if val, ok := os.LookupEnv("OTEL_LABELS"); ok { 44 | kv := collections.KeyValueSliceToMap(strings.Split(val, ",")) 45 | for k, v := range kv { 46 | attributes = append(attributes, attribute.String(k, v)) 47 | } 48 | } 49 | 50 | resources, err := resource.New(context.Background(), resource.WithAttributes(attributes...)) 51 | if err != nil { 52 | logger.Errorf("could not set opentelemetry resources: %v", err) 53 | return func(_ context.Context) error { return nil } 54 | } 55 | 56 | otel.SetTracerProvider( 57 | sdktrace.NewTracerProvider( 58 | sdktrace.WithSampler(sdktrace.AlwaysSample()), 59 | sdktrace.WithBatcher(exporter), 60 | sdktrace.WithResource(resources), 61 | ), 62 | ) 63 | 64 | // Register the TraceContext propagator globally. 65 | otel.SetTextMapPropagator(propagation.TraceContext{}) 66 | 67 | return exporter.Shutdown 68 | } 69 | -------------------------------------------------------------------------------- /testdata/.gitignore: -------------------------------------------------------------------------------- 1 | *.json 2 | -------------------------------------------------------------------------------- /utils/concurrency.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "sync" 4 | 5 | // MergeChannels merges multiple channels into one 6 | // 7 | // Source: https://go.dev/blog/pipelines 8 | func MergeChannels[T any](cs ...<-chan T) <-chan T { 9 | var wg sync.WaitGroup 10 | out := make(chan T) 11 | 12 | // Start an output goroutine for each input channel in cs. output 13 | // copies values from c to out until c is closed, then calls wg.Done. 14 | output := func(c <-chan T) { 15 | for n := range c { 16 | out <- n 17 | } 18 | wg.Done() 19 | } 20 | wg.Add(len(cs)) 21 | for _, c := range cs { 22 | go output(c) 23 | } 24 | 25 | // Start a goroutine to close out once all the output goroutines are 26 | // done. This must start after the wg.Add call. 27 | go func() { 28 | wg.Wait() 29 | close(out) 30 | }() 31 | return out 32 | } 33 | -------------------------------------------------------------------------------- /utils/debug.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/labstack/echo/v4" 5 | ) 6 | 7 | var TrackObject = func(name string, obj any) {} 8 | var MemsizeHandler any 9 | 10 | var MemsizeScan = func(obj any) uintptr { 11 | return 0 12 | } 13 | 14 | var MemsizeEchoHandler = func(c echo.Context) error { 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /utils/files.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | func Find(path string) ([]string, error) { 9 | return filepath.Glob(path) 10 | } 11 | 12 | // Read returns the contents of a file, the base filename and an error 13 | func Read(path string) ([]byte, string, error) { 14 | content, err := os.ReadFile(path) 15 | filename := filepath.Base(path) 16 | return content, filename, err 17 | } 18 | -------------------------------------------------------------------------------- /utils/hash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "encoding/json" 8 | ) 9 | 10 | func Hash(v interface{}) (string, error) { 11 | data, err := json.Marshal(v) 12 | if err != nil { 13 | return "", err 14 | } 15 | 16 | hash := md5.Sum(data) 17 | if err != nil { 18 | return "", err 19 | } 20 | return hex.EncodeToString(hash[:]), nil 21 | } 22 | 23 | func Sha256Hex(in string) string { 24 | hash := sha256.New() 25 | hash.Write([]byte(in)) 26 | hashVal := hash.Sum(nil) 27 | return hex.EncodeToString(hashVal[:]) 28 | } 29 | -------------------------------------------------------------------------------- /utils/hash_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestSha256Hex(t *testing.T) { 6 | tests := []struct { 7 | name string 8 | args string 9 | want string 10 | }{ 11 | {name: "first", args: "flanksource", want: "bba09cfc0321b05968bd39bb2e96e4a6bb5f5d3069dcf74ab0772118b7f7258f"}, 12 | {name: "first", args: "programmer", want: "7bd9ca7a756115eabdff2ab281ee9d8c22f44b51d97a6801169d65d90ff16327"}, 13 | } 14 | for _, tt := range tests { 15 | t.Run(tt.name, func(t *testing.T) { 16 | if got := Sha256Hex(tt.args); got != tt.want { 17 | t.Errorf("Sha256Hex() = %v, want %v", got, tt.want) 18 | } 19 | }) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /utils/json.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | func IsJSONPath(path string) bool { 10 | return strings.HasPrefix(path, "$") || strings.HasPrefix(path, "@") 11 | } 12 | 13 | func StructToJSON(v any) (string, error) { 14 | b, err := json.Marshal(&v) 15 | if err != nil { 16 | return "", err 17 | } 18 | return string(b), nil 19 | } 20 | 21 | // ToJSONMap takes an input value of struct or map type and converts it to a map[string]any representation 22 | // using JSON encoding and decoding. 23 | func ToJSONMap(s any) (map[string]any, error) { 24 | var raw []byte 25 | var err error 26 | 27 | switch s := s.(type) { 28 | case string: 29 | raw = []byte(s) 30 | case []byte: 31 | raw = s 32 | default: 33 | raw, err = json.Marshal(s) 34 | if err != nil { 35 | return nil, err 36 | } 37 | } 38 | 39 | result := make(map[string]any) 40 | if err := json.Unmarshal(raw, &result); err != nil { 41 | return nil, err 42 | } 43 | 44 | return result, nil 45 | } 46 | 47 | // LeafNode represents a leaf node in the JSON tree 48 | type LeafNode struct { 49 | path string // path of this node 50 | parent string // path of its parent 51 | } 52 | 53 | // collectLeafNodes recursively traverses the JSON tree and collects all the leaf nodes. 54 | func collectLeafNodes(root map[string]any, parentPath string, leafNodes map[LeafNode]struct{}) { 55 | for key, value := range root { 56 | currentPath := fmt.Sprintf("%s.%s", parentPath, key) 57 | if parentPath == "" { 58 | currentPath = key 59 | } 60 | 61 | if child, ok := value.(map[string]any); ok { 62 | collectLeafNodes(child, currentPath, leafNodes) 63 | } else { 64 | n := LeafNode{ 65 | path: currentPath, 66 | parent: parentPath, 67 | } 68 | leafNodes[n] = struct{}{} 69 | } 70 | } 71 | } 72 | 73 | // ExtractLeafNodesAndCommonParents takes a JSON map and returns the path of the leaf nodes. 74 | // If multiple nodes with the same parent, then the parent's path is returned. 75 | func ExtractLeafNodesAndCommonParents(data map[string]any) []string { 76 | leafNodes := make(map[LeafNode]struct{}) 77 | collectLeafNodes(data, "", leafNodes) 78 | 79 | var parents = make(map[string]int) 80 | for p := range leafNodes { 81 | parents[p.parent]++ 82 | } 83 | 84 | output := make([]string, 0, len(leafNodes)) 85 | seenPaths := make(map[string]struct{}) 86 | for node := range leafNodes { 87 | var path string 88 | if val := parents[node.parent]; val > 1 { 89 | path = node.parent 90 | } else { 91 | path = node.path 92 | } 93 | 94 | if _, ok := seenPaths[path]; ok { 95 | continue 96 | } 97 | 98 | seenPaths[path] = struct{}{} 99 | output = append(output, path) 100 | } 101 | 102 | return output 103 | } 104 | -------------------------------------------------------------------------------- /utils/kube/name.go: -------------------------------------------------------------------------------- 1 | package kube 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/flanksource/commons/console" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 10 | ) 11 | 12 | type Name struct { 13 | Name, Kind, Namespace string 14 | } 15 | 16 | func (n Name) String() string { 17 | if n.Namespace == "" { 18 | n.Namespace = "*" 19 | } 20 | 21 | return fmt.Sprintf("%s/%s/%s", console.Bluef("%s", n.Kind), console.Grayf("%s", n.Namespace), console.LightWhitef("%s", n.Name)) 22 | } 23 | 24 | func (n Name) GetName() string { 25 | return n.Name 26 | } 27 | func (n Name) GetKind() string { 28 | return n.Kind 29 | } 30 | 31 | func (n Name) GetNamespace() string { 32 | return n.Namespace 33 | } 34 | func GetName(obj interface{}) Name { 35 | name := Name{} 36 | switch object := obj.(type) { 37 | case *unstructured.Unstructured: 38 | if object == nil || object.Object == nil { 39 | return name 40 | } 41 | name.Name = object.GetName() 42 | name.Namespace = object.GetNamespace() 43 | case metav1.ObjectMetaAccessor: 44 | name.Name = object.GetObjectMeta().GetName() 45 | name.Namespace = object.GetObjectMeta().GetNamespace() 46 | } 47 | 48 | switch object := obj.(type) { 49 | case *unstructured.Unstructured: 50 | if object == nil || object.Object == nil { 51 | return name 52 | } 53 | name.Kind = object.GetKind() 54 | default: 55 | if t := reflect.TypeOf(obj); t.Kind() == reflect.Ptr { 56 | name.Kind = t.Elem().Name() 57 | } else { 58 | name.Kind = t.Name() 59 | } 60 | } 61 | 62 | return name 63 | } 64 | -------------------------------------------------------------------------------- /utils/struct.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "encoding/json" 4 | 5 | func CloneWithJSON[T any](v T) (T, error) { 6 | b, err := json.Marshal(&v) 7 | if err != nil { 8 | return v, err 9 | } 10 | 11 | var v2 T 12 | return v2, json.Unmarshal(b, &v2) 13 | } 14 | --------------------------------------------------------------------------------