├── .VERSION ├── hercules-packages ├── tpch │ └── 1.0.yml ├── nyc-taxi │ └── 1.0.yml ├── snowflake │ └── 1.0.yml └── bluesky │ └── 1.0.yml ├── pkg ├── db │ ├── constant.go │ ├── sql.go │ ├── sql_test.go │ ├── runner_test.go │ ├── parser.go │ ├── macro_test.go │ ├── extension_test.go │ ├── macro.go │ ├── extension.go │ └── runner.go ├── types │ ├── package.go │ └── package_test.go ├── config │ ├── env.go │ ├── env_test.go │ ├── config_test.go │ └── config.go ├── metric │ ├── meta.go │ ├── counter.go │ ├── gauge.go │ ├── histogram.go │ ├── summary.go │ ├── metric.go │ └── metric_test.go ├── util │ └── pprint.go ├── flock │ ├── flock_test.go │ └── flock.go ├── testUtil │ └── db.go ├── middleware │ └── metric.go ├── labels │ ├── labels.go │ └── labels_test.go ├── metricRegistry │ ├── registry.go │ └── registry_test.go ├── source │ ├── source.go │ └── source_test.go └── herculesPackage │ ├── package_test.go │ └── package.go ├── .gitignore ├── assets ├── hercules.png └── snowflake_query_history.parquet ├── .vscode └── settings.json ├── hercules.yml ├── .github ├── workflows │ ├── test.yml │ ├── lint.yml │ └── release.yml └── golangci.yml ├── cmd └── hercules │ ├── main.go │ └── app.go ├── LICENSE ├── schemas ├── config.json └── package.json ├── go.mod ├── Makefile ├── README.md └── go.sum /.VERSION: -------------------------------------------------------------------------------- 1 | 0.8.0 2 | -------------------------------------------------------------------------------- /hercules-packages/tpch/1.0.yml: -------------------------------------------------------------------------------- 1 | name: tpch 2 | version: 1.0 3 | -------------------------------------------------------------------------------- /pkg/db/constant.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | const ( 4 | NULL string = "null" 5 | ) 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.db 2 | *.wal 3 | .tmp/* 4 | scratch/* 5 | testprofile.out 6 | .DS_Store 7 | -------------------------------------------------------------------------------- /assets/hercules.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakthom/hercules/HEAD/assets/hercules.png -------------------------------------------------------------------------------- /pkg/db/sql.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | // SQL represents a SQL query string. 4 | type SQL string 5 | -------------------------------------------------------------------------------- /assets/snowflake_query_history.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jakthom/hercules/HEAD/assets/snowflake_query_history.parquet -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "yaml.schemas": { 3 | "./schemas/packages.json": "./hercules-packages/*.yaml", 4 | "./schemas/config.json": "/hercules.yaml", 5 | } 6 | } -------------------------------------------------------------------------------- /hercules.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | 3 | name: edge 4 | debug: false 5 | port: 9100 6 | 7 | globalLabels: 8 | - env: $ENV # Inject prometheus labels from env var 9 | 10 | packages: 11 | - package: hercules-packages/snowflake/1.0.yml 12 | -------------------------------------------------------------------------------- /pkg/types/package.go: -------------------------------------------------------------------------------- 1 | package herculestypes 2 | 3 | // PackageName represents a unique identifier for a Hercules package. 4 | type PackageName string 5 | 6 | // MetricPrefix represents a prefix string for metrics belonging to a package. 7 | type MetricPrefix string 8 | -------------------------------------------------------------------------------- /pkg/config/env.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "os" 4 | 5 | func IsTraceMode() bool { 6 | trace := os.Getenv(TraceEnvVar) 7 | return trace == "true" || trace == "1" || trace == "True" 8 | } 9 | 10 | func IsDebugMode() bool { 11 | debug := os.Getenv(DebugEnvVar) 12 | return debug == "true" || debug == "1" || debug == "True" 13 | } 14 | -------------------------------------------------------------------------------- /pkg/db/sql_test.go: -------------------------------------------------------------------------------- 1 | // Package db_test contains tests for the db package 2 | package db_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/jakthom/hercules/pkg/db" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestSql(t *testing.T) { 12 | sql := db.SQL("test") 13 | assert.Equal(t, "test", string(sql)) 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: [push] 3 | 4 | jobs: 5 | run-tests: 6 | runs-on: ubuntu-20.04 7 | steps: 8 | - uses: actions/checkout@v4 9 | - name: Setup Go 10 | uses: actions/setup-go@v5 11 | with: 12 | go-version: 1.23 13 | - name: Test with the Go CLI 14 | run: make test 15 | -------------------------------------------------------------------------------- /pkg/metric/meta.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "github.com/jakthom/hercules/pkg/labels" 5 | herculestypes "github.com/jakthom/hercules/pkg/types" 6 | ) 7 | 8 | type Metadata struct { 9 | PackageName string `json:"packageName"` 10 | Prefix herculestypes.MetricPrefix `json:"metricPrefix"` 11 | Labels labels.Labels `json:"labels"` 12 | } 13 | -------------------------------------------------------------------------------- /pkg/types/package_test.go: -------------------------------------------------------------------------------- 1 | // Package herculestypes_test contains tests for the herculestypes package 2 | package herculestypes_test 3 | 4 | import ( 5 | "testing" 6 | 7 | herculestypes "github.com/jakthom/hercules/pkg/types" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestPackageName(t *testing.T) { 12 | pkg := herculestypes.PackageName("test") 13 | assert.Equal(t, "test", string(pkg)) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/util/pprint.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/rs/zerolog/log" 7 | ) 8 | 9 | func Pprint(i interface{}) string { 10 | payload, _ := json.MarshalIndent(i, "", "\t") 11 | stringified := string(payload) 12 | log.Debug().Msg(stringified) 13 | return stringified 14 | } 15 | 16 | func Stringify(i interface{}) string { 17 | payload, _ := json.Marshal(i) 18 | return string(payload) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/flock/flock_test.go: -------------------------------------------------------------------------------- 1 | // Package flock_test contains tests for the flock package 2 | package flock_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/jakthom/hercules/pkg/config" 8 | "github.com/jakthom/hercules/pkg/flock" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestInitializeDB(t *testing.T) { 13 | db, conn := flock.InitializeDB(config.Config{}) 14 | assert.NotNil(t, db) 15 | assert.NotNil(t, conn) 16 | } 17 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | 10 | permissions: 11 | contents: read 12 | checks: write 13 | 14 | jobs: 15 | run-lint: 16 | runs-on: ubuntu-20.04 17 | steps: 18 | - uses: actions/setup-go@v5 19 | with: 20 | go-version: 1.23 21 | - uses: actions/checkout@v4 22 | - uses: golangci/golangci-lint-action@v6 23 | with: 24 | version: v1.62.0 25 | args: --config ./.github/golangci.yml ./pkg/... ./cmd/... 26 | -------------------------------------------------------------------------------- /pkg/testUtil/db.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func GetMockedConnection() (*sql.Conn, sqlmock.Sqlmock, error) { 12 | db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) 13 | conn, _ := db.Conn(context.Background()) 14 | if err != nil { 15 | log.Fatal().Err(err).Msg("error '%s' was not expected when opening a stub database connection") 16 | return nil, nil, err 17 | } 18 | defer db.Close() 19 | return conn, mock, nil 20 | } 21 | -------------------------------------------------------------------------------- /pkg/middleware/metric.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "database/sql" 5 | "net/http" 6 | 7 | registry "github.com/jakthom/hercules/pkg/metricRegistry" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func MetricsMiddleware(conn *sql.Conn, registries []*registry.MetricRegistry, next http.Handler) http.Handler { 12 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | for _, registry := range registries { 14 | err := registry.Materialize(conn) 15 | if err != nil { 16 | log.Debug().Msg("could not materialize registry") 17 | } 18 | } 19 | next.ServeHTTP(w, r) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /pkg/labels/labels.go: -------------------------------------------------------------------------------- 1 | package labels 2 | 3 | import ( 4 | "maps" 5 | "os" 6 | ) 7 | 8 | type Labels map[string]string 9 | 10 | func InjectLabelFromEnv(labelVal string) string { 11 | if labelVal[0] == '$' { 12 | return os.Getenv(labelVal[1:]) 13 | } 14 | return labelVal 15 | } 16 | 17 | func (l Labels) LabelNames() []string { 18 | var labelNames []string 19 | for k := range l { 20 | labelNames = append(labelNames, k) 21 | } 22 | return labelNames 23 | } 24 | 25 | func Merge(labels Labels, moreLabels Labels) Labels { 26 | var merged = make(map[string]string) 27 | maps.Copy(merged, labels) 28 | maps.Copy(merged, moreLabels) 29 | return merged 30 | } 31 | -------------------------------------------------------------------------------- /cmd/hercules/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | ) 7 | 8 | // version is set during build. 9 | var version string 10 | 11 | func main() { 12 | // Parse command line flags 13 | configPath := flag.String("config", "", "Path to the configuration file") 14 | flag.Parse() 15 | 16 | // Set environment variable for config path if specified via flag 17 | if *configPath != "" { 18 | if err := os.Setenv("HERCULES_CONFIG_PATH", *configPath); err != nil { 19 | panic("Failed to set HERCULES_CONFIG_PATH environment variable: " + err.Error()) 20 | } 21 | } 22 | 23 | app := Hercules{ 24 | version: version, 25 | } 26 | app.Initialize() 27 | app.Run() 28 | } 29 | -------------------------------------------------------------------------------- /pkg/flock/flock.go: -------------------------------------------------------------------------------- 1 | package flock 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | 7 | "github.com/jakthom/hercules/pkg/config" 8 | "github.com/marcboeker/go-duckdb/v2" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | func InitializeDB(conf config.Config) (*sql.DB, *sql.Conn) { 13 | // Open a connection to DuckDB using the new API 14 | connector, err := duckdb.NewConnector(conf.DB, nil) 15 | if err != nil { 16 | log.Fatal().Err(err).Msg("could not initialize duckdb database") 17 | } 18 | 19 | db := sql.OpenDB(connector) 20 | conn, err := db.Conn(context.Background()) 21 | if err != nil { 22 | log.Fatal().Err(err).Msg("could not initialize duckdb connection") 23 | } 24 | defer db.Close() 25 | return db, conn 26 | } 27 | -------------------------------------------------------------------------------- /pkg/db/runner_test.go: -------------------------------------------------------------------------------- 1 | // Package db_test contains tests for the db package 2 | package db_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jakthom/hercules/pkg/db" 9 | "github.com/jakthom/hercules/pkg/testutil" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestRunSqlQuery(t *testing.T) { 15 | conn, mock, _ := testutil.GetMockedConnection() 16 | qr := db.SQL("test") 17 | 18 | mock.ExpectQuery(string(qr)).WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{"col1"}).AddRow(1)) 19 | rows, err := db.RunSQLQuery(conn, qr) 20 | require.NoError(t, err) 21 | defer rows.Close() 22 | 23 | assert.True(t, rows.Next()) 24 | 25 | // Check rows.Err 26 | assert.NoError(t, rows.Err()) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/config/env_test.go: -------------------------------------------------------------------------------- 1 | // Package config_test contains tests for config package 2 | package config_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/jakthom/hercules/pkg/config" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestIsTraceMode(t *testing.T) { 12 | assert.False(t, config.IsTraceMode()) 13 | t.Setenv(config.TraceEnvVar, "true") 14 | assert.True(t, config.IsTraceMode()) 15 | t.Setenv(config.TraceEnvVar, "True") 16 | assert.True(t, config.IsTraceMode()) 17 | t.Setenv(config.TraceEnvVar, "1") 18 | assert.True(t, config.IsTraceMode()) 19 | } 20 | 21 | func TestIsDebugMode(t *testing.T) { 22 | assert.False(t, config.IsDebugMode()) 23 | t.Setenv(config.DebugEnvVar, "true") 24 | assert.True(t, config.IsDebugMode()) 25 | t.Setenv(config.DebugEnvVar, "True") 26 | assert.True(t, config.IsDebugMode()) 27 | t.Setenv(config.DebugEnvVar, "1") 28 | assert.True(t, config.IsDebugMode()) 29 | } 30 | -------------------------------------------------------------------------------- /hercules-packages/nyc-taxi/1.0.yml: -------------------------------------------------------------------------------- 1 | name: nyc_taxi 2 | version: '1.0' 3 | 4 | sources: 5 | - name: nyc_yellow_taxi_june_2024 6 | type: parquet 7 | source: https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-07.parquet 8 | materialize: true 9 | refreshIntervalSeconds: 100 10 | 11 | metrics: 12 | gauge: 13 | - name: pickup_location_fare_total 14 | help: Total NYC fares for the month of July by pickup location 15 | enabled: True 16 | sql: select PULocationID as pickupLocation, sum(fare_amount) as value from nyc_yellow_taxi_june_2024 group by all 17 | 18 | summary: 19 | - name: pickup_location_fares # Note this uses prometheus to do the histogram calculation. For better performance histograms can be pre-calculated and represented as a gauge. 20 | help: Total NYC fares for the month of July by pickup location 21 | enabled: True 22 | sql: select PULocationID as pickupLocation, fare_amount as value from nyc_yellow_taxi_june_2024 23 | objectives: 24 | - 0.001 25 | - 0.05 26 | - 0.01 27 | - 0.5 28 | - 0.9 29 | - 0.99 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Database Embedding Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pkg/db/parser.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "strings" 6 | ) 7 | 8 | // GetLabelNamesFromQuery uses DuckDB's own parser to extract all column names from a query. 9 | func GetLabelNamesFromQuery(conn *sql.Conn, query SQL) ([]string, error) { 10 | parseSQL := SQL(`select coalesce(nullif(row->>'alias', ''), row->>'$.column_names[-1]') 11 | from (select unnest::json as row 12 | from unnest(json_serialize_sql('` + strings.ReplaceAll(string(query), "'", "''") + 13 | `')->>'$.statements[0].node.select_list[*]'));`) 14 | 15 | rows, err := RunSQLQuery(conn, parseSQL) 16 | if err != nil { 17 | return nil, err 18 | } 19 | defer rows.Close() 20 | 21 | var columns []string 22 | for rows.Next() { 23 | var column string 24 | scanErr := rows.Scan(&column) 25 | if scanErr != nil { 26 | return columns, scanErr 27 | } 28 | if column != "value" && column != "val" && column != "v" && column != "" { 29 | columns = append(columns, column) 30 | } 31 | } 32 | 33 | // Check for any errors during iteration. 34 | rowsErr := rows.Err() 35 | if rowsErr != nil { 36 | return columns, rowsErr 37 | } 38 | 39 | return columns, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/db/macro_test.go: -------------------------------------------------------------------------------- 1 | // Package db_test contains tests for the db package 2 | package db_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jakthom/hercules/pkg/db" 9 | "github.com/jakthom/hercules/pkg/testutil" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestMacro(t *testing.T) { 14 | conn, mock, _ := testutil.GetMockedConnection() 15 | 16 | macroSQL := db.SQL("test() as (select 1)") 17 | 18 | macro := db.Macro{ 19 | Name: "test", 20 | SQL: macroSQL, 21 | } 22 | 23 | // Ensure creation/replacement sql 24 | assert.Equal(t, "create or replace macro "+string(macroSQL), string(macro.CreateOrReplaceSQL())) 25 | // Ensure query is executed appropriately 26 | mock.ExpectQuery(string(macro.CreateOrReplaceSQL())).WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 27 | db.TestHookEnsureMacro(conn, macro) 28 | } 29 | 30 | func TestEnsureMacrosWithConnection(_ *testing.T) { 31 | conn, mock, _ := testutil.GetMockedConnection() 32 | 33 | macros := []db.Macro{ 34 | { 35 | Name: "test", 36 | SQL: db.SQL("test() as (select 1)"), 37 | }, 38 | } 39 | 40 | mock.ExpectQuery(string(macros[0].CreateOrReplaceSQL())).WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 41 | db.EnsureMacrosWithConnection(macros, conn) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/labels/labels_test.go: -------------------------------------------------------------------------------- 1 | // Package labels_test contains tests for the labels package 2 | package labels_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/jakthom/hercules/pkg/labels" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInjectLabelFromEnv(t *testing.T) { 12 | label := "$HERCULES" 13 | value := "notprometheus" 14 | t.Setenv("HERCULES", value) 15 | 16 | labelValue := labels.InjectLabelFromEnv(label) 17 | assert.Equal(t, value, labelValue) 18 | } 19 | 20 | // func TestLabelNames(t *testing.T) { 21 | // labelNames := labels.Labels{ 22 | // "cell": "ausw1", 23 | // "fromEnv": "testing", 24 | // "hercules": "fake", 25 | // }.LabelNames() 26 | // assert.Equal(t, 3, len(labelNames)) 27 | // assert.Equal(t, "cell", labelNames[0]) 28 | // assert.Equal(t, "fromEnv", labelNames[1]) 29 | // assert.Equal(t, "hercules", labelNames[2]) 30 | // } 31 | 32 | func TestMerge(t *testing.T) { 33 | someLabels := labels.Labels{ 34 | "cell": "ausw1", 35 | "fromEnv": "testing", 36 | } 37 | moreLabels := labels.Labels{ 38 | "hercules": "testing", 39 | } 40 | 41 | want := labels.Labels{ 42 | "cell": "ausw1", 43 | "fromEnv": "testing", 44 | "hercules": "testing", 45 | } 46 | got := labels.Merge(someLabels, moreLabels) 47 | assert.Equal(t, want, got) 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | tag-release: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: write 13 | pull-requests: write 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 0 19 | 20 | - name: Get Version from file 21 | id: get-version 22 | uses: juliangruber/read-file-action@v1.1.6 23 | with: 24 | path: ./.VERSION 25 | 26 | - name: Configure Git 27 | run: | 28 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 29 | git config user.name "$GITHUB_ACTOR" 30 | 31 | - name: Set Reftag 32 | id: tag-version 33 | uses: mathieudutour/github-tag-action@v6.1 34 | with: 35 | github_token: ${{ secrets.GITHUB_TOKEN }} 36 | custom_tag: ${{ steps.get-version.outputs.content }} 37 | tag_prefix: "" 38 | 39 | cut-release: 40 | runs-on: ubuntu-latest 41 | needs: tag-release 42 | permissions: 43 | contents: write 44 | pull-requests: write 45 | steps: 46 | - name: Checkout repository 47 | uses: actions/checkout@v4 48 | with: 49 | fetch-depth: 0 50 | 51 | - name: Configure Git 52 | run: | 53 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 54 | git config user.name "$GITHUB_ACTOR" 55 | 56 | - name: Set up Go 57 | uses: actions/setup-go@v5 58 | 59 | # DO THE REST :grin: 60 | -------------------------------------------------------------------------------- /pkg/db/extension_test.go: -------------------------------------------------------------------------------- 1 | // Package db_test contains tests for the db package 2 | package db_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jakthom/hercules/pkg/db" 9 | "github.com/jakthom/hercules/pkg/testutil" 10 | ) 11 | 12 | func TestEnsureExtension(_ *testing.T) { 13 | conn, mock, _ := testutil.GetMockedConnection() 14 | 15 | mock.ExpectQuery("install test;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 16 | mock.ExpectQuery("load test;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 17 | db.TestHookEnsureExtension(conn, "test", db.CoreExtensionType) 18 | 19 | mock.ExpectQuery("install z from community;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 20 | mock.ExpectQuery("load z;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 21 | db.TestHookEnsureExtension(conn, "z", db.CommunityExtensionType) 22 | } 23 | 24 | func TestEnsureExtensionsWithConnection(_ *testing.T) { 25 | conn, mock, _ := testutil.GetMockedConnection() 26 | 27 | extensions := db.Extensions{ 28 | Core: []db.CoreExtension{ 29 | { 30 | Name: "testcore", 31 | }, 32 | }, 33 | Community: []db.CommunityExtension{ 34 | { 35 | Name: "testcommunity", 36 | }, 37 | }, 38 | } 39 | 40 | mock.ExpectQuery("install testcore;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 41 | mock.ExpectQuery("load testcore;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 42 | mock.ExpectQuery("install testcommunity from community;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 43 | mock.ExpectQuery("load testcommunity;").WithoutArgs().WillReturnRows(sqlmock.NewRows([]string{})) 44 | db.EnsureExtensionsWithConnection(extensions, conn) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/db/macro.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/rs/zerolog/log" 7 | ) 8 | 9 | type Macro struct { 10 | Name string `json:"name"` // No-op. Probably overkill. Nice for future reasons. 11 | SQL SQL `json:"sql"` 12 | } 13 | 14 | // CreateOrReplaceSQL returns the SQL statement to create or replace the macro. 15 | func (m *Macro) CreateOrReplaceSQL() SQL { 16 | // TODO -> be more flexible with how these are handled - allow "create", "create or replace", nameless macros, etc. 17 | return SQL("create or replace macro " + string(m.SQL)) 18 | } 19 | 20 | func (m *Macro) ensureWithConnection(conn *sql.Conn) { 21 | rows, err := RunSQLQuery(conn, m.CreateOrReplaceSQL()) 22 | if err != nil { 23 | log.Error().Err(err).Msg("could not ensure macro") 24 | panic("Failed to ensure macro: " + err.Error()) 25 | } 26 | defer func() { 27 | if closeErr := rows.Close(); closeErr != nil { 28 | log.Error().Err(closeErr).Msg("error closing rows after macro creation") 29 | } 30 | }() 31 | 32 | // Check for errors 33 | rowsErr := rows.Err() 34 | if rowsErr != nil { 35 | log.Error().Err(rowsErr).Msg("error during macro creation") 36 | panic("Error during macro creation: " + rowsErr.Error()) 37 | } 38 | 39 | log.Debug().Interface("macro", m.SQL).Msg("macro ensured") 40 | } 41 | 42 | // TestHookEnsureMacro exposes the ensureWithConnection method for testing. 43 | func TestHookEnsureMacro(conn *sql.Conn, macro Macro) { 44 | macro.ensureWithConnection(conn) 45 | } 46 | 47 | // EnsureMacrosWithConnection creates and ensures all macros are properly set up in the database. 48 | func EnsureMacrosWithConnection(macros []Macro, conn *sql.Conn) { 49 | for _, macro := range macros { 50 | macro.ensureWithConnection(conn) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/metric/counter.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "database/sql" 5 | 6 | db "github.com/jakthom/hercules/pkg/db" 7 | "github.com/jakthom/hercules/pkg/labels" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | type Counter struct { 13 | Definition Definition 14 | Collector *prometheus.CounterVec 15 | } 16 | 17 | func NewCounter(definition Definition) Counter { 18 | metric := Counter{ 19 | Definition: definition, 20 | } 21 | err := metric.register() 22 | if err != nil { 23 | log.Error().Err(err).Interface("metric", definition.FullName()).Msg("could not register metric") 24 | } 25 | return metric 26 | } 27 | 28 | func (m *Counter) AsVec() *prometheus.CounterVec { 29 | v := prometheus.NewCounterVec(prometheus.CounterOpts{ 30 | Name: m.Definition.FullName(), 31 | Help: m.Definition.Help, 32 | }, m.Definition.LabelNames()) 33 | return v 34 | } 35 | 36 | func (m *Counter) register() error { 37 | collector := m.AsVec() 38 | err := prometheus.Register(collector) 39 | m.Collector = collector 40 | return err 41 | } 42 | 43 | func (m *Counter) reregister() error { 44 | // godd this is ugly, but it's the only way I've found to make a collector go back to zero (so data isn't dup'd per request) 45 | prometheus.Unregister(m.Collector) 46 | return m.register() 47 | } 48 | 49 | func (m *Counter) Materialize(conn *sql.Conn) error { 50 | err := m.reregister() 51 | if err != nil { 52 | log.Error().Err(err).Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 53 | } 54 | results, err := db.Materialize(conn, m.Definition.SQL) 55 | if err != nil { 56 | log.Error().Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 57 | return err 58 | } 59 | for _, r := range results { 60 | l := labels.Merge(r.StringifiedLabels(), m.Definition.Metadata.Labels) 61 | m.Collector.With(map[string]string(l)).Inc() 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /schemas/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft-07/schema#", 3 | "type": "object", 4 | "properties": { 5 | "version": { 6 | "type": "string", 7 | "description": "Config version", 8 | "default": "1" 9 | }, 10 | "name": { 11 | "type": "string", 12 | "description": "Name of the hercules instance", 13 | "default": "hercules" 14 | }, 15 | "debug": { 16 | "type": "boolean", 17 | "description": "Enable debug logging", 18 | "default": false 19 | }, 20 | "port": { 21 | "type": "integer", 22 | "description": "Port to listen on", 23 | "default": "9999" 24 | }, 25 | "globalLabels": { 26 | "type": "array", 27 | "description": "Global labels injected from config", 28 | "items": { 29 | "type": "object" 30 | } 31 | }, 32 | "packages": { 33 | "type": "array", 34 | "description": "Packages to load", 35 | "items": { 36 | "type": "object", 37 | "properties": { 38 | "package": { 39 | "type": "string" 40 | }, 41 | "variables": { 42 | "type": "object", 43 | "additionalProperties": { 44 | "type": "string" 45 | } 46 | }, 47 | "metricPrefix": { 48 | "type": "string" 49 | } 50 | }, 51 | "required": [ 52 | "package" 53 | ] 54 | } 55 | } 56 | }, 57 | "required": [ 58 | "version", 59 | "name", 60 | "debug", 61 | "db", 62 | "port", 63 | "packages" 64 | ] 65 | } -------------------------------------------------------------------------------- /pkg/metric/gauge.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "database/sql" 5 | 6 | db "github.com/jakthom/hercules/pkg/db" 7 | "github.com/jakthom/hercules/pkg/labels" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | type Gauge struct { 13 | Definition Definition 14 | Collector *prometheus.GaugeVec 15 | } 16 | 17 | func NewGauge(definition Definition) Gauge { 18 | // TODO! Turn this into a generic function instead of copy/pasta 19 | metric := Gauge{ 20 | Definition: definition, 21 | } 22 | err := metric.register() 23 | if err != nil { 24 | log.Error().Err(err).Interface("metric", definition.FullName()).Msg("could not register metric") 25 | } 26 | return metric 27 | } 28 | 29 | func (m *Gauge) asVec() *prometheus.GaugeVec { 30 | v := prometheus.NewGaugeVec(prometheus.GaugeOpts{ 31 | Name: m.Definition.FullName(), 32 | Help: m.Definition.Help, 33 | }, m.Definition.LabelNames()) 34 | return v 35 | } 36 | 37 | func (m *Gauge) register() error { 38 | collector := m.asVec() 39 | err := prometheus.Register(collector) 40 | m.Collector = collector 41 | return err 42 | } 43 | 44 | func (m *Gauge) reregister() error { 45 | // godd this is ugly, but it's the only way I've found to make a collector go back to zero (so data isn't dup'd per request) 46 | prometheus.Unregister(m.Collector) 47 | return m.register() 48 | } 49 | 50 | func (m *Gauge) Materialize(conn *sql.Conn) error { 51 | err := m.reregister() 52 | if err != nil { 53 | log.Error().Err(err).Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 54 | } 55 | 56 | results, err := db.Materialize(conn, m.Definition.SQL) 57 | if err != nil { 58 | log.Error().Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 59 | return err 60 | } 61 | for _, r := range results { 62 | l := labels.Merge(r.StringifiedLabels(), m.Definition.Metadata.Labels) 63 | m.Collector.With(map[string]string(l)).Set(r.Value) 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/metricRegistry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/jakthom/hercules/pkg/metric" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | type MetricRegistry struct { 11 | Gauge map[string]metric.Gauge 12 | Counter map[string]metric.Counter 13 | Summary map[string]metric.Summary 14 | Histogram map[string]metric.Histogram 15 | } 16 | 17 | func NewMetricRegistry(definitions metric.Definitions) *MetricRegistry { 18 | r := MetricRegistry{} 19 | r.Gauge = make(map[string]metric.Gauge) 20 | r.Histogram = make(map[string]metric.Histogram) 21 | r.Summary = make(map[string]metric.Summary) 22 | r.Counter = make(map[string]metric.Counter) 23 | 24 | for _, definition := range definitions.Gauge { 25 | g := metric.NewGauge(*definition) 26 | r.Gauge[g.Definition.FullName()] = g 27 | } 28 | for _, definition := range definitions.Histogram { 29 | h := metric.NewHistogram(*definition) 30 | r.Histogram[h.Definition.FullName()] = h 31 | } 32 | for _, definition := range definitions.Summary { 33 | s := metric.NewSummary(*definition) 34 | r.Summary[s.Definition.FullName()] = s 35 | } 36 | for _, definition := range definitions.Counter { 37 | c := metric.NewCounter(*definition) 38 | r.Counter[c.Definition.FullName()] = c 39 | } 40 | return &r 41 | } 42 | 43 | func (mr *MetricRegistry) Materialize(conn *sql.Conn) error { 44 | // TODO: Make this return a list of "materialization errors" 45 | // if something fails 46 | var m []metric.Materializeable 47 | for _, metric := range mr.Gauge { 48 | m = append(m, &metric) 49 | } 50 | for _, metric := range mr.Histogram { 51 | m = append(m, &metric) 52 | } 53 | for _, metric := range mr.Summary { 54 | m = append(m, &metric) 55 | } 56 | for _, metric := range mr.Counter { 57 | m = append(m, &metric) 58 | } 59 | for _, materializable := range m { 60 | err := materializable.Materialize(conn) 61 | if err != nil { 62 | log.Error().Err(err).Msg("could not materialize metric") 63 | } 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/metric/histogram.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "database/sql" 5 | 6 | db "github.com/jakthom/hercules/pkg/db" 7 | "github.com/jakthom/hercules/pkg/labels" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | type Histogram struct { 13 | Definition Definition 14 | Collector *prometheus.HistogramVec 15 | } 16 | 17 | func NewHistogram(definition Definition) Histogram { 18 | metric := Histogram{ 19 | Definition: definition, 20 | } 21 | err := metric.register() 22 | if err != nil { 23 | log.Error().Err(err).Interface("metric", definition.FullName()).Msg("could not register metric") 24 | } 25 | return metric 26 | } 27 | 28 | func (m *Histogram) AsVec() *prometheus.HistogramVec { 29 | v := prometheus.NewHistogramVec(prometheus.HistogramOpts{ 30 | Name: m.Definition.FullName(), 31 | Help: m.Definition.Help, 32 | Buckets: m.Definition.Buckets, 33 | }, m.Definition.LabelNames()) 34 | return v 35 | } 36 | 37 | func (m *Histogram) register() error { 38 | collector := m.AsVec() 39 | err := prometheus.Register(collector) 40 | m.Collector = collector 41 | return err 42 | } 43 | 44 | func (m *Histogram) reregister() error { 45 | // godd this is ugly, but it's the only way I've found to make a collector go back to zero (so data isn't dup'd per request) 46 | prometheus.Unregister(m.Collector) 47 | return m.register() 48 | } 49 | 50 | func (m *Histogram) Materialize(conn *sql.Conn) error { 51 | err := m.reregister() 52 | if err != nil { 53 | log.Error().Err(err).Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 54 | } 55 | results, err := db.Materialize(conn, m.Definition.SQL) 56 | if err != nil { 57 | log.Error().Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 58 | return err 59 | } 60 | for _, r := range results { 61 | l := labels.Merge(r.StringifiedLabels(), m.Definition.Metadata.Labels) 62 | m.Collector.With(map[string]string(l)).Observe(r.Value) 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/metric/summary.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "database/sql" 5 | 6 | db "github.com/jakthom/hercules/pkg/db" 7 | "github.com/jakthom/hercules/pkg/labels" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | type Summary struct { 13 | Definition Definition 14 | Collector *prometheus.SummaryVec 15 | } 16 | 17 | func NewSummary(definition Definition) Summary { 18 | metric := Summary{ 19 | Definition: definition, 20 | } 21 | err := metric.register() 22 | if err != nil { 23 | log.Error().Err(err).Interface("metric", definition.FullName()).Msg("could not register metric") 24 | } 25 | return metric 26 | } 27 | 28 | func (m *Summary) AsVec() *prometheus.SummaryVec { 29 | objectives := make(map[float64]float64) 30 | for _, o := range m.Definition.Objectives { 31 | objectives[o] = o 32 | } 33 | v := prometheus.NewSummaryVec(prometheus.SummaryOpts{ 34 | Name: m.Definition.FullName(), 35 | Help: m.Definition.Help, 36 | Objectives: objectives, 37 | }, m.Definition.LabelNames()) 38 | return v 39 | } 40 | 41 | func (m *Summary) register() error { 42 | collector := m.AsVec() 43 | err := prometheus.Register(collector) 44 | m.Collector = collector 45 | return err 46 | } 47 | 48 | func (m *Summary) reregister() error { 49 | // godd this is ugly, but it's the only way I've found to make a collector go back to zero (so data isn't dup'd per request) 50 | prometheus.Unregister(m.Collector) 51 | return m.register() 52 | } 53 | 54 | func (m *Summary) Materialize(conn *sql.Conn) error { 55 | err := m.reregister() 56 | if err != nil { 57 | log.Error().Err(err).Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 58 | } 59 | results, err := db.Materialize(conn, m.Definition.SQL) 60 | if err != nil { 61 | log.Error().Interface("metric", m.Definition.FullName()).Msg("could not materialize metric") 62 | return err 63 | } 64 | for _, r := range results { 65 | l := labels.Merge(r.StringifiedLabels(), m.Definition.Metadata.Labels) 66 | m.Collector.With(map[string]string(l)).Observe(r.Value) 67 | } 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /pkg/config/config_test.go: -------------------------------------------------------------------------------- 1 | // Package config_test contains tests for the config package 2 | package config_test 3 | 4 | import ( 5 | "bytes" 6 | "testing" 7 | 8 | "github.com/jakthom/hercules/pkg/config" 9 | herculespackage "github.com/jakthom/hercules/pkg/herculesPackage" 10 | "github.com/jakthom/hercules/pkg/labels" 11 | "github.com/rs/zerolog" 12 | "github.com/rs/zerolog/log" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func buildTestConfig() config.Config { 17 | c := config.Config{ 18 | Name: "fake", 19 | Debug: false, 20 | Port: "9999", 21 | DB: "hercules.db", 22 | GlobalLabels: labels.Labels{ 23 | "cell": "ausw1", 24 | "fromEnv": "$FROMENV", 25 | }, 26 | Packages: []herculespackage.PackageConfig{}, 27 | } 28 | return c 29 | } 30 | 31 | func TestInstanceLabels(t *testing.T) { 32 | env := "testing" 33 | t.Setenv("FROMENV", env) 34 | conf := buildTestConfig() 35 | got := conf.InstanceLabels() 36 | want := labels.Labels{ 37 | "cell": "ausw1", 38 | "fromEnv": env, 39 | "hercules": "fake", 40 | } 41 | assert.Equal(t, want, got) 42 | } 43 | 44 | func TestGetConfigNoFile(t *testing.T) { 45 | // Ensure a non-existent config path is used. 46 | t.Setenv(config.HerculesConfigPath, "non_existent_config.yml") 47 | 48 | // Temporarily capture logs to prevent error message in test output. 49 | var buf bytes.Buffer 50 | oldLogger := log.Logger 51 | log.Logger = zerolog.New(&buf).With().Timestamp().Logger() 52 | defer func() { log.Logger = oldLogger }() 53 | 54 | // Call GetConfig and capture the error. 55 | conf, err := config.GetConfig() 56 | 57 | // Verify an error was returned. 58 | assert.NotNil(t, err, "Expected an error when config file doesn't exist") 59 | assert.Contains(t, err.Error(), "config file not found") 60 | 61 | // Verify the error was logged. 62 | assert.Contains(t, buf.String(), "config file does not exist") 63 | 64 | // Verify default values are still set correctly. 65 | assert.Equal(t, config.DefaultDebug, conf.Debug) 66 | assert.Equal(t, config.DefaultPort, conf.Port) 67 | assert.Equal(t, config.DefaultDB, conf.DB) 68 | 69 | // Validate should not panic even with default config. 70 | conf.Validate() 71 | } 72 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jakthom/hercules 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/DATA-DOG/go-sqlmock v1.5.2 7 | github.com/marcboeker/go-duckdb/v2 v2.2.0 8 | github.com/prometheus/client_golang v1.22.0 9 | github.com/rs/zerolog v1.34.0 10 | github.com/spf13/cast v1.7.1 11 | github.com/spf13/viper v1.20.1 12 | github.com/stretchr/testify v1.10.0 13 | sigs.k8s.io/yaml v1.4.0 14 | ) 15 | 16 | require ( 17 | github.com/duckdb/duckdb-go-bindings v0.1.14 // indirect 18 | github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9 // indirect 19 | github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9 // indirect 20 | github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9 // indirect 21 | github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9 // indirect 22 | github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9 // indirect 23 | github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 // indirect 24 | github.com/marcboeker/go-duckdb/mapping v0.0.7 // indirect 25 | ) 26 | 27 | require ( 28 | github.com/apache/arrow-go/v18 v18.2.0 // indirect 29 | github.com/beorn7/perks v1.0.1 // indirect 30 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 31 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 32 | github.com/fsnotify/fsnotify v1.9.0 // indirect 33 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 34 | github.com/goccy/go-json v0.10.5 // indirect 35 | github.com/google/flatbuffers v25.2.10+incompatible // indirect 36 | github.com/google/uuid v1.6.0 // indirect 37 | github.com/klauspost/compress v1.18.0 // indirect 38 | github.com/klauspost/cpuid/v2 v2.2.10 // indirect 39 | github.com/mattn/go-colorable v0.1.14 // indirect 40 | github.com/mattn/go-isatty v0.0.20 // indirect 41 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 42 | github.com/pelletier/go-toml/v2 v2.2.4 // indirect 43 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 44 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 45 | github.com/prometheus/client_model v0.6.2 // indirect 46 | github.com/prometheus/common v0.63.0 // indirect 47 | github.com/prometheus/procfs v0.16.1 // indirect 48 | github.com/sagikazarmark/locafero v0.9.0 // indirect 49 | github.com/sourcegraph/conc v0.3.0 // indirect 50 | github.com/spf13/afero v1.14.0 // indirect 51 | github.com/spf13/pflag v1.0.6 // indirect 52 | github.com/subosito/gotenv v1.6.0 // indirect 53 | github.com/zeebo/xxh3 v1.0.2 // indirect 54 | go.uber.org/multierr v1.11.0 // indirect 55 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect 56 | golang.org/x/mod v0.24.0 // indirect 57 | golang.org/x/sync v0.13.0 58 | golang.org/x/sys v0.32.0 // indirect 59 | golang.org/x/text v0.24.0 // indirect 60 | golang.org/x/tools v0.32.0 // indirect 61 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect 62 | google.golang.org/protobuf v1.36.6 // indirect 63 | gopkg.in/yaml.v3 v3.0.1 // indirect 64 | ) 65 | -------------------------------------------------------------------------------- /pkg/metric/metric.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import ( 4 | "database/sql" 5 | "strings" 6 | 7 | "github.com/jakthom/hercules/pkg/db" 8 | ) 9 | 10 | // Definition defines a metric with its SQL query and metadata. 11 | type Definition struct { 12 | Name string `json:"name"` 13 | Enabled bool `json:"enabled"` 14 | Help string `json:"help"` 15 | SQL db.SQL `json:"sql"` 16 | Labels []string `json:"labels"` 17 | Buckets []float64 `json:"buckets,omitempty"` // If the metric is a histogram. 18 | Objectives []float64 `json:"objectives,omitempty"` // If the metric is a summary. 19 | // Internal. 20 | Metadata Metadata `json:"metadata"` 21 | } 22 | 23 | func (m *Definition) LabelNames() []string { 24 | names := []string{} 25 | names = append(names, m.Labels...) 26 | for k := range m.Metadata.Labels { 27 | names = append(names, k) 28 | } 29 | return names 30 | } 31 | 32 | func (m *Definition) injectLabels(conn *sql.Conn) error { 33 | labels, err := db.GetLabelNamesFromQuery(conn, m.SQL) 34 | if err != nil { 35 | return err 36 | } 37 | m.Labels = labels 38 | return nil 39 | } 40 | 41 | // InjectLabels fetches the labels from the SQL query. 42 | func (m *Definition) InjectLabels(conn *sql.Conn) error { 43 | return m.injectLabels(conn) 44 | } 45 | 46 | func (m *Definition) injectMetadata(metadata Metadata) { 47 | m.Metadata = metadata 48 | } 49 | 50 | func (m *Definition) FullName() string { 51 | prefix := string(m.Metadata.Prefix) + strings.ReplaceAll(m.Metadata.PackageName, "-", "_") + "_" 52 | return prefix + m.Name 53 | } 54 | 55 | // Definitions holds collections of different metric types. 56 | type Definitions struct { 57 | Gauge []*Definition `json:"gauge"` 58 | Counter []*Definition `json:"counter"` 59 | Summary []*Definition `json:"summary"` 60 | Histogram []*Definition `json:"histogram"` 61 | } 62 | 63 | func (m *Definitions) InjectMetadata(conn *sql.Conn, metadata Metadata) error { 64 | for _, metricDefinition := range m.Gauge { 65 | if err := metricDefinition.injectLabels(conn); err != nil { 66 | return err 67 | } 68 | metricDefinition.injectMetadata(metadata) 69 | } 70 | for _, metricDefinition := range m.Counter { 71 | if err := metricDefinition.injectLabels(conn); err != nil { 72 | return err 73 | } 74 | metricDefinition.injectMetadata(metadata) 75 | } 76 | for _, metricDefinition := range m.Summary { 77 | if err := metricDefinition.injectLabels(conn); err != nil { 78 | return err 79 | } 80 | metricDefinition.injectMetadata(metadata) 81 | } 82 | for _, metricDefinition := range m.Histogram { 83 | if err := metricDefinition.injectLabels(conn); err != nil { 84 | return err 85 | } 86 | metricDefinition.injectMetadata(metadata) 87 | } 88 | return nil 89 | } 90 | 91 | func (m *Definitions) Merge(definitions Definitions) { 92 | m.Gauge = append(m.Gauge, definitions.Gauge...) 93 | m.Counter = append(m.Counter, definitions.Counter...) 94 | m.Summary = append(m.Summary, definitions.Summary...) 95 | m.Histogram = append(m.Histogram, definitions.Histogram...) 96 | } 97 | 98 | type Materializeable interface { 99 | Materialize(conn *sql.Conn) error 100 | } 101 | -------------------------------------------------------------------------------- /pkg/db/extension.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/rs/zerolog/log" 7 | ) 8 | 9 | // Function to update constants to follow Go naming conventions. 10 | const ( 11 | // CommunityExtension represents a community extension type. 12 | CommunityExtensionType string = "community" 13 | // CoreExtension represents a core extension type. 14 | CoreExtensionType string = "core" 15 | ) 16 | 17 | func ensureExtension(conn *sql.Conn, extensionName string, extensionType string) { 18 | var installSQL SQL 19 | var loadSQL = SQL("load " + extensionName + ";") 20 | if extensionType == CommunityExtensionType { 21 | installSQL = SQL("install " + extensionName + " from community;") 22 | } else { 23 | installSQL = SQL("install " + extensionName + ";") 24 | } 25 | 26 | // Run installation query 27 | rows, err := RunSQLQuery(conn, installSQL) 28 | if err != nil { 29 | // Assume that the world depends on indicated extensions installing and loading properly 30 | log.Error().Err(err). 31 | Interface("extension", extensionName). 32 | Msg("unable to install " + extensionType + " extension") 33 | panic("Failed to install extension: " + err.Error()) 34 | } 35 | defer func() { 36 | if closeErr := rows.Close(); closeErr != nil { 37 | log.Error().Err(closeErr). 38 | Interface("extension", extensionName). 39 | Msg("error closing rows after extension installation") 40 | } 41 | }() 42 | 43 | // Check for installation errors 44 | rowsErr := rows.Err() 45 | if rowsErr != nil { 46 | log.Error().Err(rowsErr). 47 | Interface("extension", extensionName). 48 | Msg("error during installation of " + extensionType + " extension") 49 | panic("Error during extension installation: " + rowsErr.Error()) 50 | } 51 | 52 | // Run load query 53 | loadRows, err := RunSQLQuery(conn, loadSQL) 54 | if err != nil { 55 | log.Error().Err(err). 56 | Interface("extension", extensionName). 57 | Msg("unable to load " + extensionType + " extension") 58 | panic("Failed to load extension: " + err.Error()) 59 | } 60 | defer func() { 61 | if closeErr := loadRows.Close(); closeErr != nil { 62 | log.Error().Err(closeErr). 63 | Interface("extension", extensionName). 64 | Msg("error closing rows after extension loading") 65 | } 66 | }() 67 | 68 | // Check for loading errors 69 | loadRowsErr := loadRows.Err() 70 | if loadRowsErr != nil { 71 | log.Error().Err(loadRowsErr). 72 | Interface("extension", extensionName). 73 | Msg("error during loading of " + extensionType + " extension") 74 | panic("Error during extension loading: " + loadRowsErr.Error()) 75 | } 76 | 77 | log.Debug().Interface("extension", extensionName).Msg(extensionType + " extension ensured") 78 | } 79 | 80 | // TestHookEnsureExtension exposes the ensureExtension function for testing. 81 | func TestHookEnsureExtension(conn *sql.Conn, extensionName string, extensionType string) { 82 | ensureExtension(conn, extensionName, extensionType) 83 | } 84 | 85 | type CoreExtension struct { 86 | Name string 87 | } 88 | 89 | func (ce *CoreExtension) ensureWithConnection(conn *sql.Conn) { 90 | ensureExtension(conn, ce.Name, CoreExtensionType) 91 | } 92 | 93 | type CommunityExtension struct { 94 | Name string 95 | } 96 | 97 | func (e *CommunityExtension) ensureWithConnection(conn *sql.Conn) { 98 | ensureExtension(conn, e.Name, CommunityExtensionType) 99 | } 100 | 101 | type Extensions struct { 102 | Core []CoreExtension 103 | Community []CommunityExtension 104 | } 105 | 106 | func EnsureExtensionsWithConnection(extensions Extensions, conn *sql.Conn) { 107 | for _, coreExtension := range extensions.Core { 108 | coreExtension.ensureWithConnection(conn) 109 | } 110 | for _, communityExtension := range extensions.Community { 111 | communityExtension.ensureWithConnection(conn) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/jakthom/hercules/pkg/db" 8 | herculespackage "github.com/jakthom/hercules/pkg/herculesPackage" 9 | "github.com/jakthom/hercules/pkg/labels" 10 | "github.com/jakthom/hercules/pkg/metric" 11 | "github.com/jakthom/hercules/pkg/source" 12 | "github.com/rs/zerolog/log" 13 | "github.com/spf13/viper" 14 | ) 15 | 16 | const ( 17 | // HerculesConfigPath is the environment variable name for the configuration file path. 18 | HerculesConfigPath string = "HERCULES_CONFIG_PATH" 19 | // DefaultHerculesConfigPath is the default path for the Hercules configuration file. 20 | DefaultHerculesConfigPath string = "hercules.yml" 21 | // DebugEnvVar is the environment variable name to enable debug mode. 22 | DebugEnvVar string = "DEBUG" 23 | // TraceEnvVar is the environment variable name to enable trace mode. 24 | TraceEnvVar string = "TRACE" 25 | // YamlConfigType is the configuration type for YAML files. 26 | YamlConfigType string = "yaml" 27 | 28 | // DefaultDebug is the default debug mode setting. 29 | DefaultDebug bool = false 30 | // DefaultPort is the default port for the Hercules server. 31 | DefaultPort string = "9999" 32 | // DefaultDB is the default database file path. 33 | DefaultDB string = "h.db" 34 | 35 | // HerculesNameLabel is the label name used for Hercules instance identification. 36 | HerculesNameLabel = "hercules" 37 | ) 38 | 39 | type Config struct { 40 | Name string `json:"name"` 41 | Debug bool `json:"debug"` 42 | Port string `json:"port"` 43 | DB string `json:"db"` 44 | GlobalLabels labels.Labels `json:"globalLabels"` 45 | Packages []herculespackage.PackageConfig `json:"packages"` 46 | Extensions db.Extensions `json:"extensions"` 47 | Macros []db.Macro `json:"macros"` 48 | Sources []source.Source `json:"sources"` 49 | Metrics metric.Definitions `json:"metrics"` 50 | } 51 | 52 | func (c *Config) InstanceLabels() labels.Labels { 53 | globalLabels := labels.Labels{} 54 | globalLabels[HerculesNameLabel] = c.Name 55 | for k, v := range c.GlobalLabels { 56 | globalLabels[k] = labels.InjectLabelFromEnv(v) 57 | } 58 | return globalLabels 59 | } 60 | 61 | func (c *Config) Validate() { 62 | // Passthrough for now - stubbed for config validation 63 | } 64 | 65 | // GetConfig retrieves the application configuration from file or returns defaults. 66 | // If the specified file cannot be read, it will fall back to sane defaults. 67 | func GetConfig() (Config, error) { 68 | // Load app config from file 69 | confPath := os.Getenv(HerculesConfigPath) 70 | if confPath == "" { 71 | confPath = DefaultHerculesConfigPath 72 | } 73 | 74 | config := &Config{} 75 | 76 | // Check if the file exists before attempting to read it 77 | if _, err := os.Stat(confPath); os.IsNotExist(err) { 78 | log.Error().Str("path", confPath).Msg("config file does not exist") 79 | config.Debug = DefaultDebug 80 | config.Port = DefaultPort 81 | config.DB = DefaultDB 82 | return *config, fmt.Errorf("config file not found at path: %s", confPath) 83 | } 84 | 85 | log.Info().Str("path", confPath).Msg("loading config from file") 86 | 87 | // Try to get configuration from file 88 | viper.SetConfigFile(confPath) 89 | viper.SetConfigType(YamlConfigType) 90 | err := viper.ReadInConfig() 91 | if err != nil { 92 | log.Error().Err(err).Str("path", confPath).Msg("could not read config - using defaults") 93 | config.Debug = DefaultDebug 94 | config.Port = DefaultPort 95 | } else { 96 | log.Debug().Str("path", confPath).Msg("config file loaded successfully") 97 | } 98 | 99 | // Mandatory defaults 100 | config.DB = DefaultDB 101 | unmarshalErr := viper.Unmarshal(config) 102 | if unmarshalErr != nil { 103 | log.Error().Err(unmarshalErr).Msg("error unmarshaling config") 104 | } 105 | 106 | return *config, nil 107 | } 108 | -------------------------------------------------------------------------------- /pkg/metricRegistry/registry_test.go: -------------------------------------------------------------------------------- 1 | // Package registry_test contains tests for the registry package 2 | package registry_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jakthom/hercules/pkg/metric" 9 | registry "github.com/jakthom/hercules/pkg/metricRegistry" 10 | "github.com/jakthom/hercules/pkg/testutil" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestNewMetricRegistry(t *testing.T) { 15 | // Create test metric definitions 16 | definitions := metric.Definitions{ 17 | Gauge: []*metric.Definition{ 18 | { 19 | Name: "test_gauge", 20 | Help: "Test gauge metric", 21 | SQL: "SELECT 1", 22 | Metadata: metric.Metadata{ 23 | PackageName: "test", 24 | Prefix: "prefix_", 25 | }, 26 | }, 27 | }, 28 | Counter: []*metric.Definition{ 29 | { 30 | Name: "test_counter", 31 | Help: "Test counter metric", 32 | SQL: "SELECT 1", 33 | Metadata: metric.Metadata{ 34 | PackageName: "test", 35 | Prefix: "prefix_", 36 | }, 37 | }, 38 | }, 39 | Summary: []*metric.Definition{ 40 | { 41 | Name: "test_summary", 42 | Help: "Test summary metric", 43 | SQL: "SELECT 1", 44 | Objectives: []float64{0.5, 0.9, 0.99}, 45 | Metadata: metric.Metadata{ 46 | PackageName: "test", 47 | Prefix: "prefix_", 48 | }, 49 | }, 50 | }, 51 | Histogram: []*metric.Definition{ 52 | { 53 | Name: "test_histogram", 54 | Help: "Test histogram metric", 55 | SQL: "SELECT 1", 56 | Buckets: []float64{0.1, 0.5, 1.0, 5.0}, 57 | Metadata: metric.Metadata{ 58 | PackageName: "test", 59 | Prefix: "prefix_", 60 | }, 61 | }, 62 | }, 63 | } 64 | 65 | // Create a new registry 66 | reg := registry.NewMetricRegistry(definitions) 67 | 68 | // Check that all metrics were registered correctly 69 | assert.Len(t, reg.Gauge, 1, "Should have 1 gauge metric") 70 | assert.Len(t, reg.Counter, 1, "Should have 1 counter metric") 71 | assert.Len(t, reg.Summary, 1, "Should have 1 summary metric") 72 | assert.Len(t, reg.Histogram, 1, "Should have 1 histogram metric") 73 | 74 | // Check the metrics are stored with the correct keys (full names) 75 | _, exists := reg.Gauge["prefix_test_test_gauge"] 76 | assert.True(t, exists, "Gauge metric should be stored with its full name") 77 | _, exists = reg.Counter["prefix_test_test_counter"] 78 | assert.True(t, exists, "Counter metric should be stored with its full name") 79 | _, exists = reg.Summary["prefix_test_test_summary"] 80 | assert.True(t, exists, "Summary metric should be stored with its full name") 81 | _, exists = reg.Histogram["prefix_test_test_histogram"] 82 | assert.True(t, exists, "Histogram metric should be stored with its full name") 83 | } 84 | 85 | func TestMetricRegistry_Materialize(t *testing.T) { 86 | // Create test metric definitions 87 | definitions := metric.Definitions{ 88 | Gauge: []*metric.Definition{ 89 | { 90 | Name: "test_gauge", 91 | Help: "Test gauge metric", 92 | SQL: "SELECT 1 AS value", 93 | Metadata: metric.Metadata{ 94 | PackageName: "test", 95 | }, 96 | }, 97 | }, 98 | Counter: []*metric.Definition{ 99 | { 100 | Name: "test_counter", 101 | Help: "Test counter metric", 102 | SQL: "SELECT 2 AS value", 103 | Metadata: metric.Metadata{ 104 | PackageName: "test", 105 | }, 106 | }}, 107 | } 108 | 109 | // Create a new registry 110 | reg := registry.NewMetricRegistry(definitions) 111 | 112 | // Setup mock connection 113 | conn, mock, _ := testutil.GetMockedConnection() 114 | 115 | // We expect gauge and counter metrics to query the database 116 | mock.ExpectQuery("SELECT 1 AS value"). 117 | WillReturnRows(sqlmock.NewRows([]string{"value"}).AddRow(1)) 118 | mock.ExpectQuery("SELECT 2 AS value"). 119 | WillReturnRows(sqlmock.NewRows([]string{"value"}).AddRow(2)) 120 | 121 | // Materialize metrics 122 | err := reg.Materialize(conn) 123 | 124 | // Verify 125 | assert.NoError(t, err) 126 | assert.NoError(t, mock.ExpectationsWereMet()) 127 | } 128 | -------------------------------------------------------------------------------- /hercules-packages/snowflake/1.0.yml: -------------------------------------------------------------------------------- 1 | # Note - this package relies on Snowflake account usage views being snapshot to S3 as parquet. 2 | # The following snapshots will need to be in S3 3 | # - SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY 4 | # - SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY 5 | # - SNOWFLAKE.ACCOUNT_USAGE.TASK_HISTORY 6 | # - SNOWFLAKE.ACCOUNT_USAGE.METERING_HISTORY 7 | # - SNOWFLAKE.ACCOUNT_USAGE.ALERT_HISTORY 8 | # - SNOWFLAKE.ACCOUNT_USAGE.COPY_HISTORY 9 | # - SNOWFLAKE.ACCOUNT_USAGE.LOAD_HISTORY 10 | # - SNOWFLAKE.ACCOUNT_USAGE.MATERIALIZED_VIEW_REFRESH_HISTORY 11 | # - SNOWFLAKE.ACCOUNT_USAGE.SERVERLESS_TASK_HISTORY 12 | # - SNOWFLAKE.ACCOUNT_USAGE.DATA_TRANSFER_HISTORY 13 | # - SNOWFLAKE.ACCOUNT_USAGE.DOCUMENT_AI_USAGE_HISTORY 14 | 15 | name: snowflake 16 | version: '1.0' 17 | 18 | macros: 19 | - sql: one() AS (SELECT 1); 20 | 21 | sources: 22 | - name: snowflake_query_history 23 | type: parquet 24 | source: assets/snowflake_query_history.parquet 25 | materialize: true 26 | refreshIntervalSeconds: 5 27 | 28 | metrics: 29 | gauge: 30 | - name: query_status_count 31 | help: Queries executed and their associated status by user and warehouse 32 | enabled: true 33 | sql: from snowflake_query_history select user_name as user, warehouse_name as warehouse, lower(execution_status) as status, count(*) as val group by all; 34 | 35 | - name: queries_this_week_total 36 | help: Queries this week total, by user and warehouse 37 | enabled: true 38 | sql: select user_name as user, warehouse_name as warehouse, count(*) as v from snowflake_query_history group by all; 39 | 40 | - name: avg_query_duration_seconds 41 | help: The average query duration for a particular user, using a particular warehouse 42 | enabled: true 43 | sql: select user_name as user, warehouse_name as warehouse, avg(TOTAL_ELAPSED_TIME) as value from snowflake_query_history group by all; 44 | 45 | - name: table_operations_count 46 | help: The number of operations on each table over the last week 47 | enabled: true 48 | sql: select user_name as user, query_type as query_type, count(*) as value from snowflake_query_history group by all; 49 | 50 | - name: avg_virtual_warehouse_spill_to_local_storage_bytes 51 | help: The average bytes spilled to disk for queries on a specific warehouse 52 | enabled: true 53 | sql: select user_name as user, warehouse_name as warehouse, avg(BYTES_SPILLED_TO_LOCAL_STORAGE) as value from snowflake_query_history group by all; 54 | 55 | - name: avg_virtual_warehouse_spill_to_remote_storage_bytes 56 | help: The average bytes spilled to remote disk for queries on a specific warehouse 57 | enabled: true 58 | sql: select user_name as user, warehouse_name as warehouse, avg(BYTES_SPILLED_TO_REMOTE_STORAGE) as value from snowflake_query_history group by all; 59 | 60 | histogram: 61 | - name: query_duration_seconds 62 | help: Histogram of query duration seconds 63 | sql: select user_name as user, warehouse_name as warehouse, total_elapsed_time as value from snowflake_query_history; 64 | buckets: 65 | - 0.1 66 | - 0.5 67 | - 1 68 | - 2 69 | - 4 70 | - 8 71 | - 16 72 | - 32 73 | - 64 74 | - 128 75 | - 256 76 | - 512 77 | - 1024 78 | - 2048 79 | - 4096 80 | - 8192 81 | - 16384 82 | - 32768 83 | 84 | summary: 85 | - name: virtual_warehouse_query_duration_seconds 86 | help: Summary of query duration seconds 87 | sql: select user_name as user, warehouse_name as warehouse, total_elapsed_time as value from snowflake_query_history; 88 | objectives: 89 | - 0.001 90 | - 0.05 91 | - 0.01 92 | - 0.5 93 | - 0.9 94 | - 0.99 95 | 96 | counter: 97 | - name: queries_executed_count 98 | help: The count of queries executed by user and warehouse 99 | sql: select user_name as user, warehouse_name as warehouse, 1 as value from snowflake_query_history; 100 | -------------------------------------------------------------------------------- /pkg/db/runner.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "math/big" 8 | "strings" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/spf13/cast" 12 | ) 13 | 14 | type QueryResult struct { 15 | Value float64 16 | Labels map[string]string 17 | } 18 | 19 | func toFloat64(v any) float64 { 20 | switch v := v.(type) { 21 | case *big.Int: 22 | val, accuracy := v.Float64() 23 | if accuracy != big.Exact { 24 | log.Debug().Interface("value", v).Msg("inexact conversion from big.Int to float64") 25 | } 26 | return val 27 | case *big.Float: 28 | val, accuracy := v.Float64() 29 | if accuracy != big.Exact { 30 | log.Debug().Interface("value", v).Msg("inexact conversion from big.Float to float64") 31 | } 32 | return val 33 | default: 34 | return cast.ToFloat64(v) 35 | } 36 | } 37 | 38 | func isFunctionColumn(column string) bool { 39 | return strings.Contains(column, "(") && strings.Contains(column, ")") 40 | } 41 | 42 | func (qr *QueryResult) StringifiedLabels() map[string]string { 43 | r := make(map[string]string) 44 | for k, v := range qr.Labels { 45 | r[k] = v 46 | } 47 | return r 48 | } 49 | 50 | // RunSQLQuery executes a SQL query on the provided connection. 51 | func RunSQLQuery(conn *sql.Conn, query SQL) (*sql.Rows, error) { 52 | log.Trace().Interface("query", query).Msg("running query") 53 | rows, err := conn.QueryContext(context.Background(), string(query)) 54 | if err != nil { 55 | log.Error().Err(err).Interface("query", query).Msg("could not run query") 56 | return nil, err 57 | } 58 | return rows, nil 59 | } 60 | 61 | func processRow(columns []string, vals []interface{}) QueryResult { 62 | queryResult := QueryResult{ 63 | Labels: make(map[string]string), 64 | } 65 | 66 | for i := range vals { 67 | columnName := columns[i] 68 | valuePtr, ok := vals[i].(*interface{}) 69 | if !ok { 70 | log.Error().Str("column", columnName).Msg("failed to cast column value pointer") 71 | continue 72 | } 73 | 74 | value := *valuePtr 75 | if isValueColumn(columnName) { 76 | queryResult.Value = toFloat64(value) 77 | } else { 78 | strValue := convertToString(columnName, value) 79 | queryResult.Labels[columnName] = strValue 80 | } 81 | } 82 | 83 | return queryResult 84 | } 85 | 86 | func isValueColumn(columnName string) bool { 87 | return columnName == "value" || columnName == "val" || columnName == "v" || isFunctionColumn(columnName) 88 | } 89 | 90 | func convertToString(columnName string, value interface{}) string { 91 | if value == nil { 92 | return "NULL" 93 | } 94 | 95 | strValue, ok := value.(string) 96 | if !ok { 97 | log.Warn(). 98 | Str("column", columnName). 99 | Interface("value", value). 100 | Msg("converting non-string value to string") 101 | strValue = fmt.Sprintf("%v", value) 102 | } 103 | return strValue 104 | } 105 | 106 | func Materialize(conn *sql.Conn, query SQL) ([]QueryResult, error) { 107 | rows, err := RunSQLQuery(conn, query) 108 | if err != nil { 109 | return nil, err 110 | } 111 | // rows is guaranteed to be non-nil if err is nil 112 | defer rows.Close() 113 | 114 | var queryResults []QueryResult 115 | 116 | // Get column names and column count from query results 117 | columns, err := rows.Columns() 118 | if err != nil { 119 | log.Error().Err(err).Msg("could not get columns") 120 | return nil, err 121 | } 122 | 123 | // Initialize values as interface{} pointers 124 | vals := prepareValueSlice(len(columns)) 125 | 126 | // Process each row 127 | for rows.Next() { 128 | scanErr := rows.Scan(vals...) 129 | if scanErr != nil { 130 | log.Error().Err(scanErr).Msg("could not scan row") 131 | continue // Skip this row and continue with the next one 132 | } 133 | 134 | queryResult := processRow(columns, vals) 135 | queryResults = append(queryResults, queryResult) 136 | } 137 | 138 | // Check for any errors during iteration 139 | rowsErr := rows.Err() 140 | if rowsErr != nil { 141 | log.Error().Err(rowsErr).Msg("error during rows iteration") 142 | return queryResults, rowsErr 143 | } 144 | 145 | return queryResults, nil 146 | } 147 | 148 | func prepareValueSlice(columnCount int) []interface{} { 149 | vals := make([]interface{}, columnCount) 150 | for i := range vals { 151 | var ii interface{} 152 | vals[i] = &ii 153 | } 154 | return vals 155 | } 156 | -------------------------------------------------------------------------------- /pkg/source/source.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/jakthom/hercules/pkg/db" 9 | "github.com/rs/zerolog/log" 10 | ) 11 | 12 | // Type represents the type of data source. 13 | type Type string 14 | 15 | const ( 16 | SQLSourceType Type = "sql" 17 | ParquetSourceType Type = "parquet" 18 | JSONSourceType Type = "json" 19 | CSVSourceType Type = "csv" 20 | ) 21 | 22 | type Source struct { 23 | Name string `json:"name"` 24 | Type Type `json:"type"` 25 | Source string `json:"source"` 26 | Materialize bool `json:"materialize"` // Whether or not to materialize as a table. 27 | RefreshIntervalSeconds int `json:"refreshIntervalSeconds"` 28 | stopChan chan bool // Channel to stop the refresh goroutine. 29 | } 30 | 31 | // Cleanup stops the background refresh process if it's running. 32 | func (s *Source) Cleanup() { 33 | if s.stopChan != nil { 34 | s.stopChan <- true 35 | close(s.stopChan) 36 | s.stopChan = nil 37 | } 38 | } 39 | 40 | // SQL returns the SQL representation of the source. 41 | func (s *Source) SQL() db.SQL { 42 | switch s.Type { 43 | case ParquetSourceType: 44 | return db.SQL(fmt.Sprintf("select * from read_parquet('%s')", s.Source)) 45 | case CSVSourceType: 46 | return db.SQL(fmt.Sprintf("select * from read_csv_auto('%s')", s.Source)) 47 | case JSONSourceType: 48 | return db.SQL(fmt.Sprintf("select * from read_json_auto('%s')", s.Source)) 49 | case SQLSourceType: 50 | return db.SQL(s.Source) 51 | default: // Default to sql 52 | return db.SQL(s.Source) 53 | } 54 | } 55 | 56 | func (s *Source) createOrReplaceTableSQL() db.SQL { 57 | return db.SQL("create or replace table " + s.Name + " as " + string(s.SQL()) + ";") 58 | } 59 | 60 | func (s *Source) createOrReplaceViewSQL() db.SQL { 61 | return db.SQL("create or replace view " + s.Name + " as " + string(s.SQL()) + ";") 62 | } 63 | 64 | func (s *Source) refreshWithConn(conn *sql.Conn) error { 65 | if s.Materialize { 66 | rows, err := db.RunSQLQuery(conn, s.createOrReplaceTableSQL()) 67 | if err != nil { 68 | return err 69 | } 70 | defer rows.Close() 71 | 72 | if rowsErr := rows.Err(); rowsErr != nil { 73 | log.Error().Err(rowsErr).Interface("source", s.Name).Msg("error during table materialization") 74 | return rowsErr 75 | } 76 | 77 | log.Debug().Interface("source", s.Name).Msg("source refreshed") 78 | return nil 79 | } 80 | 81 | rows, err := db.RunSQLQuery(conn, s.createOrReplaceViewSQL()) 82 | if err != nil { 83 | return err 84 | } 85 | defer rows.Close() 86 | 87 | if rowsErr := rows.Err(); rowsErr != nil { 88 | log.Error().Err(rowsErr).Interface("source", s.Name).Msg("error during view creation") 89 | return rowsErr 90 | } 91 | 92 | log.Debug().Interface("source", s.Name).Msg("source refreshed") 93 | return nil 94 | } 95 | 96 | func (s *Source) initializeWithConnection(conn *sql.Conn) error { 97 | err := s.refreshWithConn(conn) 98 | if err != nil { 99 | log.Fatal().Err(err).Interface("source", s.Name).Msg("could not refresh source") 100 | } 101 | // If the source is a table, materialize it on the predefined frequency. Views do not need to be refreshed. 102 | if s.Materialize && s.RefreshIntervalSeconds > 0 { 103 | // Start a ticker to continously update the source according to the predefined interval. 104 | ticker := time.NewTicker(time.Duration(s.RefreshIntervalSeconds) * time.Second) 105 | s.stopChan = make(chan bool) 106 | go func() { 107 | for { 108 | select { 109 | case <-s.stopChan: 110 | ticker.Stop() 111 | return 112 | case <-ticker.C: 113 | go func(conn *sql.Conn, source *Source) { 114 | refreshErr := source.refreshWithConn(conn) 115 | if refreshErr != nil { 116 | log.Debug().Interface("source", source.Name).Msg("could not refresh source") 117 | } 118 | }(conn, s) 119 | } 120 | } 121 | }() 122 | } 123 | return nil 124 | } 125 | 126 | // CleanupSources stops background refresh processes for all sources. 127 | func CleanupSources(sources []Source) { 128 | for i := range sources { 129 | sources[i].Cleanup() 130 | } 131 | } 132 | 133 | func InitializeSourcesWithConnection(sources []Source, conn *sql.Conn) error { 134 | for i := range sources { 135 | err := sources[i].initializeWithConnection(conn) 136 | if err != nil { 137 | log.Error().Err(err).Interface("source", sources[i].Name).Msg("could not initialize source") 138 | return err 139 | } 140 | } 141 | return nil 142 | } 143 | 144 | // CreateOrReplaceTableSQL generates SQL to create or replace a table for this source. 145 | func (s *Source) CreateOrReplaceTableSQL() db.SQL { 146 | return s.createOrReplaceTableSQL() 147 | } 148 | 149 | // CreateOrReplaceViewSQL generates SQL to create or replace a view for this source. 150 | func (s *Source) CreateOrReplaceViewSQL() db.SQL { 151 | return s.createOrReplaceViewSQL() 152 | } 153 | 154 | // TestHookRefreshWithConn exposes the refreshWithConn method for testing. 155 | func TestHookRefreshWithConn(s *Source, conn *sql.Conn) error { 156 | return s.refreshWithConn(conn) 157 | } 158 | -------------------------------------------------------------------------------- /pkg/source/source_test.go: -------------------------------------------------------------------------------- 1 | // Package source_test contains tests for the source package 2 | package source_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jakthom/hercules/pkg/source" 9 | "github.com/jakthom/hercules/pkg/testutil" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestSource_Sql(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | source source.Source 17 | expected string 18 | }{ 19 | { 20 | name: "SQL source type", 21 | source: source.Source{ 22 | Name: "test_source", 23 | Type: source.SQLSourceType, 24 | Source: "SELECT * FROM test_table", 25 | }, 26 | expected: "SELECT * FROM test_table", 27 | }, 28 | { 29 | name: "Parquet source type", 30 | source: source.Source{ 31 | Name: "test_parquet", 32 | Type: source.ParquetSourceType, 33 | Source: "/path/to/file.parquet", 34 | }, 35 | expected: "select * from read_parquet('/path/to/file.parquet')", 36 | }, 37 | { 38 | name: "JSON source type", 39 | source: source.Source{ 40 | Name: "test_json", 41 | Type: source.JSONSourceType, 42 | Source: "/path/to/file.json", 43 | }, 44 | expected: "select * from read_json_auto('/path/to/file.json')", 45 | }, 46 | { 47 | name: "CSV source type", 48 | source: source.Source{ 49 | Name: "test_csv", 50 | Type: source.CSVSourceType, 51 | Source: "/path/to/file.csv", 52 | }, 53 | expected: "select * from read_csv_auto('/path/to/file.csv')", 54 | }, 55 | } 56 | 57 | for _, tt := range tests { 58 | t.Run(tt.name, func(t *testing.T) { 59 | result := tt.source.SQL() 60 | assert.Equal(t, tt.expected, string(result)) 61 | }) 62 | } 63 | } 64 | 65 | func TestSource_CreateOrReplaceTableSql(t *testing.T) { 66 | src := source.Source{ 67 | Name: "test_table", 68 | Type: source.SQLSourceType, 69 | Source: "SELECT * FROM source_data", 70 | } 71 | 72 | expected := "create or replace table test_table as SELECT * FROM source_data;" 73 | result := src.CreateOrReplaceTableSQL() 74 | assert.Equal(t, expected, string(result)) 75 | } 76 | 77 | func TestSource_CreateOrReplaceViewSql(t *testing.T) { 78 | src := source.Source{ 79 | Name: "test_view", 80 | Type: source.SQLSourceType, 81 | Source: "SELECT * FROM source_data", 82 | } 83 | 84 | expected := "create or replace view test_view as SELECT * FROM source_data;" 85 | result := src.CreateOrReplaceViewSQL() 86 | assert.Equal(t, expected, string(result)) 87 | } 88 | 89 | func TestSource_RefreshWithConn_Table(t *testing.T) { 90 | src := source.Source{ 91 | Name: "test_table", 92 | Type: source.SQLSourceType, 93 | Source: "SELECT * FROM source_data", 94 | Materialize: true, 95 | } 96 | 97 | conn, mock, _ := testutil.GetMockedConnection() 98 | 99 | // Setup mock expectations - using Query instead of Exec to match RunSqlQuery implementation 100 | mock.ExpectQuery("create or replace table test_table as SELECT * FROM source_data;"). 101 | WillReturnRows(sqlmock.NewRows([]string{"result"})) 102 | 103 | // Call the method through the test hook 104 | err := source.TestHookRefreshWithConn(&src, conn) 105 | 106 | // Verify 107 | assert.NoError(t, err) 108 | assert.NoError(t, mock.ExpectationsWereMet()) 109 | } 110 | 111 | func TestSource_RefreshWithConn_View(t *testing.T) { 112 | src := source.Source{ 113 | Name: "test_view", 114 | Type: source.SQLSourceType, 115 | Source: "SELECT * FROM source_data", 116 | Materialize: false, 117 | } 118 | 119 | conn, mock, _ := testutil.GetMockedConnection() 120 | 121 | // Setup mock expectations - using Query instead of Exec to match RunSqlQuery implementation 122 | mock.ExpectQuery("create or replace view test_view as SELECT * FROM source_data;"). 123 | WillReturnRows(sqlmock.NewRows([]string{"result"})) 124 | 125 | // Call the method through the test hook 126 | err := source.TestHookRefreshWithConn(&src, conn) 127 | 128 | // Verify 129 | assert.NoError(t, err) 130 | assert.NoError(t, mock.ExpectationsWereMet()) 131 | } 132 | 133 | func TestInitializeSourcesWithConnection(t *testing.T) { 134 | sources := []source.Source{ 135 | { 136 | Name: "source1", 137 | Type: source.SQLSourceType, 138 | Source: "SELECT 1", 139 | Materialize: true, 140 | RefreshIntervalSeconds: 60, // Set a positive refresh interval 141 | }, 142 | { 143 | Name: "source2", 144 | Type: source.SQLSourceType, 145 | Source: "SELECT 2", 146 | Materialize: false, // Views don't use the ticker 147 | }, 148 | } 149 | 150 | conn, mock, _ := testutil.GetMockedConnection() 151 | 152 | // Setup mock expectations - using Query instead of Exec to match RunSqlQuery implementation 153 | mock.ExpectQuery("create or replace table source1 as SELECT 1;"). 154 | WillReturnRows(sqlmock.NewRows([]string{"result"})) 155 | mock.ExpectQuery("create or replace view source2 as SELECT 2;"). 156 | WillReturnRows(sqlmock.NewRows([]string{"result"})) 157 | 158 | // Call the function 159 | err := source.InitializeSourcesWithConnection(sources, conn) 160 | 161 | // Verify 162 | assert.NoError(t, err) 163 | assert.NoError(t, mock.ExpectationsWereMet()) 164 | } 165 | -------------------------------------------------------------------------------- /pkg/herculesPackage/package_test.go: -------------------------------------------------------------------------------- 1 | // Package herculespackage_test contains tests for the herculespackage package 2 | package herculespackage_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/jakthom/hercules/pkg/db" 8 | herculespackage "github.com/jakthom/hercules/pkg/herculesPackage" 9 | "github.com/jakthom/hercules/pkg/metric" 10 | "github.com/jakthom/hercules/pkg/source" 11 | herculestypes "github.com/jakthom/hercules/pkg/types" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | "sigs.k8s.io/yaml" 15 | ) 16 | 17 | // TestPackageLoadFromFile tests loading a package from a YAML file. 18 | func TestPackageLoadFromFile(t *testing.T) { 19 | // Use the Snowflake package which uses snowflake_query_history.parquet in the assets directory 20 | packagePath := "../../hercules-packages/snowflake/1.0.yml" 21 | 22 | // Configure and load package 23 | config := herculespackage.PackageConfig{ 24 | Package: packagePath, 25 | Variables: herculespackage.Variables{"env": "test"}, 26 | MetricPrefix: "prefix_", 27 | } 28 | 29 | pkg, err := config.GetPackage() 30 | require.NoError(t, err, "Should load package without error") 31 | 32 | // Validate core package attributes 33 | assert.Equal(t, herculestypes.PackageName("snowflake"), pkg.Name) 34 | assert.Equal(t, "1.0", pkg.Version) 35 | assert.Equal(t, "test", pkg.Variables["env"]) 36 | assert.Equal(t, herculestypes.MetricPrefix("prefix_"), pkg.MetricPrefix) 37 | 38 | // Validate macros 39 | assert.GreaterOrEqual(t, len(pkg.Macros), 1, "Should have at least one macro") 40 | assert.Contains(t, pkg.Macros[0].SQL, "one() AS (SELECT 1)") 41 | 42 | // Validate sources 43 | assert.GreaterOrEqual(t, len(pkg.Sources), 1, "Should have at least one source") 44 | assert.Equal(t, "snowflake_query_history", pkg.Sources[0].Name) 45 | assert.Equal(t, source.Type("parquet"), pkg.Sources[0].Type) 46 | assert.Equal(t, "assets/snowflake_query_history.parquet", pkg.Sources[0].Source) 47 | assert.True(t, pkg.Sources[0].Materialize) 48 | assert.Equal(t, 5, pkg.Sources[0].RefreshIntervalSeconds) 49 | 50 | // Validate metrics - test one of each type 51 | // Gauge metrics 52 | assert.GreaterOrEqual(t, len(pkg.Metrics.Gauge), 1, "Should have at least one gauge metric") 53 | assert.Equal(t, "query_status_count", pkg.Metrics.Gauge[0].Name) 54 | assert.Contains(t, pkg.Metrics.Gauge[0].Help, "Queries executed and their associated status") 55 | 56 | // Histogram metrics 57 | assert.GreaterOrEqual(t, len(pkg.Metrics.Histogram), 1, "Should have at least one histogram metric") 58 | assert.Equal(t, "query_duration_seconds", pkg.Metrics.Histogram[0].Name) 59 | assert.Contains(t, pkg.Metrics.Histogram[0].Help, "Histogram of query duration seconds") 60 | assert.GreaterOrEqual(t, len(pkg.Metrics.Histogram[0].Buckets), 1, "Should have histogram buckets") 61 | 62 | // Summary metrics 63 | assert.GreaterOrEqual(t, len(pkg.Metrics.Summary), 1, "Should have at least one summary metric") 64 | assert.Equal(t, "virtual_warehouse_query_duration_seconds", pkg.Metrics.Summary[0].Name) 65 | assert.Contains(t, pkg.Metrics.Summary[0].Help, "Summary of query duration seconds") 66 | assert.GreaterOrEqual(t, len(pkg.Metrics.Summary[0].Objectives), 1, "Should have summary objectives") 67 | 68 | // Counter metrics 69 | assert.GreaterOrEqual(t, len(pkg.Metrics.Counter), 1, "Should have at least one counter metric") 70 | assert.Equal(t, "queries_executed_count", pkg.Metrics.Counter[0].Name) 71 | assert.Contains(t, pkg.Metrics.Counter[0].Help, "The count of queries executed by user and warehouse") 72 | } 73 | 74 | // TestPackageSerialization tests package serialization and deserialization. 75 | func TestPackageSerialization(t *testing.T) { 76 | // Create test package 77 | originalPkg := createMinimalTestPackage() 78 | 79 | // Serialize to YAML 80 | serialized, err := yaml.Marshal(originalPkg) 81 | require.NoError(t, err, "Package serialization should succeed") 82 | 83 | // Deserialize from YAML 84 | var deserializedPkg herculespackage.Package 85 | err = yaml.Unmarshal(serialized, &deserializedPkg) 86 | require.NoError(t, err, "Package deserialization should succeed") 87 | 88 | // Validate core attributes 89 | assert.Equal(t, originalPkg.Name, deserializedPkg.Name, "Package name should match") 90 | assert.Equal(t, originalPkg.Version, deserializedPkg.Version, "Package version should match") 91 | 92 | // Validate component counts 93 | assert.Len(t, deserializedPkg.Extensions.Community, 1, "Should have one extension") 94 | assert.Len(t, deserializedPkg.Macros, 1, "Should have one macro") 95 | assert.Len(t, deserializedPkg.Sources, 1, "Should have one source") 96 | assert.Len(t, deserializedPkg.Metrics.Gauge, 1, "Should have one gauge metric") 97 | 98 | // Validate component details 99 | assert.Equal(t, originalPkg.Extensions.Community[0].Name, deserializedPkg.Extensions.Community[0].Name) 100 | assert.Equal(t, originalPkg.Macros[0].Name, deserializedPkg.Macros[0].Name) 101 | assert.Equal(t, originalPkg.Sources[0].Name, deserializedPkg.Sources[0].Name) 102 | assert.Equal(t, originalPkg.Metrics.Gauge[0].Name, deserializedPkg.Metrics.Gauge[0].Name) 103 | } 104 | 105 | // Helper functions 106 | 107 | // createMinimalTestPackage creates a minimal test package for testing. 108 | func createMinimalTestPackage() herculespackage.Package { 109 | return herculespackage.Package{ 110 | Name: "test-package", 111 | Version: "1.0", 112 | Extensions: db.Extensions{ 113 | Community: []db.CommunityExtension{ 114 | {Name: "test_ext"}, 115 | }, 116 | }, 117 | Macros: []db.Macro{ 118 | {Name: "test_macro", SQL: "test_macro()"}, 119 | }, 120 | Sources: []source.Source{ 121 | { 122 | Name: "test_source", 123 | Type: "sql", 124 | Source: "SELECT 1", 125 | Materialize: true, 126 | RefreshIntervalSeconds: 60, 127 | }, 128 | }, 129 | Metrics: metric.Definitions{ 130 | Gauge: []*metric.Definition{ 131 | {Name: "test_gauge", Help: "Test gauge", SQL: "SELECT 1"}, 132 | }, 133 | }, 134 | Metadata: metric.Metadata{ 135 | PackageName: "test-package", 136 | Prefix: "test_", 137 | }, 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /hercules-packages/bluesky/1.0.yml: -------------------------------------------------------------------------------- 1 | name: bluesky 2 | version: '1.0' 3 | 4 | extensions: 5 | community: 6 | - name: psql 7 | 8 | sources: 9 | - name: profile 10 | type: sql 11 | source: select * from read_json_auto('https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=duckdb.org'); 12 | materialize: true 13 | refreshIntervalSeconds: 60 14 | 15 | - name: posts 16 | type: sql 17 | source: select post.* from (select unnest(feed).post as post from (select * from read_json_auto('https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed?actor=did:plc:id67xmpji7oysb7vitsodr4v'))); 18 | materialize: true 19 | refreshIntervalSeconds: 60 20 | 21 | - name: jetstream 22 | type: sql 23 | source: select * from read_parquet('https://hive.buz.dev/bluesky/jetstream/latest.parquet'); 24 | materialize: true 25 | refreshIntervalSeconds: 600 26 | 27 | 28 | metrics: 29 | gauge: 30 | # DuckDB 31 | - name: duckdb_associated_starter_packs_total 32 | help: The number of starter packs the DuckDB Bluesky account is associated with 33 | sql: select associated.starterPacks as val from profile; 34 | 35 | - name: duckdb_followers_total 36 | help: The number of total accounts following DuckDB 37 | sql: select followersCount as val from profile; 38 | 39 | - name: duckdb_follows_total 40 | help: The number of total accounts the DuckDB Bluesky account follows 41 | sql: select followsCount as val from profile; 42 | 43 | - name: duckdb_posts_total 44 | help: The number of total DuckDB-authored Bluesky posts 45 | sql: select postsCount as val from profile; 46 | 47 | - name: duckdb_reply_count 48 | help: The number of total replies to DuckDB Bluesky posts 49 | sql: select sum(replyCount) from posts; 50 | 51 | - name: duckdb_repost_count 52 | help: The number of total reposts to DuckDB Bluesky posts 53 | sql: select sum(repostCount) from posts; 54 | 55 | - name: duckdb_like_count 56 | help: The number of total likes to DuckDB Bluesky posts 57 | sql: select sum(likeCount) from posts; 58 | 59 | - name: duckdb_quote_count 60 | help: The number of total quoted DuckDB Bluesky posts 61 | sql: select sum(quoteCount) from posts; 62 | 63 | - name: duckdb_author_posts_count 64 | help: The number of posts on the DuckDB Bluesky account by author 65 | sql: select author.handle, count(*) from posts group by 1; 66 | 67 | - name: duckdb_author_likes_count 68 | help: The number of likes on DuckDB Bluesky posts by author 69 | sql: select author.handle, sum(likeCount) from posts group by 1; 70 | 71 | # Jetstream 72 | 73 | - name: jetstream_records_count 74 | help: The total number of records in the Jetstream 75 | sql: select count(*) from jetstream; 76 | 77 | - name: jetstream_collection_count 78 | help: Bluesky collection count, by collection name 79 | sql: select coalesce(commit.collection, 'unknown') as collection, count(*) from jetstream group by 1; 80 | 81 | - name: jetstream_top_10_accounts_by_new_follows_count 82 | help: Top 10 Bluesky accounts by new follows 83 | sql: select commit.record->>'subject' as did, count(*) from jetstream where commit.collection = 'app.bsky.graph.follow' and (commit.record->>'subject' is not null) group by 1 order by 2 desc limit 10; 84 | 85 | - name: jetstream_top_10_posts_by_likes_count 86 | help: Top 10 Bluesky accounts by likes 87 | sql: select commit.record->>'subject'->>'uri' as post, count(*) from jetstream where commit.collection = 'app.bsky.feed.like' and post is not null group by 1 order by 2 desc limit 10; 88 | 89 | - name: jetstream_user_signups_count 90 | help: The number of new signups 91 | sql: select count(*) from jetstream where commit.collection = 'app.bsky.actor.profile' and commit.operation = 'create'; 92 | 93 | summary: 94 | # DuckDB 95 | - name: duckdb_author_post_likes_count 96 | help: Quantiles of likes on DuckDB Bluesky posts by author 97 | sql: select author.handle, likeCount as val from posts; 98 | objectives: 99 | - 0.5 100 | - 0.9 101 | - 0.99 102 | 103 | - name: duckdb_author_post_repost_count 104 | help: Quantiles of reposts on DuckDB Bluesky posts by author 105 | sql: select author.handle, repostCount as val from posts; 106 | objectives: 107 | - 0.5 108 | - 0.9 109 | - 0.99 110 | 111 | # Jetstream 112 | - name: jetstream_user_signups_per_minute_bucket 113 | help: Quantiles of user signups per minute 114 | sql: select date_trunc('minute', make_timestamp(time_us)), count(*) from jetstream where commit.collection = 'app.bsky.actor.profile' and commit.operation = 'create' group by 1 order by 1 asc; 115 | objectives: 116 | - 0.5 117 | - 0.9 118 | - 0.99 119 | - 0.999 120 | 121 | - name: jetstream_reposts_per_minute_bucket 122 | help: Quantiles of reposts per minute 123 | sql: select date_trunc('minute', make_timestamp(time_us)), count(*) from jetstream where commit.collection = 'app.bsky.feed.repost' group by 1; 124 | objectives: 125 | - 0.5 126 | - 0.9 127 | - 0.99 128 | - 0.999 129 | 130 | - name: jetstream_likes_per_minute_bucket 131 | help: Quantiles of likes per minute 132 | sql: select date_trunc('minute', make_timestamp(time_us)), count(*) from jetstream where commit.collection = 'app.bsky.feed.like' and commit.operation = 'create' group by 1; 133 | objectives: 134 | - 0.5 135 | - 0.9 136 | - 0.99 137 | - 0.999 138 | 139 | - name: jetstream_follows_per_minute_bucket 140 | help: Quantiles of follows per minute 141 | sql: select date_trunc('minute', make_timestamp(time_us)), count(*) from jetstream where commit.collection = 'app.bsky.graph.follow' and commit.operation = 'create' group by 1; 142 | objectives: 143 | - 0.5 144 | - 0.9 145 | - 0.99 146 | - 0.999 147 | 148 | - name: jetstream_posts_per_minute_bucket 149 | help: Quantiles of posts per minute 150 | sql: from jetstream |> where commit.collection = 'app.bsky.feed.post' and commit.operation = 'create' |> select date_trunc('minute', make_timestamp(time_us)), count(*) group by 1; 151 | objectives: 152 | - 0.5 153 | - 0.9 154 | - 0.99 155 | - 0.999 156 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Hercules Makefile 2 | # Defines all common build, test, and development tasks. 3 | 4 | # Build variables. 5 | VERSION := $(shell cat .VERSION 2>/dev/null || echo "dev") 6 | GO := go 7 | GOOS ?= $(shell $(GO) env GOOS) 8 | GOARCH ?= $(shell $(GO) env GOARCH) 9 | HERCULES_DIR := ./cmd/hercules 10 | BINARY_NAME := hercules 11 | ENV ?= $(shell whoami) 12 | TEST_PROFILE := coverage.out 13 | GOLANGCI_LINT_VERSION ?= v2.1.5 14 | # Extract port from hercules.yml, default to 9100 if not found. 15 | HERCULES_PORT := $(shell grep -o 'port: [0-9]\+' hercules.yml 2>/dev/null | awk '{print $$2}' || echo 9100) 16 | 17 | # Fix for macOS linker warnings - use a simpler approach. 18 | ifeq ($(shell uname),Darwin) 19 | # Use a single warning suppression flag that works on macOS. 20 | export CGO_LDFLAGS := -Wl,-w 21 | endif 22 | 23 | # Output colors. 24 | COLOR_RESET = \033[0m 25 | COLOR_BLUE = \033[34m 26 | COLOR_GREEN = \033[32m 27 | COLOR_RED = \033[31m 28 | 29 | ##@ General 30 | 31 | .PHONY: help 32 | help: ## Display this help. 33 | @awk 'BEGIN {FS = ":.*##"; printf "\n$(COLOR_BLUE)Hercules Makefile Help$(COLOR_RESET)\n"} \ 34 | /^##@/ { printf "\n$(COLOR_BLUE)%s$(COLOR_RESET)\n", substr($$0, 5) } \ 35 | /^[a-zA-Z0-9_-]+:.*?##/ { printf " $(COLOR_GREEN)%-20s$(COLOR_RESET) %s\n", $$1, $$2 }' $(MAKEFILE_LIST) 36 | 37 | .PHONY: all 38 | all: lint test build ## Run lint, tests, and build. 39 | 40 | ##@ Development 41 | 42 | .PHONY: run 43 | run: ## Run Hercules locally. 44 | @printf "$(COLOR_BLUE)Running Hercules in development mode$(COLOR_RESET)\n" 45 | CGO_ENABLED=1 ENV=$(ENV) $(GO) run -ldflags="-X 'main.version=$(VERSION)'" $(HERCULES_DIR) 46 | 47 | .PHONY: debug 48 | debug: ## Run Hercules with debugging enabled. 49 | @printf "$(COLOR_BLUE)Running Hercules in debug mode$(COLOR_RESET)\n" 50 | CGO_ENABLED=1 DEBUG=1 ENV=$(ENV) $(GO) run -ldflags="-X 'main.version=$(VERSION)'" $(HERCULES_DIR) 51 | 52 | .PHONY: mod 53 | mod: ## Tidy and verify Go modules. 54 | @printf "$(COLOR_BLUE)Tidying Go modules$(COLOR_RESET)\n" 55 | $(GO) mod tidy 56 | $(GO) mod verify 57 | 58 | .PHONY: fmt 59 | fmt: ## Format Go code. 60 | @printf "$(COLOR_BLUE)Formatting Go code$(COLOR_RESET)\n" 61 | $(GO) fmt ./... 62 | 63 | ##@ Build 64 | 65 | .PHONY: build 66 | build: ## Build Hercules binary. 67 | @printf "$(COLOR_BLUE)Building Hercules binary$(COLOR_RESET)\n" 68 | CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build -ldflags="-X main.version=$(VERSION)" -o $(BINARY_NAME) $(HERCULES_DIR) 69 | 70 | # Specific targets for different platforms to help with cross-compilation. 71 | .PHONY: build-darwin 72 | build-darwin: ## Build for macOS (darwin). 73 | @printf "$(COLOR_BLUE)Building Hercules for macOS$(COLOR_RESET)\n" 74 | CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 $(GO) build -ldflags="-X main.version=$(VERSION)" -o $(BINARY_NAME)-darwin-arm64 $(HERCULES_DIR) 75 | 76 | .PHONY: build-linux 77 | build-linux: ## Build for Linux. 78 | @printf "$(COLOR_BLUE)Building Hercules for Linux$(COLOR_RESET)\n" 79 | CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(GO) build -ldflags="-X main.version=$(VERSION)" -o $(BINARY_NAME)-linux-amd64 $(HERCULES_DIR) 80 | 81 | .PHONY: install 82 | install: build ## Install Hercules binary. 83 | @printf "$(COLOR_BLUE)Installing Hercules binary$(COLOR_RESET)\n" 84 | mv $(BINARY_NAME) $(GOPATH)/bin/ 85 | 86 | .PHONY: clean 87 | clean: ## Clean build artifacts. 88 | @printf "$(COLOR_BLUE)Cleaning build artifacts$(COLOR_RESET)\n" 89 | rm -f $(BINARY_NAME) 90 | rm -f $(TEST_PROFILE) 91 | rm -rf bin/ 92 | find . -type f -name "*.test" -delete 93 | find . -type f -name "*.out" -delete 94 | find . -type d -name "testdata" -exec rm -rf {} +; 2>/dev/null || true 95 | 96 | ##@ Quality 97 | 98 | .PHONY: lint 99 | lint: ## Run linters. 100 | @which golangci-lint > /dev/null 2>&1 || { \ 101 | printf "$(COLOR_BLUE)Installing golangci-lint using go install$(COLOR_RESET)\n"; \ 102 | go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION); \ 103 | printf "$(COLOR_GREEN)golangci-lint installed successfully$(COLOR_RESET)\n"; \ 104 | } 105 | @printf "$(COLOR_BLUE)Running linters$(COLOR_RESET)\n" 106 | golangci-lint run --config ./.github/golangci.yml ./pkg/... ./cmd/... 107 | 108 | .PHONY: vet 109 | vet: ## Run Go vet. 110 | @printf "$(COLOR_BLUE)Running go vet$(COLOR_RESET)\n" 111 | $(GO) vet -v ./... 112 | 113 | ##@ Testing 114 | 115 | .PHONY: test 116 | test: ## Run tests. 117 | @printf "$(COLOR_BLUE)Running tests$(COLOR_RESET)\n" 118 | $(GO) test -race ./pkg/... ./cmd/... 119 | 120 | .PHONY: test-cover 121 | test-cover: ## Run tests with coverage. 122 | @printf "$(COLOR_BLUE)Running tests with coverage$(COLOR_RESET)\n" 123 | $(GO) test -v -race -cover ./pkg/... ./cmd/... -coverprofile=$(TEST_PROFILE) 124 | $(GO) tool cover -func=$(TEST_PROFILE) 125 | 126 | .PHONY: test-cover-html 127 | test-cover-html: test-cover ## Run tests with coverage and open HTML report. 128 | @printf "$(COLOR_BLUE)Opening coverage report in browser$(COLOR_RESET)\n" 129 | $(GO) tool cover -html=$(TEST_PROFILE) 130 | 131 | ##@ Docker 132 | 133 | .PHONY: docker-build 134 | docker-build: ## Build Docker image. 135 | @printf "$(COLOR_BLUE)Building Docker image with port $(HERCULES_PORT)$(COLOR_RESET)\n" 136 | docker build --build-arg PORT=$(HERCULES_PORT) -t hercules:$(VERSION) -f build/Dockerfile.linux.arm64 . 137 | 138 | .PHONY: docker-run 139 | docker-run: ## Run Hercules in Docker with mounted configuration, assets, and packages. 140 | @printf "$(COLOR_BLUE)Running Hercules in Docker$(COLOR_RESET)\n" 141 | docker run --rm -p $(HERCULES_PORT):$(HERCULES_PORT) \ 142 | -v $(PWD)/hercules.yml:/app/config/hercules.yml \ 143 | -v $(PWD)/assets:/app/assets \ 144 | -v $(PWD)/hercules-packages:/app/hercules-packages \ 145 | hercules:$(VERSION) 146 | 147 | .PHONY: docker-debug 148 | docker-debug: ## Run Hercules in Docker with debug mode and mounted volumes. 149 | @printf "$(COLOR_BLUE)Running Hercules in Docker with debug enabled$(COLOR_RESET)\n" 150 | docker run --rm -p $(HERCULES_PORT):$(HERCULES_PORT) -e DEBUG=1 \ 151 | -v $(PWD)/hercules.yml:/app/config/hercules.yml \ 152 | -v $(PWD)/assets:/app/assets \ 153 | -v $(PWD)/hercules-packages:/app/hercules-packages \ 154 | hercules:$(VERSION) 155 | 156 | ##@ CI Tasks 157 | 158 | .PHONY: ci-test 159 | ci-test: ## Run tests for CI. 160 | $(GO) test -v -race -coverprofile=$(TEST_PROFILE) ./pkg/... ./cmd/... 161 | 162 | .PHONY: ci-build 163 | ci-build: ## Build for CI. 164 | CGO_ENABLED=1 GOOS=linux GOARCH=amd64 $(GO) build -ldflags="-X main.version=$(VERSION)" -o $(BINARY_NAME) $(HERCULES_DIR) 165 | -------------------------------------------------------------------------------- /cmd/hercules/app.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "sync" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/jakthom/hercules/pkg/config" 15 | "github.com/jakthom/hercules/pkg/flock" 16 | herculespackage "github.com/jakthom/hercules/pkg/herculesPackage" 17 | "github.com/jakthom/hercules/pkg/metric" 18 | registry "github.com/jakthom/hercules/pkg/metricRegistry" 19 | "github.com/jakthom/hercules/pkg/middleware" 20 | "github.com/prometheus/client_golang/prometheus" 21 | "github.com/prometheus/client_golang/prometheus/collectors" 22 | "github.com/prometheus/client_golang/prometheus/promhttp" 23 | "github.com/rs/zerolog" 24 | "github.com/rs/zerolog/log" 25 | ) 26 | 27 | type Hercules struct { 28 | config config.Config 29 | db *sql.DB 30 | packages []herculespackage.Package 31 | conn *sql.Conn 32 | metricRegistries []*registry.MetricRegistry 33 | debug bool 34 | version string // Added version field to the struct 35 | } 36 | 37 | func (d *Hercules) configure() { 38 | log.Debug().Msg("configuring Hercules") 39 | zerolog.SetGlobalLevel(zerolog.InfoLevel) 40 | debug := config.IsDebugMode() 41 | if debug { 42 | d.debug = true 43 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 44 | } 45 | trace := config.IsTraceMode() 46 | if trace { 47 | zerolog.SetGlobalLevel(zerolog.TraceLevel) 48 | } 49 | 50 | // Load configuration and handle errors 51 | var err error 52 | d.config, err = config.GetConfig() 53 | if err != nil { 54 | log.Warn().Err(err).Msg("using default configuration due to error") 55 | } else if !debug && d.config.Debug { 56 | d.debug = true 57 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 58 | log.Debug().Msg("debug mode enabled via config file") 59 | } 60 | } 61 | 62 | func (d *Hercules) initializeFlock() { 63 | log.Debug().Str("db", d.config.DB).Msg("initializing database") 64 | d.db, d.conn = flock.InitializeDB(d.config) 65 | } 66 | 67 | func (d *Hercules) loadPackages() { 68 | pkgs := []herculespackage.Package{} 69 | for _, pkgConfig := range d.config.Packages { 70 | pkg, err := pkgConfig.GetPackage() 71 | pkg.Metadata = metric.Metadata{ 72 | PackageName: string(pkg.Name), 73 | Prefix: pkgConfig.MetricPrefix, 74 | Labels: d.config.InstanceLabels(), 75 | } 76 | if err != nil { 77 | log.Error().Err(err).Msg("could not get package") 78 | } 79 | pkgs = append(pkgs, pkg) 80 | } 81 | // Represent core configuration via a package 82 | pkgs = append(pkgs, herculespackage.Package{ 83 | Name: "core", 84 | Version: "1.0.0", 85 | Extensions: d.config.Extensions, 86 | Macros: d.config.Macros, 87 | Sources: d.config.Sources, 88 | Metrics: d.config.Metrics, 89 | Metadata: metric.Metadata{ 90 | PackageName: "core", 91 | Labels: d.config.InstanceLabels(), 92 | }, 93 | }) 94 | d.packages = pkgs 95 | } 96 | 97 | func (d *Hercules) initializePackages() { 98 | // Use our new parallel package initialization function 99 | err := herculespackage.InitializePackagesWithConnection(d.packages, d.conn) 100 | if err != nil { 101 | log.Error().Err(err).Msg("error initializing packages") 102 | } 103 | } 104 | 105 | func (d *Hercules) initializeRegistries() { 106 | // Register a registry for each package 107 | for _, pkg := range d.packages { 108 | if d.metricRegistries == nil { 109 | d.metricRegistries = []*registry.MetricRegistry{registry.NewMetricRegistry(pkg.Metrics)} 110 | } else { 111 | d.metricRegistries = append(d.metricRegistries, registry.NewMetricRegistry(pkg.Metrics)) 112 | } 113 | } 114 | } 115 | 116 | func (d *Hercules) Initialize() { 117 | log.Debug().Msg("initializing Hercules") 118 | d.configure() 119 | d.initializeFlock() 120 | d.loadPackages() 121 | d.initializePackages() 122 | d.initializeRegistries() 123 | log.Debug().Interface("config", d.config).Msg("running with config") 124 | } 125 | 126 | func (d *Hercules) Run() { 127 | // Create server mux and configure routes 128 | mux := http.NewServeMux() 129 | prometheus.Unregister(collectors.NewGoCollector()) // Remove golang node defaults 130 | mux.Handle("/metrics", middleware.MetricsMiddleware(d.conn, d.metricRegistries, promhttp.Handler())) 131 | mux.Handle("/", http.RedirectHandler("/metrics", http.StatusSeeOther)) 132 | 133 | // Server timeout constants 134 | const ( 135 | readTimeoutSeconds = 5 136 | writeTimeoutSeconds = 10 137 | idleTimeoutSeconds = 120 138 | shutdownTimeoutSeconds = 15 139 | ) 140 | 141 | // Configure server with proper timeouts for better performance 142 | srv := &http.Server{ 143 | Addr: ":" + d.config.Port, 144 | Handler: mux, 145 | ReadTimeout: readTimeoutSeconds * time.Second, 146 | WriteTimeout: writeTimeoutSeconds * time.Second, 147 | IdleTimeout: idleTimeoutSeconds * time.Second, 148 | } 149 | 150 | // Setup graceful shutdown with proper signal handling 151 | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) 152 | defer stop() 153 | 154 | // Start server in a goroutine 155 | serverErrors := make(chan error, 1) 156 | go func() { 157 | log.Info().Msg("hercules is running with version: " + d.version) 158 | serverErrors <- srv.ListenAndServe() 159 | }() 160 | 161 | // Block until we receive a signal or server error 162 | select { 163 | case err := <-serverErrors: 164 | if err != nil && !errors.Is(err, http.ErrServerClosed) { 165 | log.Error().Err(err).Msg("server error") 166 | } 167 | case <-ctx.Done(): 168 | log.Info().Msg("shutdown initiated") 169 | } 170 | 171 | // Create a timeout context for graceful shutdown 172 | shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), shutdownTimeoutSeconds*time.Second) 173 | defer shutdownCancel() 174 | 175 | // Use a WaitGroup to ensure proper cleanup of resources 176 | var wg sync.WaitGroup 177 | 178 | // Gracefully shut down the server 179 | wg.Add(1) 180 | go func() { 181 | defer wg.Done() 182 | if err := srv.Shutdown(shutdownCtx); err != nil { 183 | log.Error().Err(err).Msg("server shutdown error") 184 | } 185 | }() 186 | 187 | // Close database connection in parallel 188 | wg.Add(1) 189 | go func() { 190 | defer wg.Done() 191 | log.Debug().Msg("closing database connection") 192 | if err := d.db.Close(); err != nil { 193 | log.Error().Err(err).Msg("error closing database connection") 194 | } 195 | if !d.debug { 196 | if err := os.Remove(d.config.DB); err != nil { 197 | log.Error().Err(err).Str("db", d.config.DB).Msg("error removing database file") 198 | } 199 | } 200 | }() 201 | 202 | // Wait for all cleanup operations to complete or timeout 203 | waitCh := make(chan struct{}) 204 | go func() { 205 | wg.Wait() 206 | close(waitCh) 207 | }() 208 | 209 | select { 210 | case <-waitCh: 211 | log.Info().Msg("shutdown completed gracefully") 212 | case <-shutdownCtx.Done(): 213 | log.Warn().Msg("shutdown timed out, forcing exit") 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /pkg/herculesPackage/package.go: -------------------------------------------------------------------------------- 1 | package herculespackage 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | "sync" 12 | "time" 13 | 14 | "github.com/jakthom/hercules/pkg/db" 15 | "github.com/jakthom/hercules/pkg/metric" 16 | "github.com/jakthom/hercules/pkg/source" 17 | herculestypes "github.com/jakthom/hercules/pkg/types" 18 | "github.com/rs/zerolog/log" 19 | "golang.org/x/sync/errgroup" 20 | "sigs.k8s.io/yaml" 21 | ) 22 | 23 | // Variables represents a map of variable values for package configuration. 24 | type Variables map[string]interface{} 25 | 26 | // Package concurrency constants. 27 | const ( 28 | // MaxConcurrentPackageInit is the maximum number of packages to initialize concurrently. 29 | MaxConcurrentPackageInit = 4 30 | // ErrorChannelSize is the size of the error channel buffer for package initialization. 31 | ErrorChannelSize = 3 32 | ) 33 | 34 | // HTTP request timeout constants. 35 | const ( 36 | // DefaultHTTPTimeoutSeconds is the default timeout for HTTP requests to package endpoints. 37 | DefaultHTTPTimeoutSeconds = 30 38 | ) 39 | 40 | // A hercules package consists of extensions, macros, sources, and metrics 41 | // It can be downloaded from remote sources or shipped alongside hercules. 42 | 43 | type Package struct { 44 | Name herculestypes.PackageName `json:"name"` 45 | Version string `json:"version"` 46 | Variables Variables `json:"variables"` 47 | Extensions db.Extensions `json:"extensions"` 48 | Macros []db.Macro `json:"macros"` 49 | Sources []source.Source `json:"sources"` 50 | Metrics metric.Definitions `json:"metrics"` 51 | MetricPrefix herculestypes.MetricPrefix `json:"-"` 52 | Metadata metric.Metadata `json:"metadata"` 53 | // TODO -> Package-level secrets 54 | } 55 | 56 | // InitializePackagesWithConnection initializes multiple packages in parallel 57 | // for better performance when starting up with many packages. 58 | func InitializePackagesWithConnection(packages []Package, conn *sql.Conn) error { 59 | // Use errgroup to handle concurrent initialization with error handling. 60 | g := new(errgroup.Group) 61 | // Limit concurrency to avoid overwhelming the database connection. 62 | sem := make(chan struct{}, MaxConcurrentPackageInit) 63 | 64 | for i := range packages { 65 | pkg := &packages[i] // Capture package by reference in the loop. 66 | g.Go(func() error { 67 | // Semaphore to limit concurrency. 68 | sem <- struct{}{} 69 | defer func() { <-sem }() 70 | 71 | return pkg.InitializeWithConnection(conn) 72 | }) 73 | } 74 | 75 | // Wait for all package initializations to complete 76 | return g.Wait() 77 | } 78 | 79 | func (p *Package) InitializeWithConnection(conn *sql.Conn) error { 80 | if len(p.Name) == 0 { 81 | log.Trace().Msg("empty package detected - skipping initialization") 82 | return nil 83 | } 84 | 85 | log.Info().Interface("package", p.Name).Msg("initializing " + string(p.Name) + " package") 86 | 87 | // Create a wait group for concurrent operations that don't return errors. 88 | var wg sync.WaitGroup 89 | 90 | // Error channel for collecting errors from goroutines. 91 | errChan := make(chan error, ErrorChannelSize) 92 | 93 | // Ensure extensions. 94 | wg.Add(1) 95 | go func() { 96 | defer wg.Done() 97 | db.EnsureExtensionsWithConnection(p.Extensions, conn) 98 | }() 99 | 100 | // Ensure macros. 101 | wg.Add(1) 102 | go func() { 103 | defer wg.Done() 104 | db.EnsureMacrosWithConnection(p.Macros, conn) 105 | }() 106 | 107 | // Ensure sources (this can return an error). 108 | go func() { 109 | err := source.InitializeSourcesWithConnection(p.Sources, conn) 110 | if err != nil { 111 | errChan <- fmt.Errorf("could not initialize sources for package %s: %w", p.Name, err) 112 | } 113 | }() 114 | 115 | // Inject metadata to all metrics. 116 | wg.Add(1) 117 | go func() { 118 | defer wg.Done() 119 | if err := p.Metrics.InjectMetadata(conn, p.Metadata); err != nil { 120 | errChan <- fmt.Errorf("could not inject metadata for package %s: %w", p.Name, err) 121 | } 122 | }() 123 | 124 | // Wait for all concurrent operations to finish. 125 | wg.Wait() 126 | 127 | // Check if any errors occurred during initialization. 128 | select { 129 | case err := <-errChan: 130 | log.Error().Err(err).Interface("package", p.Name).Msg("could not initialize package") 131 | return err 132 | default: 133 | // No error. 134 | } 135 | 136 | log.Info().Interface("package", p.Name).Msg(string(p.Name) + " package initialized") 137 | return nil 138 | } 139 | 140 | type PackageConfig struct { 141 | Package string `json:"package"` 142 | Variables Variables `json:"variables"` 143 | MetricPrefix herculestypes.MetricPrefix `json:"metricPrefix"` 144 | } 145 | 146 | func (p *PackageConfig) getFromFile() (Package, error) { 147 | log.Debug().Interface("package", p.Package).Msg("loading " + p.Package + " package from file") 148 | pkg := Package{} 149 | yamlFile, err := os.ReadFile(p.Package) 150 | if err != nil { 151 | log.Error().Err(err).Msg("could not get package from file " + p.Package) 152 | } 153 | err = yaml.Unmarshal(yamlFile, &pkg) 154 | pkg.MetricPrefix = p.MetricPrefix 155 | return pkg, err 156 | } 157 | 158 | func (p *PackageConfig) getFromEndpoint() (Package, error) { 159 | log.Debug().Interface("package", p.Package).Msg("loading " + p.Package + " package from endpoint") 160 | pkg := Package{} 161 | 162 | // Create a context with timeout for the HTTP request. 163 | ctx, cancel := context.WithTimeout(context.Background(), DefaultHTTPTimeoutSeconds*time.Second) 164 | defer cancel() 165 | 166 | // Create a request with context. 167 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.Package, nil) 168 | if err != nil { 169 | return pkg, fmt.Errorf("failed to create request: %w", err) 170 | } 171 | 172 | // Execute the request. 173 | resp, err := http.DefaultClient.Do(req) 174 | if err != nil { 175 | log.Error().Err(err).Msg("could not get package from endpoint " + p.Package) 176 | return pkg, err 177 | } 178 | defer resp.Body.Close() 179 | 180 | body, err := io.ReadAll(resp.Body) 181 | if err != nil { 182 | log.Error().Err(err).Msg("could not read response body from endpoint " + p.Package) 183 | return pkg, err 184 | } 185 | 186 | err = yaml.Unmarshal(body, &pkg) 187 | return pkg, err 188 | } 189 | 190 | func (p *PackageConfig) getFromObjectStorage() error { 191 | err := errors.New("object storage-backed packages are not yet supported") 192 | log.Fatal().Interface("package", p.Package).Err(err).Msg("failed to get package") 193 | return err 194 | } 195 | 196 | func (p *PackageConfig) GetPackage() (Package, error) { 197 | var pkg Package 198 | var err error 199 | switch p.Package[0:4] { 200 | case "http": 201 | pkg, err = p.getFromEndpoint() 202 | case "s3:/", "gcs:": 203 | err = p.getFromObjectStorage() 204 | return Package{}, err 205 | default: 206 | pkg, err = p.getFromFile() 207 | } 208 | if err != nil { 209 | log.Debug().Stack().Err(err).Msg("could not load package from location " + p.Package) 210 | return Package{}, err 211 | } 212 | pkg.Variables = p.Variables 213 | return pkg, nil 214 | } 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Hercules 3 |
4 | Hercules 5 |
6 |

7 | 8 |

Write SQL. Get Prometheus Metrics.

9 | 10 |
11 | 12 | ![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/jakthom/hercules) 13 | [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) 14 | [![test](https://github.com/jakthom/hercules/actions/workflows/test.yml/badge.svg)](https://github.com/jakthom/hercules/actions/workflows/test.yml) 15 | [![lint](https://github.com/jakthom/hercules/actions/workflows/lint.yml/badge.svg)](https://github.com/jakthom/hercules/actions/workflows/lint.yml) 16 |
17 | 18 | 19 | 20 | # ⚡ Quickstart 21 | 22 | Launching Hercules in a Codespace is the easiest way to get started. 23 | 24 | A `make run` from inside your codespace will get things spinning. 25 | 26 | [![Launch GitHub Codespace](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=873715049) 27 | 28 | 29 | 30 | 31 | # Sources 32 | 33 | Hercules generates Prometheus metrics by querying: 34 | 35 | - **Local files** (parquet, json, csv, xlsx, etc) 36 | - **Object storage** (GCS, S3, Azure Blob) 37 | - **HTTP endpoints** 38 | - **Databases** (PostgreSQL, MySQL, SQLite) 39 | - **Data lakes** (Iceberg, Delta) 40 | - **Data warehouses** (BigQuery) 41 | - **Arrow IPC buffers** 42 | 43 | Sources can be cached and periodically refreshed, or act as views to the underlying data. 44 | 45 | Metrics from multiple sources can be materialized using a single exporter. 46 | 47 | 48 | # Metrics 49 | 50 | ### Definition 51 | 52 | Metric definitions are `yml` and use `sql` in a number of supported dialects to aggregate, enrich, and materialize metric values. 53 | 54 | 55 | Hercules supports Prometheus **gauges, counters, summaries, and histograms.** 56 | 57 | ### Enrichment 58 | 59 | Sources and metrics can be *externally enriched*, leading to more ***thorough***, ***accurate*** (or is it precise?), ***properly-labeled*** metrics. 60 | 61 | Integrate, calculate, enrich, and label on the edge. 62 | 63 | 64 | 65 | # Macros 66 | 67 | Metric definitions can be kept DRY using SQL macros. 68 | 69 | Macros are useful for: 70 | 71 | - Parsing log lines 72 | - Reading useragent strings 73 | - Schematizing unstructured data 74 | - Enriching results with third-party tooling 75 | - Tokenizing attributes 76 | 77 | 78 | # Labels 79 | 80 | Hercules propagates global labels to all configured metrics. So you don't have to guess where a metric came from. 81 | 82 | Labels are propagated from configuration or sourced from environment variables. 83 | 84 | 85 | # Packages 86 | 87 | Hercules extensions, sources, metrics, and macros can be logically grouped and distributed by the use of **packages**. 88 | 89 | Examples can be found in the [hercules-packages](/hercules-packages/) directory. 90 | 91 | 92 | # Getting Started Locally 93 | 94 | This guide will help you get Hercules up and running on your local machine, including instructions for configuration, testing, and loading sample data. 95 | 96 | ## Prerequisites 97 | 98 | - Go 1.22+ installed on your system 99 | - Git to clone the repository 100 | - Basic understanding of SQL and Prometheus metrics 101 | 102 | ## Installation 103 | 104 | 1. Clone the repository: 105 | ``` 106 | git clone https://github.com/jakthom/hercules.git 107 | cd hercules 108 | ``` 109 | 110 | 2. Build the project: 111 | ``` 112 | make build 113 | ``` 114 | 115 | ## Running Hercules 116 | 117 | ### Basic Execution 118 | 119 | Start Hercules with default settings: 120 | ``` 121 | make run 122 | ``` 123 | 124 | This command starts Hercules with the following defaults: 125 | - Configuration file: `hercules.yml` in the project root 126 | - Port: 9999 (customizable in config) 127 | - SQLite database: `h.db` in the project root 128 | 129 | ### Debug Mode 130 | 131 | To run Hercules with additional debug logging: 132 | ``` 133 | make debug 134 | ``` 135 | 136 | ### Custom Configuration 137 | 138 | Hercules uses a YAML configuration file. By default, it looks for `hercules.yml` in the project root. You can specify a custom configuration path using the environment variable: 139 | ``` 140 | HERCULES_CONFIG_PATH=/path/to/config.yml make run 141 | ``` 142 | 143 | ## Configuration File Structure 144 | 145 | The `hercules.yml` configuration file defines your server settings and packages: 146 | 147 | ```yaml 148 | version: 1 149 | 150 | name: your-instance-name 151 | debug: false 152 | port: 9100 153 | 154 | globalLabels: 155 | - env: $ENV # Inject prometheus labels from environment variables 156 | - region: us-west-1 157 | 158 | packages: 159 | - package: hercules-packages/sample-package/1.0.yml 160 | ``` 161 | 162 | ## Testing 163 | 164 | Run the test suite to ensure everything is working correctly: 165 | ``` 166 | make test 167 | ``` 168 | 169 | For more detailed test coverage information: 170 | ``` 171 | make test-cover-pkg 172 | ``` 173 | 174 | ## Working with Sample Data 175 | 176 | Hercules includes several example packages in the `/hercules-packages/` directory: 177 | 178 | 1. **NYC Taxi Data**: 179 | ```yaml 180 | packages: 181 | - package: hercules-packages/nyc-taxi/1.0.yml 182 | ``` 183 | 184 | 2. **Snowflake Metrics**: 185 | ```yaml 186 | packages: 187 | - package: hercules-packages/snowflake/1.0.yml 188 | ``` 189 | 190 | 3. **TPCH Benchmarks**: 191 | ```yaml 192 | packages: 193 | - package: hercules-packages/tpch/1.0.yml 194 | ``` 195 | 196 | 4. **Bluesky Analytics**: 197 | ```yaml 198 | packages: 199 | - package: hercules-packages/bluesky/1.0.yml 200 | ``` 201 | 202 | To use any of these packages, add them to your `hercules.yml` file. 203 | 204 | ## Accessing Metrics 205 | 206 | Once Hercules is running, metrics are available at: 207 | - http://localhost:9100/metrics (if using default port 9100) 208 | 209 | You can connect Prometheus or any compatible metrics collector to this endpoint. 210 | 211 | ## Environment Variables 212 | 213 | Hercules supports several environment variables: 214 | - `HERCULES_CONFIG_PATH`: Path to configuration file 215 | - `DEBUG`: Set to enable debug logging 216 | - `TRACE`: Set to enable trace logging 217 | - `ENV`: Used for labeling metrics (defaults to current username) 218 | 219 | ## Creating Custom Packages 220 | 221 | To create your own package: 222 | 223 | 1. Create a new YAML file in a directory of your choice: 224 | ```yaml 225 | name: my-package 226 | version: 1.0 227 | 228 | extensions: 229 | # SQLite extensions, if needed 230 | 231 | macros: 232 | # SQL macros 233 | 234 | sources: 235 | # Data sources 236 | 237 | metrics: 238 | # Metric definitions 239 | ``` 240 | 241 | 2. Reference your package in the main configuration: 242 | ```yaml 243 | packages: 244 | - package: path/to/your-package.yml 245 | ``` 246 | 247 | # Bonus 248 | 249 | - Calculate prometheus-compatible metrics from geospatial data 250 | - Coerce unwieldy files to useful statistics using full-text search 251 | - Use modern [pipe sql syntax](https://research.google/pubs/sql-has-problems-we-can-fix-them-pipe-syntax-in-sql/) or [prql](https://prql-lang.org/) for defining and transforming your metrics 252 | - You don't need to [start queries with `select`](https://jvns.ca/blog/2019/10/03/sql-queries-don-t-start-with-select/). 253 | 254 | 255 | # Further Resources 256 | 257 | More to come. 258 | -------------------------------------------------------------------------------- /pkg/metric/metric_test.go: -------------------------------------------------------------------------------- 1 | // Package metric_test contains tests for the metric package 2 | package metric_test 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/jakthom/hercules/pkg/metric" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestDefinition_LabelNames(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | metric metric.Definition 17 | expected []string 18 | }{ 19 | { 20 | name: "With no labels", 21 | metric: metric.Definition{ 22 | Name: "test_metric", 23 | Labels: []string{}, 24 | Metadata: metric.Metadata{Labels: map[string]string{}}, 25 | }, 26 | expected: []string{}, 27 | }, 28 | { 29 | name: "With metric labels only", 30 | metric: metric.Definition{ 31 | Name: "test_metric", 32 | Labels: []string{"label1", "label2"}, 33 | Metadata: metric.Metadata{Labels: map[string]string{}}, 34 | }, 35 | expected: []string{"label1", "label2"}, 36 | }, 37 | { 38 | name: "With metadata labels only", 39 | metric: metric.Definition{ 40 | Name: "test_metric", 41 | Labels: []string{}, 42 | Metadata: metric.Metadata{Labels: map[string]string{ 43 | "meta1": "value1", 44 | "meta2": "value2", 45 | }}, 46 | }, 47 | expected: []string{"meta1", "meta2"}, 48 | }, 49 | { 50 | name: "With both types of labels", 51 | metric: metric.Definition{ 52 | Name: "test_metric", 53 | Labels: []string{"label1", "label2"}, 54 | Metadata: metric.Metadata{Labels: map[string]string{ 55 | "meta1": "value1", 56 | "meta2": "value2", 57 | }}, 58 | }, 59 | expected: []string{"label1", "label2", "meta1", "meta2"}, 60 | }, 61 | } 62 | 63 | for _, tt := range tests { 64 | t.Run(tt.name, func(t *testing.T) { 65 | result := tt.metric.LabelNames() 66 | assert.ElementsMatch(t, tt.expected, result, "LabelNames() should return all labels") 67 | }) 68 | } 69 | } 70 | 71 | func TestDefinition_FullName(t *testing.T) { 72 | tests := []struct { 73 | name string 74 | metric metric.Definition 75 | expected string 76 | }{ 77 | { 78 | name: "No prefix", 79 | metric: metric.Definition{ 80 | Name: "metric_name", 81 | Metadata: metric.Metadata{ 82 | PackageName: "test", 83 | Prefix: "", 84 | }, 85 | }, 86 | expected: "test_metric_name", 87 | }, 88 | { 89 | name: "With prefix", 90 | metric: metric.Definition{ 91 | Name: "metric_name", 92 | Metadata: metric.Metadata{ 93 | PackageName: "test", 94 | Prefix: "prefix_", 95 | }, 96 | }, 97 | expected: "prefix_test_metric_name", 98 | }, 99 | { 100 | name: "Package name with hyphens", 101 | metric: metric.Definition{ 102 | Name: "metric_name", 103 | Metadata: metric.Metadata{ 104 | PackageName: "test-package", 105 | Prefix: "prefix_", 106 | }, 107 | }, 108 | expected: "prefix_test_package_metric_name", 109 | }, 110 | } 111 | 112 | for _, tt := range tests { 113 | t.Run(tt.name, func(t *testing.T) { 114 | result := tt.metric.FullName() 115 | assert.Equal(t, tt.expected, result, "FullName() should return the correct full metric name") 116 | }) 117 | } 118 | } 119 | 120 | func TestDefinition_InjectLabels(t *testing.T) { 121 | // Create a mock database and connection that uses query matching by regexp 122 | db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) 123 | require.NoError(t, err) 124 | defer db.Close() 125 | 126 | conn, err := db.Conn(t.Context()) 127 | require.NoError(t, err) 128 | defer conn.Close() 129 | 130 | // Use a pattern that matches any query containing json_serialize_sql 131 | mock.ExpectQuery("json_serialize_sql"). 132 | WillReturnRows(sqlmock.NewRows([]string{"column"}). 133 | AddRow("id"). 134 | AddRow("name"). 135 | AddRow("value")) 136 | 137 | // Create a metric definition for testing 138 | metricDef := &metric.Definition{ 139 | Name: "test_metric", 140 | SQL: "SELECT * FROM test_table", 141 | } 142 | 143 | // Call the method being tested 144 | err = metricDef.InjectLabels(conn) 145 | require.NoError(t, err) 146 | 147 | // Verify "value" column isn't included in labels (special column name) 148 | assert.ElementsMatch(t, []string{"id", "name"}, metricDef.Labels) 149 | 150 | // Verify all expectations were met 151 | assert.NoError(t, mock.ExpectationsWereMet()) 152 | } 153 | 154 | func TestDefinitions_Merge(t *testing.T) { 155 | base := metric.Definitions{ 156 | Gauge: []*metric.Definition{ 157 | {Name: "gauge1"}, 158 | }, 159 | Counter: []*metric.Definition{ 160 | {Name: "counter1"}, 161 | }, 162 | } 163 | 164 | toMerge := metric.Definitions{ 165 | Gauge: []*metric.Definition{ 166 | {Name: "gauge2"}, 167 | }, 168 | Summary: []*metric.Definition{ 169 | {Name: "summary1"}, 170 | }, 171 | Histogram: []*metric.Definition{ 172 | {Name: "histogram1"}, 173 | }, 174 | } 175 | 176 | base.Merge(toMerge) 177 | 178 | assert.Len(t, base.Gauge, 2, "Should have 2 gauge metrics") 179 | assert.Len(t, base.Counter, 1, "Should have 1 counter metric") 180 | assert.Len(t, base.Summary, 1, "Should have 1 summary metric") 181 | assert.Len(t, base.Histogram, 1, "Should have 1 histogram metric") 182 | 183 | assert.Equal(t, "gauge1", base.Gauge[0].Name) 184 | assert.Equal(t, "gauge2", base.Gauge[1].Name) 185 | } 186 | 187 | func TestDefinitions_InjectMetadata(t *testing.T) { 188 | // Create a mock database and connection that uses query matching by regexp 189 | db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) 190 | require.NoError(t, err) 191 | defer db.Close() 192 | 193 | conn, err := db.Conn(t.Context()) 194 | require.NoError(t, err) 195 | defer conn.Close() 196 | 197 | // Set up expectations for SQL parsing queries (using simplified regex patterns) 198 | mock.ExpectQuery("json_serialize_sql\\('SELECT 1'\\)"). 199 | WillReturnRows(sqlmock.NewRows([]string{"column"}).AddRow("val")) 200 | 201 | mock.ExpectQuery("json_serialize_sql\\('SELECT 2'\\)"). 202 | WillReturnRows(sqlmock.NewRows([]string{"column"}).AddRow("val")) 203 | 204 | mock.ExpectQuery("json_serialize_sql\\('SELECT 3'\\)"). 205 | WillReturnRows(sqlmock.NewRows([]string{"column"}).AddRow("val")) 206 | 207 | mock.ExpectQuery("json_serialize_sql\\('SELECT 4'\\)"). 208 | WillReturnRows(sqlmock.NewRows([]string{"column"}).AddRow("val")) 209 | 210 | metrics := metric.Definitions{ 211 | Gauge: []*metric.Definition{ 212 | {Name: "gauge1", SQL: "SELECT 1"}, 213 | }, 214 | Counter: []*metric.Definition{ 215 | {Name: "counter1", SQL: "SELECT 2"}, 216 | }, 217 | Summary: []*metric.Definition{ 218 | {Name: "summary1", SQL: "SELECT 3"}, 219 | }, 220 | Histogram: []*metric.Definition{ 221 | {Name: "histogram1", SQL: "SELECT 4"}, 222 | }, 223 | } 224 | 225 | metadata := metric.Metadata{ 226 | PackageName: "test-package", 227 | Prefix: "prefix_", 228 | Labels: map[string]string{ 229 | "env": "test", 230 | }, 231 | } 232 | 233 | // Update the call to handle error return 234 | err = metrics.InjectMetadata(conn, metadata) 235 | require.NoError(t, err) 236 | 237 | // Check that metadata was injected into all metrics 238 | assert.Equal(t, metadata, metrics.Gauge[0].Metadata) 239 | assert.Equal(t, metadata, metrics.Counter[0].Metadata) 240 | assert.Equal(t, metadata, metrics.Summary[0].Metadata) 241 | assert.Equal(t, metadata, metrics.Histogram[0].Metadata) 242 | 243 | // Verify all expectations were met 244 | assert.NoError(t, mock.ExpectationsWereMet()) 245 | } 246 | -------------------------------------------------------------------------------- /schemas/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft-07/schema#", 3 | "title": "Hercules Package", 4 | "description": "Configuration for Hercules Packages", 5 | "type": "object", 6 | "properties": { 7 | "name": { 8 | "type": "string", 9 | "description": "Name of the package" 10 | }, 11 | "version": { 12 | "type": "string", 13 | "description": "Version of the package" 14 | }, 15 | "macros": { 16 | "type": "array", 17 | "description": "Package macros", 18 | "items": { 19 | "type": "object", 20 | "properties": { 21 | "sql": { 22 | "type": "string", 23 | "description": "SQL query for the macro" 24 | } 25 | }, 26 | "required": [ 27 | "sql" 28 | ] 29 | } 30 | }, 31 | "sources": { 32 | "type": "array", 33 | "description": "Package sources", 34 | "items": { 35 | "type": "object", 36 | "properties": { 37 | "name": { 38 | "type": "string", 39 | "description": "Name of the source" 40 | }, 41 | "type": { 42 | "type": "string", 43 | "description": "Type of the source (e.g. parquet)", 44 | "options": [ 45 | "sql", 46 | "parquet", 47 | "json", 48 | "csv" 49 | ] 50 | }, 51 | "source": { 52 | "type": "string", 53 | "description": "The source definition - file path or sql statement" 54 | }, 55 | "materialize": { 56 | "type": "boolean", 57 | "description": "Whether to materialize the source", 58 | "default": true 59 | }, 60 | "refreshIntervalSeconds": { 61 | "type": "integer", 62 | "description": "Refresh interval in seconds", 63 | "default": 5 64 | } 65 | }, 66 | "required": [ 67 | "name", 68 | "type", 69 | "source", 70 | "materialize" 71 | ] 72 | } 73 | }, 74 | "metrics": { 75 | "type": "object", 76 | "properties": { 77 | "gauge": { 78 | "type": "array", 79 | "items": { 80 | "type": "object", 81 | "properties": { 82 | "name": { 83 | "type": "string", 84 | "description": "Name of the gauge metric" 85 | }, 86 | "help": { 87 | "type": "string", 88 | "description": "Help text for the gauge metric" 89 | }, 90 | "enabled": { 91 | "type": "boolean", 92 | "description": "Whether the gauge metric is enabled" 93 | }, 94 | "sql": { 95 | "type": "string", 96 | "description": "SQL query for the gauge metric" 97 | }, 98 | "labels": { 99 | "type": "array", 100 | "items": { 101 | "type": "string" 102 | } 103 | } 104 | }, 105 | "required": [ 106 | "name", 107 | "help", 108 | "enabled", 109 | "sql" 110 | ] 111 | } 112 | }, 113 | "histogram": { 114 | "type": "array", 115 | "items": { 116 | "type": "object", 117 | "properties": { 118 | "name": { 119 | "type": "string", 120 | "description": "Name of the histogram metric" 121 | }, 122 | "help": { 123 | "type": "string", 124 | "description": "Help text for the histogram metric" 125 | }, 126 | "sql": { 127 | "type": "string", 128 | "description": "SQL query for the histogram metric" 129 | }, 130 | "labels": { 131 | "type": "array", 132 | "items": { 133 | "type": "string" 134 | } 135 | }, 136 | "buckets": { 137 | "type": "array", 138 | "items": { 139 | "type": "number" 140 | } 141 | } 142 | }, 143 | "required": [ 144 | "name", 145 | "help", 146 | "sql", 147 | "buckets" 148 | ] 149 | } 150 | }, 151 | "summary": { 152 | "type": "array", 153 | "items": { 154 | "type": "object", 155 | "properties": { 156 | "name": { 157 | "type": "string", 158 | "description": "Name of the summary metric" 159 | }, 160 | "help": { 161 | "type": "string", 162 | "description": "Help text for the summary metric" 163 | }, 164 | "sql": { 165 | "type": "string", 166 | "description": "SQL query for the summary metric" 167 | }, 168 | "labels": { 169 | "type": "array", 170 | "items": { 171 | "type": "string" 172 | } 173 | }, 174 | "objectives": { 175 | "type": "array", 176 | "items": { 177 | "type": "number" 178 | } 179 | } 180 | }, 181 | "required": [ 182 | "name", 183 | "help", 184 | "sql", 185 | "objectives" 186 | ] 187 | } 188 | }, 189 | "counter": { 190 | "type": "array", 191 | "items": { 192 | "type": "object", 193 | "properties": { 194 | "name": { 195 | "type": "string", 196 | "description": "Name of the counter metric" 197 | }, 198 | "help": { 199 | "type": "string", 200 | "description": "Help text for the counter metric" 201 | }, 202 | "enabled": { 203 | "type": "boolean", 204 | "description": "Whether the counter metric is enabled" 205 | }, 206 | "sql": { 207 | "type": "string", 208 | "description": "SQL query for the counter metric" 209 | }, 210 | "labels": { 211 | "type": "array", 212 | "items": { 213 | "type": "string" 214 | } 215 | } 216 | }, 217 | "required": [ 218 | "name", 219 | "help", 220 | "enabled", 221 | "sql" 222 | ] 223 | } 224 | } 225 | } 226 | } 227 | }, 228 | "required": [ 229 | "name", 230 | "version", 231 | "sources", 232 | "metrics" 233 | ] 234 | } -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= 2 | github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= 3 | github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= 4 | github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= 5 | github.com/apache/arrow-go/v18 v18.2.0 h1:QhWqpgZMKfWOniGPhbUxrHohWnooGURqL2R2Gg4SO1Q= 6 | github.com/apache/arrow-go/v18 v18.2.0/go.mod h1:Ic/01WSwGJWRrdAZcxjBZ5hbApNJ28K96jGYaxzzGUc= 7 | github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= 8 | github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 12 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 13 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 14 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 15 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/duckdb/duckdb-go-bindings v0.1.14 h1:57DCZuuKQ65gRQxFG+XGnqVQtMADKY/noozmCjYs+zE= 17 | github.com/duckdb/duckdb-go-bindings v0.1.14/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= 18 | github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9 h1:K95YlR28Fb3+n3D6RcBzdznNVGcCnrGaAZqs52JUFOs= 19 | github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.9/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= 20 | github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9 h1:wY3kXm1/GSK4ES8pfBIeRHxscZomEVFWTS4GOifrZCs= 21 | github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.9/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= 22 | github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9 h1:ypZyeNMA9oRAIBE/pVGfrsXzYqEM+ZRkbV/lxw7Cf5E= 23 | github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.9/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= 24 | github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9 h1:TVBDwDSanIttQCH76UpDJ9rQAq4cYNM4R7h5Xu0y/rA= 25 | github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.9/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= 26 | github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9 h1:okFoG+evMiXnyUK+cI67V0MpvKbstO6MaXlXXotst3k= 27 | github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.9/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= 28 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 29 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 30 | github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= 31 | github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 32 | github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= 33 | github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= 34 | github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= 35 | github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 36 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 37 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 38 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 39 | github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= 40 | github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= 41 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 42 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 43 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 44 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 45 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 46 | github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= 47 | github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= 48 | github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= 49 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 50 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 51 | github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= 52 | github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 53 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 54 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 55 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 56 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 57 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 58 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 59 | github.com/marcboeker/go-duckdb/arrowmapping v0.0.7 h1:6mq16sPGJPo8Tkkl6UIsXuaNv467LjHLBscRyJl2Qhc= 60 | github.com/marcboeker/go-duckdb/arrowmapping v0.0.7/go.mod h1:FdvmqJOwVdfFZLpV+anBFlTUOzfU/NdIRET37mIEczY= 61 | github.com/marcboeker/go-duckdb/mapping v0.0.7 h1:t0BaNmLXj76RKs/x80A/ZTe+KzZDimO2Ji8ct4YnPu4= 62 | github.com/marcboeker/go-duckdb/mapping v0.0.7/go.mod h1:EH3RSabeePOUePoYDtF0LqfruXPtVB3M+g03QydZsck= 63 | github.com/marcboeker/go-duckdb/v2 v2.2.0 h1:xxruuYD7vWvybY52xWzV0vvHKa1IjpDDOq6T846ax/s= 64 | github.com/marcboeker/go-duckdb/v2 v2.2.0/go.mod h1:B7swJ38GcOEm9PI0IdfkZYqn5CtIjRUiQG4ZBr3hnyc= 65 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 66 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 67 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 68 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 69 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 70 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 71 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 72 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= 73 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= 74 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= 75 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= 76 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 77 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 78 | github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= 79 | github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= 80 | github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= 81 | github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 82 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 83 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 84 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 85 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 86 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 87 | github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= 88 | github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= 89 | github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= 90 | github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= 91 | github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 92 | github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 93 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 94 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 95 | github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 96 | github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= 97 | github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= 98 | github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= 99 | github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= 100 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 101 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 102 | github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= 103 | github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= 104 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 105 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 106 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 107 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 108 | github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= 109 | github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= 110 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 111 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 112 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 113 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 114 | github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= 115 | github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= 116 | github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= 117 | github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= 118 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 119 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 120 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= 121 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= 122 | golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= 123 | golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= 124 | golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= 125 | golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 126 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 127 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 128 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 129 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 130 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 131 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 132 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 133 | golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= 134 | golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= 135 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= 136 | golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= 137 | gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= 138 | gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= 139 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 140 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 141 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 142 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 143 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 144 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 145 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 146 | sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= 147 | sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= 148 | -------------------------------------------------------------------------------- /.github/golangci.yml: -------------------------------------------------------------------------------- 1 | # This file is licensed under the terms of the MIT license https://opensource.org/license/mit 2 | # Copyright (c) 2021-2025 Marat Reymers 3 | 4 | ## Golden config for golangci-lint v2.1.5 5 | # 6 | # This is the best config for golangci-lint based on my experience and opinion. 7 | # It is very strict, but not extremely strict. 8 | # Feel free to adapt it to suit your needs. 9 | # If this config helps you, please consider keeping a link to this file (see the next comment). 10 | 11 | # Based on https://gist.github.com/maratori/47a4d00457a92aa426dbd48a18776322 12 | 13 | version: "2" 14 | 15 | issues: 16 | # Maximum count of issues with the same text. 17 | # Set to 0 to disable. 18 | # Default: 3 19 | max-same-issues: 50 20 | 21 | formatters: 22 | enable: 23 | - goimports # checks if the code and import statements are formatted according to the 'goimports' command 24 | - golines # checks if code is formatted, and fixes long lines 25 | 26 | ## you may want to enable 27 | #- gci # checks if code and import statements are formatted, with additional rules 28 | #- gofmt # checks if the code is formatted according to 'gofmt' command 29 | 30 | ## disabled 31 | #- gofumpt # [replaced by goimports, gofumports is not available yet] checks if code and import statements are formatted, with additional rules 32 | 33 | # All settings can be found here https://github.com/golangci/golangci-lint/blob/HEAD/.golangci.reference.yml 34 | settings: 35 | goimports: 36 | # A list of prefixes, which, if set, checks import paths 37 | # with the given prefixes are grouped after 3rd-party packages. 38 | # Default: [] 39 | local-prefixes: 40 | - github.com/my/project 41 | 42 | golines: 43 | # Target maximum line length. 44 | # Default: 100 45 | max-len: 120 46 | 47 | linters: 48 | enable: 49 | - asasalint # checks for pass []any as any in variadic func(...any) 50 | - asciicheck # checks that your code does not contain non-ASCII identifiers 51 | - bidichk # checks for dangerous unicode character sequences 52 | - bodyclose # checks whether HTTP response body is closed successfully 53 | - canonicalheader # checks whether net/http.Header uses canonical header 54 | - copyloopvar # detects places where loop variables are copied (Go 1.22+) 55 | - cyclop # checks function and package cyclomatic complexity 56 | - depguard # checks if package imports are in a list of acceptable packages 57 | - dupl # tool for code clone detection 58 | - durationcheck # checks for two durations multiplied together 59 | - errcheck # checking for unchecked errors, these unchecked errors can be critical bugs in some cases 60 | - errname # checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error 61 | - errorlint # finds code that will cause problems with the error wrapping scheme introduced in Go 1.13 62 | - exhaustive # checks exhaustiveness of enum switch statements 63 | - exptostd # detects functions from golang.org/x/exp/ that can be replaced by std functions 64 | - fatcontext # detects nested contexts in loops 65 | - forbidigo # forbids identifiers 66 | - funcorder # checks the order of functions, methods, and constructors 67 | - funlen # tool for detection of long functions 68 | - gocheckcompilerdirectives # validates go compiler directive comments (//go:) 69 | - gochecknoglobals # checks that no global variables exist 70 | - gochecknoinits # checks that no init functions are present in Go code 71 | - gochecksumtype # checks exhaustiveness on Go "sum types" 72 | - gocognit # computes and checks the cognitive complexity of functions 73 | - goconst # finds repeated strings that could be replaced by a constant 74 | - gocritic # provides diagnostics that check for bugs, performance and style issues 75 | - gocyclo # computes and checks the cyclomatic complexity of functions 76 | - godot # checks if comments end in a period 77 | - gomoddirectives # manages the use of 'replace', 'retract', and 'excludes' directives in go.mod 78 | - goprintffuncname # checks that printf-like functions are named with f at the end 79 | - gosec # inspects source code for security problems 80 | - govet # reports suspicious constructs, such as Printf calls whose arguments do not align with the format string 81 | - iface # checks the incorrect use of interfaces, helping developers avoid interface pollution 82 | - ineffassign # detects when assignments to existing variables are not used 83 | - intrange # finds places where for loops could make use of an integer range 84 | - loggercheck # checks key value pairs for common logger libraries (kitlog,klog,logr,zap) 85 | - makezero # finds slice declarations with non-zero initial length 86 | - mirror # reports wrong mirror patterns of bytes/strings usage 87 | - mnd # detects magic numbers 88 | - musttag # enforces field tags in (un)marshaled structs 89 | - nakedret # finds naked returns in functions greater than a specified function length 90 | - nestif # reports deeply nested if statements 91 | - nilerr # finds the code that returns nil even if it checks that the error is not nil 92 | - nilnesserr # reports that it checks for err != nil, but it returns a different nil value error (powered by nilness and nilerr) 93 | - nilnil # checks that there is no simultaneous return of nil error and an invalid value 94 | - noctx # finds sending http request without context.Context 95 | - nolintlint # reports ill-formed or insufficient nolint directives 96 | - nonamedreturns # reports all named returns 97 | - nosprintfhostport # checks for misuse of Sprintf to construct a host with port in a URL 98 | - perfsprint # checks that fmt.Sprintf can be replaced with a faster alternative 99 | - predeclared # finds code that shadows one of Go's predeclared identifiers 100 | - promlinter # checks Prometheus metrics naming via promlint 101 | - protogetter # reports direct reads from proto message fields when getters should be used 102 | - reassign # checks that package variables are not reassigned 103 | - recvcheck # checks for receiver type consistency 104 | - revive # fast, configurable, extensible, flexible, and beautiful linter for Go, drop-in replacement of golint 105 | - rowserrcheck # checks whether Err of rows is checked successfully 106 | - sloglint # ensure consistent code style when using log/slog 107 | - spancheck # checks for mistakes with OpenTelemetry/Census spans 108 | - sqlclosecheck # checks that sql.Rows and sql.Stmt are closed 109 | - staticcheck # is a go vet on steroids, applying a ton of static analysis checks 110 | - testableexamples # checks if examples are testable (have an expected output) 111 | - testifylint # checks usage of github.com/stretchr/testify 112 | - testpackage # makes you use a separate _test package 113 | - tparallel # detects inappropriate usage of t.Parallel() method in your Go test codes 114 | - unconvert # removes unnecessary type conversions 115 | - unparam # reports unused function parameters 116 | - unused # checks for unused constants, variables, functions and types 117 | - usestdlibvars # detects the possibility to use variables/constants from the Go standard library 118 | - usetesting # reports uses of functions with replacement inside the testing package 119 | - wastedassign # finds wasted assignment statements 120 | - whitespace # detects leading and trailing whitespace 121 | 122 | ## you may want to enable 123 | #- decorder # checks declaration order and count of types, constants, variables and functions 124 | #- exhaustruct # [highly recommend to enable] checks if all structure fields are initialized 125 | #- ginkgolinter # [if you use ginkgo/gomega] enforces standards of using ginkgo and gomega 126 | #- godox # detects usage of FIXME, TODO and other keywords inside comments 127 | #- goheader # checks is file header matches to pattern 128 | #- inamedparam # [great idea, but too strict, need to ignore a lot of cases by default] reports interfaces with unnamed method parameters 129 | #- interfacebloat # checks the number of methods inside an interface 130 | #- ireturn # accept interfaces, return concrete types 131 | #- prealloc # [premature optimization, but can be used in some cases] finds slice declarations that could potentially be preallocated 132 | #- tagalign # checks that struct tags are well aligned 133 | #- varnamelen # [great idea, but too many false positives] checks that the length of a variable's name matches its scope 134 | #- wrapcheck # checks that errors returned from external packages are wrapped 135 | #- zerologlint # detects the wrong usage of zerolog that a user forgets to dispatch zerolog.Event 136 | 137 | ## disabled 138 | #- containedctx # detects struct contained context.Context field 139 | #- contextcheck # [too many false positives] checks the function whether use a non-inherited context 140 | #- dogsled # checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) 141 | #- dupword # [useless without config] checks for duplicate words in the source code 142 | #- err113 # [too strict] checks the errors handling expressions 143 | #- errchkjson # [don't see profit + I'm against of omitting errors like in the first example https://github.com/breml/errchkjson] checks types passed to the json encoding functions. Reports unsupported types and optionally reports occasions, where the check for the returned error can be omitted 144 | #- forcetypeassert # [replaced by errcheck] finds forced type assertions 145 | #- gomodguard # [use more powerful depguard] allow and block lists linter for direct Go module dependencies 146 | #- gosmopolitan # reports certain i18n/l10n anti-patterns in your Go codebase 147 | #- grouper # analyzes expression groups 148 | #- importas # enforces consistent import aliases 149 | #- lll # [replaced by golines] reports long lines 150 | #- maintidx # measures the maintainability index of each function 151 | #- misspell # [useless] finds commonly misspelled English words in comments 152 | #- nlreturn # [too strict and mostly code is not more readable] checks for a new line before return and branch statements to increase code clarity 153 | #- paralleltest # [too many false positives] detects missing usage of t.Parallel() method in your Go test 154 | #- tagliatelle # checks the struct tags 155 | #- thelper # detects golang test helpers without t.Helper() call and checks the consistency of test helpers 156 | #- wsl # [too strict and mostly code is not more readable] whitespace linter forces you to use empty lines 157 | 158 | # All settings can be found here https://github.com/golangci/golangci-lint/blob/HEAD/.golangci.reference.yml 159 | settings: 160 | cyclop: 161 | # The maximal code complexity to report. 162 | # Default: 10 163 | max-complexity: 30 164 | # The maximal average package complexity. 165 | # If it's higher than 0.0 (float) the check is enabled. 166 | # Default: 0.0 167 | package-average: 10.0 168 | 169 | depguard: 170 | # Rules to apply. 171 | # 172 | # Variables: 173 | # - File Variables 174 | # Use an exclamation mark `!` to negate a variable. 175 | # Example: `!$test` matches any file that is not a go test file. 176 | # 177 | # `$all` - matches all go files 178 | # `$test` - matches all go test files 179 | # 180 | # - Package Variables 181 | # 182 | # `$gostd` - matches all of go's standard library (Pulled from `GOROOT`) 183 | # 184 | # Default (applies if no custom rules are defined): Only allow $gostd in all files. 185 | rules: 186 | "deprecated": 187 | # List of file globs that will match this list of settings to compare against. 188 | # By default, if a path is relative, it is relative to the directory where the golangci-lint command is executed. 189 | # The placeholder '${base-path}' is substituted with a path relative to the mode defined with `run.relative-path-mode`. 190 | # The placeholder '${config-path}' is substituted with a path relative to the configuration file. 191 | # Default: $all 192 | files: 193 | - "$all" 194 | # List of packages that are not allowed. 195 | # Entries can be a variable (starting with $), a string prefix, or an exact match (if ending with $). 196 | # Default: [] 197 | deny: 198 | - pkg: github.com/golang/protobuf 199 | desc: Use google.golang.org/protobuf instead, see https://developers.google.com/protocol-buffers/docs/reference/go/faq#modules 200 | - pkg: github.com/satori/go.uuid 201 | desc: Use github.com/google/uuid instead, satori's package is not maintained 202 | - pkg: github.com/gofrs/uuid$ 203 | desc: Use github.com/gofrs/uuid/v5 or later, it was not a go module before v5 204 | "non-test files": 205 | files: 206 | - "!$test" 207 | deny: 208 | - pkg: math/rand$ 209 | desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2 210 | "non-main files": 211 | files: 212 | - "!**/main.go" 213 | deny: 214 | - pkg: log$ 215 | desc: Use log/slog instead, see https://go.dev/blog/slog 216 | 217 | errcheck: 218 | # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. 219 | # Such cases aren't reported by default. 220 | # Default: false 221 | check-type-assertions: true 222 | 223 | exhaustive: 224 | # Program elements to check for exhaustiveness. 225 | # Default: [ switch ] 226 | check: 227 | - switch 228 | - map 229 | 230 | exhaustruct: 231 | # List of regular expressions to exclude struct packages and their names from checks. 232 | # Regular expressions must match complete canonical struct package/name/structname. 233 | # Default: [] 234 | exclude: 235 | # std libs 236 | - ^net/http.Client$ 237 | - ^net/http.Cookie$ 238 | - ^net/http.Request$ 239 | - ^net/http.Response$ 240 | - ^net/http.Server$ 241 | - ^net/http.Transport$ 242 | - ^net/url.URL$ 243 | - ^os/exec.Cmd$ 244 | - ^reflect.StructField$ 245 | # public libs 246 | - ^github.com/Shopify/sarama.Config$ 247 | - ^github.com/Shopify/sarama.ProducerMessage$ 248 | - ^github.com/mitchellh/mapstructure.DecoderConfig$ 249 | - ^github.com/prometheus/client_golang/.+Opts$ 250 | - ^github.com/spf13/cobra.Command$ 251 | - ^github.com/spf13/cobra.CompletionOptions$ 252 | - ^github.com/stretchr/testify/mock.Mock$ 253 | - ^github.com/testcontainers/testcontainers-go.+Request$ 254 | - ^github.com/testcontainers/testcontainers-go.FromDockerfile$ 255 | - ^golang.org/x/tools/go/analysis.Analyzer$ 256 | - ^google.golang.org/protobuf/.+Options$ 257 | - ^gopkg.in/yaml.v3.Node$ 258 | 259 | funcorder: 260 | # Checks if the exported methods of a structure are placed before the non-exported ones. 261 | # Default: true 262 | struct-method: false 263 | 264 | funlen: 265 | # Checks the number of lines in a function. 266 | # If lower than 0, disable the check. 267 | # Default: 60 268 | lines: 100 269 | # Checks the number of statements in a function. 270 | # If lower than 0, disable the check. 271 | # Default: 40 272 | statements: 50 273 | 274 | gochecksumtype: 275 | # Presence of `default` case in switch statements satisfies exhaustiveness, if all members are not listed. 276 | # Default: true 277 | default-signifies-exhaustive: false 278 | 279 | gocognit: 280 | # Minimal code complexity to report. 281 | # Default: 30 (but we recommend 10-20) 282 | min-complexity: 20 283 | 284 | gocritic: 285 | # Settings passed to gocritic. 286 | # The settings key is the name of a supported gocritic checker. 287 | # The list of supported checkers can be found at https://go-critic.com/overview. 288 | settings: 289 | captLocal: 290 | # Whether to restrict checker to params only. 291 | # Default: true 292 | paramsOnly: false 293 | underef: 294 | # Whether to skip (*x).method() calls where x is a pointer receiver. 295 | # Default: true 296 | skipRecvDeref: false 297 | 298 | govet: 299 | # Enable all analyzers. 300 | # Default: false 301 | enable-all: true 302 | # Disable analyzers by name. 303 | # Run `GL_DEBUG=govet golangci-lint run --enable=govet` to see default, all available analyzers, and enabled analyzers. 304 | # Default: [] 305 | disable: 306 | - fieldalignment # too strict 307 | # Settings per analyzer. 308 | settings: 309 | shadow: 310 | # Whether to be strict about shadowing; can be noisy. 311 | # Default: false 312 | strict: true 313 | 314 | inamedparam: 315 | # Skips check for interface methods with only a single parameter. 316 | # Default: false 317 | skip-single-param: true 318 | 319 | mnd: 320 | # List of function patterns to exclude from analysis. 321 | # Values always ignored: `time.Date`, 322 | # `strconv.FormatInt`, `strconv.FormatUint`, `strconv.FormatFloat`, 323 | # `strconv.ParseInt`, `strconv.ParseUint`, `strconv.ParseFloat`. 324 | # Default: [] 325 | ignored-functions: 326 | - args.Error 327 | - flag.Arg 328 | - flag.Duration.* 329 | - flag.Float.* 330 | - flag.Int.* 331 | - flag.Uint.* 332 | - os.Chmod 333 | - os.Mkdir.* 334 | - os.OpenFile 335 | - os.WriteFile 336 | - prometheus.ExponentialBuckets.* 337 | - prometheus.LinearBuckets 338 | 339 | nakedret: 340 | # Make an issue if func has more lines of code than this setting, and it has naked returns. 341 | # Default: 30 342 | max-func-lines: 0 343 | 344 | nolintlint: 345 | # Exclude following linters from requiring an explanation. 346 | # Default: [] 347 | allow-no-explanation: [ funlen, gocognit, golines ] 348 | # Enable to require an explanation of nonzero length after each nolint directive. 349 | # Default: false 350 | require-explanation: true 351 | # Enable to require nolint directives to mention the specific linter being suppressed. 352 | # Default: false 353 | require-specific: true 354 | 355 | perfsprint: 356 | # Optimizes into strings concatenation. 357 | # Default: true 358 | strconcat: false 359 | 360 | reassign: 361 | # Patterns for global variable names that are checked for reassignment. 362 | # See https://github.com/curioswitch/go-reassign#usage 363 | # Default: ["EOF", "Err.*"] 364 | patterns: 365 | - ".*" 366 | 367 | rowserrcheck: 368 | # database/sql is always checked. 369 | # Default: [] 370 | packages: 371 | - github.com/jmoiron/sqlx 372 | 373 | sloglint: 374 | # Enforce not using global loggers. 375 | # Values: 376 | # - "": disabled 377 | # - "all": report all global loggers 378 | # - "default": report only the default slog logger 379 | # https://github.com/go-simpler/sloglint?tab=readme-ov-file#no-global 380 | # Default: "" 381 | no-global: all 382 | # Enforce using methods that accept a context. 383 | # Values: 384 | # - "": disabled 385 | # - "all": report all contextless calls 386 | # - "scope": report only if a context exists in the scope of the outermost function 387 | # https://github.com/go-simpler/sloglint?tab=readme-ov-file#context-only 388 | # Default: "" 389 | context: scope 390 | 391 | staticcheck: 392 | # SAxxxx checks in https://staticcheck.dev/docs/configuration/options/#checks 393 | # Example (to disable some checks): [ "all", "-SA1000", "-SA1001"] 394 | # Default: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022"] 395 | checks: 396 | - all 397 | # Incorrect or missing package comment. 398 | # https://staticcheck.dev/docs/checks/#ST1000 399 | - -ST1000 400 | # Use consistent method receiver names. 401 | # https://staticcheck.dev/docs/checks/#ST1016 402 | - -ST1016 403 | # Omit embedded fields from selector expression. 404 | # https://staticcheck.dev/docs/checks/#QF1008 405 | - -QF1008 406 | 407 | usetesting: 408 | # Enable/disable `os.TempDir()` detections. 409 | # Default: false 410 | os-temp-dir: true 411 | 412 | exclusions: 413 | # Log a warning if an exclusion rule is unused. 414 | # Default: false 415 | warn-unused: true 416 | # Predefined exclusion rules. 417 | # Default: [] 418 | presets: 419 | - std-error-handling 420 | - common-false-positives 421 | # Excluding configuration per-path, per-linter, per-text and per-source. 422 | rules: 423 | - source: 'TODO' 424 | linters: [ godot ] 425 | - text: 'should have a package comment' 426 | linters: [ revive ] 427 | - text: 'exported \S+ \S+ should have comment( \(or a comment on this block\))? or be unexported' 428 | linters: [ revive ] 429 | - text: 'package comment should be of the form ".+"' 430 | source: '// ?(nolint|TODO)' 431 | linters: [ revive ] 432 | - text: 'comment on exported \S+ \S+ should be of the form ".+"' 433 | source: '// ?(nolint|TODO)' 434 | linters: [ revive, staticcheck ] 435 | - path: '_test\.go' 436 | linters: 437 | - bodyclose 438 | - dupl 439 | - errcheck 440 | - funlen 441 | - goconst 442 | - gosec 443 | - noctx 444 | - wrapcheck --------------------------------------------------------------------------------