5 |
6 | {{ fieldName . }}
7 |
8 | {{ if linkForType .Type }}
9 |
10 | {{ typeDisplayName .Type }}
11 |
12 | {{ else }}
13 | {{ typeDisplayName .Type }}
14 | {{ end }}
15 |
16 | |
17 |
18 | {{ if fieldEmbedded . }}
19 |
20 | (Members of {{ fieldName . }} are embedded into this type.)
21 |
22 | {{ end}}
23 |
24 | {{ if isOptionalMember .}}
25 | (Optional)
26 | {{ end }}
27 |
28 | {{ safe (renderComments .CommentLines) }}
29 |
30 | {{ if and (eq (.Type.Name.Name) "ObjectMeta") }}
31 | Refer to the Kubernetes API documentation for the fields of the
32 | metadata field.
33 | {{ end }}
34 |
35 | {{ if or (eq (fieldName .) "spec") }}
36 |
37 |
38 |
39 | {{ template "members" .Type }}
40 |
41 | {{ end }}
42 | |
43 |
44 | {{ end }}
45 | {{ end }}
46 | {{ end }}
47 |
--------------------------------------------------------------------------------
/hack/api-docs/template/pkg.tpl:
--------------------------------------------------------------------------------
1 | {{ define "packages" }}
2 | Druid API reference
3 |
4 | {{ with .packages}}
5 | Packages:
6 |
13 | {{ end}}
14 |
15 | {{ range .packages }}
16 |
17 | {{- packageDisplayName . -}}
18 |
19 |
20 | {{ with (index .GoPackages 0 )}}
21 | {{ with .DocComments }}
22 | {{ safe (renderComments .) }}
23 | {{ end }}
24 | {{ end }}
25 |
26 | Resource Types:
27 |
28 |
29 | {{- range (visibleTypes (sortedTypes .Types)) -}}
30 | {{ if isExportedType . -}}
31 | -
32 | {{ typeDisplayName . }}
33 |
34 | {{- end }}
35 | {{- end -}}
36 |
37 |
38 | {{ range (visibleTypes (sortedTypes .Types))}}
39 | {{ template "type" . }}
40 | {{ end }}
41 | {{ end }}
42 |
43 |
44 |
This page was automatically generated with gen-crd-api-reference-docs
45 |
3 | {{- .Name.Name }}
4 | {{ if eq .Kind "Alias" }}({{.Underlying}} alias){{ end -}}
5 |
6 |
7 | {{ with (typeReferences .) }}
8 |
9 | (Appears on:
10 | {{- $prev := "" -}}
11 | {{- range . -}}
12 | {{- if $prev -}}, {{ end -}}
13 | {{ $prev = . }}
14 | {{ typeDisplayName . }}
15 | {{- end -}}
16 | )
17 |
18 | {{ end }}
19 |
20 | {{ with .CommentLines }}
21 | {{ safe (renderComments .) }}
22 | {{ end }}
23 |
24 | {{ if .Members }}
25 |
59 | {{ end }}
60 | {{ end }}
61 |
--------------------------------------------------------------------------------
/hack/boilerplate.go.txt:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2023.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2023.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package main
18 |
19 | import (
20 | "flag"
21 | "os"
22 | "strings"
23 |
24 | "github.com/datainfrahq/druid-operator/controllers/druid"
25 | // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
26 | // to ensure that exec-entrypoint and run can make use of them.
27 | _ "k8s.io/client-go/plugin/pkg/client/auth"
28 | "sigs.k8s.io/controller-runtime/pkg/cache"
29 |
30 | "k8s.io/apimachinery/pkg/runtime"
31 | utilruntime "k8s.io/apimachinery/pkg/util/runtime"
32 | clientgoscheme "k8s.io/client-go/kubernetes/scheme"
33 | ctrl "sigs.k8s.io/controller-runtime"
34 | "sigs.k8s.io/controller-runtime/pkg/healthz"
35 | "sigs.k8s.io/controller-runtime/pkg/log/zap"
36 |
37 | druidv1alpha1 "github.com/datainfrahq/druid-operator/apis/druid/v1alpha1"
38 | druidingestioncontrollers "github.com/datainfrahq/druid-operator/controllers/ingestion"
39 | //+kubebuilder:scaffold:imports
40 | )
41 |
42 | var (
43 | scheme = runtime.NewScheme()
44 | setupLog = ctrl.Log.WithName("setup")
45 | watchNamespace = os.Getenv("WATCH_NAMESPACE")
46 | )
47 |
48 | func init() {
49 | utilruntime.Must(clientgoscheme.AddToScheme(scheme))
50 |
51 | utilruntime.Must(druidv1alpha1.AddToScheme(scheme))
52 | //+kubebuilder:scaffold:scheme
53 | }
54 |
55 | func main() {
56 | var metricsAddr string
57 | var enableLeaderElection bool
58 | var probeAddr string
59 | flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
60 | flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
61 | flag.BoolVar(&enableLeaderElection, "leader-elect", false,
62 | "Enable leader election for controller manager. "+
63 | "Enabling this will ensure there is only one active controller manager.")
64 | opts := zap.Options{
65 | Development: true,
66 | }
67 | opts.BindFlags(flag.CommandLine)
68 | flag.Parse()
69 |
70 | ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
71 |
72 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
73 | Scheme: scheme,
74 | MetricsBindAddress: metricsAddr,
75 | Port: 9443,
76 | HealthProbeBindAddress: probeAddr,
77 | LeaderElection: enableLeaderElection,
78 | LeaderElectionID: "e6946145.apache.org",
79 | Namespace: os.Getenv("WATCH_NAMESPACE"),
80 | NewCache: watchNamespaceCache(),
81 | // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
82 | // when the Manager ends. This requires the binary to immediately end when the
83 | // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
84 | // speeds up voluntary leader transitions as the new leader don't have to wait
85 | // LeaseDuration time first.
86 | //
87 | // In the default scaffold provided, the program ends immediately after
88 | // the manager stops, so would be fine to enable this option. However,
89 | // if you are doing or is intended to do any operation such as perform cleanups
90 | // after the manager stops then its usage might be unsafe.
91 | // LeaderElectionReleaseOnCancel: true,
92 | })
93 | if err != nil {
94 | setupLog.Error(err, "unable to start manager")
95 | os.Exit(1)
96 | }
97 |
98 | if err = (druid.NewDruidReconciler(mgr)).SetupWithManager(mgr); err != nil {
99 | setupLog.Error(err, "unable to create controller", "controller", "Druid")
100 | os.Exit(1)
101 | }
102 |
103 | if err = (druidingestioncontrollers.NewDruidIngestionReconciler(mgr)).SetupWithManager(mgr); err != nil {
104 | setupLog.Error(err, "unable to create controller", "controller", "DruidIngestion")
105 | os.Exit(1)
106 | }
107 |
108 | //+kubebuilder:scaffold:builder
109 |
110 | if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
111 | setupLog.Error(err, "unable to set up health check")
112 | os.Exit(1)
113 | }
114 | if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
115 | setupLog.Error(err, "unable to set up ready check")
116 | os.Exit(1)
117 | }
118 |
119 | setupLog.Info("starting manager")
120 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
121 | setupLog.Error(err, "problem running manager")
122 | os.Exit(1)
123 | }
124 | }
125 |
126 | func watchNamespaceCache() cache.NewCacheFunc {
127 | var managerWatchCache cache.NewCacheFunc
128 | ns := strings.Split(watchNamespace, ",")
129 |
130 | if len(ns) > 1 {
131 | for i := range ns {
132 | ns[i] = strings.TrimSpace(ns[i])
133 | }
134 | managerWatchCache = cache.MultiNamespacedCacheBuilder(ns)
135 | return managerWatchCache
136 | }
137 | managerWatchCache = (cache.NewCacheFunc)(nil)
138 | return managerWatchCache
139 | }
140 |
--------------------------------------------------------------------------------
/pkg/druidapi/druidapi.go:
--------------------------------------------------------------------------------
1 | package druidapi
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "fmt"
7 | "net/url"
8 | "path"
9 |
10 | internalhttp "github.com/datainfrahq/druid-operator/pkg/http"
11 | v1 "k8s.io/api/core/v1"
12 | "k8s.io/apimachinery/pkg/types"
13 | "sigs.k8s.io/controller-runtime/pkg/client"
14 | )
15 |
16 | const (
17 | DruidRouterPort = "8088"
18 | OperatorUserName = "OperatorUserName"
19 | OperatorPassword = "OperatorPassword"
20 | )
21 |
22 | type AuthType string
23 |
24 | const (
25 | BasicAuth AuthType = "basic-auth"
26 | )
27 |
28 | type Auth struct {
29 | // +required
30 | Type AuthType `json:"type"`
31 | // +required
32 | SecretRef v1.SecretReference `json:"secretRef"`
33 | }
34 |
35 | // GetAuthCreds retrieves basic authentication credentials from a Kubernetes secret.
36 | // If the Auth object is empty, it returns an empty BasicAuth object.
37 | // Parameters:
38 | //
39 | // ctx: The context object.
40 | // c: The Kubernetes client.
41 | // auth: The Auth object containing the secret reference.
42 | //
43 | // Returns:
44 | //
45 | // BasicAuth: The basic authentication credentials.
46 | func GetAuthCreds(
47 | ctx context.Context,
48 | c client.Client,
49 | auth Auth,
50 | ) (internalhttp.BasicAuth, error) {
51 | // Check if the mentioned secret exists
52 | if auth != (Auth{}) {
53 | secret := v1.Secret{}
54 | if err := c.Get(ctx, types.NamespacedName{
55 | Namespace: auth.SecretRef.Namespace,
56 | Name: auth.SecretRef.Name,
57 | }, &secret); err != nil {
58 | return internalhttp.BasicAuth{}, err
59 | }
60 | creds := internalhttp.BasicAuth{
61 | UserName: string(secret.Data[OperatorUserName]),
62 | Password: string(secret.Data[OperatorPassword]),
63 | }
64 |
65 | return creds, nil
66 | }
67 |
68 | return internalhttp.BasicAuth{}, nil
69 | }
70 |
71 | // MakePath constructs the appropriate path for the specified Druid API.
72 | // Parameters:
73 | //
74 | // baseURL: The base URL of the Druid cluster. For example, http://router-svc.namespace.svc.cluster.local:8088.
75 | // componentType: The type of Druid component. For example, "indexer".
76 | // apiType: The type of Druid API. For example, "worker".
77 | // additionalPaths: Additional path components to be appended to the URL.
78 | //
79 | // Returns:
80 | //
81 | // string: The constructed path.
82 | func MakePath(baseURL, componentType, apiType string, additionalPaths ...string) string {
83 | u, err := url.Parse(baseURL)
84 | if err != nil {
85 | fmt.Println("Error parsing URL:", err)
86 | return ""
87 | }
88 |
89 | // Construct the initial path
90 | u.Path = path.Join("druid", componentType, "v1", apiType)
91 |
92 | // Append additional path components
93 | for _, p := range additionalPaths {
94 | u.Path = path.Join(u.Path, p)
95 | }
96 |
97 | return u.String()
98 | }
99 |
100 | // GetRouterSvcUrl retrieves the URL of the Druid router service.
101 | // Parameters:
102 | //
103 | // namespace: The namespace of the Druid cluster.
104 | // druidClusterName: The name of the Druid cluster.
105 | // c: The Kubernetes client.
106 | //
107 | // Returns:
108 | //
109 | // string: The URL of the Druid router service.
110 | func GetRouterSvcUrl(namespace, druidClusterName string, c client.Client) (string, error) {
111 | listOpts := []client.ListOption{
112 | client.InNamespace(namespace),
113 | client.MatchingLabels(map[string]string{
114 | "druid_cr": druidClusterName,
115 | "component": "router",
116 | }),
117 | }
118 | svcList := &v1.ServiceList{}
119 | if err := c.List(context.Background(), svcList, listOpts...); err != nil {
120 | return "", err
121 | }
122 | var svcName string
123 |
124 | for range svcList.Items {
125 | svcName = svcList.Items[0].Name
126 | }
127 |
128 | if svcName == "" {
129 | return "", errors.New("router svc discovery fail")
130 | }
131 |
132 | newName := "http://" + svcName + "." + namespace + ":" + DruidRouterPort
133 |
134 | return newName, nil
135 | }
136 |
--------------------------------------------------------------------------------
/pkg/druidapi/druidapi_test.go:
--------------------------------------------------------------------------------
1 | package druidapi
2 |
3 | import (
4 | "testing"
5 | )
6 |
7 | func TestMakePath(t *testing.T) {
8 | tests := []struct {
9 | name string
10 | baseURL string
11 | componentType string
12 | apiType string
13 | additionalPaths []string
14 | expected string
15 | }{
16 | {
17 | name: "NoAdditionalPath",
18 | baseURL: "http://example-druid-service",
19 | componentType: "indexer",
20 | apiType: "task",
21 | expected: "http://example-druid-service/druid/indexer/v1/task",
22 | },
23 | {
24 | name: "OneAdditionalPath",
25 | baseURL: "http://example-druid-service",
26 | componentType: "indexer",
27 | apiType: "task",
28 | additionalPaths: []string{"extra"},
29 | expected: "http://example-druid-service/druid/indexer/v1/task/extra",
30 | },
31 | {
32 | name: "MultipleAdditionalPaths",
33 | baseURL: "http://example-druid-service",
34 | componentType: "coordinator",
35 | apiType: "rules",
36 | additionalPaths: []string{"wikipedia", "history"},
37 | expected: "http://example-druid-service/druid/coordinator/v1/rules/wikipedia/history",
38 | },
39 | {
40 | name: "EmptyBaseURL",
41 | baseURL: "",
42 | componentType: "indexer",
43 | apiType: "task",
44 | expected: "druid/indexer/v1/task",
45 | },
46 | }
47 |
48 | for _, tt := range tests {
49 | t.Run(tt.name, func(t *testing.T) {
50 | actual := MakePath(tt.baseURL, tt.componentType, tt.apiType, tt.additionalPaths...)
51 | if actual != tt.expected {
52 | t.Errorf("makePath() = %v, expected %v", actual, tt.expected)
53 | }
54 | })
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/pkg/http/http.go:
--------------------------------------------------------------------------------
1 | package http
2 |
3 | import (
4 | "bytes"
5 | "io"
6 | "net/http"
7 | )
8 |
9 | // DruidHTTP interface
10 | type DruidHTTP interface {
11 | Do(method, url string, body []byte) (*Response, error)
12 | }
13 |
14 | // HTTP client
15 | type DruidClient struct {
16 | HTTPClient *http.Client
17 | Auth *Auth
18 | }
19 |
20 | func NewHTTPClient(client *http.Client, auth *Auth) DruidHTTP {
21 | newClient := &DruidClient{
22 | HTTPClient: client,
23 | Auth: auth,
24 | }
25 |
26 | return newClient
27 | }
28 |
29 | // Auth mechanisms supported by Druid control plane to authenticate
30 | // with druid clusters
31 | type Auth struct {
32 | BasicAuth BasicAuth
33 | }
34 |
35 | // BasicAuth
36 | type BasicAuth struct {
37 | UserName string
38 | Password string
39 | }
40 |
41 | // Response passed to controller
42 | type Response struct {
43 | ResponseBody string
44 | StatusCode int
45 | }
46 |
47 | // Do method to be used schema and tenant controller.
48 | func (c *DruidClient) Do(Method, url string, body []byte) (*Response, error) {
49 |
50 | req, err := http.NewRequest(Method, url, bytes.NewBuffer(body))
51 | if err != nil {
52 | return nil, err
53 | }
54 |
55 | if c.Auth.BasicAuth != (BasicAuth{}) {
56 | req.SetBasicAuth(c.Auth.BasicAuth.UserName, c.Auth.BasicAuth.Password)
57 | }
58 |
59 | req.Header.Add("Content-Type", "application/json")
60 | resp, err := c.HTTPClient.Do(req)
61 | if err != nil {
62 | return nil, err
63 | }
64 |
65 | defer resp.Body.Close()
66 |
67 | responseBody, err := io.ReadAll(resp.Body)
68 | if err != nil {
69 | return nil, err
70 | }
71 |
72 | return &Response{ResponseBody: string(responseBody), StatusCode: resp.StatusCode}, nil
73 | }
74 |
--------------------------------------------------------------------------------
/pkg/util/util.go:
--------------------------------------------------------------------------------
1 | package util
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "reflect"
7 | )
8 |
9 | // ToJsonString marshals the given data into a JSON string.
10 | func ToJsonString(data interface{}) (string, error) {
11 | jsonData, err := json.Marshal(data)
12 | if err != nil {
13 | return "", err
14 | }
15 | return string(jsonData), nil
16 | }
17 |
18 | // IncludesJson checks if all key-value pairs in the desired JSON string are present in the current JSON string.
19 | func IncludesJson(currentJson, desiredJson string) (bool, error) {
20 | var current, desired map[string]interface{}
21 |
22 | // Parse the current JSON string
23 | if err := json.Unmarshal([]byte(currentJson), ¤t); err != nil {
24 | return false, fmt.Errorf("error parsing current JSON: %w", err)
25 | }
26 |
27 | // Parse the desired JSON string
28 | if err := json.Unmarshal([]byte(desiredJson), &desired); err != nil {
29 | return false, fmt.Errorf("error parsing desired JSON: %w", err)
30 | }
31 |
32 | // Check if all key-value pairs in desired are present in current
33 | return includes(current, desired), nil
34 | }
35 |
36 | // includes recursively checks if all key-value pairs in the desired map are present in the current map.
37 | func includes(current, desired map[string]interface{}) bool {
38 | for key, desiredValue := range desired {
39 | currentValue, exists := current[key]
40 | if !exists {
41 | return false
42 | }
43 |
44 | if !reflect.DeepEqual(desiredValue, currentValue) {
45 | switch desiredValueTyped := desiredValue.(type) {
46 | case map[string]interface{}:
47 | currentValueTyped, ok := currentValue.(map[string]interface{})
48 | if !ok || !includes(currentValueTyped, desiredValueTyped) {
49 | return false
50 | }
51 | case []interface{}:
52 | currentValueTyped, ok := currentValue.([]interface{})
53 | if !ok || !sliceIncludes(currentValueTyped, desiredValueTyped) {
54 | return false
55 | }
56 | default:
57 | return false
58 | }
59 | }
60 | }
61 | return true
62 | }
63 |
64 | // sliceIncludes checks if all elements of the desired slice are present in the current slice.
65 | func sliceIncludes(current, desired []interface{}) bool {
66 | for _, desiredItem := range desired {
67 | found := false
68 | for _, currentItem := range current {
69 | if reflect.DeepEqual(desiredItem, currentItem) {
70 | found = true
71 | break
72 | }
73 | }
74 | if !found {
75 | return false
76 | }
77 | }
78 | return true
79 | }
80 |
--------------------------------------------------------------------------------
/tutorials/druid-on-kind/README.md:
--------------------------------------------------------------------------------
1 | # Deploying Druid On KIND
2 |
3 | - In this tutorial, we are going to deploy an Apache Druid cluster on KIND.
4 | - This tutorial can easily run on your local machine.
5 |
6 | ## Prerequisites
7 | To follow this tutorial you will need:
8 |
9 | - The [KIND CLI](https://kind.sigs.k8s.io/) installed.
10 | - The KUBECTL CLI installed.
11 | - Docker up and Running.
12 |
13 | ## Install Kind Cluster
14 | Create kind cluster on your machine.
15 |
16 | ```kind create cluster --name druid```
17 |
18 | ## Install Druid Operator
19 |
20 | - Add Helm Repo
21 | ```
22 | helm repo add datainfra https://charts.datainfra.io
23 | helm repo update
24 | ```
25 |
26 | - Install Operator
27 | ```
28 | # Install Druid operator using Helm
29 | helm -n druid-operator-system upgrade -i --create-namespace cluster-druid-operator datainfra/druid-operator
30 | ```
31 |
32 | ## Apply Druid Customer Resource
33 |
34 | - This druid CR runs druid without zookeeper, using druid k8s extension.
35 | - MM less deployment.
36 | - Derby for metadata.
37 | - Minio for deepstorage.
38 |
39 | - Run ```make helm-minio-install ```. This will deploy minio using minio operator.
40 |
41 | - Once the minio pod is up and running in druid namespace, apply the druid CR.
42 | - ```kubectl apply -f tutorials/druid-on-kind/druid-mmless.yaml -n druid```
43 |
44 | Here's a view of the druid namespace.
45 |
46 | ```
47 | NAMESPACE NAME READY STATUS RESTARTS AGE
48 | druid druid-tiny-cluster-brokers-5ddcb655cf-plq6x 1/1 Running 0 2d
49 | druid druid-tiny-cluster-cold-0 1/1 Running 0 2d
50 | druid druid-tiny-cluster-coordinators-846df8f545-9qrsw 1/1 Running 1 2d
51 | druid druid-tiny-cluster-hot-0 1/1 Running 0 2d
52 | druid druid-tiny-cluster-routers-5c9677bf9d-qk9q7 1/1 Running 0 2d
53 | druid myminio-ss-0-0 2/2 Running 0 2d
54 |
55 | ```
56 |
57 | ## Access Router Console
58 |
59 | - Port forward router
60 | - ```kubectl port-forward svc/druid-tiny-cluster-routers 8088 -n druid```
61 |
--------------------------------------------------------------------------------
4 |