├── config
├── prometheus
│ ├── kustomization.yaml
│ └── monitor.yaml
├── certmanager
│ ├── kustomization.yaml
│ ├── kustomizeconfig.yaml
│ └── certificate.yaml
├── webhook
│ ├── kustomization.yaml
│ ├── service.yaml
│ └── kustomizeconfig.yaml
├── scorecard
│ ├── bases
│ │ └── config.yaml
│ ├── patches
│ │ ├── basic.config.yaml
│ │ └── olm.config.yaml
│ └── kustomization.yaml
├── samples
│ ├── minio_credentials.yaml
│ ├── kustomization.yaml
│ ├── pipelines_v1_transform.yaml
│ └── pipelines_v1_splittransform.yaml
├── rbac
│ ├── auth_proxy_client_clusterrole.yaml
│ ├── role_binding.yaml
│ ├── auth_proxy_role_binding.yaml
│ ├── leader_election_role_binding.yaml
│ ├── auth_proxy_role.yaml
│ ├── auth_proxy_service.yaml
│ ├── job_viewer_role.yaml
│ ├── kustomization.yaml
│ ├── transform_viewer_role.yaml
│ ├── splittransform_viewer_role.yaml
│ ├── job_editor_role.yaml
│ ├── transform_editor_role.yaml
│ ├── splittransform_editor_role.yaml
│ ├── leader_election_role.yaml
│ └── role.yaml
├── manager
│ ├── kustomization.yaml
│ └── manager.yaml
├── crd
│ ├── patches
│ │ ├── cainjection_in_jobs.yaml
│ │ ├── cainjection_in_transforms.yaml
│ │ ├── cainjection_in_splittransforms.yaml
│ │ ├── webhook_in_jobs.yaml
│ │ ├── webhook_in_transforms.yaml
│ │ └── webhook_in_splittransforms.yaml
│ ├── kustomizeconfig.yaml
│ └── kustomization.yaml
└── default
│ ├── manager_webhook_patch.yaml
│ ├── webhookcainjection_patch.yaml
│ ├── manager_auth_proxy_patch.yaml
│ └── kustomization.yaml
├── apis
├── meta
│ ├── v1
│ │ ├── doc.go
│ │ ├── config.go
│ │ ├── meta.go
│ │ ├── groupversion_info.go
│ │ ├── object.go
│ │ ├── launch_config.go
│ │ ├── elements.go
│ │ ├── constants.go
│ │ ├── pipeline.go
│ │ ├── zz_generated.deepcopy.go
│ │ └── minio.go
│ └── group.go
└── pipelines
│ ├── v1
│ ├── doc.go
│ ├── groupversion_info.go
│ ├── job_util.go
│ ├── transform_util.go
│ ├── common.go
│ ├── transform_types.go
│ ├── splittransform_util.go
│ ├── job_types.go
│ └── splittransform_types.go
│ └── group.go
├── gst
└── plugins
│ ├── go.mod
│ └── minio
│ ├── plugin.go
│ ├── properties.go
│ ├── common.go
│ ├── seek_writer.go
│ ├── miniosrc.go
│ └── miniosink.go
├── PROJECT
├── .gitignore
├── hack
├── boilerplate.go.txt
└── update-api-docs.sh
├── doc
├── refdocs.json
├── meta_template
│ ├── members.tpl
│ ├── type.tpl
│ └── pkg.tpl
└── pipelines_template
│ ├── members.tpl
│ ├── type.tpl
│ └── pkg.tpl
├── pkg
├── version
│ └── version.go
├── types
│ └── pipelines.go
├── util
│ └── minio.go
└── managers
│ └── manager.go
├── go.mod
├── Dockerfile
├── controllers
└── pipelines
│ ├── util.go
│ ├── job_controller.go
│ ├── suite_test.go
│ ├── splittransform_controller.go
│ ├── transform_controller.go
│ └── jobs.go
├── .github
└── workflows
│ └── build.yaml
├── cmd
└── runner
│ ├── Dockerfile
│ ├── elements.go
│ ├── main.go
│ └── parse_cr_pipeline.go
├── testbin
└── setup-envtest.sh
├── main.go
├── README.md
└── Makefile
/config/prometheus/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - monitor.yaml
3 |
--------------------------------------------------------------------------------
/config/certmanager/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - certificate.yaml
3 |
4 | configurations:
5 | - kustomizeconfig.yaml
6 |
--------------------------------------------------------------------------------
/config/webhook/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - manifests.yaml
3 | - service.yaml
4 |
5 | configurations:
6 | - kustomizeconfig.yaml
7 |
--------------------------------------------------------------------------------
/apis/meta/v1/doc.go:
--------------------------------------------------------------------------------
1 | // Package v1 contains API Schema definitions for the pipelines v1 API group
2 | // +groupName=meta.gst.io
3 | package v1
4 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/doc.go:
--------------------------------------------------------------------------------
1 | // Package v1 file doc.go required for the doc generator to register this as an API
2 | //
3 | // +groupName=pipelines.gst.io
4 | package v1
5 |
--------------------------------------------------------------------------------
/config/scorecard/bases/config.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: scorecard.operatorframework.io/v1alpha3
2 | kind: Configuration
3 | metadata:
4 | name: config
5 | stages:
6 | - parallel: true
7 | tests: []
8 |
--------------------------------------------------------------------------------
/config/samples/minio_credentials.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | apiVersion: v1
3 | kind: Secret
4 | metadata:
5 | name: minio-credentials
6 | data:
7 | access-key-id: YWNjZXNza2V5
8 | secret-access-key: c2VjcmV0a2V5
--------------------------------------------------------------------------------
/config/rbac/auth_proxy_client_clusterrole.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | name: metrics-reader
5 | rules:
6 | - nonResourceURLs: ["/metrics"]
7 | verbs: ["get"]
8 |
--------------------------------------------------------------------------------
/config/samples/kustomization.yaml:
--------------------------------------------------------------------------------
1 | ## Append samples you want in your CSV to this file as resources ##
2 | resources:
3 | - pipelines_v1_transform.yaml
4 | - pipelines_v1_splittransform.yaml
5 | # +kubebuilder:scaffold:manifestskustomizesamples
6 |
--------------------------------------------------------------------------------
/config/manager/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - manager.yaml
3 | apiVersion: kustomize.config.k8s.io/v1beta1
4 | kind: Kustomization
5 | images:
6 | - name: controller
7 | newName: ghcr.io/tinyzimmer/gst-pipeline-operator/controller
8 | newTag: latest
9 |
--------------------------------------------------------------------------------
/gst/plugins/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/tinyzimmer/gst-pipeline-operator/gst/plugins
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/minio/minio-go/v7 v7.0.7
7 | github.com/tinyzimmer/go-glib v0.0.19
8 | github.com/tinyzimmer/go-gst v0.2.12
9 | )
10 |
--------------------------------------------------------------------------------
/apis/meta/group.go:
--------------------------------------------------------------------------------
1 | // Package meta contains meta API versions.
2 | //
3 | // This file ensures Go source parsers acknowledge the kvdi package
4 | // and any child packages. It can be removed if any other Go source files are
5 | // added to this package.
6 | package meta
7 |
--------------------------------------------------------------------------------
/config/webhook/service.yaml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: v1
3 | kind: Service
4 | metadata:
5 | name: webhook-service
6 | namespace: system
7 | spec:
8 | ports:
9 | - port: 443
10 | targetPort: 9443
11 | selector:
12 | control-plane: controller-manager
13 |
--------------------------------------------------------------------------------
/apis/pipelines/group.go:
--------------------------------------------------------------------------------
1 | // Package pipelines contains pipelines API versions.
2 | //
3 | // This file ensures Go source parsers acknowledge the kvdi package
4 | // and any child packages. It can be removed if any other Go source files are
5 | // added to this package.
6 | package pipelines
7 |
--------------------------------------------------------------------------------
/config/scorecard/patches/basic.config.yaml:
--------------------------------------------------------------------------------
1 | - op: add
2 | path: /stages/0/tests/-
3 | value:
4 | entrypoint:
5 | - scorecard-test
6 | - basic-check-spec
7 | image: quay.io/operator-framework/scorecard-test:v1.3.0
8 | labels:
9 | suite: basic
10 | test: basic-check-spec-test
11 |
--------------------------------------------------------------------------------
/config/rbac/role_binding.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRoleBinding
3 | metadata:
4 | name: manager-rolebinding
5 | roleRef:
6 | apiGroup: rbac.authorization.k8s.io
7 | kind: ClusterRole
8 | name: manager-role
9 | subjects:
10 | - kind: ServiceAccount
11 | name: default
12 | namespace: system
13 |
--------------------------------------------------------------------------------
/config/rbac/auth_proxy_role_binding.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRoleBinding
3 | metadata:
4 | name: proxy-rolebinding
5 | roleRef:
6 | apiGroup: rbac.authorization.k8s.io
7 | kind: ClusterRole
8 | name: proxy-role
9 | subjects:
10 | - kind: ServiceAccount
11 | name: default
12 | namespace: system
13 |
--------------------------------------------------------------------------------
/config/rbac/leader_election_role_binding.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: RoleBinding
3 | metadata:
4 | name: leader-election-rolebinding
5 | roleRef:
6 | apiGroup: rbac.authorization.k8s.io
7 | kind: Role
8 | name: leader-election-role
9 | subjects:
10 | - kind: ServiceAccount
11 | name: default
12 | namespace: system
13 |
--------------------------------------------------------------------------------
/config/rbac/auth_proxy_role.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | name: proxy-role
5 | rules:
6 | - apiGroups: ["authentication.k8s.io"]
7 | resources:
8 | - tokenreviews
9 | verbs: ["create"]
10 | - apiGroups: ["authorization.k8s.io"]
11 | resources:
12 | - subjectaccessreviews
13 | verbs: ["create"]
14 |
--------------------------------------------------------------------------------
/config/rbac/auth_proxy_service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | labels:
5 | control-plane: controller-manager
6 | name: controller-manager-metrics-service
7 | namespace: system
8 | spec:
9 | ports:
10 | - name: https
11 | port: 8443
12 | targetPort: https
13 | selector:
14 | control-plane: controller-manager
15 |
--------------------------------------------------------------------------------
/PROJECT:
--------------------------------------------------------------------------------
1 | domain: gst.io
2 | layout: go.kubebuilder.io/v3
3 | multigroup: true
4 | projectName: gst-pipeline-operator
5 | repo: github.com/tinyzimmer/gst-pipeline-operator
6 | resources:
7 | - group: pipelines
8 | kind: Transform
9 | version: v1
10 | - group: pipelines
11 | kind: Job
12 | version: v1
13 | version: 3-alpha
14 | plugins:
15 | go.sdk.operatorframework.io/v2-alpha: {}
16 |
--------------------------------------------------------------------------------
/config/crd/patches/cainjection_in_jobs.yaml:
--------------------------------------------------------------------------------
1 | # The following patch adds a directive for certmanager to inject CA into the CRD
2 | # CRD conversion requires k8s 1.13 or later.
3 | apiVersion: apiextensions.k8s.io/v1beta1
4 | kind: CustomResourceDefinition
5 | metadata:
6 | annotations:
7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
8 | name: jobs.pipelines.gst.io
9 |
--------------------------------------------------------------------------------
/config/crd/patches/cainjection_in_transforms.yaml:
--------------------------------------------------------------------------------
1 | # The following patch adds a directive for certmanager to inject CA into the CRD
2 | # CRD conversion requires k8s 1.13 or later.
3 | apiVersion: apiextensions.k8s.io/v1beta1
4 | kind: CustomResourceDefinition
5 | metadata:
6 | annotations:
7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
8 | name: transforms.pipelines.gst.io
9 |
--------------------------------------------------------------------------------
/config/crd/patches/cainjection_in_splittransforms.yaml:
--------------------------------------------------------------------------------
1 | # The following patch adds a directive for certmanager to inject CA into the CRD
2 | # CRD conversion requires k8s 1.13 or later.
3 | apiVersion: apiextensions.k8s.io/v1beta1
4 | kind: CustomResourceDefinition
5 | metadata:
6 | annotations:
7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
8 | name: splittransforms.pipelines.gst.io
9 |
--------------------------------------------------------------------------------
/config/rbac/job_viewer_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions for end users to view jobs.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: ClusterRole
4 | metadata:
5 | name: job-viewer-role
6 | rules:
7 | - apiGroups:
8 | - pipelines.gst.io
9 | resources:
10 | - jobs
11 | verbs:
12 | - get
13 | - list
14 | - watch
15 | - apiGroups:
16 | - pipelines.gst.io
17 | resources:
18 | - jobs/status
19 | verbs:
20 | - get
21 |
--------------------------------------------------------------------------------
/config/prometheus/monitor.yaml:
--------------------------------------------------------------------------------
1 |
2 | # Prometheus Monitor Service (Metrics)
3 | apiVersion: monitoring.coreos.com/v1
4 | kind: ServiceMonitor
5 | metadata:
6 | labels:
7 | control-plane: controller-manager
8 | name: controller-manager-metrics-monitor
9 | namespace: system
10 | spec:
11 | endpoints:
12 | - path: /metrics
13 | port: https
14 | selector:
15 | matchLabels:
16 | control-plane: controller-manager
17 |
--------------------------------------------------------------------------------
/config/rbac/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - role.yaml
3 | - role_binding.yaml
4 | - leader_election_role.yaml
5 | - leader_election_role_binding.yaml
6 | # Comment the following 4 lines if you want to disable
7 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
8 | # which protects your /metrics endpoint.
9 | - auth_proxy_service.yaml
10 | - auth_proxy_role.yaml
11 | - auth_proxy_role_binding.yaml
12 | - auth_proxy_client_clusterrole.yaml
13 |
--------------------------------------------------------------------------------
/config/rbac/transform_viewer_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions for end users to view transforms.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: ClusterRole
4 | metadata:
5 | name: transform-viewer-role
6 | rules:
7 | - apiGroups:
8 | - pipelines.gst.io
9 | resources:
10 | - transforms
11 | verbs:
12 | - get
13 | - list
14 | - watch
15 | - apiGroups:
16 | - pipelines.gst.io
17 | resources:
18 | - transforms/status
19 | verbs:
20 | - get
21 |
--------------------------------------------------------------------------------
/config/rbac/splittransform_viewer_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions for end users to view splittransforms.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: ClusterRole
4 | metadata:
5 | name: splittransform-viewer-role
6 | rules:
7 | - apiGroups:
8 | - pipelines.gst.io
9 | resources:
10 | - splittransforms
11 | verbs:
12 | - get
13 | - list
14 | - watch
15 | - apiGroups:
16 | - pipelines.gst.io
17 | resources:
18 | - splittransforms/status
19 | verbs:
20 | - get
21 |
--------------------------------------------------------------------------------
/config/scorecard/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - bases/config.yaml
3 | patchesJson6902:
4 | - path: patches/basic.config.yaml
5 | target:
6 | group: scorecard.operatorframework.io
7 | version: v1alpha3
8 | kind: Configuration
9 | name: config
10 | - path: patches/olm.config.yaml
11 | target:
12 | group: scorecard.operatorframework.io
13 | version: v1alpha3
14 | kind: Configuration
15 | name: config
16 | # +kubebuilder:scaffold:patchesJson6902
17 |
--------------------------------------------------------------------------------
/config/certmanager/kustomizeconfig.yaml:
--------------------------------------------------------------------------------
1 | # This configuration is for teaching kustomize how to update name ref and var substitution
2 | nameReference:
3 | - kind: Issuer
4 | group: cert-manager.io
5 | fieldSpecs:
6 | - kind: Certificate
7 | group: cert-manager.io
8 | path: spec/issuerRef/name
9 |
10 | varReference:
11 | - kind: Certificate
12 | group: cert-manager.io
13 | path: spec/commonName
14 | - kind: Certificate
15 | group: cert-manager.io
16 | path: spec/dnsNames
17 |
--------------------------------------------------------------------------------
/config/rbac/job_editor_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions for end users to edit jobs.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: ClusterRole
4 | metadata:
5 | name: job-editor-role
6 | rules:
7 | - apiGroups:
8 | - pipelines.gst.io
9 | resources:
10 | - jobs
11 | verbs:
12 | - create
13 | - delete
14 | - get
15 | - list
16 | - patch
17 | - update
18 | - watch
19 | - apiGroups:
20 | - pipelines.gst.io
21 | resources:
22 | - jobs/status
23 | verbs:
24 | - get
25 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Binaries for programs and plugins
3 | *.exe
4 | *.exe~
5 | *.dll
6 | *.so
7 | *.dylib
8 | bin
9 |
10 | # Test binary, build with `go test -c`
11 | *.test
12 |
13 | # Output of the go coverage tool, specifically when used with LiteIDE
14 | *.out
15 |
16 | # Kubernetes Generated files - skip generated files, except for vendored files
17 |
18 | !vendor/**/zz_generated.*
19 |
20 | # editor and IDE paraphernalia
21 | .idea
22 | *.swp
23 | *.swo
24 | *~
25 | bin
26 | kubeconfig.yaml
27 | vendor
--------------------------------------------------------------------------------
/config/rbac/transform_editor_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions for end users to edit transforms.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: ClusterRole
4 | metadata:
5 | name: transform-editor-role
6 | rules:
7 | - apiGroups:
8 | - pipelines.gst.io
9 | resources:
10 | - transforms
11 | verbs:
12 | - create
13 | - delete
14 | - get
15 | - list
16 | - patch
17 | - update
18 | - watch
19 | - apiGroups:
20 | - pipelines.gst.io
21 | resources:
22 | - transforms/status
23 | verbs:
24 | - get
25 |
--------------------------------------------------------------------------------
/config/rbac/splittransform_editor_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions for end users to edit splittransforms.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: ClusterRole
4 | metadata:
5 | name: splittransform-editor-role
6 | rules:
7 | - apiGroups:
8 | - pipelines.gst.io
9 | resources:
10 | - splittransforms
11 | verbs:
12 | - create
13 | - delete
14 | - get
15 | - list
16 | - patch
17 | - update
18 | - watch
19 | - apiGroups:
20 | - pipelines.gst.io
21 | resources:
22 | - splittransforms/status
23 | verbs:
24 | - get
25 |
--------------------------------------------------------------------------------
/config/crd/kustomizeconfig.yaml:
--------------------------------------------------------------------------------
1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD
2 | nameReference:
3 | - kind: Service
4 | version: v1
5 | fieldSpecs:
6 | - kind: CustomResourceDefinition
7 | group: apiextensions.k8s.io
8 | path: spec/conversion/webhookClientConfig/service/name
9 |
10 | namespace:
11 | - kind: CustomResourceDefinition
12 | group: apiextensions.k8s.io
13 | path: spec/conversion/webhookClientConfig/service/namespace
14 | create: false
15 |
16 | varReference:
17 | - path: metadata/annotations
18 |
--------------------------------------------------------------------------------
/config/rbac/leader_election_role.yaml:
--------------------------------------------------------------------------------
1 | # permissions to do leader election.
2 | apiVersion: rbac.authorization.k8s.io/v1
3 | kind: Role
4 | metadata:
5 | name: leader-election-role
6 | rules:
7 | - apiGroups:
8 | - ""
9 | resources:
10 | - configmaps
11 | verbs:
12 | - get
13 | - list
14 | - watch
15 | - create
16 | - update
17 | - patch
18 | - delete
19 | - apiGroups:
20 | - ""
21 | resources:
22 | - configmaps/status
23 | verbs:
24 | - get
25 | - update
26 | - patch
27 | - apiGroups:
28 | - ""
29 | resources:
30 | - events
31 | verbs:
32 | - create
33 | - patch
34 |
--------------------------------------------------------------------------------
/hack/boilerplate.go.txt:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
--------------------------------------------------------------------------------
/config/default/manager_webhook_patch.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: controller-manager
5 | namespace: system
6 | spec:
7 | template:
8 | spec:
9 | containers:
10 | - name: manager
11 | ports:
12 | - containerPort: 9443
13 | name: webhook-server
14 | protocol: TCP
15 | volumeMounts:
16 | - mountPath: /tmp/k8s-webhook-server/serving-certs
17 | name: cert
18 | readOnly: true
19 | volumes:
20 | - name: cert
21 | secret:
22 | defaultMode: 420
23 | secretName: webhook-server-cert
24 |
--------------------------------------------------------------------------------
/config/crd/patches/webhook_in_jobs.yaml:
--------------------------------------------------------------------------------
1 | # The following patch enables conversion webhook for CRD
2 | # CRD conversion requires k8s 1.13 or later.
3 | apiVersion: apiextensions.k8s.io/v1beta1
4 | kind: CustomResourceDefinition
5 | metadata:
6 | name: jobs.pipelines.gst.io
7 | spec:
8 | conversion:
9 | strategy: Webhook
10 | webhookClientConfig:
11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
13 | caBundle: Cg==
14 | service:
15 | namespace: system
16 | name: webhook-service
17 | path: /convert
18 |
--------------------------------------------------------------------------------
/config/default/webhookcainjection_patch.yaml:
--------------------------------------------------------------------------------
1 | # This patch add annotation to admission webhook config and
2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
3 | apiVersion: admissionregistration.k8s.io/v1beta1
4 | kind: MutatingWebhookConfiguration
5 | metadata:
6 | name: mutating-webhook-configuration
7 | annotations:
8 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
9 | ---
10 | apiVersion: admissionregistration.k8s.io/v1beta1
11 | kind: ValidatingWebhookConfiguration
12 | metadata:
13 | name: validating-webhook-configuration
14 | annotations:
15 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
16 |
--------------------------------------------------------------------------------
/config/crd/patches/webhook_in_transforms.yaml:
--------------------------------------------------------------------------------
1 | # The following patch enables conversion webhook for CRD
2 | # CRD conversion requires k8s 1.13 or later.
3 | apiVersion: apiextensions.k8s.io/v1beta1
4 | kind: CustomResourceDefinition
5 | metadata:
6 | name: transforms.pipelines.gst.io
7 | spec:
8 | conversion:
9 | strategy: Webhook
10 | webhookClientConfig:
11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
13 | caBundle: Cg==
14 | service:
15 | namespace: system
16 | name: webhook-service
17 | path: /convert
18 |
--------------------------------------------------------------------------------
/doc/refdocs.json:
--------------------------------------------------------------------------------
1 | {
2 | "hideMemberFields": [
3 | "TypeMeta"
4 | ],
5 | "hideTypePatterns": [
6 | "ParseError$",
7 | "List$",
8 | ".*?Status.*?"
9 | ],
10 | "externalPackages": [
11 | {
12 | "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/",
13 | "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}"
14 | }
15 | ],
16 | "typeDisplayNamePrefixOverrides": {
17 | "k8s.io/api/": "Kubernetes ",
18 | "k8s.io/apimachinery/pkg/apis/": "Kubernetes "
19 | },
20 | "markdownDisabled": false
21 | }
--------------------------------------------------------------------------------
/config/crd/patches/webhook_in_splittransforms.yaml:
--------------------------------------------------------------------------------
1 | # The following patch enables conversion webhook for CRD
2 | # CRD conversion requires k8s 1.13 or later.
3 | apiVersion: apiextensions.k8s.io/v1beta1
4 | kind: CustomResourceDefinition
5 | metadata:
6 | name: splittransforms.pipelines.gst.io
7 | spec:
8 | conversion:
9 | strategy: Webhook
10 | webhookClientConfig:
11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
13 | caBundle: Cg==
14 | service:
15 | namespace: system
16 | name: webhook-service
17 | path: /convert
18 |
--------------------------------------------------------------------------------
/pkg/version/version.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package version
18 |
19 | var (
20 | // Version is the current tag version
21 | Version string = ""
22 | // GitCommit is the current git commit
23 | GitCommit string = ""
24 | )
25 |
--------------------------------------------------------------------------------
/config/default/manager_auth_proxy_patch.yaml:
--------------------------------------------------------------------------------
1 | # This patch inject a sidecar container which is a HTTP proxy for the
2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
3 | apiVersion: apps/v1
4 | kind: Deployment
5 | metadata:
6 | name: controller-manager
7 | namespace: system
8 | spec:
9 | template:
10 | spec:
11 | containers:
12 | - name: kube-rbac-proxy
13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
14 | args:
15 | - "--secure-listen-address=0.0.0.0:8443"
16 | - "--upstream=http://127.0.0.1:8080/"
17 | - "--logtostderr=true"
18 | - "--v=10"
19 | ports:
20 | - containerPort: 8443
21 | name: https
22 | - name: manager
23 | args:
24 | - "--metrics-addr=127.0.0.1:8080"
25 | - "--enable-leader-election"
26 |
--------------------------------------------------------------------------------
/apis/meta/v1/config.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | // SourceSinkConfig is used to declare configurations related to the retrieval or
20 | // saving of pipeline objects.
21 | type SourceSinkConfig struct {
22 | // Configurations for a MinIO source or sink
23 | MinIO *MinIOConfig `json:"minio,omitempty"`
24 | }
25 |
--------------------------------------------------------------------------------
/config/webhook/kustomizeconfig.yaml:
--------------------------------------------------------------------------------
1 | # the following config is for teaching kustomize where to look at when substituting vars.
2 | # It requires kustomize v2.1.0 or newer to work properly.
3 | nameReference:
4 | - kind: Service
5 | version: v1
6 | fieldSpecs:
7 | - kind: MutatingWebhookConfiguration
8 | group: admissionregistration.k8s.io
9 | path: webhooks/clientConfig/service/name
10 | - kind: ValidatingWebhookConfiguration
11 | group: admissionregistration.k8s.io
12 | path: webhooks/clientConfig/service/name
13 |
14 | namespace:
15 | - kind: MutatingWebhookConfiguration
16 | group: admissionregistration.k8s.io
17 | path: webhooks/clientConfig/service/namespace
18 | create: true
19 | - kind: ValidatingWebhookConfiguration
20 | group: admissionregistration.k8s.io
21 | path: webhooks/clientConfig/service/namespace
22 | create: true
23 |
24 | varReference:
25 | - path: metadata/annotations
26 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/tinyzimmer/gst-pipeline-operator
2 |
3 | go 1.13
4 |
5 | require (
6 | github.com/Masterminds/goutils v1.1.0 // indirect
7 | github.com/Masterminds/semver v1.5.0 // indirect
8 | github.com/Masterminds/sprig v2.22.0+incompatible
9 | github.com/go-logr/logr v0.3.0
10 | github.com/goccy/go-graphviz v0.0.9
11 | github.com/huandu/xstrings v1.3.2 // indirect
12 | github.com/minio/minio-go/v7 v7.0.7
13 | github.com/mitchellh/copystructure v1.0.0 // indirect
14 | github.com/onsi/ginkgo v1.14.1
15 | github.com/onsi/gomega v1.10.2
16 | github.com/pkg/errors v0.9.1
17 | github.com/russross/blackfriday/v2 v2.0.1
18 | github.com/tinyzimmer/go-glib v0.0.19
19 | github.com/tinyzimmer/go-gst v0.2.12
20 | k8s.io/api v0.20.2
21 | k8s.io/apimachinery v0.20.2
22 | k8s.io/client-go v0.20.1
23 | k8s.io/gengo v0.0.0-20201113003025-83324d819ded
24 | k8s.io/klog v1.0.0
25 | sigs.k8s.io/controller-runtime v0.8.0
26 | )
27 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | # Build the manager binary
2 | FROM golang:1.15 as builder
3 |
4 | WORKDIR /workspace
5 | # Copy the Go Modules manifests
6 | COPY go.mod go.mod
7 | COPY go.sum go.sum
8 | # cache deps before building and copying source so that we don't need to re-download as much
9 | # and so that source changes don't invalidate our downloaded layer
10 | RUN go mod download
11 |
12 | # Copy the go source
13 | COPY main.go main.go
14 | COPY apis/ apis/
15 | COPY pkg/ pkg/
16 | COPY controllers/ controllers/
17 |
18 | # Build
19 | ARG LDFLAGS
20 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager -ldflags="${LDFLAGS}" main.go
21 |
22 | # Use distroless as minimal base image to package the manager binary
23 | # Refer to https://github.com/GoogleContainerTools/distroless for more details
24 | FROM gcr.io/distroless/static:nonroot
25 | WORKDIR /
26 | COPY --from=builder /workspace/manager .
27 | USER nonroot:nonroot
28 |
29 | ENTRYPOINT ["/manager"]
30 |
--------------------------------------------------------------------------------
/config/manager/manager.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Namespace
3 | metadata:
4 | labels:
5 | control-plane: controller-manager
6 | name: system
7 | ---
8 | apiVersion: apps/v1
9 | kind: Deployment
10 | metadata:
11 | name: controller-manager
12 | namespace: system
13 | labels:
14 | control-plane: controller-manager
15 | spec:
16 | selector:
17 | matchLabels:
18 | control-plane: controller-manager
19 | replicas: 1
20 | template:
21 | metadata:
22 | labels:
23 | control-plane: controller-manager
24 | spec:
25 | containers:
26 | - command:
27 | - /manager
28 | args:
29 | - --enable-leader-election
30 | image: controller:latest
31 | name: manager
32 | resources:
33 | limits:
34 | cpu: 100m
35 | memory: 30Mi
36 | requests:
37 | cpu: 100m
38 | memory: 20Mi
39 | terminationGracePeriodSeconds: 10
40 |
--------------------------------------------------------------------------------
/config/samples/pipelines_v1_transform.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | apiVersion: pipelines.gst.io/v1
3 | kind: Transform
4 | metadata:
5 | name: mp4-converter
6 | spec:
7 | globals:
8 | minio:
9 | endpoint: "minio.default.svc.cluster.local:9000"
10 | insecureNoTLS: true
11 | region: us-east-1
12 | bucket: gst-processing
13 | credentialsSecret:
14 | name: minio-credentials
15 | src:
16 | minio:
17 | key: drop/
18 | sink:
19 | minio:
20 | key: "mp4/{{ .SrcName }}.mp4"
21 | pipeline:
22 | debug:
23 | dot:
24 | path: debug/
25 | render: png
26 | elements:
27 | - name: decodebin
28 | alias: dbin
29 |
30 | - goto: dbin
31 | - name: queue
32 | - name: audioconvert
33 | - name: audioresample
34 | - name: voaacenc
35 | - linkto: mux
36 |
37 | - goto: dbin
38 | - name: queue
39 | - name: videoconvert
40 | - name: x264enc
41 |
42 | - name: mp4mux
43 | alias: mux
44 |
--------------------------------------------------------------------------------
/controllers/pipelines/util.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package pipelines
18 |
19 | import pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
20 |
21 | func statusObservedForGeneration(status, reason string, job *pipelinesv1.Job) bool {
22 | for _, cond := range job.Status.Conditions {
23 | if cond.Type == status && cond.Reason == reason && cond.ObservedGeneration == job.GetGeneration() {
24 | return true
25 | }
26 | }
27 | return false
28 | }
29 |
--------------------------------------------------------------------------------
/config/certmanager/certificate.yaml:
--------------------------------------------------------------------------------
1 | # The following manifests contain a self-signed issuer CR and a certificate CR.
2 | # More document can be found at https://docs.cert-manager.io
3 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for
4 | # breaking changes
5 | apiVersion: cert-manager.io/v1alpha2
6 | kind: Issuer
7 | metadata:
8 | name: selfsigned-issuer
9 | namespace: system
10 | spec:
11 | selfSigned: {}
12 | ---
13 | apiVersion: cert-manager.io/v1alpha2
14 | kind: Certificate
15 | metadata:
16 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
17 | namespace: system
18 | spec:
19 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
20 | dnsNames:
21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
22 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
23 | issuerRef:
24 | kind: Issuer
25 | name: selfsigned-issuer
26 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
27 |
--------------------------------------------------------------------------------
/config/samples/pipelines_v1_splittransform.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | apiVersion: pipelines.gst.io/v1
3 | kind: SplitTransform
4 | metadata:
5 | name: video-splitter
6 | spec:
7 | globals:
8 | minio:
9 | endpoint: "minio.default.svc.cluster.local:9000"
10 | insecureNoTLS: true
11 | region: us-east-1
12 | bucket: gst-processing
13 | credentialsSecret:
14 | name: minio-credentials
15 | src:
16 | minio:
17 | key: split/
18 | video:
19 | minio:
20 | key: split_video/{{ .SrcName }}.mp4
21 | audio:
22 | minio:
23 | key: split_audio/{{ .SrcName }}.mp3
24 | pipeline:
25 | debug:
26 | dot:
27 | path: split_debug/
28 | render: png
29 | elements:
30 | - name: decodebin
31 | alias: dbin
32 |
33 | - goto: dbin
34 | - name: queue
35 | - name: audioconvert
36 | - name: audioresample
37 | - name: lamemp3enc
38 | - linkto: audio-out
39 |
40 | - goto: dbin
41 | - name: queue
42 | - name: videoconvert
43 | - name: x264enc
44 | - name: mp4mux
45 | - linkto: video-out
46 |
--------------------------------------------------------------------------------
/config/crd/kustomization.yaml:
--------------------------------------------------------------------------------
1 | # This kustomization.yaml is not intended to be run by itself,
2 | # since it depends on service name and namespace that are out of this kustomize package.
3 | # It should be run by config/default
4 | resources:
5 | - bases/pipelines.gst.io_transforms.yaml
6 | - bases/pipelines.gst.io_jobs.yaml
7 | - bases/pipelines.gst.io_splittransforms.yaml
8 | # +kubebuilder:scaffold:crdkustomizeresource
9 |
10 | patchesStrategicMerge:
11 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
12 | # patches here are for enabling the conversion webhook for each CRD
13 | #- patches/webhook_in_transforms.yaml
14 | #- patches/webhook_in_jobs.yaml
15 | #- patches/webhook_in_splittransforms.yaml
16 | # +kubebuilder:scaffold:crdkustomizewebhookpatch
17 |
18 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
19 | # patches here are for enabling the CA injection for each CRD
20 | #- patches/cainjection_in_transforms.yaml
21 | #- patches/cainjection_in_jobs.yaml
22 | #- patches/cainjection_in_splittransforms.yaml
23 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch
24 |
25 | # the following config is for teaching kustomize how to do kustomization for CRDs.
26 | configurations:
27 | - kustomizeconfig.yaml
28 |
--------------------------------------------------------------------------------
/apis/meta/v1/meta.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | // PipelineKind represents a type of pipeline.
20 | type PipelineKind string
21 |
22 | // PipelineReference is used to refer to the pipeline that holds the configuration
23 | // for a given job.
24 | type PipelineReference struct {
25 | // Name is the name of the Pipeline CR
26 | Name string `json:"name"`
27 | // Kind is the type of the Pipeline CR
28 | Kind PipelineKind `json:"kind"`
29 | }
30 |
31 | // PipelineState represents the state of a Pipeline CR.
32 | type PipelineState string
33 |
34 | const (
35 | // PipelineInSync represents that the pipeline configuration is in sync with the watchers.
36 | PipelineInSync PipelineState = "InSync"
37 | )
38 |
--------------------------------------------------------------------------------
/.github/workflows/build.yaml:
--------------------------------------------------------------------------------
1 | name: Build
2 |
3 | on:
4 | create:
5 | tags:
6 | push:
7 | branches: [ main ]
8 | pull_request:
9 | branches: [ main ]
10 |
11 | jobs:
12 |
13 | build:
14 | name: Build Images
15 | runs-on: ubuntu-latest
16 | steps:
17 |
18 | - uses: actions/checkout@v2
19 |
20 | - name: Login to container reigstry
21 | run: echo ${{ secrets.GHCR_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
22 |
23 | - name: Get image version
24 | shell: bash
25 | run: |
26 | echo ::set-output name=tag::$([[ "${GITHUB_REF##*/}" == "main" ]] && echo latest || echo ${GITHUB_REF##*/})
27 | id: version
28 |
29 | - name: Build the manager docker image
30 | run: VERSION=${{ steps.version.outputs.tag }} make docker-build
31 |
32 | - name: Build the gstreamer docker image
33 | run: VERSION=${{ steps.version.outputs.tag }} make docker-gst-build
34 |
35 | - name: Push the manager docker image
36 | run: VERSION=${{ steps.version.outputs.tag }} make docker-push
37 | if: ${{ github.event_name != 'pull_request' }}
38 |
39 | - name: Push the gstreamer docker image
40 | run: VERSION=${{ steps.version.outputs.tag }} make docker-gst-push
41 | if: ${{ github.event_name != 'pull_request' }}
42 |
--------------------------------------------------------------------------------
/doc/meta_template/members.tpl:
--------------------------------------------------------------------------------
1 | {{ define "members" }}
2 |
3 | {{ range .Members }}
4 | {{ if not (hiddenMember .)}}
5 |
6 |
7 | {{ fieldName . }}
8 |
9 | {{ if linkForType .Type }}
10 |
11 | {{ typeDisplayName .Type }}
12 |
13 | {{ else }}
14 | {{ typeDisplayName .Type }}
15 | {{ end }}
16 |
17 | |
18 |
19 | {{ if fieldEmbedded . }}
20 |
21 | (Members of {{ fieldName . }} are embedded into this type.)
22 |
23 | {{ end}}
24 |
25 | {{ if isOptionalMember .}}
26 | (Optional)
27 | {{ end }}
28 |
29 | {{ safe (renderComments .CommentLines) }}
30 |
31 | {{ if and (eq (.Type.Name.Name) "ObjectMeta") }}
32 | Refer to the Kubernetes API documentation for the fields of the
33 | metadata field.
34 | {{ end }}
35 |
36 | {{ if or (eq (fieldName .) "spec") }}
37 |
38 |
39 |
40 | {{ template "members" .Type }}
41 |
42 | {{ end }}
43 | |
44 |
45 | {{ end }}
46 | {{ end }}
47 |
48 | {{ end }}
--------------------------------------------------------------------------------
/apis/meta/v1/groupversion_info.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | // Package v1 contains API Schema definitions for the pipelines v1 API group
18 | // +kubebuilder:object:generate=true
19 | // +groupName=meta.gst.io
20 | package v1
21 |
22 | import (
23 | "k8s.io/apimachinery/pkg/runtime/schema"
24 | "sigs.k8s.io/controller-runtime/pkg/scheme"
25 | )
26 |
27 | var (
28 | // GroupVersion is group version used to register these objects
29 | GroupVersion = schema.GroupVersion{Group: "meta.gst.io", Version: "v1"}
30 |
31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme
32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
33 |
34 | // AddToScheme adds the types in this group-version to the given scheme.
35 | AddToScheme = SchemeBuilder.AddToScheme
36 | )
37 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/groupversion_info.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | // Package v1 contains API Schema definitions for the pipelines v1 API group
18 | // +kubebuilder:object:generate=true
19 | // +groupName=pipelines.gst.io
20 | package v1
21 |
22 | import (
23 | "k8s.io/apimachinery/pkg/runtime/schema"
24 | "sigs.k8s.io/controller-runtime/pkg/scheme"
25 | )
26 |
27 | var (
28 | // GroupVersion is group version used to register these objects
29 | GroupVersion = schema.GroupVersion{Group: "pipelines.gst.io", Version: "v1"}
30 |
31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme
32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
33 |
34 | // AddToScheme adds the types in this group-version to the given scheme.
35 | AddToScheme = SchemeBuilder.AddToScheme
36 | )
37 |
--------------------------------------------------------------------------------
/doc/pipelines_template/members.tpl:
--------------------------------------------------------------------------------
1 | {{ define "members" }}
2 |
3 | {{ range .Members }}
4 | {{ if not (hiddenMember .)}}
5 |
6 |
7 | {{ fieldName . }}
8 |
9 | {{ if linkForType .Type }}
10 |
11 | {{ typeDisplayName .Type }}
12 |
13 | {{ else }}
14 |
15 | {{ shortName .Type 2 }}
16 |
17 | {{ end }}
18 |
19 | |
20 |
21 | {{ if fieldEmbedded . }}
22 |
23 | (Members of {{ fieldName . }} are embedded into this type.)
24 |
25 | {{ end}}
26 |
27 | {{ if isOptionalMember .}}
28 | (Optional)
29 | {{ end }}
30 |
31 | {{ safe (renderComments .CommentLines) }}
32 |
33 | {{ if and (eq (.Type.Name.Name) "ObjectMeta") }}
34 | Refer to the Kubernetes API documentation for the fields of the
35 | metadata field.
36 | {{ end }}
37 |
38 | {{ if or (eq (fieldName .) "spec") }}
39 |
40 |
41 |
42 | {{ template "members" .Type }}
43 |
44 | {{ end }}
45 | |
46 |
47 | {{ end }}
48 | {{ end }}
49 |
50 | {{ end }}
--------------------------------------------------------------------------------
/cmd/runner/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM ubuntu:20.10 as build-base
2 |
3 | RUN mkdir -p /build \
4 | && apt-get update \
5 | && DEBIAN_FRONTEND=noninteractive apt-get install -y \
6 | golang git \
7 | libgstreamer1.0 libgstreamer1.0-dev \
8 | libgstreamer-plugins-bad1.0-dev libgstreamer-plugins-base1.0-dev
9 |
10 | COPY go.mod /build/go.mod
11 | RUN cd /build && go mod download
12 |
13 | ##
14 |
15 | FROM build-base as plugin-build
16 |
17 | COPY gst/plugins/go.mod /build/plugins/go.mod
18 | RUN cd /build/plugins && go mod download
19 |
20 | COPY gst/plugins/minio /build/plugins/minio
21 | RUN cd /build/plugins/minio && go build -o libgstminio.so -buildmode c-shared .
22 |
23 | ##
24 |
25 | FROM build-base as runner-build
26 |
27 | COPY apis /build/apis
28 | COPY pkg /build/pkg
29 | COPY cmd/runner /build/runner
30 | RUN cd /build/runner && go build -o runner .
31 |
32 | ##
33 |
34 | FROM ubuntu:20.10
35 |
36 | RUN apt-get update \
37 | && DEBIAN_FRONTEND=noninteractive apt-get install -y \
38 | libgstreamer1.0 gstreamer1.0-plugins-base \
39 | gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \
40 | gstreamer1.0-libav gstreamer1.0-tools
41 |
42 | COPY --from=plugin-build /build/plugins/minio/libgstminio.so /usr/lib/x86_64-linux-gnu/gstreamer-1.0/libgstminio.so
43 | COPY --from=runner-build /build/runner/runner /usr/local/bin/runner
44 |
45 | CMD /usr/local/bin/runner
46 |
--------------------------------------------------------------------------------
/config/scorecard/patches/olm.config.yaml:
--------------------------------------------------------------------------------
1 | - op: add
2 | path: /stages/0/tests/-
3 | value:
4 | entrypoint:
5 | - scorecard-test
6 | - olm-bundle-validation
7 | image: quay.io/operator-framework/scorecard-test:v1.3.0
8 | labels:
9 | suite: olm
10 | test: olm-bundle-validation-test
11 | - op: add
12 | path: /stages/0/tests/-
13 | value:
14 | entrypoint:
15 | - scorecard-test
16 | - olm-crds-have-validation
17 | image: quay.io/operator-framework/scorecard-test:v1.3.0
18 | labels:
19 | suite: olm
20 | test: olm-crds-have-validation-test
21 | - op: add
22 | path: /stages/0/tests/-
23 | value:
24 | entrypoint:
25 | - scorecard-test
26 | - olm-crds-have-resources
27 | image: quay.io/operator-framework/scorecard-test:v1.3.0
28 | labels:
29 | suite: olm
30 | test: olm-crds-have-resources-test
31 | - op: add
32 | path: /stages/0/tests/-
33 | value:
34 | entrypoint:
35 | - scorecard-test
36 | - olm-spec-descriptors
37 | image: quay.io/operator-framework/scorecard-test:v1.3.0
38 | labels:
39 | suite: olm
40 | test: olm-spec-descriptors-test
41 | - op: add
42 | path: /stages/0/tests/-
43 | value:
44 | entrypoint:
45 | - scorecard-test
46 | - olm-status-descriptors
47 | image: quay.io/operator-framework/scorecard-test:v1.3.0
48 | labels:
49 | suite: olm
50 | test: olm-status-descriptors-test
51 |
--------------------------------------------------------------------------------
/doc/meta_template/type.tpl:
--------------------------------------------------------------------------------
1 | {{ define "type" }}
2 |
3 |
4 | {{- .Name.Name }}
5 | {{ if eq .Kind "Alias" }}({{.Underlying}} alias){{ end -}}
6 |
7 | {{ with (typeReferences .) }}
8 |
9 | (Appears on:
10 | {{- $prev := "" -}}
11 | {{- range . -}}
12 | {{- if $prev -}}, {{ end -}}
13 | {{ $prev = . }}
14 | {{ typeDisplayName . }}
15 | {{- end -}}
16 | )
17 |
18 | {{ end }}
19 |
20 |
21 |
22 | {{ safe (renderComments .CommentLines) }}
23 |
24 |
25 | {{ if .Members }}
26 |
27 |
28 |
29 | | Field |
30 | Description |
31 |
32 |
33 |
34 | {{ if isExportedType . }}
35 |
36 |
37 | apiVersion
38 | string |
39 |
40 |
41 | {{apiGroup .}}
42 |
43 | |
44 |
45 |
46 |
47 | kind
48 | string
49 | |
50 | {{.Name.Name}} |
51 |
52 | {{ end }}
53 | {{ template "members" .}}
54 |
55 |
56 | {{ end }}
57 |
58 | {{ end }}
--------------------------------------------------------------------------------
/doc/pipelines_template/type.tpl:
--------------------------------------------------------------------------------
1 | {{ define "type" }}
2 |
3 |
4 | {{- .Name.Name }}
5 | {{ if eq .Kind "Alias" }}({{.Underlying}} alias){{ end -}}
6 |
7 | {{ with (typeReferences .) }}
8 |
9 | (Appears on:
10 | {{- $prev := "" -}}
11 | {{- range . -}}
12 | {{- if $prev -}}, {{ end -}}
13 | {{ $prev = . }}
14 | {{ typeDisplayName . }}
15 | {{- end -}}
16 | )
17 |
18 | {{ end }}
19 |
20 |
21 |
22 | {{ safe (renderComments .CommentLines) }}
23 |
24 |
25 | {{ if .Members }}
26 |
27 |
28 |
29 | | Field |
30 | Description |
31 |
32 |
33 |
34 | {{ if isExportedType . }}
35 |
36 |
37 | apiVersion
38 | string |
39 |
40 |
41 | {{apiGroup .}}
42 |
43 | |
44 |
45 |
46 |
47 | kind
48 | string
49 | |
50 | {{.Name.Name}} |
51 |
52 | {{ end }}
53 | {{ template "members" .}}
54 |
55 |
56 | {{ end }}
57 |
58 | {{ end }}
--------------------------------------------------------------------------------
/doc/meta_template/pkg.tpl:
--------------------------------------------------------------------------------
1 | {{ define "packages" }}
2 |
3 | GST Pipelines Meta CRD Reference
4 |
5 | {{ with .packages}}
6 | Packages:
7 |
14 | {{ end}}
15 |
16 | Types
17 |
18 | {{ range .packages }}
19 | {{ range (visibleTypes (sortedTypes .Types)) }}
20 | -
21 | {{ typeDisplayName . }}
22 |
23 | {{ end }}
24 | {{ end }}
25 |
26 |
27 | {{ range .packages }}
28 |
29 | {{- packageDisplayName . -}}
30 |
31 |
32 | {{ with (index .GoPackages 0 )}}
33 | {{ with .DocComments }}
34 |
35 | {{ safe (renderComments .) }}
36 |
37 | {{ end }}
38 | {{ end }}
39 |
40 | Resource Types:
41 |
42 | {{- range (visibleTypes (sortedTypes .Types)) -}}
43 | {{ if isExportedType . -}}
44 | -
45 | {{ typeDisplayName . }}
46 |
47 | {{- end }}
48 | {{- end -}}
49 |
50 |
51 | {{ range (visibleTypes (sortedTypes .Types))}}
52 | {{ template "type" . }}
53 | {{ end }}
54 |
55 | {{ end }}
56 |
57 |
58 | Generated with gen-crd-api-reference-docs
59 | {{ with .gitCommit }} on git commit {{ . }}{{end}}.
60 |
61 |
62 | {{ end }}
--------------------------------------------------------------------------------
/doc/pipelines_template/pkg.tpl:
--------------------------------------------------------------------------------
1 | {{ define "packages" }}
2 |
3 | GST Pipelines CRD Reference
4 |
5 | {{ with .packages}}
6 | Packages:
7 |
14 | {{ end}}
15 |
16 | Types
17 |
18 | {{ range .packages }}
19 | {{ range (visibleTypes (sortedTypes .Types)) }}
20 | -
21 | {{ typeDisplayName . }}
22 |
23 | {{ end }}
24 | {{ end }}
25 |
26 |
27 | {{ range .packages }}
28 |
29 | {{- packageDisplayName . -}}
30 |
31 |
32 | {{ with (index .GoPackages 0 )}}
33 | {{ with .DocComments }}
34 |
35 | {{ safe (renderComments .) }}
36 |
37 | {{ end }}
38 | {{ end }}
39 |
40 | Resource Types:
41 |
42 | {{- range (visibleTypes (sortedTypes .Types)) -}}
43 | {{ if isExportedType . -}}
44 | -
45 | {{ typeDisplayName . }}
46 |
47 | {{- end }}
48 | {{- end -}}
49 |
50 |
51 | {{ range (visibleTypes (sortedTypes .Types))}}
52 | {{ template "type" . }}
53 | {{ end }}
54 |
55 | {{ end }}
56 |
57 |
58 | Generated with gen-crd-api-reference-docs
59 | {{ with .gitCommit }} on git commit {{ . }}{{end}}.
60 |
61 |
62 | {{ end }}
--------------------------------------------------------------------------------
/config/rbac/role.yaml:
--------------------------------------------------------------------------------
1 |
2 | ---
3 | apiVersion: rbac.authorization.k8s.io/v1
4 | kind: ClusterRole
5 | metadata:
6 | creationTimestamp: null
7 | name: manager-role
8 | rules:
9 | - apiGroups:
10 | - ""
11 | resources:
12 | - secrets
13 | verbs:
14 | - get
15 | - list
16 | - watch
17 | - apiGroups:
18 | - batch
19 | resources:
20 | - jobs
21 | verbs:
22 | - create
23 | - delete
24 | - get
25 | - list
26 | - patch
27 | - update
28 | - watch
29 | - apiGroups:
30 | - coordination.k8s.io
31 | resources:
32 | - leases
33 | verbs:
34 | - create
35 | - delete
36 | - get
37 | - list
38 | - patch
39 | - update
40 | - watch
41 | - apiGroups:
42 | - pipelines.gst.io
43 | resources:
44 | - jobs
45 | verbs:
46 | - create
47 | - delete
48 | - get
49 | - list
50 | - patch
51 | - update
52 | - watch
53 | - apiGroups:
54 | - pipelines.gst.io
55 | resources:
56 | - jobs/status
57 | verbs:
58 | - get
59 | - patch
60 | - update
61 | - apiGroups:
62 | - pipelines.gst.io
63 | resources:
64 | - splittransforms
65 | verbs:
66 | - create
67 | - delete
68 | - get
69 | - list
70 | - patch
71 | - update
72 | - watch
73 | - apiGroups:
74 | - pipelines.gst.io
75 | resources:
76 | - splittransforms/status
77 | verbs:
78 | - get
79 | - patch
80 | - update
81 | - apiGroups:
82 | - pipelines.gst.io
83 | resources:
84 | - transforms
85 | verbs:
86 | - create
87 | - delete
88 | - get
89 | - list
90 | - patch
91 | - update
92 | - watch
93 | - apiGroups:
94 | - pipelines.gst.io
95 | resources:
96 | - transforms/status
97 | verbs:
98 | - get
99 | - patch
100 | - update
101 |
--------------------------------------------------------------------------------
/apis/meta/v1/object.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | // Object represents either a source or destination object for a job.
20 | type Object struct {
21 | // The actual name for the object being read or written to. In the context of
22 | // a source object this is pulled from a watch event. In the context of a destination
23 | // this is computed by the controller from the user supplied configuration.
24 | Name string `json:"name"`
25 | // The endpoint and bucket configurations for the object.
26 | Config *SourceSinkConfig `json:"config"`
27 | // The type of the stream for this object. Only applies to sinks. For a split transform
28 | // pipeline there will be an Object for each stream. Otherwise there will be a single
29 | // object with a StreamTypeAll.
30 | StreamType StreamType `json:"streamType"`
31 | }
32 |
33 | // StreamType represents a type of stream found in a source input, or designated for an output.
34 | type StreamType string
35 |
36 | const (
37 | // StreamTypeAll represents all possible output streams.
38 | StreamTypeAll StreamType = "all"
39 | // StreamTypeVideo represents a video stream.
40 | StreamTypeVideo StreamType = "video"
41 | // StreamTypeAudio represents an audio stream.
42 | StreamTypeAudio StreamType = "audio"
43 | )
44 |
--------------------------------------------------------------------------------
/pkg/types/pipelines.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package types
18 |
19 | import (
20 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22 | )
23 |
24 | // Pipeline is a generic interface implemented by the different Pipeline types.
25 | type Pipeline interface {
26 | // Extends the metav1.Object interface
27 | metav1.Object
28 |
29 | // OwnerReferences should return the owner references that can be used to apply
30 | // ownership to this pipeline.
31 | OwnerReferences() []metav1.OwnerReference
32 | // GetPipelineKind should return the type of the pipeline.
33 | GetPipelineKind() pipelinesmeta.PipelineKind
34 | // GetPipelineConfig should return the element configurations for the pipeline.
35 | GetPipelineConfig() *pipelinesmeta.PipelineConfig
36 | // GetSrcConfig should return the source configuration for the pipeline.
37 | GetSrcConfig() *pipelinesmeta.SourceSinkConfig
38 | // GetSinkConfig should return a sink configuration for the pipeline. This method
39 | // is primarily used to retrieve any required credentials when constructing a
40 | // pipeline job.
41 | GetSinkConfig() *pipelinesmeta.SourceSinkConfig
42 | // GetSinkObjects should compute the sink objects for a pipeline based on a given
43 | // source key.
44 | GetSinkObjects(srcKey string) []*pipelinesmeta.Object
45 | }
46 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/job_util.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | "context"
21 |
22 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24 | "k8s.io/apimachinery/pkg/types"
25 | "sigs.k8s.io/controller-runtime/pkg/client"
26 | )
27 |
28 | // OwnerReferences returns the OwnerReferences for this job.
29 | func (j *Job) OwnerReferences() []metav1.OwnerReference { return ownerReferences(j) }
30 |
31 | // GetPipelineKind returns the type of the pipeline.
32 | func (j *Job) GetPipelineKind() pipelinesmeta.PipelineKind { return j.Spec.PipelineReference.Kind }
33 |
34 | // GetTransformPipeline returns the transform pipeline for this job spec.
35 | func (j *Job) GetTransformPipeline(ctx context.Context, client client.Client) (*Transform, error) {
36 | nn := types.NamespacedName{
37 | Name: j.Spec.PipelineReference.Name,
38 | Namespace: j.GetNamespace(),
39 | }
40 | var pipeline Transform
41 | return &pipeline, client.Get(ctx, nn, &pipeline)
42 | }
43 |
44 | // GetSplitTransformPipeline returns the splittransform pipeline for this job spec.
45 | func (j *Job) GetSplitTransformPipeline(ctx context.Context, client client.Client) (*SplitTransform, error) {
46 | nn := types.NamespacedName{
47 | Name: j.Spec.PipelineReference.Name,
48 | Namespace: j.GetNamespace(),
49 | }
50 | var pipeline SplitTransform
51 | return &pipeline, client.Get(ctx, nn, &pipeline)
52 | }
53 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/transform_util.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21 |
22 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
23 | )
24 |
25 | // OwnerReferences returns the OwnerReferences for this pipeline to be placed on jobs.
26 | func (t *Transform) OwnerReferences() []metav1.OwnerReference { return ownerReferences(t) }
27 |
28 | // GetPipelineKind satisfies the Pipeline interface and returns the type of the pipeline.
29 | func (t *Transform) GetPipelineKind() pipelinesmeta.PipelineKind {
30 | return PipelineTransform
31 | }
32 |
33 | // GetPipelineConfig returns the PipelineConfig.
34 | func (t *Transform) GetPipelineConfig() *pipelinesmeta.PipelineConfig { return t.Spec.Pipeline }
35 |
36 | // GetSrcConfig will return the src config for this pipeline merged with the globals.
37 | func (t *Transform) GetSrcConfig() *pipelinesmeta.SourceSinkConfig {
38 | return mergeConfigs(t.Spec.Globals, t.Spec.Src)
39 | }
40 |
41 | // GetSinkConfig will return the sink config for this pipeline merged with the globals.
42 | func (t *Transform) GetSinkConfig() *pipelinesmeta.SourceSinkConfig {
43 | return mergeConfigs(t.Spec.Globals, t.Spec.Sink)
44 | }
45 |
46 | // GetSinkObjects returns the sink objects for a pipeline.
47 | func (t *Transform) GetSinkObjects(srcKey string) []*pipelinesmeta.Object {
48 | return []*pipelinesmeta.Object{
49 | {
50 | Name: t.GetSinkConfig().MinIO.GetDestinationKey(srcKey), // TODO
51 | Config: t.GetSinkConfig(),
52 | StreamType: pipelinesmeta.StreamTypeAll,
53 | },
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/hack/update-api-docs.sh:
--------------------------------------------------------------------------------
1 | set -o errexit
2 | set -o nounset
3 | set -o pipefail
4 |
5 | REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}"
6 |
7 | if ! command -v go &>/dev/null; then
8 | echo "Ensure go command is installed"
9 | exit 1
10 | fi
11 |
12 | tmpdir="$(mktemp -d)"
13 | cleanup() {
14 | export GO111MODULE="auto"
15 | echo "+++ Cleaning up temporary GOPATH"
16 | go clean -modcache
17 | rm -rf "${tmpdir}"
18 | }
19 | trap cleanup EXIT
20 |
21 | # Create fake GOPATH
22 | echo "+++ Creating temporary GOPATH"
23 | export GOPATH="${tmpdir}/go"
24 | echo "+++ Using temporary GOPATH ${GOPATH}"
25 | export GO111MODULE="on"
26 | GOROOT="$(go env GOROOT)"
27 | export GOROOT
28 | mkdir -p "${GOPATH}/src/github.com/tinyzimmer"
29 | gitdir="${GOPATH}/src/github.com/tinyzimmer/kvdi"
30 | cp -r "${REPO_ROOT}" "${gitdir}"
31 | cd "$gitdir"
32 |
33 | "${REPO_ROOT}/bin/refdocs" \
34 | --config "${REPO_ROOT}/doc/refdocs.json" \
35 | --template-dir "${REPO_ROOT}/doc/pipelines_template" \
36 | --api-dir "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1" \
37 | --out-file "${GOPATH}/out.html"
38 |
39 | pandoc --from html --to markdown_strict "${GOPATH}/out.html" -o "${REPO_ROOT}/doc/pipelines.md"
40 | sed -i 's/#pipelines\.gst\.io\/v1\./#/g' "${REPO_ROOT}/doc/pipelines.md"
41 | sed -i 's/#%23pipelines\.gst\.io%2fv1\./#/g' "${REPO_ROOT}/doc/pipelines.md"
42 | sed -i 's:#\*github\.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1\.:#:g' "${REPO_ROOT}/doc/pipelines.md"
43 | sed -i 's:meta%2fv1\.::g' "${REPO_ROOT}/doc/pipelines.md"
44 | sed -i 's:meta/v1:metav1:g' "${REPO_ROOT}/doc/pipelines.md"
45 |
46 | "${REPO_ROOT}/bin/refdocs" \
47 | --config "${REPO_ROOT}/doc/refdocs.json" \
48 | --template-dir "${REPO_ROOT}/doc/meta_template" \
49 | --api-dir "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1" \
50 | --out-file "${GOPATH}/out.html"
51 |
52 | pandoc --from html --to markdown_strict "${GOPATH}/out.html" -o "${REPO_ROOT}/doc/meta.md"
53 | sed -i 's/#meta\.gst\.io\/v1\./#/g' "${REPO_ROOT}/doc/meta.md"
54 | sed -i 's/#%23meta\.gst\.io%2fv1\./#/g' "${REPO_ROOT}/doc/meta.md"
55 | sed -i 's:#\*github\.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1\.:#:g' "${REPO_ROOT}/doc/meta.md"
56 |
57 | echo "Generated reference documentation"
--------------------------------------------------------------------------------
/gst/plugins/minio/plugin.go:
--------------------------------------------------------------------------------
1 | // This example demonstrates a src element that reads from objects in a minio bucket.
2 | // Since minio implements the S3 API this plugin could also be used for S3 buckets by
3 | // setting the correct endpoints and credentials.
4 | //
5 | // By default this plugin will use the credentials set in the environment at MINIO_ACCESS_KEY_ID
6 | // and MINIO_SECRET_ACCESS_KEY however these can also be set on the element directly.
7 | //
8 | //
9 | // In order to build the plugin for use by GStreamer, you can do the following:
10 | //
11 | // $ go build -o libgstminio.so -buildmode c-shared .
12 | //
13 | package main
14 |
15 | import "C"
16 |
17 | import (
18 | "unsafe"
19 |
20 | "github.com/tinyzimmer/go-gst/gst"
21 | "github.com/tinyzimmer/go-gst/gst/base"
22 | )
23 |
24 | // The metadata for this plugin
25 | var pluginMeta = &gst.PluginMetadata{
26 | MajorVersion: gst.VersionMajor,
27 | MinorVersion: gst.VersionMinor,
28 | Name: "minio-plugins",
29 | Description: "GStreamer plugins for reading and writing from Minio",
30 | Version: "v0.0.1",
31 | License: gst.LicenseLGPL,
32 | Source: "gst-pipeline-operator",
33 | Package: "plugins",
34 | Origin: "https://github.com/tinyzimmer/gst-pipeline-operator",
35 | ReleaseDate: "2021-01-12",
36 | // The init function is called to register elements provided by the plugin.
37 | Init: func(plugin *gst.Plugin) bool {
38 | if ok := gst.RegisterElement(
39 | plugin,
40 | // The name of the element
41 | "miniosrc",
42 | // The rank of the element
43 | gst.RankNone,
44 | // The GoElement implementation for the element
45 | &minioSrc{},
46 | // The base subclass this element extends
47 | base.ExtendsBaseSrc,
48 | ); !ok {
49 | return ok
50 | }
51 |
52 | if ok := gst.RegisterElement(
53 | plugin,
54 | // The name of the element
55 | "miniosink",
56 | // The rank of the element
57 | gst.RankNone,
58 | // The GoElement implementation for the element
59 | &minioSink{},
60 | // The base subclass this element extends
61 | base.ExtendsBaseSink,
62 | ); !ok {
63 | return ok
64 | }
65 |
66 | return true
67 | },
68 | }
69 |
70 | func main() {}
71 |
72 | //export gst_plugin_minio_get_desc
73 | func gst_plugin_minio_get_desc() unsafe.Pointer { return pluginMeta.Export() }
74 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/common.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | "crypto/md5"
21 | "encoding/json"
22 | "fmt"
23 | "io"
24 |
25 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
26 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/types"
27 |
28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29 | runtime "k8s.io/apimachinery/pkg/runtime"
30 | )
31 |
32 | func mergeConfigs(into, from *pipelinesmeta.SourceSinkConfig) *pipelinesmeta.SourceSinkConfig {
33 | if into == nil {
34 | return from
35 | }
36 | cfgBody, err := json.Marshal(from)
37 | if err != nil {
38 | fmt.Println("Failed to marshal global config to json:", err)
39 | return from
40 | }
41 | intoCopy := into.DeepCopy()
42 | if err := json.Unmarshal(cfgBody, intoCopy); err != nil {
43 | fmt.Println("Error unmarshaling configuration on top of globals:", err)
44 | return from
45 | }
46 | return intoCopy
47 | }
48 |
49 | // GetJobLabels returns the labels to apply to a new job issued from this pipeline.
50 | func GetJobLabels(pipeline types.Pipeline, key string) map[string]string {
51 | h := md5.New()
52 | io.WriteString(h, key)
53 | labels := map[string]string{
54 | pipelinesmeta.JobPipelineLabel: pipeline.GetName(),
55 | pipelinesmeta.JobPipelineKindLabel: string(pipeline.GetPipelineKind()),
56 | pipelinesmeta.JobObjectLabel: fmt.Sprintf("%x", h.Sum(nil)),
57 | }
58 | src := pipeline.GetSrcConfig()
59 | if src != nil && src.MinIO != nil {
60 | labels[pipelinesmeta.JobBucketLabel] = src.MinIO.GetBucket()
61 | }
62 | return labels
63 | }
64 |
65 | func ownerReferences(obj runtime.Object) []metav1.OwnerReference {
66 | return []metav1.OwnerReference{*metav1.NewControllerRef(obj.(metav1.Object), obj.GetObjectKind().GroupVersionKind())}
67 | }
68 |
--------------------------------------------------------------------------------
/apis/meta/v1/launch_config.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | // GstLaunchConfig is a slice of ElementConfigs that contain internal fields used for
20 | // dynamic linking.
21 | type GstLaunchConfig []*GstElementConfig
22 |
23 | // GetElements returns the elements for this pipeline.
24 | func (p *PipelineConfig) GetElements() GstLaunchConfig {
25 | out := make(GstLaunchConfig, len(p.Elements))
26 | for idx, elem := range p.Elements {
27 | out[idx] = &GstElementConfig{ElementConfig: elem}
28 | }
29 | return out
30 | }
31 |
32 | // GetByAlias returns the configuration for the element at the given alias
33 | func (g GstLaunchConfig) GetByAlias(alias string) *GstElementConfig {
34 | for _, elem := range g {
35 | if elem.Alias == alias {
36 | return elem
37 | }
38 | }
39 | return nil
40 | }
41 |
42 | // GstElementConfig is an extension of the ElementConfig struct providing
43 | // private fields for internal tracking while building a dynamic pipeline.
44 | type GstElementConfig struct {
45 | *ElementConfig
46 | pipelineName string
47 | peers []*GstElementConfig
48 | }
49 |
50 | // SetPipelineName sets the name that was assigned to this element by the pipeline for
51 | // later reference.
52 | func (e *GstElementConfig) SetPipelineName(name string) { e.pipelineName = name }
53 |
54 | // GetPipelineName returns the name that was assigned to this element by the pipeline.
55 | func (e *GstElementConfig) GetPipelineName() string { return e.pipelineName }
56 |
57 | // AddPeer will add a peer to this configuration. It is used for determining which
58 | // sink pads to pair with dynamically added src pads.
59 | func (e *GstElementConfig) AddPeer(peer *GstElementConfig) {
60 | if e.peers == nil {
61 | e.peers = make([]*GstElementConfig, 0)
62 | }
63 | e.peers = append(e.peers, peer)
64 | }
65 |
66 | // GetPeers returns the peers registered for this element.
67 | func (e *GstElementConfig) GetPeers() []*GstElementConfig { return e.peers }
68 |
--------------------------------------------------------------------------------
/apis/meta/v1/elements.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | // ElementConfig represents the configuration of a single element in a transform pipeline.
20 | type ElementConfig struct {
21 | // The name of the element. See the GStreamer plugin documentation for a comprehensive
22 | // list of all the plugins available. Custom pipeline images can also be used that are
23 | // prebaked with additional plugins.
24 | Name string `json:"name,omitempty"`
25 | // Applies an alias to this element in the pipeline configuration. This allows you to specify an
26 | // element block with this value as the name and have it act as a "goto" or "linkto" while building
27 | // the pipeline. Note that the aliases "video-out" and "audio-out" are reserved for internal use.
28 | Alias string `json:"alias,omitempty"`
29 | // The alias to an element to treat as this configuration. Useful for directing the output of elements
30 | // with multiple src pads, such as decodebin.
31 | GoTo string `json:"goto,omitempty"`
32 | // The alias to an element to link the previous element's sink pad to. Useful for directing the branches of
33 | // a multi-stream pipeline to a muxer. A linkto almost always needs to be followed by a goto, except when
34 | // the element being linked to is next in the pipeline, in which case you can omit the linkto entirely.
35 | LinkTo string `json:"linkto,omitempty"`
36 | // Optional properties to apply to this element. To not piss off the CRD generator values are
37 | // declared as a string, but almost anything that can be passed to gst-launch-1.0 will work.
38 | // Caps will be parsed from their string representation.
39 | Properties map[string]string `json:"properties,omitempty"`
40 | }
41 |
42 | // LinkToVideoOut is used during split pipelines to designate the src of a video sink
43 | const LinkToVideoOut = "video-out"
44 |
45 | // LinkToAudioOut is used during split pipelines to designate the src of an audio sink
46 | const LinkToAudioOut = "audio-out"
47 |
--------------------------------------------------------------------------------
/config/default/kustomization.yaml:
--------------------------------------------------------------------------------
1 | # Adds namespace to all resources.
2 | namespace: gst-system
3 |
4 | # Value of this field is prepended to the
5 | # names of all resources, e.g. a deployment named
6 | # "wordpress" becomes "alices-wordpress".
7 | # Note that it should also match with the prefix (text before '-') of the namespace
8 | # field above.
9 | namePrefix: gst-
10 |
11 | # Labels to add to all resources and selectors.
12 | #commonLabels:
13 | # someName: someValue
14 |
15 | bases:
16 | - ../crd
17 | - ../rbac
18 | - ../manager
19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
20 | # crd/kustomization.yaml
21 | #- ../webhook
22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
23 | #- ../certmanager
24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
25 | #- ../prometheus
26 |
27 | patchesStrategicMerge:
28 | # Protect the /metrics endpoint by putting it behind auth.
29 | # If you want your controller-manager to expose the /metrics
30 | # endpoint w/o any authn/z, please comment the following line.
31 | - manager_auth_proxy_patch.yaml
32 |
33 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
34 | # crd/kustomization.yaml
35 | #- manager_webhook_patch.yaml
36 |
37 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
38 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
39 | # 'CERTMANAGER' needs to be enabled to use ca injection
40 | #- webhookcainjection_patch.yaml
41 |
42 | # the following config is for teaching kustomize how to do var substitution
43 | vars:
44 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
45 | #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
46 | # objref:
47 | # kind: Certificate
48 | # group: cert-manager.io
49 | # version: v1alpha2
50 | # name: serving-cert # this name should match the one in certificate.yaml
51 | # fieldref:
52 | # fieldpath: metadata.namespace
53 | #- name: CERTIFICATE_NAME
54 | # objref:
55 | # kind: Certificate
56 | # group: cert-manager.io
57 | # version: v1alpha2
58 | # name: serving-cert # this name should match the one in certificate.yaml
59 | #- name: SERVICE_NAMESPACE # namespace of the service
60 | # objref:
61 | # kind: Service
62 | # version: v1
63 | # name: webhook-service
64 | # fieldref:
65 | # fieldpath: metadata.namespace
66 | #- name: SERVICE_NAME
67 | # objref:
68 | # kind: Service
69 | # version: v1
70 | # name: webhook-service
71 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/transform_types.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22 | )
23 |
24 | const (
25 | // PipelineTransform represents a transform pipeline
26 | PipelineTransform pipelinesmeta.PipelineKind = "Transform"
27 | )
28 |
29 | // TransformSpec defines the desired state of Transform
30 | type TransformSpec struct {
31 | // Global configurations to apply when omitted from the src or sink configurations.
32 | Globals *pipelinesmeta.SourceSinkConfig `json:"globals,omitempty"`
33 | // Configurations for src object to the pipeline.
34 | Src *pipelinesmeta.SourceSinkConfig `json:"src"`
35 | // Configurations for sink objects from the pipeline.
36 | Sink *pipelinesmeta.SourceSinkConfig `json:"sink"`
37 | // The configuration for the processing pipeline
38 | Pipeline *pipelinesmeta.PipelineConfig `json:"pipeline"`
39 | }
40 |
41 | // TransformStatus defines the observed state of Transform
42 | type TransformStatus struct {
43 | // Conditions represent the latest available observations of a transform's state
44 | Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
45 | }
46 |
47 | // +kubebuilder:object:root=true
48 | // +kubebuilder:subresource:status
49 | // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=`.metadata.creationTimestamp`
50 | // +kubebuilder:printcolumn:name="Status",type="string",priority=1,JSONPath=`.status.conditions[-1].message`
51 |
52 | // Transform is the Schema for the transforms API
53 | type Transform struct {
54 | metav1.TypeMeta `json:",inline"`
55 | metav1.ObjectMeta `json:"metadata,omitempty"`
56 |
57 | Spec TransformSpec `json:"spec,omitempty"`
58 | Status TransformStatus `json:"status,omitempty"`
59 | }
60 |
61 | // +kubebuilder:object:root=true
62 |
63 | // TransformList contains a list of Transform
64 | type TransformList struct {
65 | metav1.TypeMeta `json:",inline"`
66 | metav1.ListMeta `json:"metadata,omitempty"`
67 | Items []Transform `json:"items"`
68 | }
69 |
70 | func init() {
71 | SchemeBuilder.Register(&Transform{}, &TransformList{})
72 | }
73 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/splittransform_util.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
21 |
22 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
23 | )
24 |
25 | // OwnerReferences returns the OwnerReferences for this pipeline to be placed on jobs.
26 | func (t *SplitTransform) OwnerReferences() []metav1.OwnerReference { return ownerReferences(t) }
27 |
28 | // GetPipelineKind satisfies the Pipeline interface and returns the type of the pipeline.
29 | func (t *SplitTransform) GetPipelineKind() pipelinesmeta.PipelineKind {
30 | return PipelineSplitTransform
31 | }
32 |
33 | // GetPipelineConfig returns the PipelineConfig.
34 | func (t *SplitTransform) GetPipelineConfig() *pipelinesmeta.PipelineConfig { return t.Spec.Pipeline }
35 |
36 | // GetSrcConfig will return the src config for this pipeline merged with the globals.
37 | func (t *SplitTransform) GetSrcConfig() *pipelinesmeta.SourceSinkConfig {
38 | return mergeConfigs(t.Spec.Globals, t.Spec.Src)
39 | }
40 |
41 | // GetSinkConfig will return the first non-dropped sink config found in this pipeline.
42 | func (t *SplitTransform) GetSinkConfig() *pipelinesmeta.SourceSinkConfig {
43 | if t.Spec.Video != nil {
44 | return mergeConfigs(t.Spec.Globals, t.Spec.Video)
45 | }
46 | if t.Spec.Audio != nil {
47 | return mergeConfigs(t.Spec.Globals, t.Spec.Audio)
48 | }
49 | return nil
50 | }
51 |
52 | // GetSinkObjects returns the sink objects for a pipeline.
53 | func (t *SplitTransform) GetSinkObjects(srcKey string) []*pipelinesmeta.Object {
54 | objs := make([]*pipelinesmeta.Object, 0)
55 | if t.Spec.Video != nil {
56 | videoCfg := mergeConfigs(t.Spec.Globals, t.Spec.Video)
57 | objs = append(objs, &pipelinesmeta.Object{
58 | Name: videoCfg.MinIO.GetDestinationKey(srcKey), // TODO
59 | Config: videoCfg,
60 | StreamType: pipelinesmeta.StreamTypeVideo,
61 | })
62 | }
63 | if t.Spec.Audio != nil {
64 | audioCfg := mergeConfigs(t.Spec.Globals, t.Spec.Audio)
65 | objs = append(objs, &pipelinesmeta.Object{
66 | Name: audioCfg.MinIO.GetDestinationKey(srcKey), // TODO
67 | Config: audioCfg,
68 | StreamType: pipelinesmeta.StreamTypeAudio,
69 | })
70 | }
71 | return objs
72 | }
73 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/job_types.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22 | )
23 |
24 | // JobState represents the state of a pipeline job.
25 | type JobState string
26 |
27 | const (
28 | // JobPending means the pipeline is waiting to be started.
29 | JobPending JobState = "Pending"
30 | // JobInProgress means the pipeline is currently running.
31 | JobInProgress JobState = "InProgress"
32 | // JobFinished means the pipeline completed without error.
33 | JobFinished JobState = "Finished"
34 | // JobFailed means the pipeline completed with an error.
35 | JobFailed JobState = "Failed"
36 | )
37 |
38 | // JobSpec defines the desired state of Job
39 | type JobSpec struct {
40 | // A reference to the pipeline for this job's configuration.
41 | PipelineReference pipelinesmeta.PipelineReference `json:"pipelineRef"`
42 | // The source object for the pipeline.
43 | Source *pipelinesmeta.Object `json:"src"`
44 | // The output objects for the pipeline.
45 | Sinks []*pipelinesmeta.Object `json:"sinks"`
46 | }
47 |
48 | // JobStatus defines the observed state of Job
49 | type JobStatus struct {
50 | // Conditions represent the latest available observations of a job's state
51 | Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
52 | }
53 |
54 | // +kubebuilder:object:root=true
55 | // +kubebuilder:subresource:status
56 | // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=`.metadata.creationTimestamp`
57 | // +kubebuilder:printcolumn:name="Src",type="string",JSONPath=`.spec.src.name`
58 | // +kubebuilder:printcolumn:name="Sinks",type="string",JSONPath=`.spec.sinks[*].name`
59 | // +kubebuilder:printcolumn:name="Status",type="string",priority=1,JSONPath=`.status.conditions[-1].message`
60 |
61 | // Job is the Schema for the jobs API
62 | type Job struct {
63 | metav1.TypeMeta `json:",inline"`
64 | metav1.ObjectMeta `json:"metadata,omitempty"`
65 |
66 | Spec JobSpec `json:"spec,omitempty"`
67 | Status JobStatus `json:"status,omitempty"`
68 | }
69 |
70 | // +kubebuilder:object:root=true
71 |
72 | // JobList contains a list of Job
73 | type JobList struct {
74 | metav1.TypeMeta `json:",inline"`
75 | metav1.ListMeta `json:"metadata,omitempty"`
76 | Items []Job `json:"items"`
77 | }
78 |
79 | func init() {
80 | SchemeBuilder.Register(&Job{}, &JobList{})
81 | }
82 |
--------------------------------------------------------------------------------
/controllers/pipelines/job_controller.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package pipelines
18 |
19 | import (
20 | "context"
21 | "fmt"
22 |
23 | "github.com/go-logr/logr"
24 | batchv1 "k8s.io/api/batch/v1"
25 | "k8s.io/apimachinery/pkg/runtime"
26 | ctrl "sigs.k8s.io/controller-runtime"
27 | "sigs.k8s.io/controller-runtime/pkg/client"
28 |
29 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
30 | pipelinetypes "github.com/tinyzimmer/gst-pipeline-operator/pkg/types"
31 | )
32 |
33 | // JobReconciler reconciles a Job object
34 | type JobReconciler struct {
35 | client.Client
36 | Log logr.Logger
37 | Scheme *runtime.Scheme
38 | }
39 |
40 | // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
41 | // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
42 | // +kubebuilder:rbac:groups=pipelines.gst.io,resources=jobs,verbs=get;list;watch;create;update;patch;delete
43 | // +kubebuilder:rbac:groups=pipelines.gst.io,resources=jobs/status,verbs=get;update;patch
44 |
45 | // Reconcile reconciles a Job
46 | func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
47 | reqLogger := r.Log.WithValues("job", req.NamespacedName)
48 |
49 | job := &pipelinesv1.Job{}
50 | err := r.Client.Get(ctx, req.NamespacedName, job)
51 | if err != nil {
52 | if client.IgnoreNotFound(err) == nil {
53 | return ctrl.Result{}, nil
54 | }
55 | return ctrl.Result{}, err
56 | }
57 |
58 | var pipeline pipelinetypes.Pipeline
59 | switch job.GetPipelineKind() {
60 | case pipelinesv1.PipelineTransform:
61 | reqLogger.Info("Fetching Transform pipeline from Job")
62 | pipeline, err = job.GetTransformPipeline(ctx, r.Client)
63 | case pipelinesv1.PipelineSplitTransform:
64 | pipeline, err = job.GetSplitTransformPipeline(ctx, r.Client)
65 | default:
66 | err = fmt.Errorf("Unknown pipeline kind: %s", string(job.GetPipelineKind()))
67 | }
68 | if err != nil {
69 | return ctrl.Result{}, err
70 | }
71 |
72 | batchjob, err := newPipelineJob(job, pipeline)
73 | if err != nil {
74 | return ctrl.Result{}, err
75 | }
76 |
77 | return ctrl.Result{}, reconcileJob(ctx, reqLogger, r.Client, job, batchjob)
78 | }
79 |
80 | // SetupWithManager adds the Job reconciler to the given manager.
81 | func (r *JobReconciler) SetupWithManager(mgr ctrl.Manager) error {
82 | return ctrl.NewControllerManagedBy(mgr).
83 | For(&pipelinesv1.Job{}).
84 | Owns(&batchv1.Job{}).
85 | Complete(r)
86 | }
87 |
--------------------------------------------------------------------------------
/testbin/setup-envtest.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | # Copyright 2020 The Kubernetes Authors.
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 |
17 | set -o errexit
18 | set -o pipefail
19 |
20 | # Turn colors in this script off by setting the NO_COLOR variable in your
21 | # environment to any value:
22 | #
23 | # $ NO_COLOR=1 test.sh
24 | NO_COLOR=${NO_COLOR:-""}
25 | if [ -z "$NO_COLOR" ]; then
26 | header=$'\e[1;33m'
27 | reset=$'\e[0m'
28 | else
29 | header=''
30 | reset=''
31 | fi
32 |
33 | function header_text {
34 | echo "$header$*$reset"
35 | }
36 |
37 | function setup_envtest_env {
38 | header_text "setting up env vars"
39 |
40 | # Setup env vars
41 | KUBEBUILDER_ASSETS=${KUBEBUILDER_ASSETS:-""}
42 | if [[ -z "${KUBEBUILDER_ASSETS}" ]]; then
43 | export KUBEBUILDER_ASSETS=$1/bin
44 | fi
45 | }
46 |
47 | # fetch k8s API gen tools and make it available under envtest_root_dir/bin.
48 | #
49 | # Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable
50 | # in your environment to any value:
51 | #
52 | # $ SKIP_FETCH_TOOLS=1 ./check-everything.sh
53 | #
54 | # If you skip fetching tools, this script will use the tools already on your
55 | # machine.
56 | function fetch_envtest_tools {
57 | SKIP_FETCH_TOOLS=${SKIP_FETCH_TOOLS:-""}
58 | if [ -n "$SKIP_FETCH_TOOLS" ]; then
59 | return 0
60 | fi
61 |
62 | tmp_root=/tmp
63 | envtest_root_dir=$tmp_root/envtest
64 |
65 | k8s_version="${ENVTEST_K8S_VERSION:-1.16.4}"
66 | goarch="$(go env GOARCH)"
67 | goos="$(go env GOOS)"
68 |
69 | if [[ "$goos" != "linux" && "$goos" != "darwin" ]]; then
70 | echo "OS '$goos' not supported. Aborting." >&2
71 | return 1
72 | fi
73 |
74 | local dest_dir="${1}"
75 |
76 | # use the pre-existing version in the temporary folder if it matches our k8s version
77 | if [[ -x "${dest_dir}/bin/kube-apiserver" ]]; then
78 | version=$("${dest_dir}"/bin/kube-apiserver --version)
79 | if [[ $version == *"${k8s_version}"* ]]; then
80 | header_text "Using cached envtest tools from ${dest_dir}"
81 | return 0
82 | fi
83 | fi
84 |
85 | header_text "fetching envtest tools@${k8s_version} (into '${dest_dir}')"
86 | envtest_tools_archive_name="kubebuilder-tools-$k8s_version-$goos-$goarch.tar.gz"
87 | envtest_tools_download_url="https://storage.googleapis.com/kubebuilder-tools/$envtest_tools_archive_name"
88 |
89 | envtest_tools_archive_path="$tmp_root/$envtest_tools_archive_name"
90 | if [ ! -f $envtest_tools_archive_path ]; then
91 | curl -sL ${envtest_tools_download_url} -o "$envtest_tools_archive_path"
92 | fi
93 |
94 | mkdir -p "${dest_dir}"
95 | tar -C "${dest_dir}" --strip-components=1 -zvxf "$envtest_tools_archive_path"
96 | }
97 |
--------------------------------------------------------------------------------
/apis/meta/v1/constants.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | // Default Values
20 | const (
21 | // DefaultRegion if none is provided for any configurations
22 | DefaultRegion = "us-east-1"
23 | // AccessKeyIDKey is the key in secrets where the access key ID is stored.
24 | AccessKeyIDKey = "access-key-id"
25 | // SecretAccessKeyKey is the key in secrets where the secret access key is stored.
26 | SecretAccessKeyKey = "secret-access-key"
27 | // DefaultGSTDebug is the default GST_DEBUG value set in the environment for pipelines.
28 | DefaultGSTDebug = "4"
29 | // DefaultDotInterval is the default interval to query a pipeline for graphs.
30 | DefaultDotInterval = 3
31 | )
32 |
33 | // Annotations
34 | const (
35 | JobCreationSpecAnnotation = "pipelines.gst.io/creation-spec"
36 | )
37 |
38 | // Labels
39 | const (
40 | // JobPipelineLabel is the label on a job to denote the Transform pipeline that initiated
41 | // it.
42 | JobPipelineLabel = "pipelines.gst.io/pipeline"
43 | // JobPipelineKindLabel is the label where the type of the pipeline is stored.
44 | JobPipelineKindLabel = "pipelines.gst.io/kind"
45 | // JobObjectLabel is the label on a job to denote the object key it is processing.
46 | JobObjectLabel = "pipelines.gst.io/object"
47 | // JobBucketLabel is the label on a job to denote the bucket where the object is that
48 | // is being processed.
49 | JobBucketLabel = "pipelines.gst.io/bucket"
50 | )
51 |
52 | // Environment Variables
53 | const (
54 | // The environment variable where the access key id for the src bucket is stored.
55 | MinIOSrcAccessKeyIDEnvVar = "MINIO_SRC_ACCESS_KEY_ID"
56 | // The environment variable where the secret access key for the src bucket is stored.
57 | MinIOSrcSecretAccessKeyEnvVar = "MINIO_SRC_SECRET_ACCESS_KEY"
58 | // The environment variable where the access key id for the sink bucket is stored.
59 | MinIOSinkAccessKeyIDEnvVar = "MINIO_SINK_ACCESS_KEY_ID"
60 | // The environment variable where the secret access key for the sink bucket is stored.
61 | MinIOSinkSecretAccessKeyEnvVar = "MINIO_SINK_SECRET_ACCESS_KEY"
62 | // The environment variable where the pipeline config is serialized and set.
63 | JobPipelineConfigEnvVar = "GST_PIPELINE_CONFIG"
64 | // The environment variable where the source object is serialized and set.
65 | JobSrcObjectsEnvVar = "GST_PIPELINE_SRC_OBJECT"
66 | // The environment variable where the sink objects are serialized and set.
67 | JobSinkObjectsEnvVar = "GST_PIPELINE_SINK_OBJECTS"
68 | // The environment variable where the name of the pipeline being watched is set for watcher
69 | // processes.
70 | WatcherPipelineNameEnvVar = "GST_WATCH_PIPELINE_NAME"
71 | // The environment variable where the pipeline kind is set for the watcher processes.
72 | WatcherPipelineKindEnvVar = "GST_WATCH_PIPELINE_KIND"
73 | )
74 |
--------------------------------------------------------------------------------
/apis/pipelines/v1/splittransform_types.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
22 | )
23 |
24 | const (
25 | // PipelineSplitTransform represents a splittransform pipeline
26 | PipelineSplitTransform pipelinesmeta.PipelineKind = "SplitTransform"
27 | )
28 |
29 | // SplitTransformSpec defines the desired state of SplitTransform. Note that due to current
30 | // implementation, the various streams can be directed to different buckets, but they have to
31 | // be buckets accessible via the same MinIO/S3 server(s).
32 | type SplitTransformSpec struct {
33 | // Global configurations to apply when omitted from the src or sink configurations.
34 | Globals *pipelinesmeta.SourceSinkConfig `json:"globals,omitempty"`
35 | // Configurations for src object to the pipeline.
36 | Src *pipelinesmeta.SourceSinkConfig `json:"src"`
37 | // Configurations for video stream outputs. The linkto field in the pipeline config
38 | // should be present with the value `video-out` to direct an element to this output.
39 | Video *pipelinesmeta.SourceSinkConfig `json:"video,omitempty"`
40 | // Configurations for audio stream outputs. The linkto field in the pipeline config
41 | // should be present with the value `audio-out` to direct an element to this output.
42 | Audio *pipelinesmeta.SourceSinkConfig `json:"audio,omitempty"`
43 | // The configuration for the processing pipeline
44 | Pipeline *pipelinesmeta.PipelineConfig `json:"pipeline"`
45 | }
46 |
47 | // SplitTransformStatus defines the observed state of SplitTransform
48 | type SplitTransformStatus struct {
49 | // Conditions represent the latest available observations of a splittransform's state
50 | Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
51 | }
52 |
53 | // +kubebuilder:object:root=true
54 | // +kubebuilder:subresource:status
55 | // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=`.metadata.creationTimestamp`
56 | // +kubebuilder:printcolumn:name="Status",type="string",priority=1,JSONPath=`.status.conditions[-1].message`
57 |
58 | // SplitTransform is the Schema for the splittransforms API
59 | // +kubebuilder:resource:path="splittransforms",scope=Namespaced
60 | type SplitTransform struct {
61 | metav1.TypeMeta `json:",inline"`
62 | metav1.ObjectMeta `json:"metadata,omitempty"`
63 |
64 | Spec SplitTransformSpec `json:"spec,omitempty"`
65 | Status SplitTransformStatus `json:"status,omitempty"`
66 | }
67 |
68 | // +kubebuilder:object:root=true
69 |
70 | // SplitTransformList contains a list of SplitTransform
71 | type SplitTransformList struct {
72 | metav1.TypeMeta `json:",inline"`
73 | metav1.ListMeta `json:"metadata,omitempty"`
74 | Items []SplitTransform `json:"items"`
75 | }
76 |
77 | func init() {
78 | SchemeBuilder.Register(&SplitTransform{}, &SplitTransformList{})
79 | }
80 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package main
18 |
19 | import (
20 | "flag"
21 | "os"
22 |
23 | "k8s.io/apimachinery/pkg/runtime"
24 | utilruntime "k8s.io/apimachinery/pkg/util/runtime"
25 | clientgoscheme "k8s.io/client-go/kubernetes/scheme"
26 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
27 | ctrl "sigs.k8s.io/controller-runtime"
28 | "sigs.k8s.io/controller-runtime/pkg/log/zap"
29 |
30 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
31 | controllers "github.com/tinyzimmer/gst-pipeline-operator/controllers/pipelines"
32 | pipelinescontroller "github.com/tinyzimmer/gst-pipeline-operator/controllers/pipelines"
33 | // +kubebuilder:scaffold:imports
34 | )
35 |
36 | var (
37 | scheme = runtime.NewScheme()
38 | setupLog = ctrl.Log.WithName("setup")
39 | )
40 |
41 | func init() {
42 | utilruntime.Must(clientgoscheme.AddToScheme(scheme))
43 |
44 | utilruntime.Must(pipelinesv1.AddToScheme(scheme))
45 | // +kubebuilder:scaffold:scheme
46 | }
47 |
48 | func main() {
49 | var metricsAddr string
50 | var enableLeaderElection bool
51 | flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
52 | flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
53 | "Enable leader election for controller manager. "+
54 | "Enabling this will ensure there is only one active controller manager.")
55 | flag.Parse()
56 |
57 | ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
58 |
59 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
60 | Scheme: scheme,
61 | MetricsBindAddress: metricsAddr,
62 | Port: 9443,
63 | LeaderElection: enableLeaderElection,
64 | LeaderElectionID: "59ec2575.gst.io",
65 | })
66 | if err != nil {
67 | setupLog.Error(err, "unable to start manager")
68 | os.Exit(1)
69 | }
70 |
71 | if err = (&controllers.TransformReconciler{
72 | Client: mgr.GetClient(),
73 | Log: ctrl.Log.WithName("controllers").WithName("Transform"),
74 | Scheme: mgr.GetScheme(),
75 | }).SetupWithManager(mgr); err != nil {
76 | setupLog.Error(err, "unable to create controller", "controller", "Transform")
77 | os.Exit(1)
78 | }
79 | if err = (&pipelinescontroller.JobReconciler{
80 | Client: mgr.GetClient(),
81 | Log: ctrl.Log.WithName("controllers").WithName("Job"),
82 | Scheme: mgr.GetScheme(),
83 | }).SetupWithManager(mgr); err != nil {
84 | setupLog.Error(err, "unable to create controller", "controller", "Job")
85 | os.Exit(1)
86 | }
87 | if err = (&pipelinescontroller.SplitTransformReconciler{
88 | Client: mgr.GetClient(),
89 | Log: ctrl.Log.WithName("controllers").WithName("SplitTransform"),
90 | Scheme: mgr.GetScheme(),
91 | }).SetupWithManager(mgr); err != nil {
92 | setupLog.Error(err, "unable to create controller", "controller", "SplitTransform")
93 | os.Exit(1)
94 | }
95 | // +kubebuilder:scaffold:builder
96 |
97 | setupLog.Info("starting manager")
98 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
99 | setupLog.Error(err, "problem running manager")
100 | os.Exit(1)
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/controllers/pipelines/suite_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package pipelines
18 |
19 | import (
20 | "path"
21 | "testing"
22 |
23 | . "github.com/onsi/ginkgo"
24 | . "github.com/onsi/gomega"
25 | "k8s.io/client-go/kubernetes/scheme"
26 | "k8s.io/client-go/rest"
27 | ctrl "sigs.k8s.io/controller-runtime"
28 | "sigs.k8s.io/controller-runtime/pkg/client"
29 | "sigs.k8s.io/controller-runtime/pkg/envtest"
30 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer"
31 | logf "sigs.k8s.io/controller-runtime/pkg/log"
32 | "sigs.k8s.io/controller-runtime/pkg/log/zap"
33 |
34 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
35 | // +kubebuilder:scaffold:imports
36 | )
37 |
38 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to
39 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
40 |
41 | var cfg *rest.Config
42 | var k8sClient client.Client
43 | var k8sManager ctrl.Manager
44 | var testEnv *envtest.Environment
45 |
46 | func TestAPIs(t *testing.T) {
47 | RegisterFailHandler(Fail)
48 |
49 | RunSpecsWithDefaultAndCustomReporters(t,
50 | "Controller Suite",
51 | []Reporter{printer.NewlineReporter{}})
52 | }
53 |
54 | var _ = BeforeSuite(func(done Done) {
55 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
56 | var err error
57 |
58 | err = pipelinesv1.AddToScheme(scheme.Scheme)
59 | Expect(err).NotTo(HaveOccurred())
60 |
61 | By("bootstrapping test environment")
62 | testEnv = &envtest.Environment{
63 | CRDDirectoryPaths: []string{path.Join("..", "..", "config", "crd", "bases")},
64 | ErrorIfCRDPathMissing: true,
65 | }
66 |
67 | cfg, err = testEnv.Start()
68 | Expect(err).ToNot(HaveOccurred())
69 | Expect(cfg).ToNot(BeNil())
70 |
71 | // +kubebuilder:scaffold:scheme
72 |
73 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
74 | Expect(err).ToNot(HaveOccurred())
75 | Expect(k8sClient).ToNot(BeNil())
76 |
77 | k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
78 | Scheme: scheme.Scheme,
79 | })
80 | Expect(err).ToNot(HaveOccurred())
81 |
82 | err = (&TransformReconciler{
83 | Client: k8sManager.GetClient(),
84 | Log: ctrl.Log.WithName("controllers").WithName("transform"),
85 | }).SetupWithManager(k8sManager)
86 | Expect(err).ToNot(HaveOccurred())
87 |
88 | err = (&SplitTransformReconciler{
89 | Client: k8sManager.GetClient(),
90 | Log: ctrl.Log.WithName("controllers").WithName("splittransform"),
91 | }).SetupWithManager(k8sManager)
92 | Expect(err).ToNot(HaveOccurred())
93 |
94 | err = (&JobReconciler{
95 | Client: k8sManager.GetClient(),
96 | Log: ctrl.Log.WithName("controllers").WithName("job"),
97 | }).SetupWithManager(k8sManager)
98 | Expect(err).ToNot(HaveOccurred())
99 |
100 | go func() {
101 | defer GinkgoRecover()
102 | Eventually(func() error {
103 | return k8sManager.Start(ctrl.SetupSignalHandler())
104 | }, "10s", "1s").ShouldNot(HaveOccurred())
105 | }()
106 |
107 | Eventually(func() interface{} {
108 | for range k8sManager.Elected() {
109 | return nil
110 | }
111 | return nil
112 | }, "1s", "10s").Should(BeNil())
113 |
114 | k8sClient = k8sManager.GetClient()
115 | Expect(k8sClient).ToNot(BeNil())
116 |
117 | close(done)
118 | }, 60)
119 |
120 | var _ = AfterSuite(func() {
121 | By("tearing down the test environment")
122 | err := testEnv.Stop()
123 | Expect(err).ToNot(HaveOccurred())
124 | })
125 |
--------------------------------------------------------------------------------
/gst/plugins/minio/properties.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "math"
5 |
6 | "github.com/tinyzimmer/go-glib/glib"
7 | )
8 |
9 | // Even though there is overlap in properties, they have to be declared twice.
10 | // This is because the GType system doesn't allow for GObjects to share pointers
11 | // to the exact same GParamSpecs.
12 |
13 | const defaultPartSize = 1024 * 1024 * 10
14 | const minPartSize = 1024 * 1024 * 5
15 |
16 | var sinkProperties = []*glib.ParamSpec{
17 | glib.NewStringParam(
18 | "endpoint",
19 | "S3 API Endpoint",
20 | "The endpoint for the S3 API server",
21 | &defaultEndpoint,
22 | glib.ParameterReadWrite,
23 | ),
24 | glib.NewBoolParam(
25 | "use-tls",
26 | "Use TLS",
27 | "Use HTTPS for API requests",
28 | defaultUseTLS,
29 | glib.ParameterReadWrite,
30 | ),
31 | glib.NewBoolParam(
32 | "tls-skip-verify",
33 | "Disable TLS Verification",
34 | "Don't verify the signature of the MinIO server certificate",
35 | defaultInsecureSkipVerify,
36 | glib.ParameterReadWrite,
37 | ),
38 | glib.NewStringParam(
39 | "ca-cert-file",
40 | "PEM CA Cert Bundle",
41 | "A file containing a PEM certificate bundle to use to verify the MinIO certificate",
42 | nil,
43 | glib.ParameterReadWrite,
44 | ),
45 | glib.NewStringParam(
46 | "region",
47 | "Bucket region",
48 | "The region where the bucket is",
49 | &defaultRegion,
50 | glib.ParameterReadWrite,
51 | ),
52 | glib.NewStringParam(
53 | "bucket",
54 | "Bucket name",
55 | "The name of the MinIO bucket",
56 | nil,
57 | glib.ParameterReadWrite,
58 | ),
59 | glib.NewStringParam(
60 | "key",
61 | "Object key",
62 | "The key of the object inside the bucket",
63 | nil,
64 | glib.ParameterReadWrite,
65 | ),
66 | glib.NewStringParam(
67 | "access-key-id",
68 | "Access Key ID",
69 | "The access key ID to use for authentication",
70 | nil,
71 | glib.ParameterReadWrite,
72 | ),
73 | glib.NewStringParam(
74 | "secret-access-key",
75 | "Secret Access Key",
76 | "The secret access key to use for authentication",
77 | nil,
78 | glib.ParameterReadWrite,
79 | ),
80 | glib.NewUint64Param(
81 | "part-size",
82 | "Part Size",
83 | "Size for each part in the multi-part upload",
84 | minPartSize, math.MaxInt64, defaultPartSize,
85 | glib.ParameterReadWrite,
86 | ),
87 | }
88 |
89 | var srcProperties = []*glib.ParamSpec{
90 | glib.NewStringParam(
91 | "endpoint",
92 | "S3 API Endpoint",
93 | "The endpoint for the S3 API server",
94 | &defaultEndpoint,
95 | glib.ParameterReadWrite,
96 | ),
97 | glib.NewBoolParam(
98 | "use-tls",
99 | "Use TLS",
100 | "Use HTTPS for API requests",
101 | defaultUseTLS,
102 | glib.ParameterReadWrite,
103 | ),
104 | glib.NewBoolParam(
105 | "tls-skip-verify",
106 | "Disable TLS Verification",
107 | "Don't verify the signature of the MinIO server certificate",
108 | defaultInsecureSkipVerify,
109 | glib.ParameterReadWrite,
110 | ),
111 | glib.NewStringParam(
112 | "ca-cert-file",
113 | "PEM CA Cert Bundle",
114 | "A file containing a PEM certificate bundle to use to verify the MinIO certificate",
115 | nil,
116 | glib.ParameterReadWrite,
117 | ),
118 | glib.NewStringParam(
119 | "region",
120 | "Bucket region",
121 | "The region where the bucket is",
122 | &defaultRegion,
123 | glib.ParameterReadWrite,
124 | ),
125 | glib.NewStringParam(
126 | "bucket",
127 | "Bucket name",
128 | "The name of the MinIO bucket",
129 | nil,
130 | glib.ParameterReadWrite,
131 | ),
132 | glib.NewStringParam(
133 | "key",
134 | "Object key",
135 | "The key of the object inside the bucket",
136 | nil,
137 | glib.ParameterReadWrite,
138 | ),
139 | glib.NewStringParam(
140 | "access-key-id",
141 | "Access Key ID",
142 | "The access key ID to use for authentication. Use env: prefix to denote an environment variable.",
143 | nil,
144 | glib.ParameterReadWrite,
145 | ),
146 | glib.NewStringParam(
147 | "secret-access-key",
148 | "Secret Access Key",
149 | "The secret access key to use for authentication. Use env: prefix to denote an environment variable.",
150 | nil,
151 | glib.ParameterReadWrite,
152 | ),
153 | }
154 |
--------------------------------------------------------------------------------
/pkg/util/minio.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package util
18 |
19 | import (
20 | "crypto/tls"
21 | "errors"
22 | "net/http"
23 | "os"
24 |
25 | "github.com/minio/minio-go/v7"
26 | "github.com/minio/minio-go/v7/pkg/credentials"
27 | "sigs.k8s.io/controller-runtime/pkg/client"
28 |
29 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
30 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/types"
31 | )
32 |
33 | // MinIOCredentialsGetter is a credential getter for minio clients. Various
34 | // credentials sources are implemented in this package.
35 | type MinIOCredentialsGetter interface {
36 | GetCredentials() (*credentials.Credentials, error)
37 | }
38 |
39 | // MinIOSinkCredentialsFromEnv returns a credentials getter that retrieves the credentials
40 | // from the environment variables configured by the controller for the sink.
41 | func MinIOSinkCredentialsFromEnv() MinIOCredentialsGetter { return &sinkCredentialsFromEnv{} }
42 |
43 | type sinkCredentialsFromEnv struct{}
44 |
45 | func (s *sinkCredentialsFromEnv) GetCredentials() (*credentials.Credentials, error) {
46 | return credentials.NewStaticV4(os.Getenv(pipelinesmeta.MinIOSinkAccessKeyIDEnvVar), os.Getenv(pipelinesmeta.MinIOSinkSecretAccessKeyEnvVar), ""), nil
47 | }
48 |
49 | // MinIOSrcCredentialsFromEnv returns a credentials getter that retrieves the credentials
50 | // from the environment variables configured by the controller for the src.
51 | func MinIOSrcCredentialsFromEnv() MinIOCredentialsGetter { return &srcCredentialsFromEnv{} }
52 |
53 | type srcCredentialsFromEnv struct{}
54 |
55 | func (s *srcCredentialsFromEnv) GetCredentials() (*credentials.Credentials, error) {
56 | return credentials.NewStaticV4(os.Getenv(pipelinesmeta.MinIOSrcAccessKeyIDEnvVar), os.Getenv(pipelinesmeta.MinIOSrcSecretAccessKeyEnvVar), ""), nil
57 | }
58 |
59 | // MinIOWatchCredentialsFromCR returns a credentials getter that uses the given client
60 | // and CR to produce credentials to the bucket being watched for transformations.
61 | func MinIOWatchCredentialsFromCR(client client.Client, cr types.Pipeline) MinIOCredentialsGetter {
62 | return &pipelineWatchCredentials{
63 | client: client,
64 | cr: cr,
65 | }
66 | }
67 |
68 | type pipelineWatchCredentials struct {
69 | client client.Client
70 | cr types.Pipeline
71 | }
72 |
73 | func (p *pipelineWatchCredentials) GetCredentials() (*credentials.Credentials, error) {
74 | srcConfig := p.cr.GetSrcConfig()
75 | if srcConfig == nil || srcConfig.MinIO == nil {
76 | return nil, errors.New("There is no MinIO configuration for this source")
77 | }
78 | return srcConfig.MinIO.GetStaticCredentials(p.client, p.cr.GetNamespace())
79 | }
80 |
81 | // GetMinIOClient is a utility function for returning a MinIO client to the given
82 | // configuration.
83 | func GetMinIOClient(cfg *pipelinesmeta.MinIOConfig, credsGetter MinIOCredentialsGetter) (*minio.Client, error) {
84 | transport := http.DefaultTransport.(*http.Transport).Clone()
85 |
86 | if cfg.GetSecure() {
87 | certPool, err := cfg.GetRootCAs()
88 | if err != nil {
89 | return nil, err
90 | }
91 | if transport.TLSClientConfig == nil {
92 | transport.TLSClientConfig = &tls.Config{}
93 | }
94 | transport.TLSClientConfig.RootCAs = certPool
95 | transport.TLSClientConfig.InsecureSkipVerify = cfg.GetSkipVerify()
96 | }
97 |
98 | creds, err := credsGetter.GetCredentials()
99 | if err != nil {
100 | return nil, err
101 | }
102 |
103 | return minio.New(cfg.GetEndpoint(), &minio.Options{
104 | Creds: creds,
105 | Secure: cfg.GetSecure(),
106 | Region: cfg.GetRegion(),
107 | Transport: transport,
108 | })
109 | }
110 |
--------------------------------------------------------------------------------
/controllers/pipelines/splittransform_controller.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package pipelines
18 |
19 | import (
20 | "context"
21 |
22 | "github.com/go-logr/logr"
23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24 | "k8s.io/apimachinery/pkg/runtime"
25 | ctrl "sigs.k8s.io/controller-runtime"
26 | "sigs.k8s.io/controller-runtime/pkg/client"
27 |
28 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
29 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
30 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/managers"
31 | )
32 |
33 | // SplitTransformReconciler reconciles a SplitTransform object
34 | type SplitTransformReconciler struct {
35 | client.Client
36 | Log logr.Logger
37 | Scheme *runtime.Scheme
38 | }
39 |
40 | // +kubebuilder:rbac:groups=pipelines.gst.io,resources=splittransforms,verbs=get;list;watch;create;update;patch;delete
41 | // +kubebuilder:rbac:groups=pipelines.gst.io,resources=splittransforms/status,verbs=get;update;patch
42 |
43 | // Reconcile reconciles a splittransform pipeline.
44 | func (r *SplitTransformReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
45 | reqLogger := r.Log.WithValues("splittransform", req.NamespacedName)
46 |
47 | // Fetch the object for the request
48 | pipeline := &pipelinesv1.SplitTransform{}
49 | err := r.Client.Get(ctx, req.NamespacedName, pipeline)
50 | if err != nil {
51 | if client.IgnoreNotFound(err) == nil {
52 | // Object was deleted
53 | return ctrl.Result{}, nil
54 | }
55 | // Requeue any other error
56 | return ctrl.Result{}, err
57 | }
58 |
59 | // Get the controller for this pipeline
60 | controller := managers.GetManagerForPipeline(r.Client, pipeline)
61 |
62 | // Check if we are running finalizers
63 | if pipeline.GetDeletionTimestamp() != nil {
64 | if controller.IsRunning() {
65 | controller.Stop()
66 | }
67 | return ctrl.Result{}, r.removeFinalizers(ctx, reqLogger, pipeline)
68 | }
69 |
70 | if !controller.IsRunning() {
71 | reqLogger.Info("Starting PipelineManager")
72 | if err := controller.Start(); err != nil {
73 | return ctrl.Result{}, err
74 | }
75 | } else {
76 | reqLogger.Info("PipelineManager is already running, reloading config")
77 | controller.Reload(pipeline)
78 | }
79 |
80 | if err := r.ensureFinalizers(ctx, reqLogger, pipeline); err != nil {
81 | return ctrl.Result{}, nil
82 | }
83 |
84 | if !r.generationObserved(pipeline) {
85 | pipeline.Status.Conditions = append(pipeline.Status.Conditions, metav1.Condition{
86 | Type: string(pipelinesmeta.PipelineInSync),
87 | Status: metav1.ConditionTrue,
88 | ObservedGeneration: pipeline.GetGeneration(),
89 | LastTransitionTime: metav1.Now(),
90 | Reason: string(pipelinesmeta.PipelineInSync),
91 | Message: "The pipeline configuration is in-sync",
92 | })
93 | if err := r.Client.Status().Update(ctx, pipeline); err != nil {
94 | return ctrl.Result{}, err
95 | }
96 | }
97 |
98 | reqLogger.Info("Reconcile finished")
99 | return ctrl.Result{}, nil
100 | }
101 |
102 | // SetupWithManager adds the SplitTransformReconciler to the given manager.
103 | func (r *SplitTransformReconciler) SetupWithManager(mgr ctrl.Manager) error {
104 | return ctrl.NewControllerManagedBy(mgr).
105 | For(&pipelinesv1.SplitTransform{}).
106 | Complete(r)
107 | }
108 |
109 | func (r *SplitTransformReconciler) generationObserved(pipeline *pipelinesv1.SplitTransform) bool {
110 | for _, cond := range pipeline.Status.Conditions {
111 | if cond.ObservedGeneration == pipeline.GetGeneration() {
112 | return true
113 | }
114 | }
115 | return false
116 | }
117 |
118 | func (r *SplitTransformReconciler) removeFinalizers(ctx context.Context, reqLogger logr.Logger, pipeline *pipelinesv1.SplitTransform) error {
119 | pipeline.SetFinalizers([]string{})
120 | return r.Client.Update(ctx, pipeline)
121 | }
122 |
123 | func (r *SplitTransformReconciler) ensureFinalizers(ctx context.Context, reqLogger logr.Logger, pipeline *pipelinesv1.SplitTransform) error {
124 | finalizers := pipeline.GetFinalizers()
125 | if !contains(finalizers, pipelineFinalizer) {
126 | reqLogger.Info("Setting finalizers on pipeline")
127 | pipeline.SetFinalizers([]string{pipelineFinalizer})
128 | return r.Client.Update(ctx, pipeline)
129 | }
130 | return nil
131 | }
132 |
--------------------------------------------------------------------------------
/gst/plugins/minio/common.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "crypto/tls"
5 | "crypto/x509"
6 | "fmt"
7 | "io/ioutil"
8 | "net/http"
9 | "os"
10 |
11 | minio "github.com/minio/minio-go/v7"
12 | "github.com/minio/minio-go/v7/pkg/credentials"
13 | "github.com/tinyzimmer/go-glib/glib"
14 | "github.com/tinyzimmer/go-gst/gst"
15 | )
16 |
17 | const (
18 | accessKeyIDEnvVar = "MINIO_ACCESS_KEY_ID"
19 | secretAccessKeyEnvVar = "MINIO_SECRET_ACCESS_KEY"
20 | )
21 |
22 | var (
23 | defaultEndpoint = "play.min.io"
24 | defaultUseTLS = true
25 | defaultRegion = "us-east-1"
26 | defaultInsecureSkipVerify = false
27 | )
28 |
29 | type settings struct {
30 | endpoint string
31 | useTLS bool
32 | region string
33 | bucket string
34 | key string
35 | accessKeyID string
36 | secretAccessKey string
37 | insecureSkipVerify bool
38 | caCertFile string
39 | partSize uint64
40 | }
41 |
42 | func (s *settings) safestring() string {
43 | return fmt.Sprintf("%+v", &settings{
44 | endpoint: s.endpoint,
45 | useTLS: s.useTLS,
46 | region: s.region,
47 | bucket: s.bucket,
48 | key: s.key,
49 | insecureSkipVerify: s.insecureSkipVerify,
50 | caCertFile: s.caCertFile,
51 | })
52 | }
53 |
54 | func defaultSettings() *settings {
55 | return &settings{
56 | endpoint: defaultEndpoint,
57 | useTLS: defaultUseTLS,
58 | region: defaultRegion,
59 | accessKeyID: os.Getenv(accessKeyIDEnvVar),
60 | secretAccessKey: os.Getenv(secretAccessKeyEnvVar),
61 | insecureSkipVerify: defaultInsecureSkipVerify,
62 | partSize: defaultPartSize,
63 | }
64 | }
65 |
66 | func getMinIOClient(settings *settings) (*minio.Client, error) {
67 | transport := http.DefaultTransport.(*http.Transport).Clone()
68 |
69 | if settings.useTLS {
70 | if transport.TLSClientConfig == nil {
71 | transport.TLSClientConfig = &tls.Config{}
72 | }
73 | if settings.caCertFile != "" {
74 | certPool := x509.NewCertPool()
75 | body, err := ioutil.ReadFile(settings.caCertFile)
76 | if err != nil {
77 | return nil, err
78 | }
79 | certPool.AppendCertsFromPEM(body)
80 | transport.TLSClientConfig.RootCAs = certPool
81 | }
82 | transport.TLSClientConfig.InsecureSkipVerify = settings.insecureSkipVerify
83 | }
84 | return minio.New(settings.endpoint, &minio.Options{
85 | Creds: credentials.NewStaticV4(settings.accessKeyID, settings.secretAccessKey, ""),
86 | Secure: settings.useTLS,
87 | Region: settings.region,
88 | })
89 | }
90 |
91 | func setProperty(elem *gst.Element, properties []*glib.ParamSpec, settings *settings, id uint, value *glib.Value) {
92 | prop := properties[id]
93 |
94 | val, err := value.GoValue()
95 | if err != nil {
96 | elem.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorSettings,
97 | fmt.Sprintf("Could not coerce %v to go value", value), err.Error())
98 | }
99 |
100 | switch prop.Name() {
101 | case "endpoint":
102 | settings.endpoint = val.(string)
103 | case "use-tls":
104 | settings.useTLS = val.(bool)
105 | case "tls-skip-verify":
106 | settings.insecureSkipVerify = val.(bool)
107 | case "ca-cert-file":
108 | settings.caCertFile = val.(string)
109 | case "region":
110 | settings.region = val.(string)
111 | case "bucket":
112 | settings.bucket = val.(string)
113 | case "key":
114 | settings.key = val.(string)
115 | case "access-key-id":
116 | settings.accessKeyID = val.(string)
117 | case "secret-access-key":
118 | settings.secretAccessKey = val.(string)
119 | case "part-size":
120 | settings.partSize = val.(uint64)
121 | }
122 | }
123 |
124 | func getProperty(elem *gst.Element, properties []*glib.ParamSpec, settings *settings, id uint) *glib.Value {
125 | prop := properties[id]
126 |
127 | var localVal interface{}
128 |
129 | switch prop.Name() {
130 | case "endpoint":
131 | localVal = settings.endpoint
132 | case "use-tls":
133 | localVal = settings.useTLS
134 | case "tls-skip-verify":
135 | localVal = settings.insecureSkipVerify
136 | case "ca-cert-file":
137 | localVal = settings.caCertFile
138 | case "region":
139 | localVal = settings.region
140 | case "bucket":
141 | localVal = settings.bucket
142 | case "key":
143 | localVal = settings.key
144 | case "access-key-id":
145 | localVal = settings.accessKeyID
146 | case "secret-access-key":
147 | localVal = ""
148 | case "part-size":
149 | localVal = settings.partSize
150 |
151 | default:
152 | elem.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorSettings,
153 | fmt.Sprintf("Cannot get invalid property %s", prop.Name()), "")
154 | return nil
155 | }
156 |
157 | val, err := glib.GValue(localVal)
158 | if err != nil {
159 | elem.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorFailed,
160 | fmt.Sprintf("Could not convert %v to GValue", localVal),
161 | err.Error(),
162 | )
163 | return nil
164 | }
165 |
166 | return val
167 | }
168 |
--------------------------------------------------------------------------------
/controllers/pipelines/transform_controller.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package pipelines
18 |
19 | import (
20 | "context"
21 |
22 | "github.com/go-logr/logr"
23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24 | "k8s.io/apimachinery/pkg/runtime"
25 | ctrl "sigs.k8s.io/controller-runtime"
26 | "sigs.k8s.io/controller-runtime/pkg/client"
27 |
28 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
29 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
30 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/managers"
31 | )
32 |
33 | var pipelineFinalizer = "pipelines.gst.io/finalize"
34 |
35 | // TransformReconciler reconciles a Transform object
36 | type TransformReconciler struct {
37 | client.Client
38 | Log logr.Logger
39 | Scheme *runtime.Scheme
40 | }
41 |
42 | // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
43 | // +kubebuilder:rbac:groups=pipelines.gst.io,resources=transforms,verbs=get;list;watch;create;update;patch;delete
44 | // +kubebuilder:rbac:groups=pipelines.gst.io,resources=transforms/status,verbs=get;update;patch
45 |
46 | // Reconcile reconciles a Transform pipeline
47 | func (r *TransformReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
48 | reqLogger := r.Log.WithValues("transform", req.NamespacedName)
49 |
50 | // Fetch the object for the request
51 | pipeline := &pipelinesv1.Transform{}
52 | err := r.Client.Get(ctx, req.NamespacedName, pipeline)
53 | if err != nil {
54 | if client.IgnoreNotFound(err) == nil {
55 | // Object was deleted
56 | return ctrl.Result{}, nil
57 | }
58 | // Requeue any other error
59 | return ctrl.Result{}, err
60 | }
61 |
62 | // Get the controller for this pipeline
63 | controller := managers.GetManagerForPipeline(r.Client, pipeline)
64 |
65 | // Check if we are running finalizers
66 | if pipeline.GetDeletionTimestamp() != nil {
67 | if controller.IsRunning() {
68 | controller.Stop()
69 | }
70 | return ctrl.Result{}, r.removeFinalizers(ctx, reqLogger, pipeline)
71 | }
72 |
73 | if !controller.IsRunning() {
74 | reqLogger.Info("Starting PipelineManager")
75 | if err := controller.Start(); err != nil {
76 | return ctrl.Result{}, err
77 | }
78 | } else {
79 | reqLogger.Info("PipelineManager is already running, reloading config")
80 | controller.Reload(pipeline)
81 | }
82 |
83 | if err := r.ensureFinalizers(ctx, reqLogger, pipeline); err != nil {
84 | return ctrl.Result{}, nil
85 | }
86 |
87 | if !r.generationObserved(pipeline) {
88 | pipeline.Status.Conditions = append(pipeline.Status.Conditions, metav1.Condition{
89 | Type: string(pipelinesmeta.PipelineInSync),
90 | Status: metav1.ConditionTrue,
91 | ObservedGeneration: pipeline.GetGeneration(),
92 | LastTransitionTime: metav1.Now(),
93 | Reason: string(pipelinesmeta.PipelineInSync),
94 | Message: "The pipeline configuration is in-sync",
95 | })
96 | if err := r.Client.Status().Update(ctx, pipeline); err != nil {
97 | return ctrl.Result{}, err
98 | }
99 | }
100 |
101 | reqLogger.Info("Reconcile finished")
102 | return ctrl.Result{}, nil
103 | }
104 |
105 | func (r *TransformReconciler) generationObserved(pipeline *pipelinesv1.Transform) bool {
106 | for _, cond := range pipeline.Status.Conditions {
107 | if cond.ObservedGeneration == pipeline.GetGeneration() {
108 | return true
109 | }
110 | }
111 | return false
112 | }
113 |
114 | func (r *TransformReconciler) removeFinalizers(ctx context.Context, reqLogger logr.Logger, pipeline *pipelinesv1.Transform) error {
115 | pipeline.SetFinalizers([]string{})
116 | return r.Client.Update(ctx, pipeline)
117 | }
118 |
119 | func (r *TransformReconciler) ensureFinalizers(ctx context.Context, reqLogger logr.Logger, pipeline *pipelinesv1.Transform) error {
120 | finalizers := pipeline.GetFinalizers()
121 | if !contains(finalizers, pipelineFinalizer) {
122 | reqLogger.Info("Setting finalizers on pipeline")
123 | pipeline.SetFinalizers([]string{pipelineFinalizer})
124 | return r.Client.Update(ctx, pipeline)
125 | }
126 | return nil
127 | }
128 |
129 | func contains(ss []string, s string) bool {
130 | for _, x := range ss {
131 | if x == s {
132 | return true
133 | }
134 | }
135 | return false
136 | }
137 |
138 | // SetupWithManager adds the Transform pipeline controller to the manager.
139 | func (r *TransformReconciler) SetupWithManager(mgr ctrl.Manager) error {
140 | return ctrl.NewControllerManagedBy(mgr).
141 | For(&pipelinesv1.Transform{}).
142 | Owns(&pipelinesv1.Job{}).
143 | Complete(r)
144 | }
145 |
--------------------------------------------------------------------------------
/apis/meta/v1/pipeline.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | "fmt"
21 | "path"
22 | "strconv"
23 | "time"
24 |
25 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/version"
26 | corev1 "k8s.io/api/core/v1"
27 | )
28 |
29 | // PipelineConfig represents a series of elements through which to pass the contents of
30 | // processed objects.
31 | type PipelineConfig struct {
32 | // The image to use to run a/v processing pipelines.
33 | Image string `json:"image,omitempty"`
34 | // Debug configurations for the pipeline
35 | Debug *DebugConfig `json:"debug,omitempty"`
36 | // A list of element configurations in the order they will be used in the pipeline.
37 | // Using these is mutually exclusive with a decodebin configuration. This only really
38 | // works for linear pipelines. That is to say, not the syntax used by `gst-launch-1.0` that
39 | // allows naming elements and referencing them later in the pipeline. For complex handling
40 | // of multiple streams decodebin will still be better to work with for now, despite its
41 | // shortcomings.
42 | Elements []*ElementConfig `json:"elements,omitempty"`
43 | // Resource restraints to place on jobs created for this pipeline.
44 | Resources corev1.ResourceRequirements `json:"resources,omitempty"`
45 | }
46 |
47 | // DebugConfig represents debug configurations for a GStreamer pipeline.
48 | type DebugConfig struct {
49 | // The level of log output to produce from the gstreamer process. This value gets set to
50 | // the GST_DEBUG variable. Defaults to INFO level (4). Higher numbers mean more output.
51 | LogLevel int `json:"logLevel,omitempty"`
52 | // Dot specifies to dump a dot file of the pipeline layout for debugging. The extending
53 | // object allows for additional configurations to the output.
54 | Dot *DotConfig `json:"dot,omitempty"`
55 | }
56 |
57 | // DotConfig represents a configuration for the dot output of a pipeline.
58 | type DotConfig struct {
59 | // The path to save files. The configuration other than the path is assumed to be that of
60 | // the source of the pipeline. For example, for a MinIO source, this should be a prefix in
61 | // the same bucket as the source (but not overlapping with the watch prefix otherwise an infinite
62 | // loop will happen). The files will be saved in directories matching the source object's name with
63 | // the _debug suffix.
64 | Path string `json:"path,omitempty"`
65 | // Specify to also render the pipeline graph to images in the given format. Accepted formats are
66 | // png, svg, or jpg.
67 | Render string `json:"render,omitempty"`
68 | // Whether to save timestamped versions of the pipeline layout. This will produce a new graph for every
69 | // interval specified by Interval. The default is to only keep the latest graph.
70 | Timestamped bool `json:"timestamped,omitempty"`
71 | // The interval in seconds to save pipeline graphs. Defaults to every 3 seconds.
72 | Interval int `json:"interval,omitempty"`
73 | }
74 |
75 | // GetGSTDebug returns the string value of the level to set to GST_DEBUG.
76 | func (p *PipelineConfig) GetGSTDebug() string {
77 | if p.Debug == nil || p.Debug.LogLevel == 0 {
78 | return DefaultGSTDebug
79 | }
80 | return strconv.Itoa(p.Debug.LogLevel)
81 | }
82 |
83 | // DoDotDump returns true if the pipeline has DOT debugging enabled.
84 | func (p *PipelineConfig) DoDotDump() bool {
85 | return p.Debug != nil && p.Debug.Dot != nil
86 | }
87 |
88 | // TimestampDotGraphs returns true if timestamped dot images should be saved.
89 | func (p *PipelineConfig) TimestampDotGraphs() bool {
90 | if p.Debug == nil || p.Debug.Dot == nil {
91 | return false
92 | }
93 | return p.Debug.Dot.Timestamped
94 | }
95 |
96 | // GetDotInterval returns the interval in seconds to query for pipeline graphs.
97 | func (p *PipelineConfig) GetDotInterval() time.Duration {
98 | if p.Debug == nil || p.Debug.Dot == nil || p.Debug.Dot.Interval == 0 {
99 | return time.Duration(DefaultDotInterval) * time.Second
100 | }
101 | return time.Duration(p.Debug.Dot.Interval) * time.Second
102 | }
103 |
104 | // GetDotPath returns the path to save dot graphs based on the given source key.
105 | func (p *PipelineConfig) GetDotPath(srcKey string) string {
106 | if p.Debug == nil || p.Debug.Dot == nil {
107 | return ""
108 | }
109 | return path.Join(p.Debug.Dot.Path, fmt.Sprintf("%s_debug", path.Base(srcKey)))
110 | }
111 |
112 | // GetDotRenderFormat returns the image format that the dot graphs should be encoded to
113 | // when uploading alongside the raw format.
114 | func (p *PipelineConfig) GetDotRenderFormat() string {
115 | if p.Debug == nil || p.Debug.Dot == nil {
116 | return ""
117 | }
118 | return p.Debug.Dot.Render
119 | }
120 |
121 | // GetImage returns the container image to use for the gstreamer pipelines.
122 | func (p *PipelineConfig) GetImage() string {
123 | if p.Image != "" {
124 | return p.Image
125 | }
126 | return fmt.Sprintf("ghcr.io/tinyzimmer/gst-pipeline-operator/gstreamer:%s", version.Version)
127 | }
128 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # gst-pipeline-operator
2 |
3 | A Kubernetes operator for running audio/video processing pipelines
4 |
5 | - [API Reference](doc/pipelines.md)
6 |
7 | ## Quickstart
8 |
9 | This project is very young and does not have a ton of features yet. For the POC it can watch MinIO buckets
10 | for files to process, and feeds them through GStreamer pipelines defined in Custom Resources.
11 | The below guide walks you through setting up a cluster to process your own A/V in the same way.
12 |
13 | ### Setting up the Cluster
14 |
15 | First you'll need a running Kubernetes cluster. For the purpose of the quickstart we'll use [`k3d`](https://github.com/rancher/k3d) to create the cluster.
16 |
17 | ```bash
18 | k3d cluster create -p 9000:9000@loadbalancer # Expose port 9000 for minio later
19 | ```
20 |
21 | The operator currently only supports MinIO as a source or destination for pipeline objects.
22 | The intention is to extend this to support other event sources/destinations (e.g. NFS, PVs, etc.).
23 | For testing purposes, you can spin up a single node MinIO server inside the Kubernetes cluster using `helm`.
24 |
25 | ```bash
26 | helm repo add minio https://helm.min.io/
27 | helm repo update
28 |
29 | # You may want to change some of these configurations for yourself. The examples later in the Quickstart
30 | # will assume these values, and you should substitute for the ones you chose instead.
31 | helm install minio minio/minio \
32 | --set service.type=LoadBalancer \
33 | --set accessKey=accesskey \
34 | --set secretKey=secretkey \
35 | --set buckets[0].name=gst-processing \
36 | --set buckets[0].policy=public
37 | ```
38 |
39 | Finally you need to create a **Secret** holding the credentials for the operator and GStreamer pipelines to connect to MinIO.
40 |
41 | ```bash
42 | cat < s.partSize {
118 | sinkCAT.Log(gst.LevelTrace, fmt.Sprintf("Resizing part %d buffer to %d", currentPart, s.partSize))
119 | newbuf := make([]byte, s.partSize)
120 | copy(newbuf, buf)
121 | s.parts[currentPart] = newbuf
122 | buf = newbuf
123 | } else if lenToWrite+writeat > int64(len(buf)) {
124 | size := lenToWrite + writeat
125 | sinkCAT.Log(gst.LevelTrace, fmt.Sprintf("Resizing part %d buffer to %d", currentPart, size))
126 | newbuf := make([]byte, size)
127 | copy(newbuf, buf)
128 | s.parts[currentPart] = newbuf
129 | buf = newbuf
130 | }
131 |
132 | wrote := copy(buf[writeat:], p)
133 |
134 | s.currentPosition += int64(wrote)
135 |
136 | if int64(wrote) != lenToWrite {
137 | sinkCAT.Log(gst.LevelLog, fmt.Sprintf("Only wrote %d, continuing to next part", wrote))
138 | return s.buffer(from+wrote, p[wrote:])
139 | }
140 |
141 | return from + wrote, nil
142 | }
143 |
144 | func (s *seekWriter) flush(all bool) error {
145 | for part, buf := range s.parts {
146 | if all || int64(len(buf)) == s.partSize {
147 | if err := s.uploadPart(part, buf); err != nil {
148 | return err
149 | }
150 | continue
151 | }
152 | if !all {
153 | continue
154 | }
155 | if err := s.uploadPart(part, buf); err != nil {
156 | return err
157 | }
158 | }
159 | return nil
160 | }
161 |
162 | func (s *seekWriter) uploadPart(part int64, data []byte) error {
163 | h := sha256.New()
164 | if _, err := h.Write(data); err != nil {
165 | return err
166 | }
167 | datasum := fmt.Sprintf("%x", h.Sum(nil))
168 | if sum, ok := s.uploadedParts[part]; ok && sum == datasum {
169 | sinkCAT.Log(gst.LevelDebug, fmt.Sprintf("Checksum for part %d unchanged, skipping upload", part))
170 | delete(s.parts, part)
171 | return nil
172 | }
173 | sinkCAT.Log(gst.LevelInfo, fmt.Sprintf("Uploading part %d to %s/%s", part, s.bucket, s.keyForPart(part)))
174 | _, err := s.client.PutObject(context.Background(),
175 | s.bucket, s.keyForPart(part),
176 | bytes.NewReader(data), int64(len(data)),
177 | minio.PutObjectOptions{
178 | ContentType: "application/octet-stream",
179 | },
180 | )
181 | if err != nil {
182 | return err
183 | }
184 | delete(s.parts, part)
185 | s.uploadedParts[part] = datasum
186 | return nil
187 | }
188 |
189 | func (s *seekWriter) fetchRemotePart(part int64) ([]byte, error) {
190 | object, err := s.client.GetObject(context.Background(), s.bucket, s.keyForPart(part), minio.GetObjectOptions{})
191 | if err != nil {
192 | return nil, err
193 | }
194 | body, err := ioutil.ReadAll(object)
195 | if err != nil {
196 | return nil, err
197 | }
198 | s.parts[part] = body
199 | return s.parts[part], nil
200 | }
201 |
202 | func (s *seekWriter) keyForPart(part int64) string {
203 | if path.Dir(s.key) == "" {
204 | return fmt.Sprintf("%s_tmp/%d", s.key, part)
205 | }
206 | return path.Join(
207 | path.Dir(s.key),
208 | fmt.Sprintf("%s_tmp/%d", path.Base(s.key), part),
209 | )
210 | }
211 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | # Current Operator version
2 | VERSION ?= latest
3 | # Default bundle image tag
4 | BUNDLE_IMG ?= $(REPO)/controller-bundle:$(VERSION)
5 | # Options for 'bundle-build'
6 | ifneq ($(origin CHANNELS), undefined)
7 | BUNDLE_CHANNELS := --channels=$(CHANNELS)
8 | endif
9 | ifneq ($(origin DEFAULT_CHANNEL), undefined)
10 | BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
11 | endif
12 | BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
13 |
14 | REPO ?= ghcr.io/tinyzimmer/gst-pipeline-operator
15 |
16 | # Image URL to use all building/pushing image targets
17 | IMG ?= $(REPO)/controller:$(VERSION)
18 | CRD_OPTIONS ?= "crd:crdVersions=v1"
19 |
20 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
21 | ifeq (,$(shell go env GOBIN))
22 | GOBIN=$(shell go env GOPATH)/bin
23 | else
24 | GOBIN=$(shell go env GOBIN)
25 | endif
26 |
27 | all: manager
28 |
29 | # Run tests
30 | ENVTEST_ASSETS_DIR = $(shell pwd)/testbin
31 | test: generate fmt vet manifests
32 | mkdir -p $(ENVTEST_ASSETS_DIR)
33 | test -f $(ENVTEST_ASSETS_DIR)/setup-envtest.sh || curl -sSLo $(ENVTEST_ASSETS_DIR)/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.6.3/hack/setup-envtest.sh
34 | source $(ENVTEST_ASSETS_DIR)/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out
35 |
36 | LDFLAGS ?= -X github.com/tinyzimmer/gst-pipeline-operator/pkg/version.Version=$(VERSION) \
37 | -X github.com/tinyzimmer/gst-pipeline-operator/pkg/version.GitCommit=$(shell git rev-parse HEAD)
38 |
39 | # Build manager binary
40 | manager: generate fmt vet
41 | go build -o bin/manager -ldflags="$(LDFLAGS)" main.go
42 |
43 | # Run against the configured Kubernetes cluster in ~/.kube/config
44 | run: generate fmt vet manifests
45 | go run ./main.go
46 |
47 | # Install CRDs into a cluster
48 | install: manifests kustomize
49 | $(KUSTOMIZE) build config/crd | kubectl apply -f -
50 |
51 | # Uninstall CRDs from a cluster
52 | uninstall: manifests kustomize
53 | $(KUSTOMIZE) build config/crd | kubectl delete -f -
54 |
55 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
56 | deploy: manifests kustomize
57 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
58 | $(KUSTOMIZE) build config/default | kubectl apply -f -
59 |
60 | deploy-manifest: manifests kustomize
61 | mkdir -p deploy/manifests
62 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
63 | $(KUSTOMIZE) build config/default > deploy/manifests/gst-pipeline-operator-full.yaml
64 |
65 | # Generate manifests e.g. CRD, RBAC etc.
66 | manifests: controller-gen
67 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
68 |
69 | # Run go fmt against code
70 | fmt:
71 | go fmt ./...
72 |
73 | # Run go vet against code
74 | vet:
75 | go vet ./...
76 |
77 | # Generate code
78 | generate: controller-gen
79 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
80 |
81 | # Build the docker image
82 | docker-build:
83 | docker build . -t ${IMG} --build-arg LDFLAGS="$(LDFLAGS)"
84 |
85 | # Push the docker image
86 | docker-push:
87 | docker push ${IMG}
88 |
89 | # find or download controller-gen
90 | # download controller-gen if necessary
91 | controller-gen:
92 | ifeq (, $(shell which controller-gen 2> /dev/null))
93 | @{ \
94 | set -e ;\
95 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
96 | cd $$CONTROLLER_GEN_TMP_DIR ;\
97 | go mod init tmp ;\
98 | go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.0 ;\
99 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
100 | }
101 | CONTROLLER_GEN=$(GOBIN)/controller-gen
102 | else
103 | CONTROLLER_GEN=$(shell which controller-gen 2> /dev/null)
104 | endif
105 |
106 | kustomize:
107 | ifeq (, $(shell which kustomize 2> /dev/null))
108 | @{ \
109 | set -e ;\
110 | KUSTOMIZE_GEN_TMP_DIR=$$(mktemp -d) ;\
111 | cd $$KUSTOMIZE_GEN_TMP_DIR ;\
112 | go mod init tmp ;\
113 | go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4 ;\
114 | rm -rf $$KUSTOMIZE_GEN_TMP_DIR ;\
115 | }
116 | KUSTOMIZE=$(GOBIN)/kustomize
117 | else
118 | KUSTOMIZE=$(shell which kustomize 2> /dev/null)
119 | endif
120 |
121 | # Generate bundle manifests and metadata, then validate generated files.
122 | .PHONY: bundle
123 | bundle: manifests kustomize
124 | operator-sdk generate kustomize manifests -q
125 | cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
126 | $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
127 | operator-sdk bundle validate ./bundle
128 |
129 | # Build the bundle image.
130 | .PHONY: bundle-build
131 | bundle-build:
132 | docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) .
133 |
134 | ## Custom Targets
135 |
136 | GST_IMAGE ?= $(REPO)/gstreamer:$(VERSION)
137 | docker-gst-build:
138 | docker build -f cmd/runner/Dockerfile -t $(GST_IMAGE) .
139 |
140 | docker-gst-push:
141 | docker push $(GST_IMAGE)
142 |
143 | docker-build-all: docker-build docker-gst-build
144 |
145 | K3D ?= $(GOBIN)/k3d
146 | $(K3D):
147 | curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | K3D_INSTALL_DIR=$(GOBIN) bash -s -- --no-sudo
148 |
149 | CLUSTER_NAME ?= gst
150 | CLUSTER_KUBECONFIG ?= $(CURDIR)/kubeconfig.yaml
151 | local-cluster:
152 | $(K3D) cluster create $(CLUSTER_NAME) \
153 | --update-default-kubeconfig=false \
154 | -p 9000:9000@loadbalancer
155 | $(K3D) kubeconfig get $(CLUSTER_NAME) > $(CLUSTER_KUBECONFIG)
156 |
157 | local-import: docker-build-all
158 | $(K3D) image import --cluster=$(CLUSTER_NAME) $(IMG) $(GST_IMAGE)
159 |
160 | local-deploy: local-import deploy-manifest
161 | KUBECONFIG=$(CLUSTER_KUBECONFIG) kubectl apply -f deploy/manifests/gst-pipeline-operator-full.yaml
162 |
163 | TEST_HELM ?= KUBECONFIG=$(CLUSTER_KUBECONFIG) helm
164 | local-minio:
165 | $(TEST_HELM) repo add minio https://helm.min.io/
166 | $(TEST_HELM) repo update
167 | $(TEST_HELM) install \
168 | --set service.type=LoadBalancer \
169 | --set accessKey=accesskey \
170 | --set secretKey=secretkey \
171 | --set buckets[0].name=gst-processing \
172 | --set buckets[0].policy=public \
173 | minio minio/minio
174 |
175 | local-samples:
176 | KUBECONFIG=$(CLUSTER_KUBECONFIG) kubectl apply -f config/samples/minio_credentials.yaml
177 | KUBECONFIG=$(CLUSTER_KUBECONFIG) kubectl apply -f config/samples/pipelines_v1_transform.yaml
178 | KUBECONFIG=$(CLUSTER_KUBECONFIG) kubectl apply -f config/samples/pipelines_v1_splittransform.yaml
179 |
180 | local-full: local-cluster local-minio local-deploy
181 |
182 | delete-local-cluster:
183 | $(K3D) cluster delete $(CLUSTER_NAME)
184 |
185 | REFDOCS = bin/refdocs
186 | $(REFDOCS):
187 | cd hack && go build -o $(REFDOCS) .
188 |
189 | api-docs: $(REFDOCS)
190 | go mod vendor
191 | bash hack/update-api-docs.sh
--------------------------------------------------------------------------------
/gst/plugins/minio/miniosrc.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "io"
7 | "os"
8 | "strings"
9 | "sync"
10 |
11 | minio "github.com/minio/minio-go/v7"
12 |
13 | "github.com/tinyzimmer/go-glib/glib"
14 | "github.com/tinyzimmer/go-gst/gst"
15 | "github.com/tinyzimmer/go-gst/gst/base"
16 | )
17 |
18 | var srcCAT = gst.NewDebugCategory(
19 | "miniosrc",
20 | gst.DebugColorNone,
21 | "MinIOSrc Element",
22 | )
23 |
24 | type minioSrc struct {
25 | settings *settings
26 | state *srcstate
27 | }
28 |
29 | type srcstate struct {
30 | started bool
31 | object *minio.Object
32 | objInfo minio.ObjectInfo
33 |
34 | mux sync.Mutex
35 | }
36 |
37 | func (m *minioSrc) New() glib.GoObjectSubclass {
38 | srcCAT.Log(gst.LevelLog, "Creating new minioSrc object")
39 | return &minioSrc{
40 | settings: defaultSettings(),
41 | state: &srcstate{},
42 | }
43 | }
44 |
45 | func (m *minioSrc) ClassInit(klass *glib.ObjectClass) {
46 | class := gst.ToElementClass(klass)
47 | srcCAT.Log(gst.LevelLog, "Initializing miniosrc class")
48 | class.SetMetadata(
49 | "MinIO Source",
50 | "Source/File",
51 | "Read stream from a MinIO object",
52 | "Avi Zimmerman ",
53 | )
54 | srcCAT.Log(gst.LevelLog, "Adding src pad template and properties to class")
55 | class.AddPadTemplate(gst.NewPadTemplate(
56 | "src",
57 | gst.PadDirectionSource,
58 | gst.PadPresenceAlways,
59 | gst.NewAnyCaps(),
60 | ))
61 | class.InstallProperties(srcProperties)
62 | }
63 |
64 | func (m *minioSrc) SetProperty(self *glib.Object, id uint, value *glib.Value) {
65 | setProperty(gst.ToElement(self), srcProperties, m.settings, id, value)
66 | }
67 |
68 | func (m *minioSrc) GetProperty(self *glib.Object, id uint) *glib.Value {
69 | return getProperty(gst.ToElement(self), srcProperties, m.settings, id)
70 | }
71 |
72 | func (m *minioSrc) Constructed(self *glib.Object) {
73 | base.ToGstBaseSrc(self).Log(srcCAT, gst.LevelLog, "Setting format of GstBaseSrc to bytes")
74 | base.ToGstBaseSrc(self).SetFormat(gst.FormatBytes)
75 | }
76 |
77 | func (m *minioSrc) IsSeekable(*base.GstBaseSrc) bool { return true }
78 |
79 | func (m *minioSrc) GetSize(self *base.GstBaseSrc) (bool, int64) {
80 | if !m.state.started {
81 | return false, 0
82 | }
83 | return true, m.state.objInfo.Size
84 | }
85 |
86 | func (m *minioSrc) Start(self *base.GstBaseSrc) bool {
87 |
88 | if m.state.started {
89 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed, "MinIOSrc is already started", "")
90 | return false
91 | }
92 |
93 | if m.settings.bucket == "" {
94 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed, "No source bucket defined", "")
95 | return false
96 | }
97 |
98 | if m.settings.key == "" {
99 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed, "No object key defined", "")
100 | return false
101 | }
102 |
103 | m.state.mux.Lock()
104 |
105 | if strings.HasPrefix(m.settings.accessKeyID, "env:") {
106 | spl := strings.Split(m.settings.accessKeyID, "env:")
107 | m.settings.accessKeyID = os.Getenv(spl[len(spl)-1])
108 | }
109 |
110 | if strings.HasPrefix(m.settings.secretAccessKey, "env:") {
111 | spl := strings.Split(m.settings.secretAccessKey, "env:")
112 | m.settings.secretAccessKey = os.Getenv(spl[len(spl)-1])
113 | }
114 |
115 | client, err := getMinIOClient(m.settings)
116 | if err != nil {
117 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed,
118 | fmt.Sprintf("Failed to connect to MinIO endpoint %s", m.settings.endpoint), err.Error())
119 | m.state.mux.Unlock()
120 | return false
121 | }
122 |
123 | self.Log(srcCAT, gst.LevelInfo, fmt.Sprintf("Requesting %s/%s from %s", m.settings.bucket, m.settings.key, m.settings.endpoint))
124 | m.state.object, err = client.GetObject(context.Background(), m.settings.bucket, m.settings.key, minio.GetObjectOptions{})
125 | if err != nil {
126 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorOpenRead,
127 | fmt.Sprintf("Failed to retrieve object %q from bucket %q", m.settings.key, m.settings.bucket), err.Error())
128 | m.state.mux.Unlock()
129 | return false
130 | }
131 |
132 | self.Log(srcCAT, gst.LevelInfo, "Getting HEAD for object")
133 | m.state.objInfo, err = m.state.object.Stat()
134 | if err != nil {
135 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorOpenRead,
136 | fmt.Sprintf("Failed to stat object %q in bucket %q: %s", m.settings.key, m.settings.bucket, err.Error()), "")
137 | m.state.mux.Unlock()
138 | return false
139 | }
140 | self.Log(srcCAT, gst.LevelInfo, fmt.Sprintf("%+v", m.state.objInfo))
141 |
142 | m.state.started = true
143 | m.state.mux.Unlock()
144 |
145 | self.StartComplete(gst.FlowOK)
146 |
147 | self.Log(srcCAT, gst.LevelInfo, "MinIOSrc has started")
148 | return true
149 | }
150 |
151 | func (m *minioSrc) Stop(self *base.GstBaseSrc) bool {
152 | self.Log(srcCAT, gst.LevelInfo, "Stopping MinIOSrc")
153 | m.state.mux.Lock()
154 | defer m.state.mux.Unlock()
155 |
156 | if !m.state.started {
157 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorSettings, "MinIOSrc is not started", "")
158 | return false
159 | }
160 |
161 | if err := m.state.object.Close(); err != nil {
162 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorClose, "Failed to close the bucket object", err.Error())
163 | return false
164 | }
165 |
166 | m.state.object = nil
167 | m.state.started = false
168 |
169 | self.Log(srcCAT, gst.LevelInfo, "MinIOSrc has stopped")
170 | return true
171 | }
172 |
173 | func (m *minioSrc) Fill(self *base.GstBaseSrc, offset uint64, size uint, buffer *gst.Buffer) gst.FlowReturn {
174 |
175 | if !m.state.started || m.state.object == nil {
176 | self.ErrorMessage(gst.DomainCore, gst.CoreErrorFailed, "MinIOSrc is not started yet", "")
177 | return gst.FlowError
178 | }
179 |
180 | self.Log(srcCAT, gst.LevelLog, fmt.Sprintf("Request to fill buffer from offset %v with size %v", offset, size))
181 |
182 | m.state.mux.Lock()
183 | defer m.state.mux.Unlock()
184 |
185 | data := make([]byte, size)
186 | read, err := m.state.object.ReadAt(data, int64(offset))
187 | if err != nil && err != io.EOF {
188 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorRead,
189 | fmt.Sprintf("Failed to read %d bytes from object at offset %d", size, offset), err.Error())
190 | return gst.FlowError
191 | }
192 |
193 | if read < int(size) {
194 | self.Log(srcCAT, gst.LevelDebug, fmt.Sprintf("Only read %d bytes from object, trimming", read))
195 | trim := make([]byte, read)
196 | copy(trim, data)
197 | data = trim
198 | }
199 |
200 | bufmap := buffer.Map(gst.MapWrite)
201 | if bufmap == nil {
202 | self.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorFailed, "Failed to map buffer", "")
203 | return gst.FlowError
204 | }
205 | defer buffer.Unmap()
206 |
207 | bufmap.WriteData(data)
208 | buffer.SetSize(int64(read))
209 |
210 | return gst.FlowOK
211 | }
212 |
--------------------------------------------------------------------------------
/cmd/runner/main.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package main
18 |
19 | import (
20 | "bytes"
21 | "context"
22 | "encoding/json"
23 | "fmt"
24 | "os"
25 | "path"
26 | "strings"
27 | "time"
28 |
29 | "github.com/goccy/go-graphviz"
30 | "github.com/minio/minio-go/v7"
31 | "github.com/tinyzimmer/go-glib/glib"
32 | "github.com/tinyzimmer/go-gst/gst"
33 | "sigs.k8s.io/controller-runtime/pkg/log/zap"
34 |
35 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
36 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/util"
37 | )
38 |
39 | var log = zap.New(zap.UseDevMode(true)).WithName("gst-runner")
40 |
41 | func init() { gst.Init(nil) }
42 |
43 | func main() {
44 | mainLoop := glib.NewMainLoop(glib.MainContextDefault(), false)
45 |
46 | cfg, srcobject, sinkobjects, err := getPipelineCfgAndObjects()
47 | if err != nil {
48 | log.Error(err, "Failed to retrieve job spec from environment")
49 | os.Exit(1)
50 | }
51 |
52 | pipeline, err := buildPipelineFromCR(cfg, srcobject, sinkobjects)
53 | if err != nil {
54 | log.Error(err, "Failed to build pipeline from job spec")
55 | os.Exit(2)
56 | }
57 |
58 | pipeline.GetPipelineBus().AddWatch(func(msg *gst.Message) bool {
59 | switch msg.Type() {
60 | case gst.MessageEOS:
61 | log.Info("Received EOS, setting pipeline state to NULL")
62 | pipeline.BlockSetState(gst.StateNull)
63 | mainLoop.Quit()
64 | return false
65 | case gst.MessageError:
66 | err := msg.ParseError()
67 | log.Error(err, err.DebugString())
68 | os.Exit(3)
69 | }
70 |
71 | log.Info(msg.String())
72 | return true
73 | })
74 |
75 | pipeline.BlockSetState(gst.StatePlaying)
76 |
77 | go func() {
78 | var mc *minio.Client
79 | var err error
80 | var outbucket string
81 | var outpath string
82 | var g *graphviz.Graphviz
83 |
84 | for cfg.DoDotDump() {
85 | mc, err = util.GetMinIOClient(srcobject.Config.MinIO, util.MinIOSrcCredentialsFromEnv())
86 | if err != nil {
87 | log.Error(err, "Could not create src minio client for dot graph debugging")
88 | mc = nil
89 | break
90 | }
91 | outbucket = srcobject.Config.MinIO.GetBucket()
92 | outpath = cfg.GetDotPath(srcobject.Name)
93 | break
94 | }
95 |
96 | for range time.NewTicker(cfg.GetDotInterval()).C {
97 |
98 | DebugDotData:
99 | for cfg.DoDotDump() && mc != nil {
100 | if g == nil {
101 | g = graphviz.New()
102 | }
103 | var dotname, imgname string
104 | var dotdata, imgdata []byte
105 |
106 | dotdata = []byte(pipeline.DebugBinToDotData(gst.DebugGraphShowAll))
107 |
108 | if cfg.TimestampDotGraphs() {
109 | ts := time.Now().UTC().Format(time.RFC3339)
110 | dotname = path.Join(outpath, fmt.Sprintf("pipeline_%s.dot", ts))
111 | if render := cfg.GetDotRenderFormat(); render != "" {
112 | imgname = path.Join(outpath, fmt.Sprintf("pipeline_%s.%s", ts, strings.ToLower(render)))
113 | }
114 | } else {
115 | dotname = path.Join(outpath, "pipeline.dot")
116 | if render := cfg.GetDotRenderFormat(); render != "" {
117 | imgname = path.Join(outpath, fmt.Sprintf("pipeline.%s", strings.ToLower(render)))
118 | }
119 | }
120 |
121 | graph, err := graphviz.ParseBytes(dotdata)
122 | if err != nil {
123 | log.Error(err, "Failed to parse pipeline dot data")
124 | break DebugDotData
125 | }
126 | var buf bytes.Buffer
127 | switch strings.ToLower(cfg.GetDotRenderFormat()) {
128 | case "png":
129 | if err := g.Render(graph, graphviz.PNG, &buf); err != nil {
130 | log.Error(err, "Failed to convert dotdata to PNG")
131 | break DebugDotData
132 | }
133 | imgdata = buf.Bytes()
134 | case "svg":
135 | if err := g.Render(graph, graphviz.SVG, &buf); err != nil {
136 | log.Error(err, "Failed to convert dotdata to SVG")
137 | break DebugDotData
138 | }
139 | imgdata = buf.Bytes()
140 | case "jpg":
141 | if err := g.Render(graph, graphviz.JPG, &buf); err != nil {
142 | log.Error(err, "Failed to convert dotdata to JPG")
143 | break DebugDotData
144 | }
145 | imgdata = buf.Bytes()
146 | }
147 |
148 | _, err = mc.PutObject(context.Background(), outbucket, dotname, bytes.NewBuffer(dotdata), int64(len([]byte(dotdata))), minio.PutObjectOptions{
149 | ContentType: "application/octet-stream",
150 | })
151 | if err != nil {
152 | log.Error(err, "Failed to upload pipeline dot data")
153 | break DebugDotData
154 | }
155 |
156 | if imgdata != nil {
157 | if _, err := mc.PutObject(context.Background(), outbucket, imgname, bytes.NewBuffer(imgdata), int64(len(imgdata)), minio.PutObjectOptions{
158 | ContentType: "application/octet-stream",
159 | }); err != nil {
160 | log.Error(err, "Failed to upload rendered image of dot graph")
161 | }
162 | }
163 |
164 | break DebugDotData
165 | }
166 |
167 | posquery := gst.NewPositionQuery(gst.FormatTime)
168 | durquery := gst.NewDurationQuery(gst.FormatTime)
169 | pok := pipeline.Query(posquery)
170 | pipeline.Query(durquery) // we don't care if we don't have a return for this
171 | if !pok {
172 | log.Info("Failed to query the pipeline for the current position")
173 | }
174 | if pok {
175 | _, position := posquery.ParsePosition()
176 | _, duration := durquery.ParseDuration()
177 | log.Info(fmt.Sprintf("Current position %v/%v", time.Duration(position), time.Duration(duration)))
178 | }
179 | }
180 | }()
181 |
182 | mainLoop.Run()
183 |
184 | log.Info("Main loop has returned, ensuring pipeline has reached null state")
185 | for pipeline.GetState() == gst.StatePlaying {
186 | }
187 |
188 | log.Info("Pipeline finished", "State", pipeline.GetState())
189 | }
190 |
191 | func getPipelineCfgAndObjects() (cfg *pipelinesmeta.PipelineConfig, src *pipelinesmeta.Object, sinks []*pipelinesmeta.Object, err error) {
192 | cfg = &pipelinesmeta.PipelineConfig{}
193 | src = &pipelinesmeta.Object{}
194 | sinks = []*pipelinesmeta.Object{}
195 | if err = json.Unmarshal([]byte(os.Getenv(pipelinesmeta.JobPipelineConfigEnvVar)), cfg); err != nil {
196 | return
197 | }
198 | if err = json.Unmarshal([]byte(os.Getenv(pipelinesmeta.JobSrcObjectsEnvVar)), src); err != nil {
199 | return
200 | }
201 | if err = json.Unmarshal([]byte(os.Getenv(pipelinesmeta.JobSinkObjectsEnvVar)), &sinks); err != nil {
202 | return
203 | }
204 | return
205 | }
206 |
--------------------------------------------------------------------------------
/cmd/runner/parse_cr_pipeline.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package main
18 |
19 | import (
20 | "errors"
21 | "fmt"
22 |
23 | "github.com/tinyzimmer/go-gst/gst"
24 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
25 | )
26 |
27 | func buildPipelineFromCR(cfg *pipelinesmeta.PipelineConfig, srcObject *pipelinesmeta.Object, sinkObjects []*pipelinesmeta.Object) (*gst.Pipeline, error) {
28 | // Create a new pipeline
29 | pipeline, err := gst.NewPipeline("")
30 | if err != nil {
31 | return nil, err
32 | }
33 |
34 | pipelineCfg := cfg.GetElements()
35 |
36 | // Create the source element
37 | src, err := makeSrcElement(srcObject)
38 | if err != nil {
39 | return nil, err
40 | }
41 |
42 | pipeline.Add(src)
43 |
44 | var last *gst.Element = src
45 | var lastCfg *pipelinesmeta.GstElementConfig
46 | var staticSinks bool
47 |
48 | for _, elementCfg := range pipelineCfg {
49 |
50 | // If we are jumping in the pipeline - set the last pointers to the appropriate element and config
51 | if elementCfg.GoTo != "" {
52 | // We are jumping in the pipeline
53 | lastCfg = pipelineCfg.GetByAlias(elementCfg.GoTo)
54 | if lastCfg == nil {
55 | return nil, fmt.Errorf("No configuration referenced by alias %s", elementCfg.GoTo)
56 | }
57 | // Set the last element to this one
58 | last, err = elementForPipeline(pipeline, lastCfg)
59 | if err != nil {
60 | return nil, err
61 | }
62 | continue
63 | }
64 |
65 | // If we are linking the previous element - perform the links depending on the alias.
66 | // Sets the last pointers as well, but at this point the user is probably doing a goto
67 | // next or this is the end of the pipeline.
68 | if elementCfg.LinkTo != "" {
69 | var thisElem *gst.Element
70 | var thisCfg *pipelinesmeta.GstElementConfig
71 | var err error
72 | if elementCfg.LinkTo == pipelinesmeta.LinkToVideoOut {
73 | // Check if this is a split pipeline and we are creating a sink for video
74 | staticSinks = true
75 | sinkobj := objectByStreamType(pipelinesmeta.StreamTypeVideo, sinkObjects)
76 | if sinkobj == nil {
77 | return nil, errors.New("No video sink configured for pipeline")
78 | }
79 | thisElem, thisCfg, err = makeSinkElement(sinkobj)
80 | if err != nil {
81 | return nil, err
82 | }
83 | pipeline.Add(thisElem)
84 | } else if elementCfg.LinkTo == pipelinesmeta.LinkToAudioOut {
85 | // Check if this is a split pipeline and we are creating a sink for audio
86 | staticSinks = true
87 | sinkobj := objectByStreamType(pipelinesmeta.StreamTypeAudio, sinkObjects)
88 | if sinkobj == nil {
89 | return nil, errors.New("No audio sink configured for pipeline")
90 | }
91 | thisElem, thisCfg, err = makeSinkElement(sinkobj)
92 | if err != nil {
93 | return nil, err
94 | }
95 | pipeline.Add(thisElem)
96 | } else {
97 | thisCfg = pipelineCfg.GetByAlias(elementCfg.LinkTo)
98 | thisElem, err = elementForPipeline(pipeline, thisCfg)
99 | if err != nil {
100 | return nil, err
101 | }
102 | }
103 | if err := linkLast(pipeline, last, lastCfg, thisElem, thisCfg); err != nil {
104 | return nil, err
105 | }
106 |
107 | last = thisElem
108 | lastCfg = thisCfg
109 | continue
110 | }
111 |
112 | // Neither of the conditions apply we are creating a new element and linking
113 | // the previous one to it.
114 | element, err := elementForPipeline(pipeline, elementCfg)
115 | if err != nil {
116 | return nil, err
117 | }
118 |
119 | if err := linkLast(pipeline, last, lastCfg, element, elementCfg); err != nil {
120 | return nil, err
121 | }
122 |
123 | last = element
124 | lastCfg = elementCfg
125 | }
126 |
127 | // If we did not add static sinks while building the pipeline (i.e. this is a regular Transform pipeline)
128 | // then create a link to the assumed only sink object
129 | if !staticSinks {
130 | sinkobj := objectByStreamType(pipelinesmeta.StreamTypeAll, sinkObjects)
131 | if sinkobj == nil {
132 | return nil, errors.New("No sink configured for pipeline")
133 | }
134 | sink, sinkCfg, err := makeSinkElement(sinkobj)
135 | if err != nil {
136 | return nil, err
137 | }
138 | pipeline.Add(sink)
139 | if err := linkLast(pipeline, last, lastCfg, sink, sinkCfg); err != nil {
140 | return nil, err
141 | }
142 | }
143 |
144 | return pipeline, nil
145 | }
146 |
147 | func linkLast(pipeline *gst.Pipeline, last *gst.Element, lastCfg *pipelinesmeta.GstElementConfig, element *gst.Element, elementCfg *pipelinesmeta.GstElementConfig) error {
148 | // If the last element has a static src pad, link it to this element
149 | // and continue
150 | if srcpad := last.GetStaticPad("src"); srcpad != nil {
151 | return last.Link(element)
152 | }
153 |
154 | // The last element provides dynamic src pads (we hope - user will find out quick if they messed up)
155 | lastCfg.AddPeer(elementCfg)
156 | // weakLastCfg := lastCfg
157 | last.Connect("no-more-pads", func(self *gst.Element) {
158 | pads, err := self.GetPads()
159 | if err != nil {
160 | self.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorFailed, err.Error(), "")
161 | return
162 | }
163 | Pads:
164 | for _, srcpad := range pads {
165 | // Skip already linked and non-src pads
166 | if srcpad.IsLinked() || srcpad.Direction() != gst.PadDirectionSource {
167 | continue Pads
168 | }
169 | for _, peer := range lastCfg.GetPeers() {
170 | peerElem, err := elementForPipeline(pipeline, peer)
171 | if err != nil {
172 | self.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorFailed, err.Error(), "")
173 | return
174 | }
175 | peersink := peerElem.GetStaticPad("sink")
176 | if peersink == nil {
177 | self.ErrorMessage(gst.DomainLibrary, gst.LibraryErrorFailed, fmt.Sprintf("peer %s does not have a static sink pad", peer.Name), "")
178 | return
179 | }
180 | if srcpad.CanLink(peersink) {
181 | srcpad.Link(peersink)
182 | continue Pads
183 | }
184 | }
185 | }
186 | })
187 |
188 | return nil
189 | }
190 |
191 | func elementForPipeline(pipeline *gst.Pipeline, cfg *pipelinesmeta.GstElementConfig) (thiselem *gst.Element, err error) {
192 | // Ensure the element is added to the pipeline
193 | if name := cfg.GetPipelineName(); name != "" {
194 | // the element was already created because it was referenced elsewhere
195 | thiselem, err = pipeline.GetElementByName(name)
196 | if err != nil {
197 | return
198 | }
199 | } else {
200 | thiselem, err = makeElement(cfg)
201 | if err != nil {
202 | return
203 | }
204 | pipeline.Add(thiselem)
205 | cfg.SetPipelineName(thiselem.GetName())
206 | }
207 | return
208 | }
209 |
210 | func objectByStreamType(t pipelinesmeta.StreamType, objs []*pipelinesmeta.Object) *pipelinesmeta.Object {
211 | for _, o := range objs {
212 | if o.StreamType == t {
213 | return o
214 | }
215 | }
216 | return nil
217 | }
218 |
--------------------------------------------------------------------------------
/gst/plugins/minio/miniosink.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "os"
7 | "strings"
8 | "sync"
9 |
10 | "github.com/tinyzimmer/go-glib/glib"
11 | "github.com/tinyzimmer/go-gst/gst"
12 | "github.com/tinyzimmer/go-gst/gst/base"
13 | )
14 |
15 | var sinkCAT = gst.NewDebugCategory(
16 | "miniosink",
17 | gst.DebugColorNone,
18 | "MinIOSink Element",
19 | )
20 |
21 | type minioSink struct {
22 | settings *settings
23 | state *sinkstate
24 |
25 | writer *seekWriter
26 | mux sync.Mutex
27 | }
28 |
29 | type sinkstate struct {
30 | started bool
31 | }
32 |
33 | func (m *minioSink) New() glib.GoObjectSubclass {
34 | srcCAT.Log(gst.LevelLog, "Creating new minioSink object")
35 | return &minioSink{
36 | settings: defaultSettings(),
37 | state: &sinkstate{},
38 | }
39 | }
40 |
41 | func (m *minioSink) ClassInit(klass *glib.ObjectClass) {
42 | class := gst.ToElementClass(klass)
43 | sinkCAT.Log(gst.LevelLog, "Initializing miniosink class")
44 | class.SetMetadata(
45 | "MinIO Sink",
46 | "Sink/File",
47 | "Write stream to a MinIO object",
48 | "Avi Zimmerman ",
49 | )
50 | sinkCAT.Log(gst.LevelLog, "Adding sink pad template and properties to class")
51 | class.AddPadTemplate(gst.NewPadTemplate(
52 | "sink",
53 | gst.PadDirectionSink,
54 | gst.PadPresenceAlways,
55 | gst.NewAnyCaps(),
56 | ))
57 | class.InstallProperties(sinkProperties)
58 | }
59 |
60 | func (m *minioSink) Constructed(obj *glib.Object) { base.ToGstBaseSink(obj).SetSync(false) }
61 |
62 | func (m *minioSink) SetProperty(self *glib.Object, id uint, value *glib.Value) {
63 | setProperty(gst.ToElement(self), sinkProperties, m.settings, id, value)
64 | }
65 |
66 | func (m *minioSink) GetProperty(self *glib.Object, id uint) *glib.Value {
67 | return getProperty(gst.ToElement(self), sinkProperties, m.settings, id)
68 | }
69 |
70 | func (m *minioSink) Query(self *base.GstBaseSink, query *gst.Query) bool {
71 | switch query.Type() {
72 |
73 | case gst.QuerySeeking:
74 | self.Log(sinkCAT, gst.LevelDebug, "Answering seeking query")
75 | query.SetSeeking(gst.FormatTime, true, 0, -1)
76 | return true
77 |
78 | }
79 | return false
80 | }
81 |
82 | func (m *minioSink) Start(self *base.GstBaseSink) bool {
83 | m.mux.Lock()
84 | defer m.mux.Unlock()
85 |
86 | if m.state.started {
87 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorSettings,
88 | "MinIOSink is already started", "")
89 | return false
90 | }
91 |
92 | if m.settings.bucket == "" {
93 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorSettings,
94 | "No bucket configured on the miniosink", "")
95 | return false
96 | }
97 |
98 | if m.settings.key == "" {
99 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorSettings,
100 | "No bucket configured on the miniosink", "")
101 | return false
102 | }
103 |
104 | self.Log(sinkCAT, gst.LevelDebug, m.settings.safestring())
105 |
106 | if strings.HasPrefix(m.settings.accessKeyID, "env:") {
107 | spl := strings.Split(m.settings.accessKeyID, "env:")
108 | m.settings.accessKeyID = os.Getenv(spl[len(spl)-1])
109 | }
110 |
111 | if strings.HasPrefix(m.settings.secretAccessKey, "env:") {
112 | spl := strings.Split(m.settings.secretAccessKey, "env:")
113 | m.settings.secretAccessKey = os.Getenv(spl[len(spl)-1])
114 | }
115 |
116 | self.Log(sinkCAT, gst.LevelInfo, fmt.Sprintf("Creating new MinIO client for %s", m.settings.endpoint))
117 | client, err := getMinIOClient(m.settings)
118 | if err != nil {
119 | self.Log(sinkCAT, gst.LevelError, err.Error())
120 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed,
121 | fmt.Sprintf("Failed to connect to MinIO endpoint %s", m.settings.endpoint), err.Error())
122 | return false
123 | }
124 |
125 | self.Log(sinkCAT, gst.LevelInfo, "Initializing new MinIO writer")
126 | m.writer = newSeekWriter(client, int64(m.settings.partSize), m.settings.bucket, m.settings.key)
127 |
128 | m.state.started = true
129 | self.Log(sinkCAT, gst.LevelInfo, "MinIOSink has started")
130 | return true
131 | }
132 |
133 | func (m *minioSink) Stop(self *base.GstBaseSink) bool {
134 | self.Log(sinkCAT, gst.LevelInfo, "Stopping MinIOSink")
135 | m.mux.Lock()
136 | defer m.mux.Unlock()
137 |
138 | if !m.state.started {
139 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorSettings, "MinIOSink is not started", "")
140 | return false
141 | }
142 |
143 | m.writer = nil
144 | m.state.started = false
145 |
146 | self.Log(sinkCAT, gst.LevelInfo, "MinIOSink has stopped")
147 | return true
148 | }
149 |
150 | func (m *minioSink) Render(self *base.GstBaseSink, buffer *gst.Buffer) gst.FlowReturn {
151 | m.mux.Lock()
152 | defer m.mux.Unlock()
153 |
154 | if !m.state.started {
155 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorSettings, "MinIOSink is not started", "")
156 | return gst.FlowError
157 | }
158 |
159 | self.Log(sinkCAT, gst.LevelTrace, fmt.Sprintf("Rendering buffer %v", buffer))
160 |
161 | if _, err := m.writer.Write(buffer.Bytes()); err != nil {
162 | self.Log(sinkCAT, gst.LevelError, err.Error())
163 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorWrite, fmt.Sprintf("Failed to write data to minio buffer: %s", err.Error()), "")
164 | return gst.FlowError
165 | }
166 |
167 | return gst.FlowOK
168 | }
169 |
170 | func (m *minioSink) Event(self *base.GstBaseSink, event *gst.Event) bool {
171 |
172 | switch event.Type() {
173 |
174 | case gst.EventTypeSegment:
175 | segment := event.ParseSegment()
176 |
177 | if segment.GetFormat() == gst.FormatBytes {
178 | if uint64(m.writer.currentPosition) != segment.GetStart() {
179 | m.mux.Lock()
180 | self.Log(sinkCAT, gst.LevelInfo, fmt.Sprintf("Seeking to %d", segment.GetStart()))
181 | if _, err := m.writer.Seek(int64(segment.GetStart()), io.SeekStart); err != nil {
182 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed, err.Error(), "")
183 | m.mux.Unlock()
184 | return false
185 | }
186 | m.mux.Unlock()
187 | } else {
188 | self.Log(sinkCAT, gst.LevelDebug, "Ignored SEGMENT, no seek needed")
189 | }
190 | } else {
191 | self.Log(sinkCAT, gst.LevelDebug, fmt.Sprintf("Ignored SEGMENT event of format %s", segment.GetFormat().String()))
192 | }
193 |
194 | case gst.EventTypeFlushStop:
195 | self.Log(sinkCAT, gst.LevelInfo, "Flushing contents of writer and seeking back to start")
196 | if m.writer.currentPosition != 0 {
197 | m.mux.Lock()
198 | if err := m.writer.flush(true); err != nil {
199 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorWrite, err.Error(), "")
200 | m.mux.Unlock()
201 | return false
202 | }
203 | if _, err := m.writer.Seek(0, io.SeekStart); err != nil {
204 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorFailed, err.Error(), "")
205 | m.mux.Unlock()
206 | return false
207 | }
208 | m.mux.Unlock()
209 | }
210 |
211 | case gst.EventTypeEOS:
212 | self.Log(sinkCAT, gst.LevelInfo, "Received EOS, closing MinIO writer")
213 | m.mux.Lock()
214 | if err := m.writer.Close(); err != nil {
215 | self.Log(sinkCAT, gst.LevelError, err.Error())
216 | self.ErrorMessage(gst.DomainResource, gst.ResourceErrorClose, fmt.Sprintf("Failed to close MinIO writer: %s", err.Error()), "")
217 | m.mux.Unlock()
218 | return false
219 | }
220 | m.mux.Unlock()
221 |
222 | default:
223 | self.Log(sinkCAT, gst.LevelLog, fmt.Sprintf("Ignoring EVENT: %s", event.Type().String()))
224 | }
225 |
226 | return self.ParentEvent(event)
227 |
228 | }
229 |
--------------------------------------------------------------------------------
/pkg/managers/manager.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package managers
18 |
19 | import (
20 | "bytes"
21 | "context"
22 | "errors"
23 | "io/ioutil"
24 | "path"
25 | "sync"
26 |
27 | minio "github.com/minio/minio-go/v7"
28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
29 | "k8s.io/apimachinery/pkg/types"
30 | ctrl "sigs.k8s.io/controller-runtime"
31 | "sigs.k8s.io/controller-runtime/pkg/client"
32 |
33 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
34 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
35 | pipelinetypes "github.com/tinyzimmer/gst-pipeline-operator/pkg/types"
36 | "github.com/tinyzimmer/gst-pipeline-operator/pkg/util"
37 | )
38 |
39 | var log = ctrl.Log.WithName("pipeline-manager")
40 |
41 | var backoffLimit int32 = 5
42 |
43 | // A globally held map of the current pipeline managers running
44 | var managers = make(map[types.UID]*PipelineManager)
45 | var managersMutex sync.Mutex
46 |
47 | // GetManagerForPipeline returns a PipelineManager for the given transformation pipeline.
48 | // If one already exists globally, it is returned.
49 | func GetManagerForPipeline(client client.Client, pipeline pipelinetypes.Pipeline) *PipelineManager {
50 | managersMutex.Lock()
51 | defer managersMutex.Unlock()
52 |
53 | if manager, ok := managers[pipeline.GetUID()]; ok {
54 | return manager
55 | }
56 | managers[pipeline.GetUID()] = &PipelineManager{
57 | client: client,
58 | pipeline: pipeline,
59 | reloadChan: make(chan struct{}),
60 | stopChan: make(chan struct{}),
61 | }
62 | return managers[pipeline.GetUID()]
63 | }
64 |
65 | // PipelineManager is an object for watching MinIO buckets for changes and queuing
66 | // processing in a pipeline. It exports a method for reloading configuration changes.
67 | type PipelineManager struct {
68 | client client.Client
69 | pipeline pipelinetypes.Pipeline
70 | reloadChan chan struct{}
71 | stopChan chan struct{}
72 | running bool
73 | mux sync.Mutex
74 | }
75 |
76 | var marker = ".gst-watch"
77 |
78 | // Start starts the pipeline manager.
79 | func (p *PipelineManager) Start() error {
80 | p.mux.Lock()
81 | defer p.mux.Unlock()
82 |
83 | if p.running {
84 | return errors.New("pipeline manager is already running")
85 | }
86 |
87 | srcConfigFull := p.pipeline.GetSrcConfig()
88 | if srcConfigFull.MinIO == nil {
89 | return errors.New("Non-MinIO sources are not yet implemented")
90 | }
91 | srcConfig := srcConfigFull.MinIO
92 |
93 | client, err := util.GetMinIOClient(srcConfig, util.MinIOWatchCredentialsFromCR(p.client, p.pipeline))
94 | if err != nil {
95 | return err
96 | }
97 |
98 | markerName := path.Join(srcConfig.GetPrefix(), marker)
99 |
100 | // Check for a marker in the prefix we are watching. This checks for the existence of the
101 | // bucket as well as ensure the subsequent watch works correctly.
102 | obj, err := client.GetObject(context.TODO(), srcConfig.GetBucket(), markerName, minio.GetObjectOptions{})
103 | if err != nil {
104 | return err
105 | }
106 |
107 | // The errors we care about would be while trying to read it
108 | if _, err := ioutil.ReadAll(obj); err != nil {
109 | if resErr, ok := err.(minio.ErrorResponse); ok {
110 | switch resErr.Code {
111 | case "NoSuchKey":
112 | log.Info("Laying watch marker in bucket prefix", "Bucket", srcConfig.GetBucket(), "Prefix", srcConfig.GetPrefix())
113 | if _, err := client.PutObject(context.TODO(), srcConfig.GetBucket(), markerName, bytes.NewReader([]byte{}), 0, minio.PutObjectOptions{}); err != nil {
114 | return err
115 | }
116 | default:
117 | return err
118 | }
119 | } else {
120 | return err
121 | }
122 | }
123 |
124 | go p.watchSrcBucket(srcConfig, client)
125 | p.running = true
126 | return nil
127 | }
128 |
129 | // IsRunning returns true if the pipeline manager is already running.
130 | func (p *PipelineManager) IsRunning() bool { return p.running }
131 |
132 | // Reload reloads the bucket watchers with the given pipeline configuration.
133 | func (p *PipelineManager) Reload(cfg pipelinetypes.Pipeline) {
134 | p.mux.Lock()
135 | defer p.mux.Unlock()
136 |
137 | p.pipeline = cfg
138 | p.reloadChan <- struct{}{}
139 | }
140 |
141 | // Stop stops the bucket watching goroutine.
142 | func (p *PipelineManager) Stop() {
143 | p.mux.Lock()
144 | defer p.mux.Unlock()
145 |
146 | p.stopChan <- struct{}{}
147 | p.running = false
148 | }
149 |
150 | func (p *PipelineManager) watchSrcBucket(srcConfig *pipelinesmeta.MinIOConfig, client *minio.Client) {
151 | log.Info("Watching for object created events", "Bucket", srcConfig.GetBucket(), "Prefix", srcConfig.GetPrefix())
152 | eventChan := client.ListenBucketNotification(context.Background(), srcConfig.GetBucket(), srcConfig.GetPrefix(), "", []string{"s3:ObjectCreated:*"})
153 | excludeRegex := srcConfig.GetExcludeRegex()
154 | for {
155 | select {
156 | case event := <-eventChan:
157 | for _, record := range event.Records {
158 | log.Info("Processing record from MinIO event", "Record", record)
159 | if excludeRegex != nil && excludeRegex.MatchString(record.S3.Object.Key) {
160 | log.Info("Skipping processing for item matching exclude regex", "Object", record.S3.Object.Key)
161 | continue
162 | }
163 | p.createJob(srcConfig, record.S3.Object.Key)
164 | }
165 | case <-p.reloadChan:
166 | srcConfig = p.pipeline.GetSrcConfig().MinIO // TODO
167 | excludeRegex = srcConfig.GetExcludeRegex()
168 | log.Info("Reloading event channel", "Bucket", srcConfig.GetBucket(), "Prefix", srcConfig.GetPrefix())
169 | eventChan = client.ListenBucketNotification(context.Background(), srcConfig.GetBucket(), srcConfig.GetPrefix(), "", []string{"s3:ObjectCreated:*"})
170 | case <-p.stopChan:
171 | return
172 | }
173 | }
174 | }
175 |
176 | func (p *PipelineManager) createJob(srcConfig *pipelinesmeta.MinIOConfig, object string) {
177 | log.Info("Creating pipeline job", "Bucket", srcConfig.GetBucket(), "Key", object)
178 | job := p.newJobForObject(object)
179 | if err := p.client.Create(context.TODO(), job); err != nil {
180 | log.Error(err, "Failed to create processing job for object")
181 | }
182 | }
183 |
184 | func (p *PipelineManager) newJobForObject(key string) *pipelinesv1.Job {
185 | job := &pipelinesv1.Job{
186 | ObjectMeta: metav1.ObjectMeta{
187 | GenerateName: p.pipeline.GetName(),
188 | Namespace: p.pipeline.GetNamespace(),
189 | Labels: pipelinesv1.GetJobLabels(p.pipeline, key),
190 | OwnerReferences: p.pipeline.OwnerReferences(),
191 | },
192 | Spec: pipelinesv1.JobSpec{
193 | PipelineReference: pipelinesmeta.PipelineReference{
194 | Name: p.pipeline.GetName(),
195 | Kind: p.pipeline.GetPipelineKind(),
196 | },
197 | Source: &pipelinesmeta.Object{
198 | Name: key,
199 | Config: p.pipeline.GetSrcConfig(),
200 | },
201 | Sinks: p.pipeline.GetSinkObjects(key),
202 | },
203 | }
204 | return job
205 | }
206 |
--------------------------------------------------------------------------------
/apis/meta/v1/zz_generated.deepcopy.go:
--------------------------------------------------------------------------------
1 | // +build !ignore_autogenerated
2 |
3 | /*
4 | Copyright 2021.
5 |
6 | Licensed under the Apache License, Version 2.0 (the "License");
7 | you may not use this file except in compliance with the License.
8 | You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing, software
13 | distributed under the License is distributed on an "AS IS" BASIS,
14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | See the License for the specific language governing permissions and
16 | limitations under the License.
17 | */
18 |
19 | // Code generated by controller-gen. DO NOT EDIT.
20 |
21 | package v1
22 |
23 | import (
24 | corev1 "k8s.io/api/core/v1"
25 | )
26 |
27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
28 | func (in *DebugConfig) DeepCopyInto(out *DebugConfig) {
29 | *out = *in
30 | if in.Dot != nil {
31 | in, out := &in.Dot, &out.Dot
32 | *out = new(DotConfig)
33 | **out = **in
34 | }
35 | }
36 |
37 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DebugConfig.
38 | func (in *DebugConfig) DeepCopy() *DebugConfig {
39 | if in == nil {
40 | return nil
41 | }
42 | out := new(DebugConfig)
43 | in.DeepCopyInto(out)
44 | return out
45 | }
46 |
47 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
48 | func (in *DotConfig) DeepCopyInto(out *DotConfig) {
49 | *out = *in
50 | }
51 |
52 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DotConfig.
53 | func (in *DotConfig) DeepCopy() *DotConfig {
54 | if in == nil {
55 | return nil
56 | }
57 | out := new(DotConfig)
58 | in.DeepCopyInto(out)
59 | return out
60 | }
61 |
62 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
63 | func (in *ElementConfig) DeepCopyInto(out *ElementConfig) {
64 | *out = *in
65 | if in.Properties != nil {
66 | in, out := &in.Properties, &out.Properties
67 | *out = make(map[string]string, len(*in))
68 | for key, val := range *in {
69 | (*out)[key] = val
70 | }
71 | }
72 | }
73 |
74 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ElementConfig.
75 | func (in *ElementConfig) DeepCopy() *ElementConfig {
76 | if in == nil {
77 | return nil
78 | }
79 | out := new(ElementConfig)
80 | in.DeepCopyInto(out)
81 | return out
82 | }
83 |
84 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
85 | func (in *GstElementConfig) DeepCopyInto(out *GstElementConfig) {
86 | *out = *in
87 | if in.ElementConfig != nil {
88 | in, out := &in.ElementConfig, &out.ElementConfig
89 | *out = new(ElementConfig)
90 | (*in).DeepCopyInto(*out)
91 | }
92 | if in.peers != nil {
93 | in, out := &in.peers, &out.peers
94 | *out = make([]*GstElementConfig, len(*in))
95 | for i := range *in {
96 | if (*in)[i] != nil {
97 | in, out := &(*in)[i], &(*out)[i]
98 | *out = new(GstElementConfig)
99 | (*in).DeepCopyInto(*out)
100 | }
101 | }
102 | }
103 | }
104 |
105 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GstElementConfig.
106 | func (in *GstElementConfig) DeepCopy() *GstElementConfig {
107 | if in == nil {
108 | return nil
109 | }
110 | out := new(GstElementConfig)
111 | in.DeepCopyInto(out)
112 | return out
113 | }
114 |
115 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
116 | func (in GstLaunchConfig) DeepCopyInto(out *GstLaunchConfig) {
117 | {
118 | in := &in
119 | *out = make(GstLaunchConfig, len(*in))
120 | for i := range *in {
121 | if (*in)[i] != nil {
122 | in, out := &(*in)[i], &(*out)[i]
123 | *out = new(GstElementConfig)
124 | (*in).DeepCopyInto(*out)
125 | }
126 | }
127 | }
128 | }
129 |
130 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GstLaunchConfig.
131 | func (in GstLaunchConfig) DeepCopy() GstLaunchConfig {
132 | if in == nil {
133 | return nil
134 | }
135 | out := new(GstLaunchConfig)
136 | in.DeepCopyInto(out)
137 | return *out
138 | }
139 |
140 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
141 | func (in *MinIOConfig) DeepCopyInto(out *MinIOConfig) {
142 | *out = *in
143 | if in.CredentialsSecret != nil {
144 | in, out := &in.CredentialsSecret, &out.CredentialsSecret
145 | *out = new(corev1.LocalObjectReference)
146 | **out = **in
147 | }
148 | }
149 |
150 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MinIOConfig.
151 | func (in *MinIOConfig) DeepCopy() *MinIOConfig {
152 | if in == nil {
153 | return nil
154 | }
155 | out := new(MinIOConfig)
156 | in.DeepCopyInto(out)
157 | return out
158 | }
159 |
160 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
161 | func (in *Object) DeepCopyInto(out *Object) {
162 | *out = *in
163 | if in.Config != nil {
164 | in, out := &in.Config, &out.Config
165 | *out = new(SourceSinkConfig)
166 | (*in).DeepCopyInto(*out)
167 | }
168 | }
169 |
170 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Object.
171 | func (in *Object) DeepCopy() *Object {
172 | if in == nil {
173 | return nil
174 | }
175 | out := new(Object)
176 | in.DeepCopyInto(out)
177 | return out
178 | }
179 |
180 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
181 | func (in *PipelineConfig) DeepCopyInto(out *PipelineConfig) {
182 | *out = *in
183 | if in.Debug != nil {
184 | in, out := &in.Debug, &out.Debug
185 | *out = new(DebugConfig)
186 | (*in).DeepCopyInto(*out)
187 | }
188 | if in.Elements != nil {
189 | in, out := &in.Elements, &out.Elements
190 | *out = make([]*ElementConfig, len(*in))
191 | for i := range *in {
192 | if (*in)[i] != nil {
193 | in, out := &(*in)[i], &(*out)[i]
194 | *out = new(ElementConfig)
195 | (*in).DeepCopyInto(*out)
196 | }
197 | }
198 | }
199 | in.Resources.DeepCopyInto(&out.Resources)
200 | }
201 |
202 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PipelineConfig.
203 | func (in *PipelineConfig) DeepCopy() *PipelineConfig {
204 | if in == nil {
205 | return nil
206 | }
207 | out := new(PipelineConfig)
208 | in.DeepCopyInto(out)
209 | return out
210 | }
211 |
212 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
213 | func (in *PipelineReference) DeepCopyInto(out *PipelineReference) {
214 | *out = *in
215 | }
216 |
217 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PipelineReference.
218 | func (in *PipelineReference) DeepCopy() *PipelineReference {
219 | if in == nil {
220 | return nil
221 | }
222 | out := new(PipelineReference)
223 | in.DeepCopyInto(out)
224 | return out
225 | }
226 |
227 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
228 | func (in *SourceSinkConfig) DeepCopyInto(out *SourceSinkConfig) {
229 | *out = *in
230 | if in.MinIO != nil {
231 | in, out := &in.MinIO, &out.MinIO
232 | *out = new(MinIOConfig)
233 | (*in).DeepCopyInto(*out)
234 | }
235 | }
236 |
237 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceSinkConfig.
238 | func (in *SourceSinkConfig) DeepCopy() *SourceSinkConfig {
239 | if in == nil {
240 | return nil
241 | }
242 | out := new(SourceSinkConfig)
243 | in.DeepCopyInto(out)
244 | return out
245 | }
246 |
--------------------------------------------------------------------------------
/apis/meta/v1/minio.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package v1
18 |
19 | import (
20 | "bytes"
21 | "context"
22 | "crypto/x509"
23 | "encoding/base64"
24 | "errors"
25 | "fmt"
26 | "path"
27 | "regexp"
28 | "strings"
29 | "text/template"
30 |
31 | "github.com/Masterminds/sprig"
32 | "github.com/minio/minio-go/v7/pkg/credentials"
33 | corev1 "k8s.io/api/core/v1"
34 | "k8s.io/apimachinery/pkg/types"
35 | "sigs.k8s.io/controller-runtime/pkg/client"
36 | )
37 |
38 | // MinIOConfig defines a source or sink location for pipelines.
39 | type MinIOConfig struct {
40 | // The MinIO endpoint *without* the leading `http(s)://`.
41 | Endpoint string `json:"endpoint,omitempty"`
42 | // Do not use TLS when communicating with the MinIO API.
43 | InsecureNoTLS bool `json:"insecureNoTLS,omitempty"`
44 | // A base64-endcoded PEM certificate chain to use when verifying the certificate
45 | // supplied by the MinIO server.
46 | EndpointCA string `json:"endpointCA,omitempty"`
47 | // Skip verification of the certificate supplied by the MinIO server.
48 | InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
49 | // The region to connect to in MinIO.
50 | Region string `json:"region,omitempty"`
51 | // In the context of a src config, the bucket to watch for objects to pass through
52 | // the pipeline. In the context of a sink config, the bucket to save processed objects.
53 | Bucket string `json:"bucket,omitempty"`
54 | // In the context of a src config, a directory prefix to match for objects to be sent
55 | // through the pipeline. An empty value means ALL objects in the bucket, or the equivalent of
56 | // `/`. In the context of a sink config, a go-template to use for the destination name. The
57 | // template allows sprig functions and is passed the value "SrcName" representing the base of the key
58 | // of the object that triggered the pipeline, and "SrcExt" with the extension. An empty value represents
59 | // using the same key as the source which would only work for objects being processed to different
60 | // buckets and prefixes.
61 | Prefix string `json:"key,omitempty"`
62 | // A regular expression to filter out items placed in the `key`. Only makes sense in the context of a src
63 | // config. This can be useful when chaining pipelines. You may want to exclude the "*_tmp" expression to
64 | // filter out the temporary objects created while the miniosink is rendering the output of a pipeline, since
65 | // it first creates chunked objects, and then pieces them together with the ComposeObject API.
66 | Exclude string `json:"exclude,omitempty"`
67 | // The secret that contains the credentials for connecting to MinIO. The secret must contain
68 | // two keys. The `access-key-id` key must contain the contents of the Access Key ID. The
69 | // `secret-access-key` key must contain the contents of the Secret Access Key.
70 | CredentialsSecret *corev1.LocalObjectReference `json:"credentialsSecret,omitempty"`
71 | }
72 |
73 | // GetEndpoint returns the API endpoint for this configuration.
74 | func (m *MinIOConfig) GetEndpoint() string { return m.Endpoint }
75 |
76 | // GetSecure returns whether to use HTTPS for API communication.
77 | func (m *MinIOConfig) GetSecure() bool { return !m.InsecureNoTLS }
78 |
79 | // GetSkipVerify returns where to skip TLS verification of the server certificate.
80 | func (m *MinIOConfig) GetSkipVerify() bool { return !m.InsecureSkipVerify }
81 |
82 | // GetBucket returns the bucket for this configuration.
83 | func (m *MinIOConfig) GetBucket() string { return m.Bucket }
84 |
85 | // GetPrefix returns the prefix for this configuration.
86 | func (m *MinIOConfig) GetPrefix() string { return m.Prefix }
87 |
88 | // GetRegion returns the region to connect the client to.
89 | func (m *MinIOConfig) GetRegion() string {
90 | if m.Region == "" {
91 | return DefaultRegion
92 | }
93 | return m.Region
94 | }
95 |
96 | // GetRootPEM returns the raw PEM of the root certificate included in the configuration.
97 | func (m *MinIOConfig) GetRootPEM() ([]byte, error) {
98 | if m.EndpointCA == "" {
99 | return nil, nil
100 | }
101 | return base64.StdEncoding.DecodeString(m.EndpointCA)
102 | }
103 |
104 | // GetRootCAs returns an x509.CertPool for any provided CA certificates.
105 | func (m *MinIOConfig) GetRootCAs() (*x509.CertPool, error) {
106 | certPEM, err := m.GetRootPEM()
107 | if err != nil {
108 | return nil, err
109 | }
110 | if certPEM == nil {
111 | return nil, nil
112 | }
113 | certPool := x509.NewCertPool()
114 | if ok := certPool.AppendCertsFromPEM(certPEM); !ok {
115 | return nil, errors.New("Failed to append CA PEM certificates to cert pool")
116 | }
117 | return certPool, nil
118 | }
119 |
120 | // GetCredentialsSecret returns the name of the credentials secret.
121 | func (m *MinIOConfig) GetCredentialsSecret() (string, error) {
122 | if m.CredentialsSecret == nil {
123 | return "", errors.New("No secret reference included in the CR for endpoint credentials")
124 | }
125 | return m.CredentialsSecret.Name, nil
126 | }
127 |
128 | // GetCredentials attemps to retrieve the access key ID and secret access key for this config.
129 | func (m *MinIOConfig) GetCredentials(client client.Client, namespace string) (accessKeyID, secretAccessKey string, err error) {
130 | secretName, err := m.GetCredentialsSecret()
131 | if err != nil {
132 | return "", "", err
133 | }
134 | secret := &corev1.Secret{}
135 | if err := client.Get(context.TODO(), types.NamespacedName{Name: secretName, Namespace: namespace}, secret); err != nil {
136 | return "", "", err
137 | }
138 | accessKeyIDRaw, ok := secret.Data[AccessKeyIDKey]
139 | if !ok {
140 | return "", "", fmt.Errorf("No %s in secret %s/%s", AccessKeyIDKey, namespace, secretName)
141 | }
142 | secretAccessKeyRaw, ok := secret.Data[SecretAccessKeyKey]
143 | if !ok {
144 | return "", "", fmt.Errorf("No %s in secret %s/%s", SecretAccessKeyKey, namespace, secretName)
145 | }
146 | return string(accessKeyIDRaw), string(secretAccessKeyRaw), nil
147 | }
148 |
149 | // GetStaticCredentials attempts to return API credentials for MinIO using the given
150 | // client looking in the given namespace.
151 | func (m *MinIOConfig) GetStaticCredentials(client client.Client, namespace string) (*credentials.Credentials, error) {
152 | accessKeyID, secretAccessKey, err := m.GetCredentials(client, namespace)
153 | if err != nil {
154 | return nil, err
155 | }
156 | return credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), nil
157 | }
158 |
159 | // GetDestinationKey computes what the destination object's name should be based on the
160 | // given source object name. If a template is present and it fails to execute, it is logged
161 | // and the default behavior is returned.
162 | func (m *MinIOConfig) GetDestinationKey(objectKey string) string {
163 | tmpl := m.GetPrefix()
164 | for tmpl != "" {
165 | ext := path.Ext(objectKey)
166 | name := path.Base(strings.TrimSuffix(objectKey, ext))
167 | var buf bytes.Buffer
168 | var t *template.Template
169 | t, err := template.New("").Funcs(sprig.TxtFuncMap()).Parse(tmpl)
170 | if err != nil {
171 | fmt.Println(err)
172 | break
173 | }
174 | t.Execute(&buf, map[string]string{
175 | "SrcName": name,
176 | "SrcExt": ext,
177 | })
178 | return buf.String()
179 | }
180 | return path.Join(strings.TrimSuffix(m.GetPrefix(), "/"), path.Base(objectKey))
181 | }
182 |
183 | // GetExcludeRegex returns the regex to use for excluding objects, or nil if not present
184 | // or any error.
185 | func (m *MinIOConfig) GetExcludeRegex() *regexp.Regexp {
186 | if m.Exclude == "" {
187 | return nil
188 | }
189 | re, err := regexp.Compile(m.Exclude)
190 | if err != nil {
191 | fmt.Println("Failed to compile exclude regex", m.Exclude, "error:", err)
192 | return nil
193 | }
194 | return re
195 | }
196 |
--------------------------------------------------------------------------------
/controllers/pipelines/jobs.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2021.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package pipelines
18 |
19 | import (
20 | "context"
21 | "encoding/json"
22 |
23 | pipelinesmeta "github.com/tinyzimmer/gst-pipeline-operator/apis/meta/v1"
24 | pipelinesv1 "github.com/tinyzimmer/gst-pipeline-operator/apis/pipelines/v1"
25 | pipelinetypes "github.com/tinyzimmer/gst-pipeline-operator/pkg/types"
26 |
27 | "github.com/go-logr/logr"
28 | batchv1 "k8s.io/api/batch/v1"
29 | corev1 "k8s.io/api/core/v1"
30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31 | "k8s.io/apimachinery/pkg/types"
32 | "sigs.k8s.io/controller-runtime/pkg/client"
33 | )
34 |
35 | var backoffLimit int32 = 5
36 |
37 | func reconcileJob(ctx context.Context, reqLogger logr.Logger, c client.Client, pipelineJob *pipelinesv1.Job, job *batchv1.Job) error {
38 | nn := types.NamespacedName{
39 | Name: job.GetName(),
40 | Namespace: job.GetNamespace(),
41 | }
42 |
43 | // Check if job exists
44 | found := &batchv1.Job{}
45 | err := c.Get(ctx, nn, found)
46 | if err != nil {
47 | if client.IgnoreNotFound(err) != nil {
48 | return err
49 | }
50 | // Need to create job
51 | reqLogger.Info("Creating new Job", "Name", job.GetName(), "Namespace", job.GetNamespace())
52 | err := c.Create(ctx, job)
53 | if err != nil {
54 | return err
55 | }
56 | // Add job pending status condition
57 | reqLogger.Info("Updating pipeline job status to Pending")
58 | pipelineJob.Status.Conditions = append(pipelineJob.Status.Conditions, metav1.Condition{
59 | Type: string(pipelinesv1.JobPending),
60 | Status: metav1.ConditionTrue,
61 | ObservedGeneration: pipelineJob.GetGeneration(),
62 | LastTransitionTime: metav1.Now(),
63 | Reason: "JobCreated",
64 | Message: "The pipeline job has been created",
65 | })
66 | return c.Status().Update(ctx, pipelineJob)
67 | }
68 |
69 | // The job exists
70 |
71 | reqLogger = reqLogger.WithValues("Name", found.GetName(), "Namespace", found.GetNamespace())
72 |
73 | // Check if the job is still pending
74 | if jobPending(found) {
75 | reqLogger.Info("Job is currently pending creation")
76 | // Add job in progress status condition
77 | if !statusObservedForGeneration(string(pipelinesv1.JobPending), "JobPending", pipelineJob) {
78 | reqLogger.Info("Job is waiting for active containers")
79 | pipelineJob.Status.Conditions = append(pipelineJob.Status.Conditions, metav1.Condition{
80 | Type: string(pipelinesv1.JobPending),
81 | Status: metav1.ConditionTrue,
82 | ObservedGeneration: pipelineJob.GetGeneration(),
83 | LastTransitionTime: metav1.Now(),
84 | Reason: "JobPending",
85 | Message: "Waiting for the job to be scheduled",
86 | })
87 | return c.Status().Update(ctx, pipelineJob)
88 | }
89 | return nil
90 | }
91 |
92 | // Check if the job is still in progress
93 | if jobInProgress(found) {
94 | reqLogger.Info("Job is currently in progress")
95 | // Add job in progress status condition
96 | if !statusObservedForGeneration(string(pipelinesv1.JobInProgress), "JobInProgress", pipelineJob) {
97 | reqLogger.Info("Job is in progress, updating status")
98 | pipelineJob.Status.Conditions = append(pipelineJob.Status.Conditions, metav1.Condition{
99 | Type: string(pipelinesv1.JobInProgress),
100 | Status: metav1.ConditionTrue,
101 | ObservedGeneration: pipelineJob.GetGeneration(),
102 | LastTransitionTime: metav1.Now(),
103 | Reason: "JobInProgress",
104 | Message: "The pipeline job is currently running",
105 | })
106 | return c.Status().Update(ctx, pipelineJob)
107 | }
108 | return nil
109 | }
110 |
111 | // Check if the job succeeded
112 | if jobSucceeded(found) {
113 | // Add job finished status condition
114 | if !statusObservedForGeneration(string(pipelinesv1.JobFinished), "JobFinished", pipelineJob) {
115 | reqLogger.Info("Job finished successfully, updating status")
116 | pipelineJob.Status.Conditions = append(pipelineJob.Status.Conditions, metav1.Condition{
117 | Type: string(pipelinesv1.JobFinished),
118 | Status: metav1.ConditionTrue,
119 | ObservedGeneration: pipelineJob.GetGeneration(),
120 | LastTransitionTime: metav1.Now(),
121 | Reason: "JobFinished",
122 | Message: "The pipeline job completed successfully",
123 | })
124 | return c.Status().Update(ctx, pipelineJob)
125 | }
126 | return nil
127 | }
128 |
129 | // Check if the job failed
130 | if jobFailed(found) {
131 | // Add job failed status condition
132 | if !statusObservedForGeneration(string(pipelinesv1.JobFailed), "JobFailed", pipelineJob) {
133 | reqLogger.Info("Job failed, updating status")
134 | pipelineJob.Status.Conditions = append(pipelineJob.Status.Conditions, metav1.Condition{
135 | Type: string(pipelinesv1.JobFailed),
136 | Status: metav1.ConditionTrue,
137 | ObservedGeneration: pipelineJob.GetGeneration(),
138 | LastTransitionTime: metav1.Now(),
139 | Reason: "JobFailed",
140 | Message: "The pipeline job failed to complete",
141 | })
142 | return c.Status().Update(ctx, pipelineJob)
143 | }
144 | return nil
145 | }
146 |
147 | return nil
148 | }
149 |
150 | func jobSucceeded(job *batchv1.Job) bool { return job.Status.Succeeded == 1 }
151 | func jobFailed(job *batchv1.Job) bool { return job.Status.Failed == 1 }
152 | func jobInProgress(job *batchv1.Job) bool { return job.Status.Succeeded == 0 && job.Status.Failed == 0 }
153 | func jobPending(job *batchv1.Job) bool { return job.Status.Active == 0 && jobInProgress(job) }
154 |
155 | func newPipelineJob(pipelineJob *pipelinesv1.Job, pipeline pipelinetypes.Pipeline) (*batchv1.Job, error) {
156 | // TODO
157 | srcConfig := pipeline.GetSrcConfig().MinIO
158 | sinkConfig := pipeline.GetSinkConfig().MinIO
159 | srcSecret, err := srcConfig.GetCredentialsSecret()
160 | if err != nil {
161 | return nil, err
162 | }
163 | sinkSecret, err := sinkConfig.GetCredentialsSecret()
164 | if err != nil {
165 | return nil, err
166 | }
167 | pipelineCfg := pipeline.GetPipelineConfig()
168 | marshaledConfig, err := json.Marshal(pipelineCfg)
169 | if err != nil {
170 | return nil, err
171 | }
172 | marshaledSrc, err := json.Marshal(pipelineJob.Spec.Source)
173 | if err != nil {
174 | return nil, err
175 | }
176 | marshaledSinks, err := json.Marshal(pipelineJob.Spec.Sinks)
177 | if err != nil {
178 | return nil, err
179 | }
180 | job := &batchv1.Job{
181 | ObjectMeta: metav1.ObjectMeta{
182 | Name: pipelineJob.GetName(),
183 | Namespace: pipelineJob.GetNamespace(),
184 | Labels: pipelineJob.GetLabels(),
185 | OwnerReferences: pipelineJob.OwnerReferences(),
186 | },
187 | Spec: batchv1.JobSpec{
188 | BackoffLimit: &backoffLimit,
189 | Template: corev1.PodTemplateSpec{
190 | Spec: corev1.PodSpec{
191 | RestartPolicy: corev1.RestartPolicyOnFailure,
192 | Containers: []corev1.Container{
193 | {
194 | Name: "gstreamer",
195 | Image: pipelineCfg.GetImage(),
196 | Resources: pipelineCfg.Resources,
197 | Env: []corev1.EnvVar{
198 | {
199 | Name: "GST_DEBUG",
200 | Value: pipelineCfg.GetGSTDebug(),
201 | },
202 | {
203 | Name: pipelinesmeta.JobSrcObjectsEnvVar,
204 | Value: string(marshaledSrc),
205 | },
206 | {
207 | Name: pipelinesmeta.JobSinkObjectsEnvVar,
208 | Value: string(marshaledSinks),
209 | },
210 | {
211 | Name: pipelinesmeta.JobPipelineConfigEnvVar,
212 | Value: string(marshaledConfig),
213 | },
214 | {
215 | Name: pipelinesmeta.MinIOSrcAccessKeyIDEnvVar,
216 | ValueFrom: &corev1.EnvVarSource{
217 | SecretKeyRef: &corev1.SecretKeySelector{
218 | LocalObjectReference: corev1.LocalObjectReference{
219 | Name: srcSecret,
220 | },
221 | Key: pipelinesmeta.AccessKeyIDKey,
222 | },
223 | },
224 | },
225 | {
226 | Name: pipelinesmeta.MinIOSrcSecretAccessKeyEnvVar,
227 | ValueFrom: &corev1.EnvVarSource{
228 | SecretKeyRef: &corev1.SecretKeySelector{
229 | LocalObjectReference: corev1.LocalObjectReference{
230 | Name: srcSecret,
231 | },
232 | Key: pipelinesmeta.SecretAccessKeyKey,
233 | },
234 | },
235 | },
236 | {
237 | Name: pipelinesmeta.MinIOSinkAccessKeyIDEnvVar,
238 | ValueFrom: &corev1.EnvVarSource{
239 | SecretKeyRef: &corev1.SecretKeySelector{
240 | LocalObjectReference: corev1.LocalObjectReference{
241 | Name: sinkSecret,
242 | },
243 | Key: pipelinesmeta.AccessKeyIDKey,
244 | },
245 | },
246 | },
247 | {
248 | Name: pipelinesmeta.MinIOSinkSecretAccessKeyEnvVar,
249 | ValueFrom: &corev1.EnvVarSource{
250 | SecretKeyRef: &corev1.SecretKeySelector{
251 | LocalObjectReference: corev1.LocalObjectReference{
252 | Name: sinkSecret,
253 | },
254 | Key: pipelinesmeta.SecretAccessKeyKey,
255 | },
256 | },
257 | },
258 | },
259 | },
260 | },
261 | },
262 | },
263 | },
264 | }
265 |
266 | return job, nil
267 | }
268 |
--------------------------------------------------------------------------------