├── .code.yml ├── .github └── workflows │ └── go.yml ├── .gitignore ├── .gitmodules ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── api └── v1alpha1 │ ├── groupversion_info.go │ ├── jupytergateway_types.go │ ├── jupyterkernel_types.go │ ├── jupyterkernelspec_types.go │ ├── jupyterkerneltemplate_types.go │ ├── jupyternotebook_types.go │ └── zz_generated.deepcopy.go ├── cli ├── README.md ├── cmd │ └── root.go └── main.go ├── config ├── certmanager │ ├── certificate.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── crd │ ├── bases │ │ ├── kubeflow.tkestack.io_jupytergateways.yaml │ │ ├── kubeflow.tkestack.io_jupyterkernels.yaml │ │ ├── kubeflow.tkestack.io_jupyterkernelspecs.yaml │ │ ├── kubeflow.tkestack.io_jupyterkerneltemplates.yaml │ │ └── kubeflow.tkestack.io_jupyternotebooks.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_jupytergateways.yaml │ │ ├── cainjection_in_jupyterkernels.yaml │ │ ├── cainjection_in_jupyterkernelspecs.yaml │ │ ├── cainjection_in_jupyterkerneltemplates.yaml │ │ ├── cainjection_in_jupyternotebooks.yaml │ │ ├── webhook_in_jupytergateways.yaml │ │ ├── webhook_in_jupyterkernels.yaml │ │ ├── webhook_in_jupyterkernelspecs.yaml │ │ ├── webhook_in_jupyterkerneltemplates.yaml │ │ └── webhook_in_jupyternotebooks.yaml ├── default │ └── kustomization.yaml ├── manager │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── jupytergateway_editor_role.yaml │ ├── jupytergateway_viewer_role.yaml │ ├── jupyterkernel_editor_role.yaml │ ├── jupyterkernel_viewer_role.yaml │ ├── jupyterkernelspec_editor_role.yaml │ ├── jupyterkernelspec_viewer_role.yaml │ ├── jupyterkerneltemplate_editor_role.yaml │ ├── jupyterkerneltemplate_viewer_role.yaml │ ├── jupyternotebook_editor_role.yaml │ ├── jupyternotebook_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ └── role_binding.yaml ├── samples │ ├── kubeflow.tkestack.io_v1alpha1_jupytergateway-kernels.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupyterkernel.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupyterkernelspec-custom-launcher.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml │ └── kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml └── webhook │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── service.yaml ├── controllers ├── jupytergateway_controller.go ├── jupyterkernel_controller.go ├── jupyterkernelspec_controller.go ├── jupyterkerneltemplate_controller.go ├── jupyternotebook_controller.go ├── jupyternotebook_controller_test.go └── suite_test.go ├── docs ├── README.md ├── api │ ├── autogen │ │ ├── config.yaml │ │ └── templates │ │ │ ├── gv_details.tpl │ │ │ ├── gv_list.tpl │ │ │ ├── type.tpl │ │ │ └── type_members.tpl │ └── generated.asciidoc ├── design.md ├── images │ ├── arch.jpeg │ ├── elastic.jpeg │ ├── gateway.png │ ├── jupyter.jpeg │ ├── kubeflow.jpeg │ ├── multiuser.jpeg │ ├── overview.jpeg │ └── uml.jpeg ├── kernel.md └── quick-start.md ├── examples ├── elastic-with-custom-kernels │ ├── kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml │ ├── kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml │ └── kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml ├── elastic │ ├── kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml │ └── kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml └── simple-deployments │ └── kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml ├── go.mod ├── go.sum ├── hack ├── add-license.sh ├── boilerplate.go.txt ├── enterprise_gateway │ └── prepare.yaml └── license.txt ├── main.go └── pkg ├── gateway ├── generate.go └── reconcile.go ├── kernel ├── generate.go └── reconcile.go ├── kernelspec ├── generate.go ├── reconcile.go └── types.go └── notebook ├── generate.go ├── generate_test.go ├── reconcile.go └── reconcile_test.go /.code.yml: -------------------------------------------------------------------------------- 1 | source: 2 | third_party_source: 3 | filepath_regex: [".*/cmd/.*", 4 | ".*/examples/.*", 5 | ".*/hack/.*", 6 | ".*/installer/.*", 7 | ".*/pkg/.*", 8 | ".*/third_party/.*", 9 | ".*/build/.*", 10 | ".*/test/.*"] -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.17 20 | 21 | - name: Reuse go mod cache 22 | uses: actions/cache@v2 23 | with: 24 | path: ~/go/pkg/mod 25 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 26 | restore-keys: | 27 | ${{ runner.os }}-go- 28 | 29 | - name: Build the operator 30 | run: go build -v ./main.go 31 | 32 | - name: Build the kubeflow-launcher 33 | run: go build -v ./cli/main.go 34 | 35 | test: 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v2 39 | 40 | - name: Set up Go 41 | uses: actions/setup-go@v2 42 | with: 43 | go-version: 1.17 44 | 45 | - name: Reuse go mod cache 46 | uses: actions/cache@v2 47 | with: 48 | path: ~/go/pkg/mod 49 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 50 | restore-keys: | 51 | ${{ runner.os }}-go- 52 | 53 | - name: Test 54 | run: | 55 | curl -L -O "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.0/kubebuilder_2.3.0_linux_amd64.tar.gz" 56 | tar -zxvf kubebuilder_2.3.0_linux_amd64.tar.gz 57 | sudo mv kubebuilder_2.3.0_linux_amd64 /usr/local/kubebuilder 58 | export PATH=$PATH:/usr/local/kubebuilder/bin 59 | go test -v ./... 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Kubernetes Generated files - skip generated files, except for vendored files 17 | 18 | !vendor/**/zz_generated.* 19 | 20 | # editor and IDE paraphernalia 21 | .idea 22 | *.swp 23 | *.swo 24 | *~ 25 | 26 | vendor/ 27 | .vscode/ 28 | /elastic-jupyter-operator 29 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "enterprise_gateway"] 2 | path = enterprise_gateway 3 | url = git@github.com:skai-x/enterprise_gateway.git 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [Unreleased](https://github.com/tkestack/elastic-jupyter-operator/tree/HEAD) 4 | 5 | [Full Changelog](https://github.com/tkestack/elastic-jupyter-operator/compare/v0.2.0...HEAD) 6 | 7 | **Merged pull requests:** 8 | 9 | - \[typo\] fix typo in README.md [\#40](https://github.com/tkestack/elastic-jupyter-operator/pull/40) ([mkkb473](https://github.com/mkkb473)) 10 | 11 | ## [v0.2.0](https://github.com/tkestack/elastic-jupyter-operator/tree/v0.2.0) (2021-09-28) 12 | 13 | [Full Changelog](https://github.com/tkestack/elastic-jupyter-operator/compare/v0.1.1...v0.2.0) 14 | 15 | **Implemented enhancements:** 16 | 17 | - feat\(notebook\): Avoid token in the URL [\#34](https://github.com/tkestack/elastic-jupyter-operator/issues/34) 18 | - chore: Support CI with GitHub Actions [\#24](https://github.com/tkestack/elastic-jupyter-operator/issues/24) 19 | 20 | **Fixed bugs:** 21 | 22 | - \[feat\] Set the kernel owner reference to gateway [\#18](https://github.com/tkestack/elastic-jupyter-operator/issues/18) 23 | 24 | **Closed issues:** 25 | 26 | - kernel: Support cullout [\#28](https://github.com/tkestack/elastic-jupyter-operator/issues/28) 27 | - \[feat\] Design Kernel CRD to ease the maintanance [\#4](https://github.com/tkestack/elastic-jupyter-operator/issues/4) 28 | 29 | **Merged pull requests:** 30 | 31 | - fix: Fix the jupyter kernel spec name [\#39](https://github.com/tkestack/elastic-jupyter-operator/pull/39) ([gaocegege](https://github.com/gaocegege)) 32 | - feat: Add docs and new api doc [\#38](https://github.com/tkestack/elastic-jupyter-operator/pull/38) ([gaocegege](https://github.com/gaocegege)) 33 | - feat\(notebook\): Support auth configuration [\#36](https://github.com/tkestack/elastic-jupyter-operator/pull/36) ([gaocegege](https://github.com/gaocegege)) 34 | - feat\(examples\): Add examples for simple jupyter [\#35](https://github.com/tkestack/elastic-jupyter-operator/pull/35) ([gaocegege](https://github.com/gaocegege)) 35 | - feat: Support cullout [\#33](https://github.com/tkestack/elastic-jupyter-operator/pull/33) ([gaocegege](https://github.com/gaocegege)) 36 | - feat: Add loglevel [\#32](https://github.com/tkestack/elastic-jupyter-operator/pull/32) ([gaocegege](https://github.com/gaocegege)) 37 | - feat\(launcher\): Set owner for kernel [\#31](https://github.com/tkestack/elastic-jupyter-operator/pull/31) ([gaocegege](https://github.com/gaocegege)) 38 | - fix: Resize the image in README [\#30](https://github.com/tkestack/elastic-jupyter-operator/pull/30) ([gaocegege](https://github.com/gaocegege)) 39 | - chore: Update docs [\#29](https://github.com/tkestack/elastic-jupyter-operator/pull/29) ([gaocegege](https://github.com/gaocegege)) 40 | - chore: Add go mod cache [\#27](https://github.com/tkestack/elastic-jupyter-operator/pull/27) ([gaocegege](https://github.com/gaocegege)) 41 | - chore: Add CLI README and print events when there is any problem creating gateway CR [\#23](https://github.com/tkestack/elastic-jupyter-operator/pull/23) ([gaocegege](https://github.com/gaocegege)) 42 | 43 | ## [v0.1.1](https://github.com/tkestack/elastic-jupyter-operator/tree/v0.1.1) (2021-06-02) 44 | 45 | [Full Changelog](https://github.com/tkestack/elastic-jupyter-operator/compare/v0.1.0...v0.1.1) 46 | 47 | **Closed issues:** 48 | 49 | - \[feat\] Support Launching Notebooks with CRD Directly [\#8](https://github.com/tkestack/elastic-jupyter-operator/issues/8) 50 | 51 | **Merged pull requests:** 52 | 53 | - feat\(kernel\): Add Kernel CRD [\#19](https://github.com/tkestack/elastic-jupyter-operator/pull/19) ([gaocegege](https://github.com/gaocegege)) 54 | - feat: Add Kernel Launcher and KernelTemplate and KernelSpec CRD [\#17](https://github.com/tkestack/elastic-jupyter-operator/pull/17) ([gaocegege](https://github.com/gaocegege)) 55 | - chore: Add docs for kernel [\#11](https://github.com/tkestack/elastic-jupyter-operator/pull/11) ([gaocegege](https://github.com/gaocegege)) 56 | 57 | ## [v0.1.0](https://github.com/tkestack/elastic-jupyter-operator/tree/v0.1.0) (2021-04-22) 58 | 59 | [Full Changelog](https://github.com/tkestack/elastic-jupyter-operator/compare/v0.1.0-rc.1...v0.1.0) 60 | 61 | **Closed issues:** 62 | 63 | - \[bug\] Kernel completed when run [\#13](https://github.com/tkestack/elastic-jupyter-operator/issues/13) 64 | - \[feat\] Use 2.5.0 enterprise gateway [\#9](https://github.com/tkestack/elastic-jupyter-operator/issues/9) 65 | 66 | **Merged pull requests:** 67 | 68 | - fix: Add testcases to notebook and fix generate logic for notebook deployment [\#16](https://github.com/tkestack/elastic-jupyter-operator/pull/16) ([Mirrored90](https://github.com/Mirrored90)) 69 | - feat\(notebook\): Support Launching Notebooks with CRD Directly [\#15](https://github.com/tkestack/elastic-jupyter-operator/pull/15) ([Mirrored90](https://github.com/Mirrored90)) 70 | - feat: Add a new CRD KernelSpec [\#12](https://github.com/tkestack/elastic-jupyter-operator/pull/12) ([gaocegege](https://github.com/gaocegege)) 71 | - feat\(gateway\): change dev tag to 2.5.0 [\#10](https://github.com/tkestack/elastic-jupyter-operator/pull/10) ([Mirrored90](https://github.com/Mirrored90)) 72 | - fix: Fix readme [\#7](https://github.com/tkestack/elastic-jupyter-operator/pull/7) ([pokerfaceSad](https://github.com/pokerfaceSad)) 73 | 74 | ## [v0.1.0-rc.1](https://github.com/tkestack/elastic-jupyter-operator/tree/v0.1.0-rc.1) (2021-04-02) 75 | 76 | [Full Changelog](https://github.com/tkestack/elastic-jupyter-operator/compare/v0.1.0-rc.0...v0.1.0-rc.1) 77 | 78 | **Merged pull requests:** 79 | 80 | - fix: Fix ignore file [\#6](https://github.com/tkestack/elastic-jupyter-operator/pull/6) ([gaocegege](https://github.com/gaocegege)) 81 | 82 | ## [v0.1.0-rc.0](https://github.com/tkestack/elastic-jupyter-operator/tree/v0.1.0-rc.0) (2021-03-01) 83 | 84 | [Full Changelog](https://github.com/tkestack/elastic-jupyter-operator/compare/c194c0ee41da0d42bf156827f53066c4b259e557...v0.1.0-rc.0) 85 | 86 | **Merged pull requests:** 87 | 88 | - feat: Update docs [\#5](https://github.com/tkestack/elastic-jupyter-operator/pull/5) ([gaocegege](https://github.com/gaocegege)) 89 | - fix: Update readme [\#2](https://github.com/tkestack/elastic-jupyter-operator/pull/2) ([gaocegege](https://github.com/gaocegege)) 90 | - fix: Fix rbac issues [\#1](https://github.com/tkestack/elastic-jupyter-operator/pull/1) ([gaocegege](https://github.com/gaocegege)) 91 | 92 | 93 | 94 | \* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* 95 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the elastic-jupyter-operator binary 2 | FROM golang:1.17.6-alpine as builder 3 | 4 | WORKDIR /workspace 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | 9 | # Download libs first to use docker buildx caching 10 | RUN go mod download 11 | RUN go mod verify 12 | 13 | # Copy the go source 14 | COPY main.go main.go 15 | COPY api/ api/ 16 | COPY controllers/ controllers/ 17 | COPY pkg/ pkg/ 18 | 19 | # Build 20 | RUN CGO_ENABLED=0 go build -a -o elastic-jupyter-operator main.go 21 | 22 | # Use distroless as minimal base image to package the elastic-jupyter-operator binary 23 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 24 | FROM gcr.io/distroless/static:nonroot 25 | 26 | LABEL org.opencontainers.image.source https://github.com/tkestack/elastic-jupyter-operator 27 | 28 | WORKDIR / 29 | COPY --from=builder /workspace/elastic-jupyter-operator . 30 | USER nonroot:nonroot 31 | 32 | ENTRYPOINT ["/elastic-jupyter-operator"] 33 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= ghcr.io/skai-x/elastic-jupyter-operator:latest 4 | REGISTRY_IMG ?= ghcr.io/skai-x/enterprise-gateway:latest 5 | REGISTRY_K8S_IMG ?= ghcr.io/skai-x/enterprise-gateway-with-kernel-spec:latest 6 | KERNEL_PY_IMG ?= ghcr.io/skai-x/jupyter-kernel-py:2.6.0 7 | KERNEL_R_IMG ?= ghcr.io/skai-x/jupyter-kernel-r:2.6.0 8 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 9 | CRD_OPTIONS ?= "crd:trivialVersions=true" 10 | 11 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 12 | ifeq (,$(shell go env GOBIN)) 13 | GOBIN=$(shell go env GOPATH)/bin 14 | else 15 | GOBIN=$(shell go env GOBIN) 16 | endif 17 | 18 | all: manager 19 | 20 | # Run tests 21 | test: generate fmt vet manifests 22 | go test ./... -coverprofile cover.out 23 | 24 | # Build manager binary 25 | manager: generate fmt vet 26 | go build -o bin/elastic-jupyter-operator main.go 27 | 28 | # Run against the configured Kubernetes cluster in ~/.kube/config 29 | run: generate fmt vet manifests 30 | go run ./main.go 31 | 32 | # Install CRDs into a cluster 33 | install: manifests kustomize 34 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 35 | 36 | # Uninstall CRDs from a cluster 37 | uninstall: manifests kustomize 38 | $(KUSTOMIZE) build config/crd | kubectl delete -f - 39 | 40 | # Deploy controller in the configured Kubernetes cluster in ~/.kube/config 41 | deploy: manifests kustomize 42 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 43 | $(KUSTOMIZE) build config/default | kubectl apply -f - 44 | 45 | # Generate manifests e.g. CRD, RBAC etc. 46 | manifests: controller-gen 47 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases 48 | 49 | # Run go fmt against code 50 | fmt: 51 | go fmt ./... 52 | 53 | # Run go vet against code 54 | vet: 55 | go vet ./... 56 | 57 | # Generate code 58 | generate: controller-gen api-reference 59 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." 60 | 61 | api-reference: install-tools ## Generate API reference documentation 62 | $(GOBIN)/crd-ref-docs \ 63 | --source-path ./api/v1alpha1 \ 64 | --config ./docs/api/autogen/config.yaml \ 65 | --templates-dir ./docs/api/autogen/templates \ 66 | --output-path ./docs/api/generated.asciidoc \ 67 | --max-depth 30 68 | 69 | # Build the docker image 70 | docker-build: test 71 | docker build . -t ${IMG} 72 | 73 | # Push the docker image 74 | docker-push: 75 | docker buildx build --push --platform linux/amd64,linux/arm64 --tag ${IMG} . 76 | 77 | # find or download controller-gen 78 | # download controller-gen if necessary 79 | controller-gen: 80 | ifeq (, $(shell which controller-gen)) 81 | @{ \ 82 | set -e ;\ 83 | CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ 84 | cd $$CONTROLLER_GEN_TMP_DIR ;\ 85 | go mod init tmp ;\ 86 | go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\ 87 | rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ 88 | } 89 | CONTROLLER_GEN=$(GOBIN)/controller-gen 90 | else 91 | CONTROLLER_GEN=$(shell which controller-gen) 92 | endif 93 | 94 | kustomize: 95 | ifeq (, $(shell which kustomize)) 96 | @{ \ 97 | set -e ;\ 98 | KUSTOMIZE_GEN_TMP_DIR=$$(mktemp -d) ;\ 99 | cd $$KUSTOMIZE_GEN_TMP_DIR ;\ 100 | go mod init tmp ;\ 101 | go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4 ;\ 102 | rm -rf $$KUSTOMIZE_GEN_TMP_DIR ;\ 103 | } 104 | KUSTOMIZE=$(GOBIN)/kustomize 105 | else 106 | KUSTOMIZE=$(shell which kustomize) 107 | endif 108 | 109 | install-tools: 110 | go get github.com/elastic/crd-ref-docs 111 | 112 | enterprise-gateway: 113 | cd enterprise_gateway && python setup.py bdist_wheel \ 114 | && rm -rf *.egg-info && cd - && \ 115 | docker buildx build --push --platform linux/amd64,linux/arm64 --tag ${REGISTRY_IMG} -f enterprise_gateway/etc/docker/enterprise-gateway/Dockerfile . 116 | 117 | enterprise-gateway-k8s: 118 | cd enterprise_gateway && make kernelspecs && \ 119 | python setup.py bdist_wheel \ 120 | && rm -rf *.egg-info && cd - && \ 121 | docker buildx build --push --platform linux/amd64,linux/arm64 --tag ${REGISTRY_K8S_IMG} -f enterprise_gateway/etc/docker/enterprise-gateway-k8s/Dockerfile . 122 | 123 | kernel: kernel-py kernel-r 124 | 125 | kernel-py: 126 | docker buildx build --push --platform linux/amd64,linux/arm64 --tag ${KERNEL_PY_IMG} -f enterprise_gateway/etc/docker/kernel-py/Dockerfile . 127 | 128 | kernel-r: 129 | docker buildx build --push --platform linux/amd64,linux/arm64 --tag ${KERNEL_R_IMG} -f enterprise_gateway/etc/docker/kernel-r/Dockerfile . -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | layout: go.kubebuilder.io/v2 2 | projectName: elastic-jupyter-operator 3 | repo: github.com/tkestack/elastic-jupyter-operator 4 | resources: 5 | - group: kubeflow.tkestack.io 6 | kind: JupyterNotebook 7 | version: v1alpha1 8 | - group: kubeflow.tkestack.io 9 | kind: JupyterGateway 10 | version: v1alpha1 11 | - group: kubeflow.tkestack.io 12 | kind: JupyterKernelSpec 13 | version: v1alpha1 14 | version: 3-alpha 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elastic-jupyter-operator 2 | 3 | Elastic Jupyter Notebooks on Kubernetes 4 | 5 | ## Motivation 6 | 7 | Jupyter is a free, open-source, interactive web tool known as a computational notebook, which researchers can use to combine software code, computational output, explanatory text, and multimedia resources in a single document. 8 | 9 | For data scientists and machine learning engineers, Jupyter has emerged as a de facto standard. At the same time, there has been growing criticism that the way notebooks are being used leads to low resource utilization. 10 | 11 | GPU and other hardware resources will be bound to the specified notebooks even if the data scientists do not need them currently. This project proposes some Kubernetes CRDs to solve these problems. 12 | 13 | ## Introduction 14 | 15 | elastic-jupyter-operator provides elastic Jupyter notebook services with these features: 16 | 17 | - Provide users the out-of-box Jupyter notebooks on Kubernetes. 18 | - Autoscale Jupyter kernels when the kernels are not used within the given time frame to increase the resource utilization. 19 | - Customize the kernel configuration in runtime without restarting the notebook. 20 | 21 |

22 | Figure 1. elastic-jupyter-operator 23 | 24 |

25 | Figure 2. Other Jupyter on Kubernetes solutions 26 | 27 | ## Deploy 28 | 29 | ```bash 30 | kubectl apply -f ./hack/enterprise_gateway/prepare.yaml 31 | make deploy 32 | ``` 33 | 34 | ## Quickstart 35 | 36 | You can follow the [quickstart](./docs/quick-start.md) to create the notebook server and kernel in Kubernetes like this: 37 | 38 | ```bash 39 | NAME READY STATUS RESTARTS AGE 40 | jovyan-fd191444-b08c-4668-ba4e-3748a54a0ac1-5789574d66-tb5cm 1/1 Running 0 146m 41 | jupytergateway-sample-858bbc8d5c-xds44 1/1 Running 0 3h46m 42 | jupyternotebook-sample-5bf7d9d9fb-pdv9b 1/1 Running 10 77d 43 | ``` 44 | 45 | There are three pods running in the demo: 46 | 47 | - `jupyternotebook-sample-5bf7d9d9fb-pdv9b` is the notebook server 48 | - `jupytergateway-sample-858bbc8d5c-xds44` is the jupyter gateway to support remote kernels 49 | - `jovyan-fd191444-b08c-4668-ba4e-3748a54a0ac1-5789574d66-tb5cm` is the remote kernel 50 | 51 | The kernel will be deleted if the notebook does not use it in 10 mins. And it will be recreated if there is any new run in the notebook. 52 | 53 | ## Community 54 | 55 | Please join [![Discord][discord-badge]][discord-url] 56 | 57 | [discord-badge]: https://img.shields.io/discord/913359799058587658?logo=Discord&style=flat-square 58 | [discord-url]: https://discord.gg/NJsd4guhPM 59 | 60 | ## Design 61 | 62 | Please refer to [design doc](docs/design.md) 63 | 64 | ## API Documentation 65 | 66 | Please refer to [API doc](docs/api/generated.asciidoc) 67 | 68 | ## Special Thanks 69 | 70 | - [jupyter/enterprise_gateway](https://github.com/jupyter/enterprise_gateway) which implements the logic to run kernels remotely 71 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | // Package v1alpha1 contains API Schema definitions for the kubeflow.tkestack.io v1alpha1 API group 18 | // +kubebuilder:object:generate=true 19 | // +groupName=kubeflow.tkestack.io 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "kubeflow.tkestack.io", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /api/v1alpha1/jupytergateway_types.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | appsv1 "k8s.io/api/apps/v1" 21 | v1 "k8s.io/api/core/v1" 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | ) 24 | 25 | // JupyterGatewaySpec defines the desired state of JupyterGateway 26 | type JupyterGatewaySpec struct { 27 | // Knernels defines the kernels in the gateway. 28 | // We will add kernels at runtime, thus we do not make it a type. 29 | Kernels []string `json:"kernels,omitempty"` 30 | // DefaultKernel defines the default kernel in the gateway. 31 | DefaultKernel *string `json:"defaultKernel,omitempty"` 32 | // Timeout (in seconds) after which a kernel is considered idle and 33 | // ready to be culled. Values of 0 or lower disable culling. Very 34 | // short timeouts may result in kernels being culled for users 35 | // with poor network connections. 36 | // Ref https://jupyter-notebook.readthedocs.io/en/stable/config.html 37 | CullIdleTimeout *int32 `json:"cullIdleTimeout,omitempty"` 38 | 39 | // The interval (in seconds) on which to check for idle kernels 40 | // exceeding the cull timeout value. 41 | CullInterval *int32 `json:"cullInterval,omitempty"` 42 | 43 | LogLevel *LogLevel `json:"logLevel,omitempty"` 44 | 45 | // Compute Resources required by this container. 46 | // Cannot be updated. 47 | // More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ 48 | // +optional 49 | Resources *v1.ResourceRequirements `json:"resources,omitempty"` 50 | 51 | // Docker image name. 52 | // More info: https://kubernetes.io/docs/concepts/containers/images 53 | // This field defaults to ghcr.io/skai-x/enterprise-gateway:2.6.0 54 | // +optional 55 | Image string `json:"image,omitempty"` 56 | 57 | // ClusterRole for the gateway, which is used to create the kernel pods in the cluster. Defaults to enterprise-gateway-controller (created at startup). 58 | ClusterRole *string `json:"clusterRole,omitempty"` 59 | } 60 | 61 | type LogLevel string 62 | 63 | const ( 64 | LogLevelDebug = "DEBUG" 65 | LogLevelInfo = "INFO" 66 | LogLevelWarning = "WARNING" 67 | ) 68 | 69 | // JupyterGatewayStatus defines the observed state of JupyterGateway 70 | type JupyterGatewayStatus struct { 71 | appsv1.DeploymentStatus `json:",inline"` 72 | } 73 | 74 | // +kubebuilder:object:root=true 75 | // +kubebuilder:subresource:status 76 | 77 | // JupyterGateway is the Schema for the jupytergateways API 78 | type JupyterGateway struct { 79 | metav1.TypeMeta `json:",inline"` 80 | metav1.ObjectMeta `json:"metadata,omitempty"` 81 | 82 | Spec JupyterGatewaySpec `json:"spec,omitempty"` 83 | Status JupyterGatewayStatus `json:"status,omitempty"` 84 | } 85 | 86 | // +kubebuilder:object:root=true 87 | 88 | // JupyterGatewayList contains a list of JupyterGateway 89 | type JupyterGatewayList struct { 90 | metav1.TypeMeta `json:",inline"` 91 | metav1.ListMeta `json:"metadata,omitempty"` 92 | Items []JupyterGateway `json:"items"` 93 | } 94 | 95 | func init() { 96 | SchemeBuilder.Register(&JupyterGateway{}, &JupyterGatewayList{}) 97 | } 98 | -------------------------------------------------------------------------------- /api/v1alpha1/jupyterkernel_types.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // JupyterKernelSpec defines the desired state of JupyterKernel 25 | type JupyterKernelCRDSpec struct { 26 | Template v1.PodTemplateSpec `json:"template,omitempty"` 27 | } 28 | 29 | // JupyterKernelStatus defines the observed state of JupyterKernel 30 | type JupyterKernelStatus struct { 31 | // Conditions is an array of current observed job conditions. 32 | Conditions []JupyterKernelCondition `json:"conditions"` 33 | 34 | // Represents time when the job was acknowledged by the job controller. 35 | // It is not guaranteed to be set in happens-before order across separate operations. 36 | // It is represented in RFC3339 form and is in UTC. 37 | StartTime *metav1.Time `json:"startTime,omitempty"` 38 | 39 | // Represents time when the job was completed. It is not guaranteed to 40 | // be set in happens-before order across separate operations. 41 | // It is represented in RFC3339 form and is in UTC. 42 | CompletionTime *metav1.Time `json:"completionTime,omitempty"` 43 | 44 | // Represents last time when the job was reconciled. It is not guaranteed to 45 | // be set in happens-before order across separate operations. 46 | // It is represented in RFC3339 form and is in UTC. 47 | LastReconcileTime *metav1.Time `json:"lastReconcileTime,omitempty"` 48 | } 49 | 50 | type JupyterKernelCondition struct { 51 | // Type of job condition. 52 | Type JupyterKernelConditionType `json:"type"` 53 | // Status of the condition, one of True, False, Unknown. 54 | Status v1.ConditionStatus `json:"status"` 55 | // The reason for the condition's last transition. 56 | Reason string `json:"reason,omitempty"` 57 | // A human readable message indicating details about the transition. 58 | Message string `json:"message,omitempty"` 59 | // The last time this condition was updated. 60 | LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` 61 | // Last time the condition transitioned from one status to another. 62 | LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` 63 | } 64 | 65 | type JupyterKernelConditionType string 66 | 67 | const ( 68 | JupyterKernelRunning JupyterKernelConditionType = "Running" 69 | JupyterKernelFailed JupyterKernelConditionType = "Failed" 70 | JupyterKernelSucceeded JupyterKernelConditionType = "Succeeded" 71 | ) 72 | 73 | // +kubebuilder:object:root=true 74 | // +kubebuilder:subresource:status 75 | // +kubebuilder:pruning:PreserveUnknownFields 76 | 77 | // JupyterKernel is the Schema for the jupyterkernels API 78 | type JupyterKernel struct { 79 | metav1.TypeMeta `json:",inline"` 80 | metav1.ObjectMeta `json:"metadata,omitempty"` 81 | 82 | Spec JupyterKernelCRDSpec `json:"spec,omitempty"` 83 | Status JupyterKernelStatus `json:"status,omitempty"` 84 | } 85 | 86 | // +kubebuilder:object:root=true 87 | 88 | // JupyterKernelList contains a list of JupyterKernel 89 | type JupyterKernelList struct { 90 | metav1.TypeMeta `json:",inline"` 91 | metav1.ListMeta `json:"metadata,omitempty"` 92 | Items []JupyterKernel `json:"items"` 93 | } 94 | 95 | func init() { 96 | SchemeBuilder.Register(&JupyterKernel{}, &JupyterKernelList{}) 97 | } 98 | -------------------------------------------------------------------------------- /api/v1alpha1/jupyterkernelspec_types.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // JupyterKernelSpecSpec defines the desired state of JupyterKernelSpec 25 | type JupyterKernelSpecSpec struct { 26 | Language string `json:"language,omitempty"` 27 | DisplayName string `json:"displayName,omitempty"` 28 | Image string `json:"image,omitempty"` 29 | Env []v1.EnvVar `json:"env,omitempty"` 30 | Command []string `json:"command,omitempty"` 31 | ClassName string `json:"className,omitempty"` 32 | 33 | Template *v1.ObjectReference `json:"template,omitempty"` 34 | // TODO(gaocegege): Support resources and so on. 35 | } 36 | 37 | // JupyterKernelSpecStatus defines the observed state of JupyterKernelSpec 38 | type JupyterKernelSpecStatus struct { 39 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 40 | // Important: Run "make" to regenerate code after modifying this file 41 | } 42 | 43 | // +kubebuilder:object:root=true 44 | // +kubebuilder:subresource:status 45 | 46 | // JupyterKernelSpec is the Schema for the jupyterkernelspecs API 47 | type JupyterKernelSpec struct { 48 | metav1.TypeMeta `json:",inline"` 49 | metav1.ObjectMeta `json:"metadata,omitempty"` 50 | 51 | Spec JupyterKernelSpecSpec `json:"spec,omitempty"` 52 | Status JupyterKernelSpecStatus `json:"status,omitempty"` 53 | } 54 | 55 | // +kubebuilder:object:root=true 56 | 57 | // JupyterKernelSpecList contains a list of JupyterKernelSpec 58 | type JupyterKernelSpecList struct { 59 | metav1.TypeMeta `json:",inline"` 60 | metav1.ListMeta `json:"metadata,omitempty"` 61 | Items []JupyterKernelSpec `json:"items"` 62 | } 63 | 64 | func init() { 65 | SchemeBuilder.Register(&JupyterKernelSpec{}, &JupyterKernelSpecList{}) 66 | } 67 | -------------------------------------------------------------------------------- /api/v1alpha1/jupyterkerneltemplate_types.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // JupyterKernelTemplateSpec defines the desired state of JupyterKernelTemplate 25 | type JupyterKernelTemplateSpec struct { 26 | Template *v1.PodTemplateSpec `json:"template,omitempty"` 27 | } 28 | 29 | // JupyterKernelTemplateStatus defines the observed state of JupyterKernelTemplate 30 | type JupyterKernelTemplateStatus struct { 31 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 32 | // Important: Run "make" to regenerate code after modifying this file 33 | } 34 | 35 | // +kubebuilder:object:root=true 36 | // +kubebuilder:subresource:status 37 | 38 | // JupyterKernelTemplate is the Schema for the jupyterkerneltemplates API 39 | type JupyterKernelTemplate struct { 40 | metav1.TypeMeta `json:",inline"` 41 | metav1.ObjectMeta `json:"metadata,omitempty"` 42 | 43 | Spec JupyterKernelTemplateSpec `json:"spec,omitempty"` 44 | Status JupyterKernelTemplateStatus `json:"status,omitempty"` 45 | } 46 | 47 | // +kubebuilder:object:root=true 48 | 49 | // JupyterKernelTemplateList contains a list of JupyterKernelTemplate 50 | type JupyterKernelTemplateList struct { 51 | metav1.TypeMeta `json:",inline"` 52 | metav1.ListMeta `json:"metadata,omitempty"` 53 | Items []JupyterKernelTemplate `json:"items"` 54 | } 55 | 56 | func init() { 57 | SchemeBuilder.Register(&JupyterKernelTemplate{}, &JupyterKernelTemplateList{}) 58 | } 59 | -------------------------------------------------------------------------------- /api/v1alpha1/jupyternotebook_types.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | v1 "k8s.io/api/core/v1" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 25 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 26 | 27 | // JupyterNotebookSpec defines the desired state of JupyterNotebook 28 | type JupyterNotebookSpec struct { 29 | Gateway *v1.ObjectReference `json:"gateway,omitempty"` 30 | Auth *JupyterAuth `json:"auth,omitempty"` 31 | 32 | Template *v1.PodTemplateSpec `json:"template,omitempty"` 33 | } 34 | 35 | // JupyterAuth defines how to deal with jupyter notebook tokens or passwords. 36 | // https://jupyter-notebook.readthedocs.io/en/stable/security.html 37 | type JupyterAuth struct { 38 | // TODO(gaocegege): Is this field necessary since we make Token and Password a pointer? 39 | Mode ModeJupyterAuth `json:"mode,omitempty"` 40 | Token *string `json:"token,omitempty"` 41 | Password *string `json:"password,omitempty"` 42 | } 43 | 44 | type ModeJupyterAuth string 45 | 46 | const ( 47 | ModeJupyterAuthEnable ModeJupyterAuth = "enable" 48 | // ModeJupyterAuthDisable disables authentication altogether by setting the token 49 | // and password to empty strings, but this is NOT RECOMMENDED, unless authentication 50 | // or access restrictions are handled at a different layer in your web application 51 | ModeJupyterAuthDisable ModeJupyterAuth = "disable" 52 | ) 53 | 54 | // JupyterNotebookStatus defines the observed state of JupyterNotebook 55 | type JupyterNotebookStatus struct { 56 | } 57 | 58 | // +kubebuilder:object:root=true 59 | // +kubebuilder:subresource:status 60 | 61 | // JupyterNotebook is the Schema for the jupyternotebooks API 62 | type JupyterNotebook struct { 63 | metav1.TypeMeta `json:",inline"` 64 | metav1.ObjectMeta `json:"metadata,omitempty"` 65 | 66 | Spec JupyterNotebookSpec `json:"spec,omitempty"` 67 | Status JupyterNotebookStatus `json:"status,omitempty"` 68 | } 69 | 70 | // +kubebuilder:object:root=true 71 | 72 | // JupyterNotebookList contains a list of JupyterNotebook 73 | type JupyterNotebookList struct { 74 | metav1.TypeMeta `json:",inline"` 75 | metav1.ListMeta `json:"metadata,omitempty"` 76 | Items []JupyterNotebook `json:"items"` 77 | } 78 | 79 | func init() { 80 | SchemeBuilder.Register(&JupyterNotebook{}, &JupyterNotebookList{}) 81 | } 82 | -------------------------------------------------------------------------------- /cli/README.md: -------------------------------------------------------------------------------- 1 | # Kubeflow Launcher 2 | 3 | Kubeflow launcher is used in enterprise gateway image to launch kernels. 4 | -------------------------------------------------------------------------------- /cli/cmd/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2021 NAME HERE 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package cmd 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "os" 22 | 23 | "github.com/spf13/cobra" 24 | v1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/types" 26 | "k8s.io/client-go/kubernetes/scheme" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | "sigs.k8s.io/controller-runtime/pkg/client/config" 29 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 30 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 31 | 32 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 33 | ) 34 | 35 | const ( 36 | envKernelPodName = "KERNEL_POD_NAME" 37 | envKernelImage = "KERNEL_IMAGE" 38 | envKernelNamespace = "KERNEL_NAMESPACE" 39 | envKernelID = "KERNEL_ID" 40 | envKernelLanguage = "KERNEL_LANGUAGE" 41 | envKernelName = "KERNEL_NAME" 42 | envKernelSpark = "KERNEL_SPARK_CONTEXT_INIT_MODE" 43 | envKernelUsername = "KERNEL_USERNAME" 44 | 45 | envPortRange = "EG_PORT_RANGE" 46 | envResponseAddress = "EG_RESPONSE_ADDRESS" 47 | envPublicKey = "EG_PUBLIC_KEY" 48 | envGatewayName = "EG_NAME" 49 | envGatewayNamespace = "EG_NAMESPACE" 50 | 51 | labelKernelID = "kernel_id" 52 | ) 53 | 54 | var ( 55 | kernelID, portRange, responseAddr, 56 | publicKey, sparkContextInitMode, 57 | kernelTemplateName, kernelTemplateNamespace string 58 | gatewayName string 59 | verbose bool 60 | ) 61 | 62 | // rootCmd represents the base command when called without any subcommands 63 | var rootCmd = &cobra.Command{ 64 | Use: "kubeflow-launcher", 65 | Short: "Launch kernels", 66 | Long: `Launch kernels in the jupyter enterprise gateway`, 67 | Run: func(cmd *cobra.Command, args []string) { 68 | logger := zap.New(zap.UseDevMode(verbose)) 69 | 70 | gatewayNamespace := os.Getenv(envGatewayNamespace) 71 | gatewayName := os.Getenv(envGatewayName) 72 | 73 | if gatewayName == "" || gatewayNamespace == "" { 74 | panic(fmt.Errorf("failed to get the gateway name or namespace from the env var")) 75 | } 76 | if kernelTemplateName == "" || kernelTemplateNamespace == "" { 77 | panic(fmt.Errorf("failed to get the template's name or namespace")) 78 | } 79 | 80 | logger.Info("Launching the kernel", 81 | "kernelID", kernelID, "responseAddr", responseAddr, 82 | "kernelTemplateName", kernelTemplateName, 83 | "kernelTemplateNamespace", kernelTemplateNamespace, 84 | "gatewayName", gatewayName, 85 | "gatewayNamespace", gatewayNamespace, 86 | ) 87 | 88 | if err := v1alpha1.AddToScheme(scheme.Scheme); err != nil { 89 | panic(err) 90 | } 91 | 92 | cfg, err := config.GetConfig() 93 | if err != nil { 94 | panic(err) 95 | } 96 | 97 | cli, err := client.New(cfg, client.Options{ 98 | Scheme: scheme.Scheme, 99 | }) 100 | if err != nil { 101 | panic(err) 102 | } 103 | 104 | kt := &v1alpha1.JupyterKernelTemplate{} 105 | if err := cli.Get(context.TODO(), client.ObjectKey{ 106 | Namespace: kernelTemplateNamespace, 107 | Name: kernelTemplateName, 108 | }, kt); err != nil { 109 | panic(err) 110 | } 111 | 112 | kernel := &v1alpha1.JupyterKernel{ 113 | ObjectMeta: kt.Spec.Template.ObjectMeta, 114 | Spec: v1alpha1.JupyterKernelCRDSpec{ 115 | Template: *kt.Spec.Template, 116 | }, 117 | } 118 | 119 | // Set image from the kernel spec. 120 | image := os.Getenv(envKernelImage) 121 | if image != "" && len(kernel.Spec.Template.Spec.Containers) != 0 { 122 | kernel.Spec.Template.Spec.Containers[0].Image = image 123 | } 124 | 125 | kernel.Name = os.Getenv(envKernelPodName) 126 | kernel.Namespace = os.Getenv(envKernelNamespace) 127 | if kernel.Spec.Template.Labels == nil { 128 | kernel.Spec.Template.Labels = make(map[string]string) 129 | } 130 | // We cannot rely on it because of 131 | // https://github.com/kubernetes-sigs/controller-tools/issues/448 132 | kernel.Spec.Template.Labels[labelKernelID] = kernelID 133 | 134 | // Set the environment variables. 135 | if kernel.Spec.Template.Spec.Containers[0].Env == nil { 136 | kernel.Spec.Template.Spec.Containers[0].Env = make([]v1.EnvVar, 0) 137 | } 138 | kernel.Spec.Template.Spec.Containers[0].Env = append( 139 | kernel.Spec.Template.Spec.Containers[0].Env, 140 | v1.EnvVar{ 141 | Name: envPortRange, 142 | Value: portRange, 143 | }, 144 | v1.EnvVar{ 145 | Name: envResponseAddress, 146 | Value: responseAddr, 147 | }, 148 | v1.EnvVar{ 149 | Name: envPublicKey, 150 | Value: publicKey, 151 | }, 152 | v1.EnvVar{ 153 | Name: envKernelID, 154 | Value: kernelID, 155 | }, 156 | v1.EnvVar{ 157 | Name: envKernelLanguage, 158 | Value: os.Getenv(envKernelLanguage), 159 | }, 160 | v1.EnvVar{ 161 | Name: envKernelName, 162 | Value: os.Getenv(envKernelName), 163 | }, 164 | v1.EnvVar{ 165 | Name: envKernelNamespace, 166 | Value: os.Getenv(envKernelNamespace), 167 | }, 168 | v1.EnvVar{ 169 | Name: envKernelSpark, 170 | Value: sparkContextInitMode, 171 | }, 172 | v1.EnvVar{ 173 | Name: envKernelUsername, 174 | Value: os.Getenv(envKernelUsername), 175 | }, 176 | ) 177 | 178 | gateway := &v1alpha1.JupyterGateway{} 179 | if err := cli.Get(context.TODO(), types.NamespacedName{ 180 | Name: gatewayName, 181 | Namespace: gatewayNamespace, 182 | }, gateway); err != nil { 183 | panic(err) 184 | } 185 | 186 | if err := controllerutil.SetControllerReference( 187 | gateway, kernel, scheme.Scheme); err != nil { 188 | panic(err) 189 | } 190 | 191 | logger.Info("Creating the kernel", "kernel", kernel) 192 | if err := cli.Create(context.TODO(), kernel); err != nil { 193 | panic(err) 194 | } 195 | }, 196 | } 197 | 198 | // Execute adds all child commands to the root command and sets flags appropriately. 199 | // This is called by main.main(). It only needs to happen once to the rootCmd. 200 | func Execute() { 201 | if err := rootCmd.Execute(); err != nil { 202 | fmt.Println(err) 203 | os.Exit(1) 204 | } 205 | } 206 | 207 | func init() { 208 | // Cobra also supports local flags, which will only run 209 | // when this action is called directly. 210 | rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") 211 | 212 | rootCmd.Flags().StringVar(&kernelID, 213 | "RemoteProcessProxy.kernel-id", "", "kernel id") 214 | rootCmd.Flags().StringVar(&portRange, 215 | "RemoteProcessProxy.port-range", "", "port range") 216 | rootCmd.Flags().StringVar(&responseAddr, 217 | "RemoteProcessProxy.response-address", "", "response address") 218 | rootCmd.Flags().StringVar(&publicKey, 219 | "RemoteProcessProxy.public-key", "", "public key") 220 | rootCmd.Flags().StringVar(&sparkContextInitMode, 221 | "RemoteProcessProxy.spark-context-initialization-mode", 222 | "", "spark context init mode") 223 | 224 | rootCmd.Flags().StringVar(&kernelTemplateName, 225 | "kernel-template-name", "", "kernel template CRD name") 226 | rootCmd.Flags().StringVar(&kernelTemplateNamespace, 227 | "kernel-template-namespace", "", "kernel template CRD namesapce") 228 | 229 | rootCmd.Flags().BoolVar(&verbose, "verbose", false, "Set verbose") 230 | } 231 | -------------------------------------------------------------------------------- /cli/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2021 NAME HERE 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package main 17 | 18 | import "github.com/tkestack/elastic-jupyter-operator/cli/cmd" 19 | 20 | func main() { 21 | // kubeflow-launcher is a launcher for Kubeflow. 22 | cmd.Execute() 23 | } 24 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for 4 | # breaking changes 5 | apiVersion: cert-manager.io/v1alpha2 6 | kind: Issuer 7 | metadata: 8 | name: selfsigned-issuer 9 | namespace: system 10 | spec: 11 | selfSigned: {} 12 | --- 13 | apiVersion: cert-manager.io/v1alpha2 14 | kind: Certificate 15 | metadata: 16 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 17 | namespace: system 18 | spec: 19 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 20 | dnsNames: 21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 22 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 23 | issuerRef: 24 | kind: Issuer 25 | name: selfsigned-issuer 26 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 27 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/crd/bases/kubeflow.tkestack.io_jupytergateways.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.5.0 8 | creationTimestamp: null 9 | name: jupytergateways.kubeflow.tkestack.io 10 | spec: 11 | group: kubeflow.tkestack.io 12 | names: 13 | kind: JupyterGateway 14 | listKind: JupyterGatewayList 15 | plural: jupytergateways 16 | singular: jupytergateway 17 | scope: Namespaced 18 | versions: 19 | - name: v1alpha1 20 | schema: 21 | openAPIV3Schema: 22 | description: JupyterGateway is the Schema for the jupytergateways API 23 | properties: 24 | apiVersion: 25 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 26 | type: string 27 | kind: 28 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 29 | type: string 30 | metadata: 31 | type: object 32 | spec: 33 | description: JupyterGatewaySpec defines the desired state of JupyterGateway 34 | properties: 35 | clusterRole: 36 | description: ClusterRole for the gateway, which is used to create the kernel pods in the cluster. Defaults to enterprise-gateway-controller (created at startup). 37 | type: string 38 | cullIdleTimeout: 39 | description: Timeout (in seconds) after which a kernel is considered idle and ready to be culled. Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled for users with poor network connections. Ref https://jupyter-notebook.readthedocs.io/en/stable/config.html 40 | format: int32 41 | type: integer 42 | cullInterval: 43 | description: The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value. 44 | format: int32 45 | type: integer 46 | defaultKernel: 47 | description: DefaultKernel defines the default kernel in the gateway. 48 | type: string 49 | image: 50 | description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field defaults to ghcr.io/skai-x/enterprise-gateway:2.6.0' 51 | type: string 52 | kernels: 53 | description: Knernels defines the kernels in the gateway. We will add kernels at runtime, thus we do not make it a type. 54 | items: 55 | type: string 56 | type: array 57 | logLevel: 58 | type: string 59 | resources: 60 | description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' 61 | properties: 62 | limits: 63 | additionalProperties: 64 | anyOf: 65 | - type: integer 66 | - type: string 67 | pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ 68 | x-kubernetes-int-or-string: true 69 | description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' 70 | type: object 71 | requests: 72 | additionalProperties: 73 | anyOf: 74 | - type: integer 75 | - type: string 76 | pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ 77 | x-kubernetes-int-or-string: true 78 | description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' 79 | type: object 80 | type: object 81 | type: object 82 | status: 83 | description: JupyterGatewayStatus defines the observed state of JupyterGateway 84 | properties: 85 | availableReplicas: 86 | description: Total number of available pods (ready for at least minReadySeconds) targeted by this deployment. 87 | format: int32 88 | type: integer 89 | collisionCount: 90 | description: Count of hash collisions for the Deployment. The Deployment controller uses this field as a collision avoidance mechanism when it needs to create the name for the newest ReplicaSet. 91 | format: int32 92 | type: integer 93 | conditions: 94 | description: Represents the latest available observations of a deployment's current state. 95 | items: 96 | description: DeploymentCondition describes the state of a deployment at a certain point. 97 | properties: 98 | lastTransitionTime: 99 | description: Last time the condition transitioned from one status to another. 100 | format: date-time 101 | type: string 102 | lastUpdateTime: 103 | description: The last time this condition was updated. 104 | format: date-time 105 | type: string 106 | message: 107 | description: A human readable message indicating details about the transition. 108 | type: string 109 | reason: 110 | description: The reason for the condition's last transition. 111 | type: string 112 | status: 113 | description: Status of the condition, one of True, False, Unknown. 114 | type: string 115 | type: 116 | description: Type of deployment condition. 117 | type: string 118 | required: 119 | - status 120 | - type 121 | type: object 122 | type: array 123 | observedGeneration: 124 | description: The generation observed by the deployment controller. 125 | format: int64 126 | type: integer 127 | readyReplicas: 128 | description: Total number of ready pods targeted by this deployment. 129 | format: int32 130 | type: integer 131 | replicas: 132 | description: Total number of non-terminated pods targeted by this deployment (their labels match the selector). 133 | format: int32 134 | type: integer 135 | unavailableReplicas: 136 | description: Total number of unavailable pods targeted by this deployment. This is the total number of pods that are still required for the deployment to have 100% available capacity. They may either be pods that are running but not yet available or pods that still have not been created. 137 | format: int32 138 | type: integer 139 | updatedReplicas: 140 | description: Total number of non-terminated pods targeted by this deployment that have the desired template spec. 141 | format: int32 142 | type: integer 143 | type: object 144 | type: object 145 | served: true 146 | storage: true 147 | subresources: 148 | status: {} 149 | status: 150 | acceptedNames: 151 | kind: "" 152 | plural: "" 153 | conditions: [] 154 | storedVersions: [] 155 | -------------------------------------------------------------------------------- /config/crd/bases/kubeflow.tkestack.io_jupyterkernelspecs.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.5.0 8 | creationTimestamp: null 9 | name: jupyterkernelspecs.kubeflow.tkestack.io 10 | spec: 11 | group: kubeflow.tkestack.io 12 | names: 13 | kind: JupyterKernelSpec 14 | listKind: JupyterKernelSpecList 15 | plural: jupyterkernelspecs 16 | singular: jupyterkernelspec 17 | scope: Namespaced 18 | versions: 19 | - name: v1alpha1 20 | schema: 21 | openAPIV3Schema: 22 | description: JupyterKernelSpec is the Schema for the jupyterkernelspecs API 23 | properties: 24 | apiVersion: 25 | description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 26 | type: string 27 | kind: 28 | description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 29 | type: string 30 | metadata: 31 | type: object 32 | spec: 33 | description: JupyterKernelSpecSpec defines the desired state of JupyterKernelSpec 34 | properties: 35 | className: 36 | type: string 37 | command: 38 | items: 39 | type: string 40 | type: array 41 | displayName: 42 | type: string 43 | env: 44 | items: 45 | description: EnvVar represents an environment variable present in a Container. 46 | properties: 47 | name: 48 | description: Name of the environment variable. Must be a C_IDENTIFIER. 49 | type: string 50 | value: 51 | description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' 52 | type: string 53 | valueFrom: 54 | description: Source for the environment variable's value. Cannot be used if value is not empty. 55 | properties: 56 | configMapKeyRef: 57 | description: Selects a key of a ConfigMap. 58 | properties: 59 | key: 60 | description: The key to select. 61 | type: string 62 | name: 63 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' 64 | type: string 65 | optional: 66 | description: Specify whether the ConfigMap or its key must be defined 67 | type: boolean 68 | required: 69 | - key 70 | type: object 71 | fieldRef: 72 | description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' 73 | properties: 74 | apiVersion: 75 | description: Version of the schema the FieldPath is written in terms of, defaults to "v1". 76 | type: string 77 | fieldPath: 78 | description: Path of the field to select in the specified API version. 79 | type: string 80 | required: 81 | - fieldPath 82 | type: object 83 | resourceFieldRef: 84 | description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' 85 | properties: 86 | containerName: 87 | description: 'Container name: required for volumes, optional for env vars' 88 | type: string 89 | divisor: 90 | anyOf: 91 | - type: integer 92 | - type: string 93 | description: Specifies the output format of the exposed resources, defaults to "1" 94 | pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ 95 | x-kubernetes-int-or-string: true 96 | resource: 97 | description: 'Required: resource to select' 98 | type: string 99 | required: 100 | - resource 101 | type: object 102 | secretKeyRef: 103 | description: Selects a key of a secret in the pod's namespace 104 | properties: 105 | key: 106 | description: The key of the secret to select from. Must be a valid secret key. 107 | type: string 108 | name: 109 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' 110 | type: string 111 | optional: 112 | description: Specify whether the Secret or its key must be defined 113 | type: boolean 114 | required: 115 | - key 116 | type: object 117 | type: object 118 | required: 119 | - name 120 | type: object 121 | type: array 122 | image: 123 | type: string 124 | language: 125 | type: string 126 | template: 127 | description: 'ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". Those cannot be well described when embedded. 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple and the version of the actual struct is irrelevant. 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type will affect numerous schemas. Don''t make new APIs embed an underspecified API type they do not control. Instead of using this type, create a locally provided and used type that is well-focused on your reference. For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 .' 128 | properties: 129 | apiVersion: 130 | description: API version of the referent. 131 | type: string 132 | fieldPath: 133 | description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' 134 | type: string 135 | kind: 136 | description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 137 | type: string 138 | name: 139 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' 140 | type: string 141 | namespace: 142 | description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' 143 | type: string 144 | resourceVersion: 145 | description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' 146 | type: string 147 | uid: 148 | description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' 149 | type: string 150 | type: object 151 | type: object 152 | status: 153 | description: JupyterKernelSpecStatus defines the observed state of JupyterKernelSpec 154 | type: object 155 | type: object 156 | served: true 157 | storage: true 158 | subresources: 159 | status: {} 160 | status: 161 | acceptedNames: 162 | kind: "" 163 | plural: "" 164 | conditions: [] 165 | storedVersions: [] 166 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/kubeflow.tkestack.io_jupyternotebooks.yaml 6 | - bases/kubeflow.tkestack.io_jupytergateways.yaml 7 | - bases/kubeflow.tkestack.io_jupyterkernelspecs.yaml 8 | - bases/kubeflow.tkestack.io_jupyterkerneltemplates.yaml 9 | - bases/kubeflow.tkestack.io_jupyterkernels.yaml 10 | # +kubebuilder:scaffold:crdkustomizeresource 11 | 12 | patchesStrategicMerge: 13 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 14 | # patches here are for enabling the conversion webhook for each CRD 15 | #- patches/webhook_in_jupyternotebooks.yaml 16 | #- patches/webhook_in_jupytergateways.yaml 17 | #- patches/webhook_in_jupyterkernelspecs.yaml 18 | #- patches/webhook_in_jupyterkerneltemplates.yaml 19 | #- patches/webhook_in_jupyterkernels.yaml 20 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 21 | 22 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 23 | # patches here are for enabling the CA injection for each CRD 24 | #- patches/cainjection_in_jupyternotebooks.yaml 25 | #- patches/cainjection_in_jupytergateways.yaml 26 | #- patches/cainjection_in_jupyterkernelspecs.yaml 27 | #- patches/cainjection_in_jupyterkerneltemplates.yaml 28 | #- patches/cainjection_in_jupyterkernels.yaml 29 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 30 | 31 | # the following config is for teaching kustomize how to do kustomization for CRDs. 32 | configurations: 33 | - kustomizeconfig.yaml 34 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | group: apiextensions.k8s.io 8 | path: spec/conversion/webhookClientConfig/service/name 9 | 10 | namespace: 11 | - kind: CustomResourceDefinition 12 | group: apiextensions.k8s.io 13 | path: spec/conversion/webhookClientConfig/service/namespace 14 | create: false 15 | 16 | varReference: 17 | - path: metadata/annotations 18 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jupytergateways.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: jupytergateways.kubeflow.tkestack.io 9 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jupyterkernels.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: jupyterkernels.kubeflow.tkestack.io 9 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jupyterkernelspecs.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: jupyterkernelspecs.kubeflow.tkestack.io 9 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jupyterkerneltemplates.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: jupyterkerneltemplates.kubeflow.tkestack.io 9 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_jupyternotebooks.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: jupyternotebooks.kubeflow.tkestack.io 9 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jupytergateways.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: jupytergateways.kubeflow.tkestack.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jupyterkernels.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: jupyterkernels.kubeflow.tkestack.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jupyterkernelspecs.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: jupyterkernelspecs.kubeflow.tkestack.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jupyterkerneltemplates.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: jupyterkerneltemplates.kubeflow.tkestack.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_jupyternotebooks.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: jupyternotebooks.kubeflow.tkestack.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: elastic-jupyter-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: elastic-jupyter-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | #- ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | #- ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 28 | # crd/kustomization.yaml 29 | #- manager_webhook_patch.yaml 30 | 31 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 32 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 33 | # 'CERTMANAGER' needs to be enabled to use ca injection 34 | #- webhookcainjection_patch.yaml 35 | 36 | # the following config is for teaching kustomize how to do var substitution 37 | # vars: 38 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 39 | #- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 40 | # objref: 41 | # kind: Certificate 42 | # group: cert-manager.io 43 | # version: v1alpha2 44 | # name: serving-cert # this name should match the one in certificate.yaml 45 | # fieldref: 46 | # fieldpath: metadata.namespace 47 | #- name: CERTIFICATE_NAME 48 | # objref: 49 | # kind: Certificate 50 | # group: cert-manager.io 51 | # version: v1alpha2 52 | # name: serving-cert # this name should match the one in certificate.yaml 53 | #- name: SERVICE_NAMESPACE # namespace of the service 54 | # objref: 55 | # kind: Service 56 | # version: v1 57 | # name: webhook-service 58 | # fieldref: 59 | # fieldpath: metadata.namespace 60 | #- name: SERVICE_NAME 61 | # objref: 62 | # kind: Service 63 | # version: v1 64 | # name: webhook-service 65 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | images: 6 | - name: controller 7 | newName: ghcr.io/skai-x/elastic-jupyter-operator 8 | newTag: latest 9 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | labels: 23 | control-plane: controller-manager 24 | spec: 25 | containers: 26 | - command: 27 | - /elastic-jupyter-operator 28 | args: 29 | - --enable-leader-election 30 | image: ghcr.io/skai-x/elastic-jupyter-operator:latest 31 | name: manager 32 | resources: 33 | limits: 34 | cpu: 100m 35 | memory: 30Mi 36 | requests: 37 | cpu: 100m 38 | memory: 20Mi 39 | terminationGracePeriodSeconds: 10 40 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | selector: 15 | matchLabels: 16 | control-plane: controller-manager 17 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: ["/metrics"] 7 | verbs: ["get"] 8 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: ["authentication.k8s.io"] 7 | resources: 8 | - tokenreviews 9 | verbs: ["create"] 10 | - apiGroups: ["authorization.k8s.io"] 11 | resources: 12 | - subjectaccessreviews 13 | verbs: ["create"] 14 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /config/rbac/jupytergateway_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jupytergateways. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupytergateway-editor-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupytergateways 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - kubeflow.tkestack.io 21 | resources: 22 | - jupytergateways/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jupytergateway_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jupytergateways. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupytergateway-viewer-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupytergateways 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - kubeflow.tkestack.io 17 | resources: 18 | - jupytergateways/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/jupyterkernel_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jupyterkernels. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyterkernel-editor-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyterkernels 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - kubeflow.tkestack.io 21 | resources: 22 | - jupyterkernels/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jupyterkernel_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jupyterkernels. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyterkernel-viewer-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyterkernels 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - kubeflow.tkestack.io 17 | resources: 18 | - jupyterkernels/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/jupyterkernelspec_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jupyterkernelspecs. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyterkernelspec-editor-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyterkernelspecs 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - kubeflow.tkestack.io 21 | resources: 22 | - jupyterkernelspecs/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jupyterkernelspec_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jupyterkernelspecs. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyterkernelspec-viewer-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyterkernelspecs 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - kubeflow.tkestack.io 17 | resources: 18 | - jupyterkernelspecs/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/jupyterkerneltemplate_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jupyterkerneltemplates. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyterkerneltemplate-editor-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyterkerneltemplates 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - kubeflow.tkestack.io 21 | resources: 22 | - jupyterkerneltemplates/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jupyterkerneltemplate_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jupyterkerneltemplates. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyterkerneltemplate-viewer-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyterkerneltemplates 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - kubeflow.tkestack.io 17 | resources: 18 | - jupyterkerneltemplates/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/jupyternotebook_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit jupyternotebooks. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyternotebook-editor-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyternotebooks 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - kubeflow.tkestack.io 21 | resources: 22 | - jupyternotebooks/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/jupyternotebook_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view jupyternotebooks. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: jupyternotebook-viewer-role 6 | rules: 7 | - apiGroups: 8 | - kubeflow.tkestack.io 9 | resources: 10 | - jupyternotebooks 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - kubeflow.tkestack.io 17 | resources: 18 | - jupyternotebooks/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - role.yaml 3 | - role_binding.yaml 4 | - leader_election_role.yaml 5 | - leader_election_role_binding.yaml 6 | # Comment the following 4 lines if you want to disable 7 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 8 | # which protects your /metrics endpoint. 9 | # - auth_proxy_service.yaml 10 | # - auth_proxy_role.yaml 11 | # - auth_proxy_role_binding.yaml 12 | # - auth_proxy_client_clusterrole.yaml 13 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - configmaps/status 23 | verbs: 24 | - get 25 | - update 26 | - patch 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - events 31 | verbs: 32 | - create 33 | - patch 34 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - configmaps 13 | - events 14 | - namespaces 15 | - persistentvolumeclaims 16 | - persistentvolumes 17 | - pods 18 | - secrets 19 | - serviceaccounts 20 | - services 21 | verbs: 22 | - create 23 | - delete 24 | - get 25 | - list 26 | - patch 27 | - update 28 | - watch 29 | - apiGroups: 30 | - apps 31 | resources: 32 | - deployments 33 | verbs: 34 | - create 35 | - delete 36 | - get 37 | - list 38 | - patch 39 | - update 40 | - watch 41 | - apiGroups: 42 | - kubeflow.tkestack.io 43 | resources: 44 | - jupytergateways 45 | verbs: 46 | - create 47 | - delete 48 | - get 49 | - list 50 | - patch 51 | - update 52 | - watch 53 | - apiGroups: 54 | - kubeflow.tkestack.io 55 | resources: 56 | - jupytergateways/status 57 | verbs: 58 | - get 59 | - patch 60 | - update 61 | - apiGroups: 62 | - kubeflow.tkestack.io 63 | resources: 64 | - jupyterkernels 65 | verbs: 66 | - create 67 | - delete 68 | - get 69 | - list 70 | - patch 71 | - update 72 | - watch 73 | - apiGroups: 74 | - kubeflow.tkestack.io 75 | resources: 76 | - jupyterkernels/status 77 | verbs: 78 | - get 79 | - patch 80 | - update 81 | - apiGroups: 82 | - kubeflow.tkestack.io 83 | resources: 84 | - jupyterkernelspecs 85 | verbs: 86 | - create 87 | - delete 88 | - get 89 | - list 90 | - patch 91 | - update 92 | - watch 93 | - apiGroups: 94 | - kubeflow.tkestack.io 95 | resources: 96 | - jupyterkernelspecs/status 97 | verbs: 98 | - get 99 | - patch 100 | - update 101 | - apiGroups: 102 | - kubeflow.tkestack.io 103 | resources: 104 | - jupyterkerneltemplates 105 | verbs: 106 | - create 107 | - delete 108 | - get 109 | - list 110 | - patch 111 | - update 112 | - watch 113 | - apiGroups: 114 | - kubeflow.tkestack.io 115 | resources: 116 | - jupyterkerneltemplates/status 117 | verbs: 118 | - get 119 | - patch 120 | - update 121 | - apiGroups: 122 | - kubeflow.tkestack.io 123 | resources: 124 | - jupyternotebooks 125 | verbs: 126 | - create 127 | - delete 128 | - get 129 | - list 130 | - patch 131 | - update 132 | - watch 133 | - apiGroups: 134 | - kubeflow.tkestack.io 135 | resources: 136 | - jupyternotebooks/status 137 | verbs: 138 | - get 139 | - patch 140 | - update 141 | - apiGroups: 142 | - rbac.authorization.k8s.io 143 | resources: 144 | - rolebindings 145 | verbs: 146 | - create 147 | - delete 148 | - get 149 | - list 150 | - patch 151 | - update 152 | - watch 153 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupytergateway-kernels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterGateway 3 | metadata: 4 | name: jupytergateway-sample 5 | spec: 6 | cullIdleTimeout: 10 7 | cullInterval: 10 8 | logLevel: DEBUG 9 | image: ghcr.io/skai-x/enterprise-gateway:2.6.0 10 | kernels: 11 | - python-kubernetes 12 | resources: 13 | requests: 14 | memory: "64Mi" 15 | cpu: "250m" 16 | limits: 17 | memory: "128Mi" 18 | cpu: "500m" 19 | -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterGateway 3 | metadata: 4 | name: jupytergateway-sample 5 | spec: 6 | cullIdleTimeout: 3600 7 | resources: 8 | requests: 9 | memory: "64Mi" 10 | cpu: "250m" 11 | limits: 12 | memory: "128Mi" 13 | cpu: "500m" -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupyterkernel.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterKernel 3 | metadata: 4 | name: jupyterkernel-sample 5 | spec: 6 | template: 7 | metadata: 8 | app: enterprise-gateway 9 | component: kernel 10 | spec: 11 | restartPolicy: Never 12 | containers: 13 | - name: kernel 14 | image: ghcr.io/skai-x/jupyter-kernel-py:2.6.0 15 | -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec-custom-launcher.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterKernelSpec 3 | metadata: 4 | name: python-kubernetes 5 | spec: 6 | language: Python 7 | displayName: "Python on Kubernetes as a JupyterKernelSpec" 8 | image: ghcr.io/skai-x/jupyter-kernel-py:2.6.0 9 | className: enterprise_gateway.services.processproxies.kubeflow.KubeflowProcessProxy 10 | template: 11 | namespace: default 12 | name: jupyterkerneltemplate-sample 13 | command: 14 | # Use the default scripts to launch the kernel. 15 | - "kubeflow-launcher" 16 | - "--verbose" 17 | - "--RemoteProcessProxy.kernel-id" 18 | - "{kernel_id}" 19 | - "--RemoteProcessProxy.port-range" 20 | - "{port_range}" 21 | - "--RemoteProcessProxy.response-address" 22 | - "{response_address}" 23 | -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterKernelSpec 3 | metadata: 4 | name: python-kubernetes 5 | spec: 6 | language: Python 7 | displayName: "Python on Kubernetes as a JupyterKernelSpec" 8 | image: ghcr.io/skai-x/jupyter-kernel-py:2.6.0 9 | command: 10 | - "python" 11 | # Use the default scripts to launch the kernel. 12 | - "/usr/local/share/jupyter/kernels/python_kubernetes/scripts/launch_kubernetes.py" 13 | - "--RemoteProcessProxy.kernel-id" 14 | - "{kernel_id}" 15 | - "--RemoteProcessProxy.port-range" 16 | - "{port_range}" 17 | - "--RemoteProcessProxy.response-address" 18 | - "{response_address}" 19 | -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterKernelTemplate 3 | metadata: 4 | name: jupyterkerneltemplate-sample 5 | spec: 6 | template: 7 | metadata: 8 | app: enterprise-gateway 9 | component: kernel 10 | spec: 11 | restartPolicy: Always 12 | containers: 13 | - name: kernel 14 | -------------------------------------------------------------------------------- /config/samples/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterNotebook 3 | metadata: 4 | name: jupyternotebook-sample 5 | spec: 6 | gateway: 7 | name: jupytergateway-sample 8 | namespace: default 9 | # resources: 10 | # requests: 11 | # memory: "64Mi" 12 | # cpu: "250m" 13 | # limits: 14 | # memory: "128Mi" 15 | # cpu: "500m" 16 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 9443 11 | selector: 12 | control-plane: controller-manager 13 | -------------------------------------------------------------------------------- /controllers/jupytergateway_controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | appsv1 "k8s.io/api/apps/v1" 24 | "k8s.io/apimachinery/pkg/api/errors" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/client-go/tools/record" 27 | ctrl "sigs.k8s.io/controller-runtime" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | "sigs.k8s.io/controller-runtime/pkg/handler" 30 | "sigs.k8s.io/controller-runtime/pkg/source" 31 | 32 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 33 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 34 | "github.com/tkestack/elastic-jupyter-operator/pkg/gateway" 35 | ) 36 | 37 | // JupyterGatewayReconciler reconciles a JupyterGateway object 38 | type JupyterGatewayReconciler struct { 39 | client.Client 40 | Log logr.Logger 41 | Recorder record.EventRecorder 42 | Scheme *runtime.Scheme 43 | } 44 | 45 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupytergateways,verbs=get;list;watch;create;update;patch;delete 46 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupytergateways/status,verbs=get;update;patch 47 | // +kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete 48 | // +kubebuilder:rbac:groups="",resources=pods;namespaces;services;serviceaccounts;configmaps;secrets;persistentvolumes;persistentvolumeclaims;events,verbs=get;list;watch;create;update;create;patch;delete 49 | // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;create;update;patch;list;watch;delete 50 | 51 | func (r *JupyterGatewayReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { 52 | _ = context.Background() 53 | _ = r.Log.WithValues("jupytergateway", req.NamespacedName) 54 | 55 | original := &v1alpha1.JupyterGateway{} 56 | 57 | err := r.Get(context.TODO(), req.NamespacedName, original) 58 | if err != nil { 59 | if errors.IsNotFound(err) { 60 | // Object not found, return. Created objects are automatically garbage collected. 61 | // For additional cleanup logic use finalizers. 62 | return ctrl.Result{}, nil 63 | } 64 | // Error reading the object - requeue the request. 65 | r.Log.Error(err, "Failed to get the object, requeuing the request") 66 | return ctrl.Result{}, err 67 | } 68 | instance := original.DeepCopy() 69 | 70 | gr, err := gateway.NewReconciler(r.Client, r.Log, r.Recorder, r.Scheme, instance) 71 | if err != nil { 72 | return ctrl.Result{}, err 73 | } 74 | if err := gr.Reconcile(); err != nil { 75 | return ctrl.Result{}, err 76 | } 77 | return ctrl.Result{}, nil 78 | } 79 | 80 | func (r *JupyterGatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { 81 | return ctrl.NewControllerManagedBy(mgr). 82 | For(&kubeflowtkestackiov1alpha1.JupyterGateway{}). 83 | Watches(&source.Kind{Type: &appsv1.Deployment{}}, 84 | &handler.EnqueueRequestForOwner{ 85 | IsController: true, 86 | OwnerType: &v1alpha1.JupyterGateway{}, 87 | }). 88 | Complete(r) 89 | } 90 | -------------------------------------------------------------------------------- /controllers/jupyterkernel_controller.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/client-go/tools/record" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | 29 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 30 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 31 | "github.com/tkestack/elastic-jupyter-operator/pkg/kernel" 32 | ) 33 | 34 | // JupyterKernelReconciler reconciles a JupyterKernel object 35 | type JupyterKernelReconciler struct { 36 | client.Client 37 | Log logr.Logger 38 | Scheme *runtime.Scheme 39 | Recorder record.EventRecorder 40 | } 41 | 42 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyterkernels,verbs=get;list;watch;create;update;patch;delete 43 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyterkernels/status,verbs=get;update;patch 44 | 45 | func (r *JupyterKernelReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { 46 | _ = context.Background() 47 | _ = r.Log.WithValues("jupyterkernel", req.NamespacedName) 48 | 49 | original := &v1alpha1.JupyterKernel{} 50 | 51 | err := r.Get(context.TODO(), req.NamespacedName, original) 52 | if err != nil { 53 | if errors.IsNotFound(err) { 54 | // Object not found, return. Created objects are automatically garbage collected. 55 | // For additional cleanup logic use finalizers. 56 | return ctrl.Result{}, nil 57 | } 58 | // Error reading the object - requeue the request. 59 | r.Log.Error(err, "Failed to get the object, requeuing the request") 60 | return ctrl.Result{}, err 61 | } 62 | instance := original.DeepCopy() 63 | 64 | gr, err := kernel.NewReconciler(r.Client, r.Log, r.Recorder, r.Scheme, instance) 65 | if err != nil { 66 | return ctrl.Result{}, err 67 | } 68 | if err := gr.Reconcile(); err != nil { 69 | return ctrl.Result{}, err 70 | } 71 | 72 | return ctrl.Result{}, nil 73 | } 74 | 75 | func (r *JupyterKernelReconciler) SetupWithManager(mgr ctrl.Manager) error { 76 | return ctrl.NewControllerManagedBy(mgr). 77 | For(&kubeflowtkestackiov1alpha1.JupyterKernel{}). 78 | Complete(r) 79 | } 80 | -------------------------------------------------------------------------------- /controllers/jupyterkernelspec_controller.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | "k8s.io/client-go/tools/record" 26 | ctrl "sigs.k8s.io/controller-runtime" 27 | "sigs.k8s.io/controller-runtime/pkg/client" 28 | 29 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 30 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 31 | "github.com/tkestack/elastic-jupyter-operator/pkg/kernelspec" 32 | ) 33 | 34 | // JupyterKernelSpecReconciler reconciles a JupyterKernelSpec object 35 | type JupyterKernelSpecReconciler struct { 36 | client.Client 37 | Log logr.Logger 38 | Recorder record.EventRecorder 39 | Scheme *runtime.Scheme 40 | } 41 | 42 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyterkernelspecs,verbs=get;list;watch;create;update;patch;delete 43 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyterkernelspecs/status,verbs=get;update;patch 44 | 45 | func (r *JupyterKernelSpecReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { 46 | _ = context.Background() 47 | _ = r.Log.WithValues("jupyterkernelspec", req.NamespacedName) 48 | 49 | original := &v1alpha1.JupyterKernelSpec{} 50 | 51 | err := r.Get(context.TODO(), req.NamespacedName, original) 52 | if err != nil { 53 | if errors.IsNotFound(err) { 54 | // Object not found, return. Created objects are automatically garbage collected. 55 | // For additional cleanup logic use finalizers. 56 | return ctrl.Result{}, nil 57 | } 58 | // Error reading the object - requeue the request. 59 | r.Log.Error(err, "Failed to get the object, requeuing the request") 60 | return ctrl.Result{}, err 61 | } 62 | instance := original.DeepCopy() 63 | 64 | gr, err := kernelspec.NewReconciler(r.Client, r.Log, r.Recorder, r.Scheme, instance) 65 | if err != nil { 66 | return ctrl.Result{}, err 67 | } 68 | if err := gr.Reconcile(); err != nil { 69 | return ctrl.Result{}, err 70 | } 71 | 72 | return ctrl.Result{}, nil 73 | } 74 | 75 | func (r *JupyterKernelSpecReconciler) SetupWithManager(mgr ctrl.Manager) error { 76 | return ctrl.NewControllerManagedBy(mgr). 77 | For(&kubeflowtkestackiov1alpha1.JupyterKernelSpec{}). 78 | Complete(r) 79 | } 80 | -------------------------------------------------------------------------------- /controllers/jupyterkerneltemplate_controller.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | ctrl "sigs.k8s.io/controller-runtime" 25 | "sigs.k8s.io/controller-runtime/pkg/client" 26 | 27 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 28 | ) 29 | 30 | // JupyterKernelTemplateReconciler reconciles a JupyterKernelTemplate object 31 | type JupyterKernelTemplateReconciler struct { 32 | client.Client 33 | Log logr.Logger 34 | Scheme *runtime.Scheme 35 | } 36 | 37 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyterkerneltemplates,verbs=get;list;watch;create;update;patch;delete 38 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyterkerneltemplates/status,verbs=get;update;patch 39 | 40 | func (r *JupyterKernelTemplateReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { 41 | _ = context.Background() 42 | _ = r.Log.WithValues("jupyterkerneltemplate", req.NamespacedName) 43 | 44 | // your logic here 45 | 46 | return ctrl.Result{}, nil 47 | } 48 | 49 | func (r *JupyterKernelTemplateReconciler) SetupWithManager(mgr ctrl.Manager) error { 50 | return ctrl.NewControllerManagedBy(mgr). 51 | For(&kubeflowtkestackiov1alpha1.JupyterKernelTemplate{}). 52 | Complete(r) 53 | } 54 | -------------------------------------------------------------------------------- /controllers/jupyternotebook_controller.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | appsv1 "k8s.io/api/apps/v1" 24 | "k8s.io/apimachinery/pkg/api/errors" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/client-go/tools/record" 27 | ctrl "sigs.k8s.io/controller-runtime" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | "sigs.k8s.io/controller-runtime/pkg/handler" 30 | "sigs.k8s.io/controller-runtime/pkg/source" 31 | 32 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 33 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 34 | "github.com/tkestack/elastic-jupyter-operator/pkg/notebook" 35 | ) 36 | 37 | // JupyterNotebookReconciler reconciles a JupyterNotebook object 38 | type JupyterNotebookReconciler struct { 39 | client.Client 40 | Log logr.Logger 41 | Recorder record.EventRecorder 42 | Scheme *runtime.Scheme 43 | } 44 | 45 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyternotebooks,verbs=get;list;watch;create;update;patch;delete 46 | // +kubebuilder:rbac:groups=kubeflow.tkestack.io,resources=jupyternotebooks/status,verbs=get;update;patch 47 | 48 | func (r *JupyterNotebookReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { 49 | _ = context.Background() 50 | _ = r.Log.WithValues("jupyternotebook", req.NamespacedName) 51 | 52 | original := &v1alpha1.JupyterNotebook{} 53 | 54 | err := r.Get(context.TODO(), req.NamespacedName, original) 55 | if err != nil { 56 | if errors.IsNotFound(err) { 57 | // Object not found, return. Created objects are automatically garbage collected. 58 | // For additional cleanup logic use finalizers. 59 | return ctrl.Result{}, nil 60 | } 61 | // Error reading the object - requeue the request. 62 | r.Log.Error(err, "Failed to get the object, requeuing the request") 63 | return ctrl.Result{}, err 64 | } 65 | instance := original.DeepCopy() 66 | 67 | gr, err := notebook.NewReconciler(r.Client, r.Log, r.Recorder, r.Scheme, instance) 68 | if err != nil { 69 | return ctrl.Result{}, err 70 | } 71 | if err := gr.Reconcile(); err != nil { 72 | return ctrl.Result{}, err 73 | } 74 | return ctrl.Result{}, nil 75 | } 76 | 77 | func (r *JupyterNotebookReconciler) SetupWithManager(mgr ctrl.Manager) error { 78 | return ctrl.NewControllerManagedBy(mgr). 79 | For(&kubeflowtkestackiov1alpha1.JupyterNotebook{}). 80 | Watches(&source.Kind{Type: &appsv1.Deployment{}}, 81 | &handler.EnqueueRequestForOwner{ 82 | IsController: true, 83 | OwnerType: &v1alpha1.JupyterNotebook{}, 84 | }). 85 | Complete(r) 86 | } 87 | -------------------------------------------------------------------------------- /controllers/jupyternotebook_controller_test.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | v1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | 13 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 14 | ) 15 | 16 | var _ = Describe("JupyterNotebook controller", func() { 17 | const ( 18 | JupyterNotebookName = "jupyternotebook-sample" 19 | JupyterNotebookNamespace = "default" 20 | DefaultContainerName = "notebook" 21 | DefaultImage = "busysandbox" 22 | 23 | timeout = time.Second * 10 24 | duration = time.Second * 10 25 | interval = time.Millisecond * 250 26 | ) 27 | 28 | var ( 29 | podSpec = v1.PodSpec{ 30 | Containers: []v1.Container{ 31 | { 32 | Name: DefaultContainerName, 33 | Image: DefaultImage, 34 | ImagePullPolicy: v1.PullIfNotPresent, 35 | }, 36 | }, 37 | } 38 | 39 | notebookWithTemplate = &kubeflowtkestackiov1alpha1.JupyterNotebook{ 40 | TypeMeta: metav1.TypeMeta{ 41 | APIVersion: "kubeflow.tkestack.io/v1alpha1", 42 | Kind: "JupyterNotebook", 43 | }, 44 | ObjectMeta: metav1.ObjectMeta{ 45 | Name: JupyterNotebookName, 46 | Namespace: JupyterNotebookNamespace, 47 | }, 48 | Spec: kubeflowtkestackiov1alpha1.JupyterNotebookSpec{ 49 | Template: &v1.PodTemplateSpec{ 50 | Spec: podSpec, 51 | }, 52 | }, 53 | } 54 | 55 | key = types.NamespacedName{ 56 | Name: JupyterNotebookName, 57 | Namespace: JupyterNotebookNamespace, 58 | } 59 | ) 60 | 61 | Context("JupyterNotebook only have template", func() { 62 | It("Should create successfully", func() { 63 | Expect(k8sClient.Create(context.Background(), notebookWithTemplate)).Should(Succeed()) 64 | By("Expecting container name") 65 | Eventually(func() string { 66 | actual := &kubeflowtkestackiov1alpha1.JupyterNotebook{} 67 | if err := k8sClient.Get(context.Background(), key, actual); err == nil { 68 | return actual.Spec.Template.Spec.Containers[0].Name 69 | } 70 | return "" 71 | }, timeout, interval).Should(Equal(DefaultContainerName)) 72 | }) 73 | 74 | It("Should update successfully", func() { 75 | name := "NewName" 76 | actual := &kubeflowtkestackiov1alpha1.JupyterNotebook{} 77 | Expect(k8sClient.Get(context.Background(), key, actual)).Should(Succeed()) 78 | actual.Spec.Template.Name = name 79 | Expect(k8sClient.Update(context.Background(), actual)).Should(Succeed()) 80 | 81 | By("Expecting template name") 82 | Eventually(func() string { 83 | notebook := &kubeflowtkestackiov1alpha1.JupyterNotebook{} 84 | if err := k8sClient.Get(context.Background(), key, notebook); err == nil { 85 | return actual.Spec.Template.Name 86 | } 87 | return "" 88 | }, timeout, interval).Should(Equal(name)) 89 | }) 90 | 91 | It("Should delete successfully", func() { 92 | By("Expecting to delete successfully") 93 | Eventually(func() error { 94 | actual := &kubeflowtkestackiov1alpha1.JupyterNotebook{} 95 | k8sClient.Get(context.Background(), key, actual) 96 | return k8sClient.Delete(context.Background(), actual) 97 | }, timeout, interval).Should(Succeed()) 98 | 99 | By("Expecting to delete finish") 100 | Eventually(func() error { 101 | actual := &kubeflowtkestackiov1alpha1.JupyterNotebook{} 102 | return k8sClient.Get(context.Background(), key, actual) 103 | }, timeout, interval).ShouldNot(Succeed()) 104 | }) 105 | }) 106 | }) 107 | -------------------------------------------------------------------------------- /controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "path/filepath" 21 | "testing" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "k8s.io/client-go/kubernetes/scheme" 26 | "k8s.io/client-go/rest" 27 | ctrl "sigs.k8s.io/controller-runtime" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | "sigs.k8s.io/controller-runtime/pkg/envtest" 30 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 31 | logf "sigs.k8s.io/controller-runtime/pkg/log" 32 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 33 | 34 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 35 | // +kubebuilder:scaffold:imports 36 | ) 37 | 38 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 39 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 40 | 41 | var cfg *rest.Config 42 | var k8sClient client.Client 43 | var testEnv *envtest.Environment 44 | 45 | func TestAPIs(t *testing.T) { 46 | RegisterFailHandler(Fail) 47 | 48 | RunSpecsWithDefaultAndCustomReporters(t, 49 | "Controller Suite", 50 | []Reporter{printer.NewlineReporter{}}) 51 | } 52 | 53 | var _ = BeforeSuite(func(done Done) { 54 | logf.SetLogger(zap.New(zap.UseDevMode(true))) 55 | 56 | By("bootstrapping test environment") 57 | testEnv = &envtest.Environment{ 58 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 59 | } 60 | 61 | var err error 62 | cfg, err = testEnv.Start() 63 | Expect(err).ToNot(HaveOccurred()) 64 | Expect(cfg).ToNot(BeNil()) 65 | 66 | err = kubeflowtkestackiov1alpha1.AddToScheme(scheme.Scheme) 67 | Expect(err).NotTo(HaveOccurred()) 68 | 69 | err = kubeflowtkestackiov1alpha1.AddToScheme(scheme.Scheme) 70 | Expect(err).NotTo(HaveOccurred()) 71 | 72 | err = kubeflowtkestackiov1alpha1.AddToScheme(scheme.Scheme) 73 | Expect(err).NotTo(HaveOccurred()) 74 | 75 | err = kubeflowtkestackiov1alpha1.AddToScheme(scheme.Scheme) 76 | Expect(err).NotTo(HaveOccurred()) 77 | 78 | err = kubeflowtkestackiov1alpha1.AddToScheme(scheme.Scheme) 79 | Expect(err).NotTo(HaveOccurred()) 80 | 81 | // +kubebuilder:scaffold:scheme 82 | 83 | k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ 84 | Scheme: scheme.Scheme, 85 | }) 86 | Expect(err).ToNot(HaveOccurred()) 87 | 88 | err = (&JupyterNotebookReconciler{ 89 | Client: k8sManager.GetClient(), 90 | Log: ctrl.Log.WithName("controllers").WithName("JupyterNotebook"), 91 | Recorder: k8sManager.GetEventRecorderFor("JupyterNotebook"), 92 | Scheme: k8sManager.GetScheme(), 93 | }).SetupWithManager(k8sManager) 94 | Expect(err).ToNot(HaveOccurred()) 95 | 96 | go func() { 97 | err = k8sManager.Start(ctrl.SetupSignalHandler()) 98 | Expect(err).ToNot(HaveOccurred()) 99 | }() 100 | 101 | k8sClient = k8sManager.GetClient() 102 | Expect(k8sClient).ToNot(BeNil()) 103 | 104 | close(done) 105 | }, 60) 106 | 107 | var _ = AfterSuite(func() { 108 | By("tearing down the test environment") 109 | err := testEnv.Stop() 110 | Expect(err).ToNot(HaveOccurred()) 111 | }) 112 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Elastic Jupyter Notebooks on Kubernetes: the Cloud Native Way 2 | 3 | 4 | Running [Jupyter Notebook][] on Kubernetes is common, but it's not easy. The notebook server runs kernels on the host by default. However, it is necessary to run remote kernels if [Jupyter Notebook][] is deployed and used on Kubernetes. 5 | 6 | Deep learning model training is a good example. It requires lots of resources, usually some GPUs. Meanwhile, GPU resources are expensive to use. Thus users expect sharing GPUs between notebooks. 7 | 8 | ## State of the art 9 | 10 | There are some existing [Jupyter Notebook][] operators in Kubernetes community, such as [Kubeflow jupyter operator](https://github.com/kubeflow/kubeflow/tree/master/components/notebook-controller). These projects just deploy the [Jupyter Notebook][] as a deployment directly on Kubernetes. The GPU utilization does not meet our expectation, because the GPUs are allocated by users statically. 11 | 12 | [Jupyter Enterprise Gateway][] could help us improve the utilization by running the notebook server processes and kernel processes separately. But there are some limitations. [Jupyter Enterprise Gateway][] is designed to be used on different resource managers, e.g. Yarn, Kubernetes, etc. Thus it is not Kubernetes native. Maintaining such a gateway and multiple notebook servers/kernels is not easy. 13 | 14 |

15 | 16 | Besides this, customizing the kernel specifications requires [rebooting the enterprise gateway](https://github.com/jupyter/enterprise_gateway/blob/master/etc/docker/enterprise-gateway/Dockerfile#L30) on Kubernetes, because the kernel specifications are hard-coded in the image. 17 | 18 | Last, the resources used by the kernel can not be updated easily. The Kernel YAML template is defined as [jinja2 template](https://github.com/jupyter/enterprise_gateway/blob/master/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2). It is also hard-coded in the image. 19 | 20 | To solve these problems, we implemented a new operator [elastic-jupyter-operator][] based on Kubernetes and [Jupyter Enterprise Gateway][], to make it easy to deploy and use elastic [Jupyter Notebook][] on Kubernetes. You can manage the notebook server and kernels on Kubernetes in a declarative way via the CustomResourceDefinitions (CRDs), instead of getting trouble with the containers and networking things. 21 | 22 | ## Quick start 23 | 24 | First you need to clone the repository and install the operator. Five CustomResourceDefinitions (CRDs) are installed in the cluster: `JupyterGateway`, `JupyterNotebook`, `JupyterKernel`, `JupyterKernelTemplate` and `JupyterKernelSpec`. 25 | 26 | ```yaml 27 | git clone git@github.com:tkestack/elastic-jupyter-operator.git 28 | kubectl apply -f ./hack/enterprise_gateway/prepare.yaml 29 | make deploy 30 | ``` 31 | 32 | ### Remote kernels 33 | 34 | Users can create the elastic [Jupyter Notebook][] on Kubernetes by creating `JupyterNotebook` and `JupyterGateway`. 35 | 36 | ```bash 37 | $ cat ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 38 | apiVersion: kubeflow.tkestack.io/v1alpha1 39 | kind: JupyterNotebook 40 | metadata: 41 | name: jupyternotebook-elastic 42 | spec: 43 | gateway: 44 | name: jupytergateway-elastic 45 | namespace: default 46 | auth: 47 | mode: disable 48 | 49 | $ cat ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 50 | apiVersion: kubeflow.tkestack.io/v1alpha1 51 | kind: JupyterGateway 52 | metadata: 53 | name: jupytergateway-elastic 54 | spec: 55 | cullIdleTimeout: 3600 56 | image: ghcr.io/skai-x/enterprise-gateway:2.6.0 57 | 58 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 59 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 60 | $ kubectl port-forward deploy/jupyternotebook-elastic 8888:8888 61 | ``` 62 | 63 | When the code is executed in the notebook page, there will be a new kernel pod created in the cluster. 64 | 65 | ``` 66 | NAME READY STATUS RESTARTS AGE 67 | kernel-219cfd49-89ad-428c-8e0d-3e61e15d79a7 1/1 Running 0 170m 68 | jupytergateway-elastic-868d8f465c-8mg44 1/1 Running 0 3h 69 | jupyternotebook-elastic-787d94bb4b-xdwnc 1/1 Running 0 3h10m 70 | ``` 71 | 72 | ### Remote kernels with custom configuration 73 | 74 | If you want to custom the kernel deployment, for example. you want to update the resource requirements of the python kernel or use different images for the kernel, you can deploy the jupyter notebooks and gateways with custom kernels. 75 | 76 | First, you need to create the JupyterKernelSpec CR, which is used to generate the [Jupyter kernelspec](https://jupyter-client.readthedocs.io/en/stable/kernels.html). 77 | 78 | ```yaml 79 | $ cat examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml 80 | apiVersion: kubeflow.tkestack.io/v1alpha1 81 | kind: JupyterKernelSpec 82 | metadata: 83 | name: python-kubernetes 84 | spec: 85 | language: Python 86 | displayName: "Python on Kubernetes as a JupyterKernelSpec" 87 | image: ghcr.io/skai-x/jupyter-kernel-py:2.6.0 88 | className: enterprise_gateway.services.processproxies.kubeflow.KubeflowProcessProxy 89 | # Use the template defined in JupyterKernelTemplate CR. 90 | template: 91 | namespace: default 92 | name: jupyterkerneltemplate-elastic-with-custom-kernels 93 | command: 94 | # Use the default scripts to launch the kernel. 95 | - "kubeflow-launcher" 96 | - "--verbose" 97 | - "--RemoteProcessProxy.kernel-id" 98 | - "{kernel_id}" 99 | - "--RemoteProcessProxy.port-range" 100 | - "{port_range}" 101 | - "--RemoteProcessProxy.response-address" 102 | - "{response_address}" 103 | 104 | $ cat examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml 105 | apiVersion: kubeflow.tkestack.io/v1alpha1 106 | kind: JupyterKernelTemplate 107 | metadata: 108 | name: jupyterkerneltemplate-elastic-with-custom-kernels 109 | spec: 110 | template: 111 | metadata: 112 | app: enterprise-gateway 113 | component: kernel 114 | spec: 115 | restartPolicy: Always 116 | containers: 117 | - name: kernel 118 | 119 | $ kubectl apply -f ./examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml 120 | $ kubectl apply -f ./examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml 121 | ``` 122 | 123 | There will be a configmap created with the given CR, and it will be mounted into the gateway. 124 | 125 | ```yaml 126 | $ cat examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 127 | apiVersion: kubeflow.tkestack.io/v1alpha1 128 | kind: JupyterGateway 129 | metadata: 130 | name: jupytergateway-elastic-with-custom-kernels 131 | spec: 132 | cullIdleTimeout: 10 133 | cullInterval: 10 134 | logLevel: DEBUG 135 | image: ghcr.io/skai-x/enterprise-gateway:2.6.0 136 | # Use the kernel which is defined in JupyterKernelSpec CR. 137 | kernels: 138 | - python-kubernetes 139 | 140 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 141 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 142 | $ kubectl port-forward deploy/jupyternotebook-elastic-with-custom-kernels 8888:8888 143 | ``` 144 | 145 | ## Design and implementation 146 | 147 | [elastic-jupyter-operator][] reuses the [Jupyter Enterprise Gateway][] to support remote execution of Jupyter notebooks. The request will be sent to the notebook server process first when users execute the code in the browser. But the request cannot be processed since there is no kernel to execute it. The notebook server will issue a request then to the gateway to create a new kernel. The gateway creates the `JupyterKernel` CR via our custom `KubeflowProcessProxy` in the gateway's source code. The operator watches the `JupyterKernel` CR and creates the corresponding kernel pod in Kubernetes. Then the execution result will be sent back to the notebook server via ZeroMQ. 148 | 149 |

150 | 151 | The gateway monitors the kernels and culls the idle kernels. The operator also monitors them to restart the kernel if the kernel is not ready. 152 | 153 | ## Summary 154 | 155 | There are still too many features to be covered in this document. But the basic features are listed below: 156 | 157 | - Remote Jupyter kernel execution with custom configuration 158 | - Declarative way to manage Jupyter Notebook and Geteway 159 | - Support adding/removing kernel specs dynamically 160 | - Support custom kernel image, command, resource requirements and so on 161 | 162 | ## License 163 | 164 | - This article is licensed under [CC BY-NC-SA 3.0](https://creativecommons.org/licenses/by-nc-sa/3.0/). 165 | - Please contact me for commercial use. 166 | 167 | 168 | [Jupyter Enterprise Gateway]: https://jupyter.org/enterprise_gateway/ 169 | [Jupyter Notebook]: https://jupyter.org/ 170 | [elastic-jupyter-operator]: https://github.com/tkestack/elastic-jupyter-operator 171 | -------------------------------------------------------------------------------- /docs/api/autogen/config.yaml: -------------------------------------------------------------------------------- 1 | render: 2 | kubernetesVersion: "1.20" -------------------------------------------------------------------------------- /docs/api/autogen/templates/gv_details.tpl: -------------------------------------------------------------------------------- 1 | {{- define "gvDetails" -}} 2 | {{- $gv := . -}} 3 | [id="{{ asciidocGroupVersionID $gv | asciidocRenderAnchorID }}"] 4 | == {{ $gv.GroupVersionString }} 5 | 6 | {{ $gv.Doc }} 7 | 8 | {{- if $gv.Kinds }} 9 | .Resource Types 10 | {{- range $gv.SortedKinds }} 11 | - {{ $gv.TypeForKind . | asciidocRenderTypeLink }} 12 | {{- end }} 13 | {{ end }} 14 | 15 | === Definitions 16 | {{ range $gv.SortedTypes }} 17 | {{ template "type" . }} 18 | {{ end }} 19 | 20 | {{- end -}} -------------------------------------------------------------------------------- /docs/api/autogen/templates/gv_list.tpl: -------------------------------------------------------------------------------- 1 | {{- define "gvList" -}} 2 | {{- $groupVersions := . -}} 3 | 4 | // Generated documentation. Please do not edit. 5 | :anchor_prefix: k8s-api 6 | 7 | [id="{p}-api-reference"] 8 | = API Reference 9 | 10 | .Packages 11 | {{- range $groupVersions }} 12 | - {{ asciidocRenderGVLink . }} 13 | {{- end }} 14 | 15 | {{ range $groupVersions }} 16 | {{ template "gvDetails" . }} 17 | {{ end }} 18 | 19 | {{- end -}} -------------------------------------------------------------------------------- /docs/api/autogen/templates/type.tpl: -------------------------------------------------------------------------------- 1 | {{- define "type" -}} 2 | {{- $type := . -}} 3 | {{- if asciidocShouldRenderType $type -}} 4 | 5 | [id="{{ asciidocTypeID $type | asciidocRenderAnchorID }}"] 6 | ==== {{ $type.Name }} {{ if $type.IsAlias }}({{ asciidocRenderTypeLink $type.UnderlyingType }}) {{ end }} 7 | 8 | {{ $type.Doc }} 9 | 10 | {{ if $type.References -}} 11 | .Appears In: 12 | **** 13 | {{- range $type.SortedReferences }} 14 | - {{ asciidocRenderTypeLink . }} 15 | {{- end }} 16 | **** 17 | {{- end }} 18 | 19 | {{ if $type.Members -}} 20 | [cols="25a,75a", options="header"] 21 | |=== 22 | | Field | Description 23 | {{ if $type.GVK -}} 24 | | *`apiVersion`* __string__ | `{{ $type.GVK.Group }}/{{ $type.GVK.Version }}` 25 | | *`kind`* __string__ | `{{ $type.GVK.Kind }}` 26 | {{ end -}} 27 | 28 | {{ range $type.Members -}} 29 | | *`{{ .Name }}`* __{{ asciidocRenderType .Type }}__ | {{ template "type_members" . }} 30 | {{ end -}} 31 | |=== 32 | {{ end -}} 33 | 34 | {{- end -}} 35 | {{- end -}} -------------------------------------------------------------------------------- /docs/api/autogen/templates/type_members.tpl: -------------------------------------------------------------------------------- 1 | {{- define "type_members" -}} 2 | {{- $field := . -}} 3 | {{- if eq $field.Name "metadata" -}} 4 | Refer to Kubernetes API documentation for fields of `metadata`. 5 | {{ else -}} 6 | {{ $field.Doc }} 7 | {{- end -}} 8 | {{- end -}} -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | There are 5 CRDs defined in elastic-jupyter-operator: 2 | 3 | - jupytergateways.kubeflow.tkestack.io 4 | - jupyterkernels.kubeflow.tkestack.io 5 | - jupyterkernelspecs.kubeflow.tkestack.io 6 | - jupyterkerneltemplates.kubeflow.tkestack.io 7 | - jupyternotebooks.kubeflow.tkestack.io 8 | 9 | elastic-jupyter-operator 的架构如图所示，`JupyterGateway` 和 `JupyterNotebook` 是两个 CRD。其中 Notebook 是 Jupyter Notebook 的前端服务，负责面向用户提供用户界面，并且与后端服务通过 HTTPS 和 Websocket 进行通信，处理用户的计算请求。 10 | 11 | Gateway 是对应的后端服务。它负责处理来自 Notebook CR 的请求，通过调用 Kubernetes 的 API 按需创建出真正负责处理用户计算任务的 Kernel。 12 | 13 |

-------------------------------------------------------------------------------- /docs/images/arch.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/arch.jpeg -------------------------------------------------------------------------------- /docs/images/elastic.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/elastic.jpeg -------------------------------------------------------------------------------- /docs/images/gateway.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/gateway.png -------------------------------------------------------------------------------- /docs/images/jupyter.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/jupyter.jpeg -------------------------------------------------------------------------------- /docs/images/kubeflow.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/kubeflow.jpeg -------------------------------------------------------------------------------- /docs/images/multiuser.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/multiuser.jpeg -------------------------------------------------------------------------------- /docs/images/overview.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/overview.jpeg -------------------------------------------------------------------------------- /docs/images/uml.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skai-x/elastic-jupyter-operator/941f006c729cdaf8412e2a846a1c49ec1ce54d23/docs/images/uml.jpeg -------------------------------------------------------------------------------- /docs/kernel.md: -------------------------------------------------------------------------------- 1 | # Design Proposal for Jupyter Kernel CRD 2 | 3 | Authors: 4 | - Ce Gao 5 | 6 | ## Background 7 | 8 | There are two CustomResourceDefinitions `JupyterNotebook` and `JupyterGateway` for users. They are used to create Notebook instances and [Jupyter Enterprise Gateways](https://github.com/jupyter/enterprise_gateway). But it is still hard to configure the kernels. 9 | 10 | ## Motivation 11 | 12 | When we configure kernels with Jupyter Enterprise Gateway, we need to provide the kernel json format configburation files. Currently, these configurations is packaged with the gateway image. The kernel specification is shown here: 13 | 14 | ``` 15 | kernel.json 16 | logo-64x64.png 17 | scripts/ 18 | kernel-pod.yaml.j2 19 | launch_kubernetes.py 20 | ``` 21 | 22 | The scripts directory is copied from the source code of enterprise gateway, and the kernel.json looks like: 23 | 24 | ```json 25 | { 26 | "language": "python", 27 | "display_name": "Python on Kubernetes with Tensorflow", 28 | "metadata": { 29 | "process_proxy": { 30 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy", 31 | "config": { 32 | "image_name": "elyra/kernel-tf-py:VERSION" 33 | } 34 | } 35 | }, 36 | "env": { 37 | }, 38 | "argv": [ 39 | "python", 40 | "/usr/local/share/jupyter/kernels/python_tf_kubernetes/scripts/launch_kubernetes.py", 41 | "--RemoteProcessProxy.kernel-id", 42 | "{kernel_id}", 43 | "--RemoteProcessProxy.response-address", 44 | "{response_address}" 45 | ] 46 | } 47 | ``` 48 | 49 | It is hand-written by enterprise-gateway maintainers. All kernel specs are copied into the enterprise-gateway docker image: 50 | 51 | ```dockerfile 52 | ADD jupyter_enterprise_gateway_kernelspecs*.tar.gz /usr/local/share/jupyter/kernels/ 53 | ``` 54 | 55 | Thus it is hard to update them on the fly. 56 | 57 | ## Goals 58 | 59 | This proposal is to allow users to: 60 | 61 | - Configure kernels on the fly 62 | - Manage kernels on Kubernetes easily 63 | 64 | ## Non-goals 65 | 66 | This proposal is not to: 67 | 68 | - Make the design the default implementaion, which means that the current design and implementation will not change. 69 | 70 | ## Implementation 71 | 72 | The design proposal involves four main changes to elastic jupyter operator and enterprise gateway: 73 | 74 | - Kernel CRD (new): It is used to manage kernels on Kubernetes 75 | - KernelSpec CRD (new): It is used to configure the kernel specs in runtime 76 | - KernelTemplate CRD (new): It is used to configure kernel in runtime 77 | - JupyterGateway CRD: Changes is made to support updating kernels on the fly 78 | - Kubeflow Kernel Launcher (new): It is used to launch Kernel CRD inside the gateway 79 | - Kubeflow Process Proxy (new): It is used to manage kernels in enterprise gateway 80 | 81 | ### Kernel CRD 82 | 83 | ```yaml 84 | spec: 85 | ID: "{{ kernel_id }}" 86 | restartPolicy: Never 87 | serviceAccountName: "{{ kernel_service_account_name }}" 88 | securityContext: ... 89 | environments: 90 | respondAddress: "{{ eg_response_address }}" 91 | language: "{{ kernel_language }}" 92 | ``` 93 | 94 | ### KernelTemplate CRD 95 | 96 | KernelTemplate CRD is used to replace [etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2](https://github.com/jupyter/enterprise_gateway/blob/master/etc/kernel-launchers/kubernetes/scripts/kernel-pod.yaml.j2). Its definition looks like: 97 | 98 | ```yaml 99 | apiVersion: kubeflow.tkestack.io/v1alpha1 100 | kind: JupyterKernelTemplate 101 | metadata: 102 | name: jupyterkerneltemplate-sample 103 | spec: 104 | template: 105 | metadata: 106 | app: enterprise-gateway 107 | component: kernel 108 | template: 109 | spec: 110 | restartPolicy: Never 111 | containers: 112 | - name: kernel 113 | ``` 114 | 115 | ### KernelSpec CRD 116 | 117 | > The primary vehicle for indicating a given kernel should be handled in a different manner is the kernel specification, otherwise known as the kernel spec. Enterprise Gateway leverages the natively extensible metadata stanza to introduce a new stanza named process_proxy. 118 | > 119 | > The process_proxy stanza identifies the class that provides the kernel’s process abstraction (while allowing for future extensions). This class then provides the kernel’s lifecycle management operations relative to the managed resource or functional equivalent. 120 | > 121 | > Here’s an example of a kernel specification that uses the DistributedProcessProxy class for its abstraction: 122 | > 123 | ```json 124 | { 125 | "language": "scala", 126 | "display_name": "Spark - Scala (YARN Client Mode)", 127 | "metadata": { 128 | "process_proxy": { 129 | "class_name": "enterprise_gateway.services.processproxies.distributed.DistributedProcessProxy" 130 | } 131 | }, 132 | "env": { 133 | "SPARK_HOME": "/usr/hdp/current/spark2-client", 134 | "__TOREE_SPARK_OPTS__": "--master yarn --deploy-mode client --name ${KERNEL_ID:-ERROR__NO__KERNEL_ID}", 135 | "__TOREE_OPTS__": "", 136 | "LAUNCH_OPTS": "", 137 | "DEFAULT_INTERPRETER": "Scala" 138 | }, 139 | "argv": [ 140 | "/usr/local/share/jupyter/kernels/spark_scala_yarn_client/bin/run.sh", 141 | "--RemoteProcessProxy.kernel-id", 142 | "{kernel_id}", 143 | "--RemoteProcessProxy.response-address", 144 | "{response_address}", 145 | "--RemoteProcessProxy.public-key", 146 | "{public_key}" 147 | ] 148 | } 149 | ``` 150 | 151 | The kernel specifications are placed in the docker image at build time, which is not easy to maintain on the fly. The kernelspec CRD is defined to support dynamic update. The CRD specification looks like this: 152 | 153 | ```yaml 154 | spec: 155 | language: Python 156 | displayName: "Python on Kubernetes with Tensorflow" 157 | image: elyra/kernel-tf-py:VERSION 158 | envs: ... 159 | command: 160 | - "python", 161 | - "/usr/local/share/jupyter/scripts/launch_kubernetes.py", 162 | - "--RemoteProcessProxy.kernel-id", 163 | - "{kernel_id}", 164 | - "--RemoteProcessProxy.response-address", 165 | - "{response_address}" 166 | ``` 167 | 168 | When a JupyterKernelSpec CR is created, we will create the corresponding configmap. And the configmap will be used as a mount volume in the gateway. 169 | 170 | Besides this, The KernelSpec CRD maintains a object reference to one KernelTemplate CRD. It is used to generate commands. 171 | 172 | ```diff 173 | { 174 | "language": "Python", 175 | "display_name": "Python on Kubernetes as a JupyterKernelSpec", 176 | "metadata": { 177 | "process_proxy": { 178 | "class_name": "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy" 179 | }, 180 | "config": { 181 | "image_name": "ghcr.io/skai-x/jupyter-kernel-py:2.6.0" 182 | } 183 | }, 184 | "argv": [ 185 | "kubeflow-launcher", 186 | "--RemoteProcessProxy.kernel-id", 187 | "{kernel_id}", 188 | "--RemoteProcessProxy.port-range", 189 | "{port_range}", 190 | "--RemoteProcessProxy.response-address", 191 | "{response_address}", 192 | + "--kernel-template-name", 193 | + "jupyterkerneltemplate-sample", 194 | + "--kernel-template-namespace", 195 | + "default" 196 | ] 197 | } 198 | ``` 199 | 200 | ### JupyterGateway CRD 201 | 202 | The specification generation logic needs to be changed to support the new JupyterKernelSpec CRD. 203 | 204 | ```yaml 205 | spec: 206 | kernels: 207 | - python 208 | - r 209 | - dask 210 | ... 211 | ``` 212 | 213 | When `kernels` are defined in the spec, we should get the jupyter kernelspec CRs from the kubernetes api server, then mount the configmaps as volumes into the gateway container. 214 | 215 | ### Kernel Launcher 216 | 217 | The [kernel launcher](https://github.com/tkestack/elastic-jupyter-operator/tree/master/cli) is introduced to replace [etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py](https://github.com/jupyter/enterprise_gateway/blob/master/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py). 218 | 219 | The launcher gets the corresponding KernelTemplate CRD and creates the kernel in the cluster. 220 | 221 | ## Reference 222 | 223 | - [Jupyter Enterprise Gateway System Architecture](https://jupyter-enterprise-gateway.readthedocs.io/en/latest/system-architecture.html) 224 | -------------------------------------------------------------------------------- /docs/quick-start.md: -------------------------------------------------------------------------------- 1 | ## Quickstart 2 | 3 | ### Simple deployment 4 | 5 | You can create a simple Jupyter notebook with all components in one pod, like this: 6 | 7 | ```yaml 8 | $ cat ./examples/simple-deployments/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 9 | apiVersion: kubeflow.tkestack.io/v1alpha1 10 | kind: JupyterNotebook 11 | metadata: 12 | name: jupyternotebook-simple 13 | spec: 14 | auth: 15 | mode: disable 16 | template: 17 | metadata: 18 | labels: 19 | notebook: simple 20 | spec: 21 | containers: 22 | - name: notebook 23 | image: jupyter/base-notebook:python-3.9.7 24 | command: ["tini", "-g", "--", "start-notebook.sh"] 25 | 26 | $ kubectl apply -f ./examples/simple-deployments/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 27 | $ kubectl port-forward deploy/jupyternotebook-simple 8888:8888 28 | ``` 29 | 30 | Then you can open the URL `http://127.0.0.1:8888/` to get the simple Jupyter notebook instance. The deployment follows the architecture below: 31 | 32 |

33 | 34 | 35 | ## Elastic deployment 36 | 37 | elastic-jupyter-operator supports running Jupyter kernels in separate pods. In this example, we will create the notebook and gateway. 38 | 39 | ```yaml 40 | $ cat ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 41 | apiVersion: kubeflow.tkestack.io/v1alpha1 42 | kind: JupyterNotebook 43 | metadata: 44 | name: jupyternotebook-elastic 45 | spec: 46 | gateway: 47 | name: jupytergateway-elastic 48 | namespace: default 49 | auth: 50 | mode: disable 51 | 52 | $ cat ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 53 | apiVersion: kubeflow.tkestack.io/v1alpha1 54 | kind: JupyterGateway 55 | metadata: 56 | name: jupytergateway-elastic 57 | spec: 58 | cullIdleTimeout: 3600 59 | image: ghcr.io/skai-x/enterprise-gateway:2.6.0 60 | 61 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 62 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 63 | $ kubectl port-forward deploy/jupyternotebook-elastic 8888:8888 64 | ``` 65 | 66 | When users run the code in the browser, there will be a new kernel pod created in the cluster. 67 | 68 | ``` 69 | NAME READY STATUS RESTARTS AGE 70 | jovyan-219cfd49-89ad-428c-8e0d-3e61e15d79a7 1/1 Running 0 170m 71 | jupytergateway-elastic-868d8f465c-8mg44 1/1 Running 0 3h 72 | jupyternotebook-elastic-787d94bb4b-xdwnc 1/1 Running 0 3h10m 73 | ``` 74 | 75 | ### Elastic deployment with custom kernel 76 | 77 | If you want to custom the kernel deployment, for example. you want to update the resource requirements of the python kernel or use different images for the kernel, you can deploy the jupyter notebooks and gateways with custom kernels. 78 | 79 | First, you need to create the JupyterKernelSpec CR, which is used to generate the [Jupyter kernelspec](https://jupyter-client.readthedocs.io/en/stable/kernels.html). 80 | 81 | ```yaml 82 | $ cat examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml 83 | apiVersion: kubeflow.tkestack.io/v1alpha1 84 | kind: JupyterKernelSpec 85 | metadata: 86 | name: python-kubernetes 87 | spec: 88 | language: Python 89 | displayName: "Python on Kubernetes as a JupyterKernelSpec" 90 | image: ghcr.io/skai-x/jupyter-kernel-py:2.6.0 91 | className: enterprise_gateway.services.processproxies.kubeflow.KubeflowProcessProxy 92 | # Use the template defined in JupyterKernelTemplate CR. 93 | template: 94 | namespace: default 95 | name: jupyterkerneltemplate-elastic-with-custom-kernels 96 | command: 97 | # Use the default scripts to launch the kernel. 98 | - "kubeflow-launcher" 99 | - "--verbose" 100 | - "--RemoteProcessProxy.kernel-id" 101 | - "{kernel_id}" 102 | - "--RemoteProcessProxy.port-range" 103 | - "{port_range}" 104 | - "--RemoteProcessProxy.response-address" 105 | - "{response_address}" 106 | 107 | $ cat examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml 108 | apiVersion: kubeflow.tkestack.io/v1alpha1 109 | kind: JupyterKernelTemplate 110 | metadata: 111 | name: jupyterkerneltemplate-elastic-with-custom-kernels 112 | spec: 113 | template: 114 | metadata: 115 | app: enterprise-gateway 116 | component: kernel 117 | spec: 118 | restartPolicy: Always 119 | containers: 120 | - name: kernel 121 | 122 | $ kubectl apply -f ./examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml 123 | $ kubectl apply -f ./examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml 124 | ``` 125 | 126 | There will be a configmap created with the given CR, and it will be mounted into the gateway. 127 | 128 | ```yaml 129 | $ cat examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 130 | apiVersion: kubeflow.tkestack.io/v1alpha1 131 | kind: JupyterGateway 132 | metadata: 133 | name: jupytergateway-elastic-with-custom-kernels 134 | spec: 135 | cullIdleTimeout: 10 136 | cullInterval: 10 137 | logLevel: DEBUG 138 | image: ghcr.io/skai-x/enterprise-gateway:2.6.0 139 | # Use the kernel which is defined in JupyterKernelSpec CR. 140 | kernels: 141 | - python-kubernetes 142 | 143 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml 144 | $ kubectl apply -f ./examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml 145 | $ kubectl port-forward deploy/jupyternotebook-elastic-with-custom-kernels 8888:8888 146 | ``` 147 | -------------------------------------------------------------------------------- /examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterGateway 3 | metadata: 4 | name: jupytergateway-elastic-with-custom-kernels 5 | spec: 6 | cullIdleTimeout: 10 7 | cullInterval: 10 8 | logLevel: DEBUG 9 | image: ghcr.io/skai-x/enterprise-gateway:2.6.0 10 | # Use the kernel which is defined in JupyterKernelSpec CR. 11 | kernels: 12 | - python-kubernetes 13 | -------------------------------------------------------------------------------- /examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkernelspec.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterKernelSpec 3 | metadata: 4 | name: python-kubernetes 5 | spec: 6 | language: Python 7 | displayName: "Elastic Python Kernel on Kubernetes" 8 | image: ghcr.io/skai-x/jupyter-kernel-py:2.6.0 9 | className: enterprise_gateway.services.processproxies.kubeflow.KubeflowProcessProxy 10 | # Use the template defined in JupyterKernelTemplate CR. 11 | template: 12 | namespace: default 13 | name: jupyterkerneltemplate-elastic-with-custom-kernels 14 | command: 15 | # Use the default scripts to launch the kernel. 16 | - "kubeflow-launcher" 17 | - "--verbose" 18 | - "--RemoteProcessProxy.kernel-id" 19 | - "{kernel_id}" 20 | - "--RemoteProcessProxy.port-range" 21 | - "{port_range}" 22 | - "--RemoteProcessProxy.response-address" 23 | - "{response_address}" 24 | - "--RemoteProcessProxy.public-key" 25 | - "{public_key}" 26 | -------------------------------------------------------------------------------- /examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyterkerneltemplate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterKernelTemplate 3 | metadata: 4 | name: jupyterkerneltemplate-elastic-with-custom-kernels 5 | spec: 6 | template: 7 | metadata: 8 | app: enterprise-gateway 9 | component: kernel 10 | spec: 11 | restartPolicy: Always 12 | containers: 13 | - name: kernel 14 | -------------------------------------------------------------------------------- /examples/elastic-with-custom-kernels/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterNotebook 3 | metadata: 4 | name: jupyternotebook-elastic-with-custom-kernels 5 | spec: 6 | gateway: 7 | name: jupytergateway-elastic-with-custom-kernels 8 | namespace: default 9 | # Disable the password and token based auth in this example, 10 | # please do not do it in PROD. 11 | auth: 12 | mode: disable 13 | -------------------------------------------------------------------------------- /examples/elastic/kubeflow.tkestack.io_v1alpha1_jupytergateway.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterGateway 3 | metadata: 4 | name: jupytergateway-elastic 5 | spec: 6 | # Timeout (in seconds) after which a kernel is considered idle and ready to be culled. 7 | cullIdleTimeout: 3600 8 | image: ghcr.io/skai-x/enterprise-gateway-with-kernel-spec:latest 9 | -------------------------------------------------------------------------------- /examples/elastic/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterNotebook 3 | metadata: 4 | name: jupyternotebook-elastic 5 | spec: 6 | gateway: 7 | name: jupytergateway-elastic 8 | namespace: default 9 | # Disable the password and token based auth in this example, 10 | # please do not do it in PROD. 11 | auth: 12 | mode: disable 13 | -------------------------------------------------------------------------------- /examples/simple-deployments/kubeflow.tkestack.io_v1alpha1_jupyternotebook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubeflow.tkestack.io/v1alpha1 2 | kind: JupyterNotebook 3 | metadata: 4 | name: jupyternotebook-simple 5 | spec: 6 | # Disable the password and token based auth in this example, 7 | # please do not do it in PROD. 8 | auth: 9 | mode: disable 10 | template: 11 | metadata: 12 | labels: 13 | notebook: simple 14 | spec: 15 | containers: 16 | - name: notebook 17 | image: jupyter/base-notebook:python-3.9.7 18 | command: ["tini", "-g", "--", "start-notebook.sh"] 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tkestack/elastic-jupyter-operator 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/go-logr/logr v0.1.0 7 | github.com/onsi/ginkgo v1.12.1 8 | github.com/onsi/gomega v1.10.1 9 | github.com/spf13/cobra v0.0.5 10 | k8s.io/api v0.18.6 11 | k8s.io/apimachinery v0.18.6 12 | k8s.io/client-go v0.18.6 13 | sigs.k8s.io/controller-runtime v0.6.4 14 | ) 15 | 16 | require ( 17 | cloud.google.com/go v0.38.0 // indirect 18 | github.com/BurntSushi/toml v0.3.1 // indirect 19 | github.com/beorn7/perks v1.0.0 // indirect 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/evanphx/json-patch v4.9.0+incompatible // indirect 22 | github.com/fsnotify/fsnotify v1.4.9 // indirect 23 | github.com/go-logr/zapr v0.1.0 // indirect 24 | github.com/gogo/protobuf v1.3.1 // indirect 25 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect 26 | github.com/golang/protobuf v1.4.2 // indirect 27 | github.com/google/go-cmp v0.4.0 // indirect 28 | github.com/google/gofuzz v1.1.0 // indirect 29 | github.com/google/uuid v1.1.1 // indirect 30 | github.com/googleapis/gnostic v0.3.1 // indirect 31 | github.com/hashicorp/golang-lru v0.5.4 // indirect 32 | github.com/imdario/mergo v0.3.9 // indirect 33 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 34 | github.com/json-iterator/go v1.1.10 // indirect 35 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 36 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 37 | github.com/modern-go/reflect2 v1.0.1 // indirect 38 | github.com/nxadm/tail v1.4.4 // indirect 39 | github.com/pkg/errors v0.8.1 // indirect 40 | github.com/prometheus/client_golang v1.0.0 // indirect 41 | github.com/prometheus/client_model v0.2.0 // indirect 42 | github.com/prometheus/common v0.4.1 // indirect 43 | github.com/prometheus/procfs v0.0.11 // indirect 44 | github.com/spf13/pflag v1.0.5 // indirect 45 | go.uber.org/atomic v1.5.0 // indirect 46 | go.uber.org/multierr v1.4.0 // indirect 47 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect 48 | go.uber.org/zap v1.13.0 // indirect 49 | golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect 50 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect 51 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 // indirect 52 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect 53 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd // indirect 54 | golang.org/x/text v0.3.3 // indirect 55 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect 56 | golang.org/x/tools v0.0.0-20191114161115-faa69481e761 // indirect 57 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect 58 | gomodules.xyz/jsonpatch/v2 v2.0.1 // indirect 59 | google.golang.org/appengine v1.5.0 // indirect 60 | google.golang.org/protobuf v1.23.0 // indirect 61 | gopkg.in/inf.v0 v0.9.1 // indirect 62 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 63 | gopkg.in/yaml.v2 v2.3.0 // indirect 64 | honnef.co/go/tools v0.0.1-2019.2.3 // indirect 65 | k8s.io/apiextensions-apiserver v0.18.6 // indirect 66 | k8s.io/klog v1.0.0 // indirect 67 | k8s.io/klog/v2 v2.0.0 // indirect 68 | k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 // indirect 69 | k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 // indirect 70 | sigs.k8s.io/structured-merge-diff/v3 v3.0.0 // indirect 71 | sigs.k8s.io/yaml v1.2.0 // indirect 72 | ) 73 | -------------------------------------------------------------------------------- /hack/add-license.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. 4 | 5 | cd ${SCRIPT_ROOT} 6 | addlicense -f ./hack/license.txt *.go pkg/**/*.go 7 | cd - >/dev/null 8 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | 10 | // https://opensource.org/licenses/Apache-2.0 11 | 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- /hack/enterprise_gateway/prepare.yaml: -------------------------------------------------------------------------------- 1 | # This file defines the Kubernetes objects necessary for Enterprise Gateway to run within Kubernetes. 2 | # 3 | apiVersion: v1 4 | kind: Namespace 5 | metadata: 6 | name: enterprise-gateway 7 | labels: 8 | app: enterprise-gateway 9 | --- 10 | apiVersion: v1 11 | kind: ServiceAccount 12 | metadata: 13 | name: enterprise-gateway-sa 14 | namespace: enterprise-gateway 15 | labels: 16 | app: enterprise-gateway 17 | component: enterprise-gateway 18 | --- 19 | apiVersion: rbac.authorization.k8s.io/v1 20 | kind: ClusterRole 21 | metadata: 22 | name: enterprise-gateway-controller 23 | labels: 24 | app: enterprise-gateway 25 | component: enterprise-gateway 26 | rules: 27 | - apiGroups: [""] 28 | resources: ["pods", "namespaces", "services", "configmaps", "secrets", "persistentvolumes", "persistentvolumeclaims"] 29 | verbs: ["get", "watch", "list", "create", "delete"] 30 | - apiGroups: ["rbac.authorization.k8s.io"] 31 | resources: ["rolebindings"] 32 | verbs: ["get", "list", "create", "delete"] 33 | - apiGroups: ["kubeflow.tkestack.io"] 34 | resources: ["jupyterkerneltemplates", "jupyterkernels", "jupytergateways"] 35 | verbs: ["get", "list", "create", "delete"] 36 | --- 37 | apiVersion: rbac.authorization.k8s.io/v1 38 | kind: ClusterRole 39 | metadata: 40 | # Referenced by EG_KERNEL_CLUSTER_ROLE below 41 | name: kernel-controller 42 | labels: 43 | app: enterprise-gateway 44 | component: kernel 45 | rules: 46 | - apiGroups: [""] 47 | resources: ["pods"] 48 | verbs: ["get", "watch", "list", "create", "delete"] 49 | --- 50 | apiVersion: rbac.authorization.k8s.io/v1 51 | kind: ClusterRoleBinding 52 | metadata: 53 | name: enterprise-gateway-controller 54 | labels: 55 | app: enterprise-gateway 56 | component: enterprise-gateway 57 | subjects: 58 | - kind: ServiceAccount 59 | name: enterprise-gateway-sa 60 | namespace: enterprise-gateway 61 | roleRef: 62 | kind: ClusterRole 63 | name: enterprise-gateway-controller 64 | apiGroup: rbac.authorization.k8s.io 65 | -------------------------------------------------------------------------------- /hack/license.txt: -------------------------------------------------------------------------------- 1 | Tencent is pleased to support the open source community by making TKEStack 2 | available. 3 | 4 | Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | this file except in compliance with the License. You may obtain a copy of the 8 | License at 9 | 10 | https://opensource.org/licenses/Apache-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | "k8s.io/apimachinery/pkg/runtime" 24 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 25 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 26 | _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" 27 | ctrl "sigs.k8s.io/controller-runtime" 28 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 29 | 30 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 31 | "github.com/tkestack/elastic-jupyter-operator/controllers" 32 | // +kubebuilder:scaffold:imports 33 | ) 34 | 35 | var ( 36 | scheme = runtime.NewScheme() 37 | setupLog = ctrl.Log.WithName("setup") 38 | ) 39 | 40 | func init() { 41 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 42 | 43 | utilruntime.Must(kubeflowtkestackiov1alpha1.AddToScheme(scheme)) 44 | // +kubebuilder:scaffold:scheme 45 | } 46 | 47 | func main() { 48 | var metricsAddr string 49 | var enableLeaderElection bool 50 | flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") 51 | flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, 52 | "Enable leader election for controller manager. "+ 53 | "Enabling this will ensure there is only one active controller manager.") 54 | flag.Parse() 55 | 56 | ctrl.SetLogger(zap.New(zap.UseDevMode(true))) 57 | 58 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 59 | Scheme: scheme, 60 | MetricsBindAddress: metricsAddr, 61 | Port: 9443, 62 | LeaderElection: enableLeaderElection, 63 | LeaderElectionID: "82ec55e3.kubeflow.tkestack.io", 64 | }) 65 | if err != nil { 66 | setupLog.Error(err, "unable to start manager") 67 | os.Exit(1) 68 | } 69 | 70 | if err = (&controllers.JupyterNotebookReconciler{ 71 | Client: mgr.GetClient(), 72 | Log: ctrl.Log.WithName("controllers").WithName("JupyterNotebook"), 73 | Recorder: mgr.GetEventRecorderFor("JupyterNotebook"), 74 | Scheme: mgr.GetScheme(), 75 | }).SetupWithManager(mgr); err != nil { 76 | setupLog.Error(err, "unable to create controller", "controller", "JupyterNotebook") 77 | os.Exit(1) 78 | } 79 | if err = (&controllers.JupyterGatewayReconciler{ 80 | Client: mgr.GetClient(), 81 | Recorder: mgr.GetEventRecorderFor("JupyterGateway"), 82 | Log: ctrl.Log.WithName("controllers").WithName("JupyterGateway"), 83 | Scheme: mgr.GetScheme(), 84 | }).SetupWithManager(mgr); err != nil { 85 | setupLog.Error(err, "unable to create controller", "controller", "JupyterGateway") 86 | os.Exit(1) 87 | } 88 | if err = (&controllers.JupyterKernelSpecReconciler{ 89 | Client: mgr.GetClient(), 90 | Recorder: mgr.GetEventRecorderFor("JupyterKernelSpec"), 91 | Log: ctrl.Log.WithName("controllers").WithName("JupyterKernelSpec"), 92 | Scheme: mgr.GetScheme(), 93 | }).SetupWithManager(mgr); err != nil { 94 | setupLog.Error(err, "unable to create controller", "controller", "JupyterKernelSpec") 95 | os.Exit(1) 96 | } 97 | if err = (&controllers.JupyterKernelTemplateReconciler{ 98 | Client: mgr.GetClient(), 99 | Log: ctrl.Log.WithName("controllers").WithName("JupyterKernelTemplate"), 100 | Scheme: mgr.GetScheme(), 101 | }).SetupWithManager(mgr); err != nil { 102 | setupLog.Error(err, "unable to create controller", "controller", "JupyterKernelTemplate") 103 | os.Exit(1) 104 | } 105 | if err = (&controllers.JupyterKernelReconciler{ 106 | Client: mgr.GetClient(), 107 | Log: ctrl.Log.WithName("controllers").WithName("JupyterKernel"), 108 | Scheme: mgr.GetScheme(), 109 | }).SetupWithManager(mgr); err != nil { 110 | setupLog.Error(err, "unable to create controller", "controller", "JupyterKernel") 111 | os.Exit(1) 112 | } 113 | // +kubebuilder:scaffold:builder 114 | 115 | setupLog.Info("starting manager") 116 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 117 | setupLog.Error(err, "problem running manager") 118 | os.Exit(1) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /pkg/gateway/generate.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Tencent is pleased to support the open source community by making TKEStack 3 | * available. 4 | * 5 | * Copyright (C) 2012-2020 Tencent. All Rights Reserved. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use 8 | * this file except in compliance with the License. You may obtain a copy of the 9 | * License at 10 | * 11 | * https://opensource.org/licenses/Apache-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OF ANY KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations under the License. 17 | */ 18 | 19 | package gateway 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | "strconv" 25 | "strings" 26 | 27 | appsv1 "k8s.io/api/apps/v1" 28 | v1 "k8s.io/api/core/v1" 29 | rbacv1 "k8s.io/api/rbac/v1" 30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 31 | "k8s.io/apimachinery/pkg/types" 32 | "sigs.k8s.io/controller-runtime/pkg/client" 33 | 34 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 35 | ) 36 | 37 | const ( 38 | defaultImage = "ghcr.io/skai-x/enterprise-gateway:2.6.0" 39 | defaultContainerName = "gateway" 40 | defaultKernelImage = "ghcr.io/skai-x/jupyter-kernel-py:2.6.0" 41 | defaultPortName = "gateway" 42 | defaultKernel = "python_kubernetes" 43 | defaultPort = 8888 44 | defaultGatewayClusterRole = "enterprise-gateway-controller" 45 | defaultServiceAccount = "enterprise-gateway-sa" 46 | 47 | LabelGateway = "gateway" 48 | LabelNS = "namespace" 49 | 50 | cullTimeoutOpt = "--MappingKernelManager.cull_idle_timeout" 51 | cullInterval = "--MappingKernelManager.cull_interval" 52 | 53 | defaultKernelPath = "/usr/local/share/jupyter/kernels/" 54 | defaultKernels = "'r_kubernetes','python_kubernetes','python_tf_kubernetes','python_tf_gpu_kubernetes','scala_kubernetes','spark_r_kubernetes','spark_python_kubernetes','spark_scala_kubernetes'" 55 | ) 56 | 57 | // generator defines the generator which is used to generate 58 | // desired specs. 59 | type generator struct { 60 | gateway *v1alpha1.JupyterGateway 61 | cli client.Client 62 | } 63 | 64 | // newGenerator creates a new Generator. 65 | func newGenerator(c client.Client, gateway *v1alpha1.JupyterGateway) ( 66 | *generator, error) { 67 | if gateway == nil { 68 | return nil, fmt.Errorf("Got nil when initializing Generator") 69 | } 70 | g := &generator{ 71 | gateway: gateway, 72 | cli: c, 73 | } 74 | 75 | return g, nil 76 | } 77 | 78 | // DesiredServiceWithoutOwner returns desired service without 79 | // owner. 80 | func (g generator) DesiredServiceWithoutOwner() *v1.Service { 81 | labels := g.labels() 82 | s := &v1.Service{ 83 | ObjectMeta: metav1.ObjectMeta{ 84 | Namespace: g.gateway.Namespace, 85 | Name: g.gateway.Name, 86 | Labels: labels, 87 | }, 88 | Spec: v1.ServiceSpec{ 89 | Selector: labels, 90 | Type: v1.ServiceTypeClusterIP, 91 | SessionAffinity: v1.ServiceAffinityClientIP, 92 | Ports: []v1.ServicePort{ 93 | { 94 | Name: defaultPortName, 95 | Port: defaultPort, 96 | Protocol: v1.ProtocolTCP, 97 | }, 98 | }, 99 | }, 100 | } 101 | return s 102 | } 103 | 104 | func (g generator) DesiredRoleBinding( 105 | sa *v1.ServiceAccount) *rbacv1.RoleBinding { 106 | labels := g.labels() 107 | crb := &rbacv1.RoleBinding{ 108 | ObjectMeta: metav1.ObjectMeta{ 109 | Namespace: g.gateway.Namespace, 110 | Name: g.gateway.Name, 111 | Labels: labels, 112 | }, 113 | Subjects: []rbacv1.Subject{ 114 | { 115 | Kind: "ServiceAccount", 116 | Name: sa.Name, 117 | Namespace: sa.Namespace, 118 | }, 119 | }, 120 | RoleRef: rbacv1.RoleRef{ 121 | Name: defaultGatewayClusterRole, 122 | Kind: "ClusterRole", 123 | APIGroup: "rbac.authorization.k8s.io", 124 | }, 125 | } 126 | return crb 127 | } 128 | 129 | func (g generator) DesiredServiceAccountWithoutOwner() *v1.ServiceAccount { 130 | labels := g.labels() 131 | sa := &v1.ServiceAccount{ 132 | TypeMeta: metav1.TypeMeta{ 133 | Kind: "ServiceAccount", 134 | }, 135 | ObjectMeta: metav1.ObjectMeta{ 136 | Namespace: g.gateway.Namespace, 137 | Name: g.gateway.Name, 138 | Labels: labels, 139 | }, 140 | } 141 | return sa 142 | } 143 | 144 | // DesiredDeploymentWithoutOwner returns the desired deployment 145 | // without owner. 146 | func (g generator) DesiredDeploymentWithoutOwner( 147 | sa string) (*appsv1.Deployment, error) { 148 | // Generate volumes with the kernelspec CR. 149 | volumes, err := g.volumes() 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | labels := g.labels() 155 | selector := &metav1.LabelSelector{ 156 | MatchLabels: labels, 157 | } 158 | d := &appsv1.Deployment{ 159 | ObjectMeta: metav1.ObjectMeta{ 160 | Namespace: g.gateway.Namespace, 161 | Name: g.gateway.Name, 162 | Labels: labels, 163 | }, 164 | Spec: appsv1.DeploymentSpec{ 165 | Selector: selector, 166 | Template: v1.PodTemplateSpec{ 167 | ObjectMeta: metav1.ObjectMeta{ 168 | Labels: labels, 169 | }, 170 | Spec: v1.PodSpec{ 171 | ServiceAccountName: sa, 172 | Volumes: volumes, 173 | Containers: []v1.Container{ 174 | { 175 | Name: defaultContainerName, 176 | Image: defaultImage, 177 | ImagePullPolicy: v1.PullIfNotPresent, 178 | Ports: []v1.ContainerPort{ 179 | { 180 | Name: defaultPortName, 181 | ContainerPort: defaultPort, 182 | Protocol: v1.ProtocolTCP, 183 | }, 184 | }, 185 | Command: []string{"/usr/local/bin/start-enterprise-gateway.sh"}, 186 | VolumeMounts: g.volumeMounts(volumes), 187 | Env: []v1.EnvVar{ 188 | { 189 | Name: "EG_DEFAULT_KERNEL_NAME", 190 | Value: g.defaultKernel(), 191 | }, 192 | { 193 | Name: "EG_KERNEL_CLUSTER_ROLE", 194 | Value: g.defaultClusterRole(), 195 | }, 196 | { 197 | Name: "EG_KERNEL_WHITELIST", 198 | Value: g.kernels(), 199 | }, 200 | { 201 | Name: "EG_PORT", 202 | Value: strconv.Itoa(defaultPort), 203 | }, 204 | // --EnterpriseGatewayApp.port_range= 205 | // Specifies the lower and upper port numbers from which ports are created. The 206 | // bounded values are separated by '..' (e.g., 33245..34245 specifies a range 207 | // of 1000 ports to be randomly selected). A range of zero (e.g., 33245..33245 208 | // or 0..0) disables port-range enforcement. (EG_PORT_RANGE env var) 209 | { 210 | Name: "EG_PORT_RANGE", 211 | Value: "0..0", 212 | }, 213 | { 214 | Name: "EG_NAMESPACE", 215 | Value: g.gateway.Namespace, 216 | }, 217 | { 218 | Name: "EG_NAME", 219 | Value: g.gateway.Name, 220 | }, 221 | { 222 | // TODO(gaocegege): Make it configurable. 223 | Name: "EG_SHARED_NAMESPACE", 224 | Value: "true", 225 | }, 226 | { 227 | // TODO(gaocegege): Make it configurable. 228 | Name: "EG_MIRROR_WORKING_DIRS", 229 | Value: "false", 230 | }, 231 | { 232 | Name: "EG_CULL_IDLE_TIMEOUT", 233 | Value: "3600", 234 | }, 235 | { 236 | Name: "EG_KERNEL_LAUNCH_TIMEOUT", 237 | Value: "60", 238 | }, 239 | { 240 | Name: "EG_KERNEL_IMAGE", 241 | Value: defaultKernelImage, 242 | }, 243 | }, 244 | }, 245 | }, 246 | }, 247 | }, 248 | }, 249 | } 250 | 251 | if g.gateway.Spec.Image != "" { 252 | d.Spec.Template.Spec.Containers[0].Image = g.gateway.Spec.Image 253 | } 254 | 255 | if g.gateway.Spec.LogLevel != nil { 256 | env := v1.EnvVar{ 257 | Name: "EG_LOG_LEVEL", 258 | Value: string(*g.gateway.Spec.LogLevel), 259 | } 260 | d.Spec.Template.Spec.Containers[0].Env = append( 261 | d.Spec.Template.Spec.Containers[0].Env, env) 262 | } 263 | 264 | if g.gateway.Spec.CullIdleTimeout != nil { 265 | env := v1.EnvVar{ 266 | Name: "EG_CULL_IDLE_TIMEOUT", 267 | Value: strconv.Itoa(int(*g.gateway.Spec.CullIdleTimeout)), 268 | } 269 | d.Spec.Template.Spec.Containers[0].Env = append( 270 | d.Spec.Template.Spec.Containers[0].Env, env) 271 | } 272 | if g.gateway.Spec.CullInterval != nil { 273 | env := v1.EnvVar{ 274 | Name: "EG_CULL_INTERVAL", 275 | Value: strconv.Itoa(int(*g.gateway.Spec.CullInterval)), 276 | } 277 | d.Spec.Template.Spec.Containers[0].Env = append( 278 | d.Spec.Template.Spec.Containers[0].Env, env) 279 | } 280 | if g.gateway.Spec.Resources != nil { 281 | d.Spec.Template.Spec.Containers[0].Resources = *g.gateway.Spec.Resources 282 | } 283 | 284 | return d, nil 285 | } 286 | 287 | func (g generator) volumeMounts( 288 | volumes []v1.Volume) []v1.VolumeMount { 289 | volumeMounts := []v1.VolumeMount{} 290 | for _, v := range volumes { 291 | volumeMounts = append(volumeMounts, v1.VolumeMount{ 292 | Name: v.Name, 293 | ReadOnly: true, 294 | MountPath: fmt.Sprintf("%s/%s", defaultKernelPath, v.Name), 295 | }) 296 | } 297 | return volumeMounts 298 | } 299 | 300 | func (g generator) volumes() ([]v1.Volume, error) { 301 | volumes := []v1.Volume{} 302 | for _, k := range g.gateway.Spec.Kernels { 303 | ks := &v1alpha1.JupyterKernelSpec{} 304 | if err := g.cli.Get(context.TODO(), types.NamespacedName{ 305 | Namespace: g.gateway.Namespace, 306 | Name: k, 307 | }, ks); err != nil { 308 | return nil, err 309 | } 310 | 311 | volumes = append(volumes, v1.Volume{ 312 | Name: k, 313 | VolumeSource: v1.VolumeSource{ 314 | ConfigMap: &v1.ConfigMapVolumeSource{ 315 | LocalObjectReference: v1.LocalObjectReference{Name: k}, 316 | }, 317 | }, 318 | }) 319 | } 320 | return volumes, nil 321 | } 322 | 323 | func (g generator) defaultClusterRole() string { 324 | if g.gateway.Spec.ClusterRole != nil { 325 | return *g.gateway.Spec.ClusterRole 326 | } 327 | return defaultGatewayClusterRole 328 | } 329 | 330 | func (g generator) labels() map[string]string { 331 | return map[string]string{ 332 | LabelNS: g.gateway.Namespace, 333 | LabelGateway: g.gateway.Name, 334 | } 335 | } 336 | 337 | func (g generator) kernels() string { 338 | if g.gateway.Spec.Kernels != nil { 339 | ks := []string{} 340 | for _, k := range g.gateway.Spec.Kernels { 341 | ks = append(ks, fmt.Sprintf("'%s'", k)) 342 | } 343 | return strings.Join(ks, ",") 344 | } 345 | return defaultKernels 346 | } 347 | 348 | func (g generator) defaultKernel() string { 349 | if g.gateway.Spec.DefaultKernel != nil { 350 | return *g.gateway.Spec.DefaultKernel 351 | } 352 | return defaultKernel 353 | } 354 | -------------------------------------------------------------------------------- /pkg/gateway/reconcile.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | // 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | // 10 | // https://opensource.org/licenses/Apache-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package gateway 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | appsv1 "k8s.io/api/apps/v1" 24 | v1 "k8s.io/api/core/v1" 25 | rbacv1 "k8s.io/api/rbac/v1" 26 | "k8s.io/apimachinery/pkg/api/equality" 27 | "k8s.io/apimachinery/pkg/api/errors" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/client-go/tools/record" 31 | "sigs.k8s.io/controller-runtime/pkg/client" 32 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 33 | 34 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 35 | ) 36 | 37 | type Reconciler struct { 38 | cli client.Client 39 | log logr.Logger 40 | recorder record.EventRecorder 41 | scheme *runtime.Scheme 42 | 43 | instance *v1alpha1.JupyterGateway 44 | gen *generator 45 | } 46 | 47 | func NewReconciler(cli client.Client, l logr.Logger, 48 | r record.EventRecorder, s *runtime.Scheme, 49 | i *v1alpha1.JupyterGateway) (*Reconciler, error) { 50 | g, err := newGenerator(cli, i) 51 | if err != nil { 52 | return nil, err 53 | } 54 | return &Reconciler{ 55 | cli: cli, 56 | log: l, 57 | recorder: r, 58 | scheme: s, 59 | instance: i, 60 | gen: g, 61 | }, nil 62 | } 63 | 64 | func (r Reconciler) Reconcile() error { 65 | serviceAccountName, err := r.reconcileRBAC() 66 | if err != nil { 67 | return err 68 | } 69 | if err := r.reconcileDeployment(serviceAccountName); err != nil { 70 | return err 71 | } 72 | if err := r.reconcileService(); err != nil { 73 | return err 74 | } 75 | return nil 76 | } 77 | 78 | func (r Reconciler) reconcileRBAC() (string, error) { 79 | sa, err := r.reconcileServiceAccount() 80 | if err != nil { 81 | return "", err 82 | } 83 | if err := r.reconcileRoleBinding(sa); err != nil { 84 | return "", err 85 | } 86 | return sa.Name, nil 87 | } 88 | 89 | func (r Reconciler) reconcileRoleBinding( 90 | sa *v1.ServiceAccount) error { 91 | desired := r.gen.DesiredRoleBinding(sa) 92 | 93 | if err := controllerutil.SetControllerReference( 94 | r.instance, desired, r.scheme); err != nil { 95 | r.log.Error(err, 96 | "Set controller reference error, requeuing the request") 97 | return err 98 | } 99 | 100 | actual := &rbacv1.RoleBinding{} 101 | err := r.cli.Get(context.TODO(), 102 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 103 | if err != nil && errors.IsNotFound(err) { 104 | r.log.Info("Creating rolebinding", 105 | "namespace", desired.Namespace, "name", desired.Name) 106 | 107 | if err := r.cli.Create(context.TODO(), desired); err != nil { 108 | r.log.Error(err, "Failed to create the rolebinding", 109 | "rolebinding", desired.Name) 110 | return err 111 | } 112 | } else if err != nil { 113 | r.log.Error(err, "failed to get the expected rolebinding", 114 | "rolebinding", desired.Name) 115 | return err 116 | } 117 | return nil 118 | } 119 | 120 | func (r Reconciler) reconcileServiceAccount() (*v1.ServiceAccount, error) { 121 | desired := r.gen.DesiredServiceAccountWithoutOwner() 122 | 123 | if err := controllerutil.SetControllerReference( 124 | r.instance, desired, r.scheme); err != nil { 125 | r.log.Error(err, 126 | "Set controller reference error, requeuing the request") 127 | return nil, err 128 | } 129 | 130 | actual := &v1.ServiceAccount{} 131 | err := r.cli.Get(context.TODO(), 132 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 133 | if err != nil && errors.IsNotFound(err) { 134 | r.log.Info("Creating serviceaccount", "namespace", desired.Namespace, "name", desired.Name) 135 | 136 | if err := r.cli.Create(context.TODO(), desired); err != nil { 137 | r.log.Error(err, "Failed to create the serviceaccount", 138 | "serviceaccount", desired.Name) 139 | return nil, err 140 | } 141 | } else if err != nil { 142 | r.log.Error(err, "failed to get the expected serviceaccount", 143 | "serviceaccount", desired.Name) 144 | return nil, err 145 | } 146 | // When the sa is created, actual is nil. Thus actual cannot be used to build rolebinding. 147 | return desired, nil 148 | } 149 | 150 | func (r Reconciler) reconcileService() error { 151 | desired := r.gen.DesiredServiceWithoutOwner() 152 | 153 | if err := controllerutil.SetControllerReference( 154 | r.instance, desired, r.scheme); err != nil { 155 | r.log.Error(err, 156 | "Set controller reference error, requeuing the request") 157 | return err 158 | } 159 | 160 | actual := &v1.Service{} 161 | err := r.cli.Get(context.TODO(), 162 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 163 | if err != nil && errors.IsNotFound(err) { 164 | r.log.Info("Creating service", "namespace", desired.Namespace, "name", desired.Name) 165 | 166 | if err := r.cli.Create(context.TODO(), desired); err != nil { 167 | r.log.Error(err, "Failed to create the serivce", 168 | "service", desired.Name) 169 | return err 170 | } 171 | } else if err != nil { 172 | r.log.Error(err, "failed to get the expected service", 173 | "service", desired.Name) 174 | return err 175 | } 176 | return nil 177 | } 178 | 179 | func (r Reconciler) reconcileDeployment(sa string) error { 180 | desired, err := r.gen.DesiredDeploymentWithoutOwner(sa) 181 | if err != nil { 182 | r.recorder.Event(r.instance, v1.EventTypeWarning, "FailedToGenerate", err.Error()) 183 | return err 184 | } 185 | 186 | if err := controllerutil.SetControllerReference( 187 | r.instance, desired, r.scheme); err != nil { 188 | r.log.Error(err, 189 | "Set controller reference error, requeuing the request") 190 | return err 191 | } 192 | 193 | actual := &appsv1.Deployment{} 194 | err = r.cli.Get(context.TODO(), 195 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 196 | if err != nil && errors.IsNotFound(err) { 197 | r.log.Info("Creating deployment", "namespace", desired.Namespace, "name", desired.Name) 198 | 199 | if err := r.cli.Create(context.TODO(), desired); err != nil { 200 | r.log.Error(err, "Failed to create the deployment", 201 | "deployment", desired.Name) 202 | r.recorder.Event(r.instance, v1.EventTypeWarning, "FailedToCreate", err.Error()) 203 | return err 204 | } 205 | } else if err != nil { 206 | r.log.Error(err, "failed to get the expected deployment", 207 | "deployment", desired.Name) 208 | r.recorder.Event(r.instance, v1.EventTypeWarning, "FailedToGet", err.Error()) 209 | return err 210 | } 211 | 212 | if !equality.Semantic.DeepEqual(r.instance.Status.DeploymentStatus, actual.Status) { 213 | r.instance.Status.DeploymentStatus = actual.Status 214 | if err := r.cli.Status().Update(context.TODO(), r.instance); err != nil { 215 | r.log.Error(err, "failed to update status", 216 | "namespace", r.instance.Namespace, 217 | "jupytergateway", r.instance.Name) 218 | r.recorder.Event(r.instance, v1.EventTypeWarning, "FailedToUpdateStatus", err.Error()) 219 | return err 220 | } 221 | } 222 | return nil 223 | } 224 | -------------------------------------------------------------------------------- /pkg/kernel/generate.go: -------------------------------------------------------------------------------- 1 | package kernel 2 | 3 | import ( 4 | "fmt" 5 | 6 | appsv1 "k8s.io/api/apps/v1" 7 | v1 "k8s.io/api/core/v1" 8 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 9 | 10 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 11 | ) 12 | 13 | const ( 14 | labelNS = "namespace" 15 | labelKernel = "kernel" 16 | envKernelID = "KERNEL_ID" 17 | labelKernelID = "kernel_id" 18 | ) 19 | 20 | // generator defines the generator which is used to generate 21 | // desired specs. 22 | type generator struct { 23 | k *v1alpha1.JupyterKernel 24 | } 25 | 26 | // newGenerator creates a new Generator. 27 | func newGenerator(k *v1alpha1.JupyterKernel) ( 28 | *generator, error) { 29 | if k == nil { 30 | return nil, fmt.Errorf("Got nil when initializing Generator") 31 | } 32 | g := &generator{ 33 | k: k, 34 | } 35 | 36 | return g, nil 37 | } 38 | 39 | func (g generator) DesiredDeployment() (*appsv1.Deployment, error) { 40 | labels := g.labels() 41 | 42 | d := &appsv1.Deployment{ 43 | ObjectMeta: metav1.ObjectMeta{ 44 | Name: g.k.Name, 45 | Namespace: g.k.Namespace, 46 | Labels: labels, 47 | }, 48 | Spec: appsv1.DeploymentSpec{ 49 | Template: g.k.Spec.Template, 50 | Selector: &metav1.LabelSelector{ 51 | MatchLabels: labels, 52 | }, 53 | }, 54 | } 55 | 56 | if d.Spec.Template.Labels == nil { 57 | d.Spec.Template.Labels = make(map[string]string) 58 | } 59 | // Set the labels to the pod template. 60 | for k, v := range labels { 61 | d.Spec.Template.Labels[k] = v 62 | } 63 | 64 | // Update the metadata. 65 | g.hackLabelID(&d.Spec.Template) 66 | 67 | return d, nil 68 | } 69 | 70 | func (g generator) labels() map[string]string { 71 | return map[string]string{ 72 | labelNS: g.k.Namespace, 73 | labelKernel: g.k.Name, 74 | } 75 | } 76 | 77 | // hackLabelID copies the ID from environment variables to 78 | // metadata. 79 | // TODO(gaocegege): Use newer version of controller-tools to avoid it. 80 | // https://github.com/kubernetes-sigs/controller-tools/issues/448 81 | func (g generator) hackLabelID(pod *v1.PodTemplateSpec) { 82 | if pod.Spec.Containers == nil || len(pod.Spec.Containers) == 0 { 83 | return 84 | } 85 | for _, env := range pod.Spec.Containers[0].Env { 86 | if env.Name == envKernelID { 87 | if pod.Labels == nil { 88 | pod.Labels = make(map[string]string) 89 | } 90 | pod.Labels[labelKernelID] = env.Value 91 | return 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/kernel/reconcile.go: -------------------------------------------------------------------------------- 1 | package kernel 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-logr/logr" 7 | appsv1 "k8s.io/api/apps/v1" 8 | "k8s.io/apimachinery/pkg/api/errors" 9 | "k8s.io/apimachinery/pkg/runtime" 10 | "k8s.io/apimachinery/pkg/types" 11 | "k8s.io/client-go/tools/record" 12 | "sigs.k8s.io/controller-runtime/pkg/client" 13 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 14 | 15 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 16 | ) 17 | 18 | type Reconciler struct { 19 | cli client.Client 20 | log logr.Logger 21 | recorder record.EventRecorder 22 | scheme *runtime.Scheme 23 | 24 | instance *v1alpha1.JupyterKernel 25 | gen *generator 26 | } 27 | 28 | func NewReconciler(cli client.Client, l logr.Logger, 29 | r record.EventRecorder, s *runtime.Scheme, 30 | i *v1alpha1.JupyterKernel) (*Reconciler, error) { 31 | g, err := newGenerator(i) 32 | if err != nil { 33 | return nil, err 34 | } 35 | return &Reconciler{ 36 | cli: cli, 37 | log: l, 38 | recorder: r, 39 | scheme: s, 40 | instance: i, 41 | gen: g, 42 | }, nil 43 | } 44 | 45 | func (r Reconciler) Reconcile() error { 46 | if err := r.reconcileDeployment(); err != nil { 47 | return err 48 | } 49 | 50 | return nil 51 | } 52 | 53 | func (r Reconciler) reconcileDeployment() error { 54 | desired, err := r.gen.DesiredDeployment() 55 | if err != nil { 56 | return err 57 | } 58 | 59 | if err := controllerutil.SetControllerReference( 60 | r.instance, desired, r.scheme); err != nil { 61 | r.log.Error(err, 62 | "Set controller reference error, requeuing the request") 63 | return err 64 | } 65 | 66 | actual := &appsv1.Deployment{} 67 | err = r.cli.Get(context.TODO(), 68 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 69 | if err != nil && errors.IsNotFound(err) { 70 | r.log.Info("Creating deployment", "namespace", desired.Namespace, "name", desired.Name) 71 | 72 | if err := r.cli.Create(context.TODO(), desired); err != nil { 73 | r.log.Error(err, "Failed to create the deployment", 74 | "deployment", desired.Name) 75 | return err 76 | } 77 | } else if err != nil { 78 | r.log.Error(err, "failed to get the expected deployment", 79 | "deployment", desired.Name) 80 | return err 81 | } 82 | 83 | // TODO(gaocegege): Update status. 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/kernelspec/generate.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | // 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | // 10 | // https://opensource.org/licenses/Apache-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | // Tencent is pleased to support the open source community by making TKEStack 17 | // available. 18 | // 19 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 20 | // 21 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 22 | // this file except in compliance with the License. You may obtain a copy of the 23 | // License at 24 | // 25 | // https://opensource.org/licenses/Apache-2.0 26 | // 27 | // Unless required by applicable law or agreed to in writing, software 28 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 29 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 30 | // specific language governing permissions and limitations under the License. 31 | 32 | package kernelspec 33 | 34 | import ( 35 | "encoding/json" 36 | "fmt" 37 | 38 | v1 "k8s.io/api/core/v1" 39 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 40 | 41 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 42 | ) 43 | 44 | const ( 45 | LabelKernelSpec = "kernelspec" 46 | LabelNS = "namespace" 47 | 48 | fileName = "kernel.json" 49 | defaultClassName = "enterprise_gateway.services.processproxies.k8s.KubernetesProcessProxy" 50 | 51 | keyKernelTemplateName = "--kernel-template-name" 52 | keyKernelTemplateNamespace = "--kernel-template-namespace" 53 | ) 54 | 55 | // generator defines the generator which is used to generate 56 | // desired specs. 57 | type generator struct { 58 | kernelSpec *v1alpha1.JupyterKernelSpec 59 | } 60 | 61 | // newGenerator creates a new Generator. 62 | func newGenerator(kernelSpec *v1alpha1.JupyterKernelSpec) ( 63 | *generator, error) { 64 | if kernelSpec == nil { 65 | return nil, fmt.Errorf("Got nil when initializing Generator") 66 | } 67 | g := &generator{ 68 | kernelSpec: kernelSpec, 69 | } 70 | 71 | return g, nil 72 | } 73 | 74 | func (g generator) DesiredConfigmapWithoutOwner() (*v1.ConfigMap, error) { 75 | labels := g.labels() 76 | 77 | jsonConfig, err := g.desiredJSON() 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | cm := &v1.ConfigMap{ 83 | ObjectMeta: metav1.ObjectMeta{ 84 | Namespace: g.kernelSpec.Namespace, 85 | Name: g.kernelSpec.Name, 86 | Labels: labels, 87 | }, 88 | Data: map[string]string{ 89 | fileName: jsonConfig, 90 | }, 91 | } 92 | 93 | return cm, nil 94 | } 95 | 96 | func (g generator) desiredJSON() (string, error) { 97 | c := &kernelConfig{ 98 | Language: g.kernelSpec.Spec.Language, 99 | DisplayName: g.kernelSpec.Spec.DisplayName, 100 | Metadata: metadata{ 101 | ProcessProxy: processProxy{ 102 | ClassName: defaultClassName, 103 | Config: config{ 104 | ImageName: g.kernelSpec.Spec.Image, 105 | }, 106 | }, 107 | }, 108 | Argv: g.kernelSpec.Spec.Command, 109 | } 110 | 111 | // Set the class name to desired. 112 | if g.kernelSpec.Spec.ClassName != "" { 113 | c.Metadata.ProcessProxy.ClassName = g.kernelSpec.Spec.ClassName 114 | } 115 | // Set the namespace and name for the jupyter kernel spec. 116 | c.Argv = append(c.Argv, 117 | keyKernelTemplateName, g.kernelSpec.Spec.Template.Name, 118 | keyKernelTemplateNamespace, g.kernelSpec.Spec.Template.Namespace) 119 | v, err := json.Marshal(c) 120 | return string(v), err 121 | } 122 | 123 | func (g generator) labels() map[string]string { 124 | return map[string]string{ 125 | LabelNS: g.kernelSpec.Namespace, 126 | LabelKernelSpec: g.kernelSpec.Name, 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /pkg/kernelspec/reconcile.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | // 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | // 10 | // https://opensource.org/licenses/Apache-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package kernelspec 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | v1 "k8s.io/api/core/v1" 24 | "k8s.io/apimachinery/pkg/api/errors" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/types" 27 | "k8s.io/client-go/tools/record" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 30 | 31 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 32 | ) 33 | 34 | type Reconciler struct { 35 | cli client.Client 36 | log logr.Logger 37 | recorder record.EventRecorder 38 | scheme *runtime.Scheme 39 | 40 | instance *v1alpha1.JupyterKernelSpec 41 | gen *generator 42 | } 43 | 44 | func NewReconciler(cli client.Client, l logr.Logger, 45 | r record.EventRecorder, s *runtime.Scheme, 46 | i *v1alpha1.JupyterKernelSpec) (*Reconciler, error) { 47 | g, err := newGenerator(i) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return &Reconciler{ 52 | cli: cli, 53 | log: l, 54 | recorder: r, 55 | scheme: s, 56 | instance: i, 57 | gen: g, 58 | }, nil 59 | } 60 | 61 | func (r Reconciler) Reconcile() error { 62 | if err := r.reconcileConfigmap(); err != nil { 63 | return err 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func (r Reconciler) reconcileConfigmap() error { 70 | desired, err := r.gen.DesiredConfigmapWithoutOwner() 71 | if err != nil { 72 | return err 73 | } 74 | 75 | if err = controllerutil.SetControllerReference( 76 | r.instance, desired, r.scheme); err != nil { 77 | r.log.Error(err, 78 | "Set controller reference error, requeuing the request") 79 | return err 80 | } 81 | 82 | actual := &v1.ConfigMap{} 83 | err = r.cli.Get(context.TODO(), 84 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 85 | if err != nil && errors.IsNotFound(err) { 86 | r.log.Info("Creating confimap", "namespace", desired.Namespace, "name", desired.Name) 87 | 88 | if err := r.cli.Create(context.TODO(), desired); err != nil { 89 | r.log.Error(err, "Failed to create the confimap", 90 | "confimap", desired.Name) 91 | return err 92 | } 93 | } else if err != nil { 94 | r.log.Error(err, "failed to get the expected confimap", 95 | "confimap", desired.Name) 96 | return err 97 | } 98 | return nil 99 | } 100 | -------------------------------------------------------------------------------- /pkg/kernelspec/types.go: -------------------------------------------------------------------------------- 1 | package kernelspec 2 | 3 | type kernelConfig struct { 4 | Language string `json:"language,omitempty"` 5 | DisplayName string `json:"display_name,omitempty"` 6 | Metadata metadata `json:"metadata,omitempty"` 7 | Argv []string `json:"argv,omitempty"` 8 | } 9 | 10 | type metadata struct { 11 | ProcessProxy processProxy `json:"process_proxy,omitempty"` 12 | } 13 | 14 | type processProxy struct { 15 | ClassName string `json:"class_name,omitempty"` 16 | Config config `json:"config,omitempty"` 17 | } 18 | 19 | type config struct { 20 | ImageName string `json:"image_name,omitempty"` 21 | } 22 | -------------------------------------------------------------------------------- /pkg/notebook/generate.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | // 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | // 10 | // https://opensource.org/licenses/Apache-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package notebook 18 | 19 | import ( 20 | "fmt" 21 | 22 | appsv1 "k8s.io/api/apps/v1" 23 | v1 "k8s.io/api/core/v1" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/util/intstr" 26 | 27 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 28 | ) 29 | 30 | const ( 31 | defaultImage = "jupyter/base-notebook:python-3.9.7" 32 | defaultContainerName = "notebook" 33 | defaultPortName = "notebook" 34 | defaultPort = 8888 35 | 36 | LabelNotebook = "notebook" 37 | LabelNS = "namespace" 38 | 39 | argumentGatewayURL = "--gateway-url" 40 | argumentNotebookToken = "--NotebookApp.token" 41 | argumentNotebookPassword = "--NotebookApp.password" 42 | ) 43 | 44 | type generator struct { 45 | nb *v1alpha1.JupyterNotebook 46 | } 47 | 48 | // newGenerator creates a new Generator. 49 | func newGenerator(nb *v1alpha1.JupyterNotebook) ( 50 | *generator, error) { 51 | if nb == nil { 52 | return nil, fmt.Errorf("the notebook is null") 53 | } 54 | g := &generator{ 55 | nb: nb, 56 | } 57 | 58 | return g, nil 59 | } 60 | 61 | func (g generator) DesiredDeploymentWithoutOwner() (*appsv1.Deployment, error) { 62 | if g.nb.Spec.Template == nil && g.nb.Spec.Gateway == nil { 63 | return nil, fmt.Errorf("no gateway and template applied") 64 | } 65 | 66 | podSpec := v1.PodSpec{} 67 | podLabels := g.labels() 68 | podAnnotations := g.annotations() 69 | labels := g.labels() 70 | annotations := g.annotations() 71 | selector := &metav1.LabelSelector{ 72 | MatchLabels: labels, 73 | } 74 | terminationGracePeriodSeconds := int64(30) 75 | 76 | if g.nb.Spec.Template != nil { 77 | if g.nb.Spec.Template.Labels != nil { 78 | for k, v := range g.nb.Spec.Template.Labels { 79 | podLabels[k] = v 80 | } 81 | } 82 | if g.nb.Spec.Template.Annotations != nil { 83 | for k, v := range g.nb.Spec.Template.Annotations { 84 | podAnnotations[k] = v 85 | } 86 | } 87 | podSpec = completePodSpec(&g.nb.Spec.Template.Spec) 88 | } else { 89 | podSpec = v1.PodSpec{ 90 | Containers: []v1.Container{ 91 | { 92 | Name: defaultContainerName, 93 | Image: defaultImage, 94 | ImagePullPolicy: v1.PullIfNotPresent, 95 | TerminationMessagePath: v1.TerminationMessagePathDefault, 96 | TerminationMessagePolicy: v1.TerminationMessageReadFile, 97 | Args: []string{ 98 | "start-notebook.sh", 99 | }, 100 | Ports: []v1.ContainerPort{ 101 | { 102 | Name: defaultPortName, 103 | ContainerPort: defaultPort, 104 | Protocol: v1.ProtocolTCP, 105 | }, 106 | }, 107 | }, 108 | }, 109 | RestartPolicy: v1.RestartPolicyAlways, 110 | TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, 111 | DNSPolicy: v1.DNSClusterFirst, 112 | SecurityContext: &v1.PodSecurityContext{}, 113 | SchedulerName: v1.DefaultSchedulerName, 114 | } 115 | } 116 | 117 | replicas := int32(1) 118 | revisionHistoryLimit := int32(10) 119 | progressDeadlineSeconds := int32(600) 120 | maxUnavailable := intstr.FromInt(25) 121 | 122 | d := &appsv1.Deployment{ 123 | ObjectMeta: metav1.ObjectMeta{ 124 | Namespace: g.nb.Namespace, 125 | Name: g.nb.Name, 126 | Labels: labels, 127 | Annotations: annotations, 128 | }, 129 | Spec: appsv1.DeploymentSpec{ 130 | Replicas: &replicas, 131 | Selector: selector, 132 | Template: v1.PodTemplateSpec{ 133 | ObjectMeta: metav1.ObjectMeta{ 134 | Labels: podLabels, 135 | Annotations: podAnnotations, 136 | }, 137 | Spec: podSpec, 138 | }, 139 | Strategy: appsv1.DeploymentStrategy{ 140 | Type: appsv1.DeploymentStrategyType(appsv1.RollingUpdateDaemonSetStrategyType), 141 | RollingUpdate: &appsv1.RollingUpdateDeployment{ 142 | MaxUnavailable: &maxUnavailable, 143 | MaxSurge: &maxUnavailable, 144 | }, 145 | }, 146 | RevisionHistoryLimit: &revisionHistoryLimit, 147 | ProgressDeadlineSeconds: &progressDeadlineSeconds, 148 | }, 149 | } 150 | 151 | if g.nb.Spec.Gateway != nil { 152 | gatewayURL := fmt.Sprintf("http://%s.%s:%d", 153 | g.nb.Spec.Gateway.Name, g.nb.Spec.Gateway.Namespace, defaultPort) 154 | d.Spec.Template.Spec.Containers[0].Args = append( 155 | d.Spec.Template.Spec.Containers[0].Args, argumentGatewayURL, gatewayURL) 156 | } 157 | 158 | // Set the auth configuration to notebook instance. 159 | if g.nb.Spec.Auth != nil { 160 | auth := g.nb.Spec.Auth 161 | // Set the token and password to empty. 162 | if auth.Mode == v1alpha1.ModeJupyterAuthDisable { 163 | d.Spec.Template.Spec.Containers[0].Args = append( 164 | d.Spec.Template.Spec.Containers[0].Args, 165 | argumentNotebookToken, "", 166 | argumentNotebookPassword, "", 167 | ) 168 | } else { 169 | if auth.Token != nil { 170 | d.Spec.Template.Spec.Containers[0].Args = append( 171 | d.Spec.Template.Spec.Containers[0].Args, 172 | argumentNotebookToken, *auth.Token, 173 | ) 174 | } 175 | if auth.Password != nil { 176 | d.Spec.Template.Spec.Containers[0].Args = append( 177 | d.Spec.Template.Spec.Containers[0].Args, 178 | argumentNotebookPassword, *auth.Password, 179 | ) 180 | } 181 | } 182 | } 183 | 184 | return d, nil 185 | } 186 | 187 | func (g generator) labels() map[string]string { 188 | return map[string]string{ 189 | LabelNS: g.nb.Namespace, 190 | LabelNotebook: g.nb.Name, 191 | } 192 | } 193 | 194 | func (g generator) annotations() map[string]string { 195 | return map[string]string{} 196 | } 197 | 198 | func completePodSpec(old *v1.PodSpec) v1.PodSpec { 199 | new := old.DeepCopy() 200 | for i := range new.Containers { 201 | if new.Containers[i].TerminationMessagePath == "" { 202 | new.Containers[i].TerminationMessagePath = v1.TerminationMessagePathDefault 203 | } 204 | if new.Containers[i].TerminationMessagePolicy == v1.TerminationMessagePolicy("") { 205 | new.Containers[i].TerminationMessagePolicy = v1.TerminationMessageReadFile 206 | } 207 | if new.Containers[i].ImagePullPolicy == v1.PullPolicy("") { 208 | new.Containers[i].ImagePullPolicy = v1.PullIfNotPresent 209 | } 210 | } 211 | 212 | if new.RestartPolicy == v1.RestartPolicy("") { 213 | new.RestartPolicy = v1.RestartPolicyAlways 214 | } 215 | 216 | if new.TerminationGracePeriodSeconds == nil { 217 | d := int64(v1.DefaultTerminationGracePeriodSeconds) 218 | new.TerminationGracePeriodSeconds = &d 219 | } 220 | 221 | if new.DNSPolicy == v1.DNSPolicy("") { 222 | new.DNSPolicy = v1.DNSClusterFirst 223 | } 224 | 225 | if new.SecurityContext == nil { 226 | new.SecurityContext = &v1.PodSecurityContext{} 227 | } 228 | 229 | if new.SchedulerName == "" { 230 | new.SchedulerName = v1.DefaultSchedulerName 231 | } 232 | 233 | return *new 234 | } 235 | -------------------------------------------------------------------------------- /pkg/notebook/generate_test.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 10 | v1 "k8s.io/api/core/v1" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | ) 13 | 14 | const ( 15 | JupyterNotebookName = "jupyternotebook-sample" 16 | JupyterNotebookNamespace = "default" 17 | DefaultContainerName = "notebook" 18 | DefaultImage = "busysandbox" 19 | DefaultImageWithGateway = "jupyter/base-notebook:python-3.9.7" 20 | GatewayName = "gateway" 21 | GatewayNamespace = "default" 22 | ) 23 | 24 | var ( 25 | podSpec = v1.PodSpec{ 26 | Containers: []v1.Container{ 27 | { 28 | Name: DefaultContainerName, 29 | Image: DefaultImage, 30 | }, 31 | }, 32 | } 33 | 34 | emptyNotebook = &v1alpha1.JupyterNotebook{ 35 | ObjectMeta: metav1.ObjectMeta{ 36 | Name: JupyterNotebookName, 37 | Namespace: JupyterNotebookNamespace, 38 | }, 39 | } 40 | 41 | notebookWithTemplate = &v1alpha1.JupyterNotebook{ 42 | ObjectMeta: metav1.ObjectMeta{ 43 | Name: JupyterNotebookName, 44 | Namespace: JupyterNotebookNamespace, 45 | }, 46 | Spec: v1alpha1.JupyterNotebookSpec{ 47 | Template: &v1.PodTemplateSpec{ 48 | ObjectMeta: metav1.ObjectMeta{ 49 | Labels: map[string]string{ 50 | "app": "notebook", 51 | "custom-label": "yes", 52 | }, 53 | Annotations: map[string]string{"custom-annotation": "yes"}, 54 | }, 55 | Spec: podSpec, 56 | }, 57 | }, 58 | } 59 | 60 | notebookWithGateway = &v1alpha1.JupyterNotebook{ 61 | ObjectMeta: metav1.ObjectMeta{ 62 | Name: JupyterNotebookName, 63 | Namespace: JupyterNotebookNamespace, 64 | }, 65 | Spec: v1alpha1.JupyterNotebookSpec{ 66 | Gateway: &v1.ObjectReference{ 67 | Kind: "JupyterGateway", 68 | Namespace: GatewayNamespace, 69 | Name: GatewayName, 70 | }, 71 | }, 72 | } 73 | 74 | testPwd = "test" 75 | notebookWithAuthPassword = &v1alpha1.JupyterNotebook{ 76 | ObjectMeta: metav1.ObjectMeta{ 77 | Name: JupyterNotebookName, 78 | Namespace: JupyterNotebookNamespace, 79 | }, 80 | Spec: v1alpha1.JupyterNotebookSpec{ 81 | Auth: &v1alpha1.JupyterAuth{ 82 | Password: &testPwd, 83 | }, 84 | Template: &v1.PodTemplateSpec{ 85 | ObjectMeta: metav1.ObjectMeta{ 86 | Labels: map[string]string{"app": "notebook"}, 87 | }, 88 | Spec: podSpec, 89 | }, 90 | }, 91 | } 92 | 93 | completeNotebook = &v1alpha1.JupyterNotebook{ 94 | ObjectMeta: metav1.ObjectMeta{ 95 | Name: JupyterNotebookName, 96 | Namespace: JupyterNotebookNamespace, 97 | }, 98 | Spec: v1alpha1.JupyterNotebookSpec{ 99 | Gateway: &v1.ObjectReference{ 100 | Kind: "JupyterGateway", 101 | Namespace: GatewayNamespace, 102 | Name: GatewayName, 103 | }, 104 | Template: &v1.PodTemplateSpec{ 105 | Spec: podSpec, 106 | }, 107 | }, 108 | } 109 | ) 110 | 111 | func TestGenerate(t *testing.T) { 112 | type test struct { 113 | input *v1alpha1.JupyterNotebook 114 | expectedErr error 115 | expectedGen *generator 116 | } 117 | 118 | tests := []test{ 119 | {input: nil, expectedErr: errors.New("the notebook is null"), expectedGen: nil}, 120 | {input: notebookWithTemplate, expectedErr: nil, expectedGen: &generator{nb: notebookWithTemplate}}, 121 | {input: notebookWithGateway, expectedErr: nil, expectedGen: &generator{nb: notebookWithGateway}}, 122 | {input: completeNotebook, expectedErr: nil, expectedGen: &generator{nb: completeNotebook}}, 123 | {input: emptyNotebook, expectedErr: nil, expectedGen: &generator{nb: emptyNotebook}}, 124 | } 125 | 126 | for _, tc := range tests { 127 | gen, err := newGenerator(tc.input) 128 | if !reflect.DeepEqual(tc.expectedErr, err) { 129 | t.Errorf("expected: %v, got: %v", tc.expectedErr, err) 130 | } 131 | if err == nil && !reflect.DeepEqual(tc.expectedGen, gen) { 132 | t.Errorf("expected: %v, got: %v", tc.expectedGen, gen) 133 | } 134 | } 135 | } 136 | 137 | func TestDesiredDeploymentWithoutOwner(t *testing.T) { 138 | type test struct { 139 | gen *generator 140 | expectedErr error 141 | expectedImage string 142 | expectedLabels map[string]string 143 | expectedAnnotations map[string]string 144 | expectedArgs []string 145 | } 146 | 147 | tests := []test{ 148 | { 149 | gen: &generator{nb: notebookWithTemplate}, 150 | expectedErr: nil, 151 | expectedImage: DefaultImage, 152 | expectedLabels: map[string]string{ 153 | "app": "notebook", 154 | "custom-label": "yes", 155 | }, 156 | expectedAnnotations: map[string]string{ 157 | "custom-annotation": "yes", 158 | }, 159 | expectedArgs: nil, 160 | }, 161 | { 162 | gen: &generator{nb: notebookWithGateway}, 163 | expectedErr: nil, 164 | expectedImage: DefaultImageWithGateway, 165 | expectedArgs: []string{ 166 | "start-notebook.sh", 167 | argumentGatewayURL, 168 | fmt.Sprintf("http://%s.%s:%d", GatewayName, GatewayNamespace, 8888), 169 | }, 170 | }, 171 | { 172 | gen: &generator{nb: completeNotebook}, 173 | expectedErr: nil, expectedImage: DefaultImage, 174 | expectedArgs: []string{ 175 | argumentGatewayURL, 176 | fmt.Sprintf("http://%s.%s:%d", GatewayName, GatewayNamespace, 8888), 177 | }, 178 | }, 179 | { 180 | gen: &generator{nb: emptyNotebook}, 181 | expectedErr: errors.New("no gateway and template applied"), 182 | }, 183 | { 184 | gen: &generator{nb: notebookWithAuthPassword}, 185 | expectedErr: nil, expectedImage: DefaultImage, 186 | expectedArgs: []string{argumentNotebookPassword, *notebookWithAuthPassword.Spec.Auth.Password}, 187 | }, 188 | } 189 | 190 | for i, tc := range tests { 191 | d, err := tc.gen.DesiredDeploymentWithoutOwner() 192 | if !reflect.DeepEqual(tc.expectedErr, err) { 193 | t.Errorf("expected: %v, got: %v", tc.expectedErr, err) 194 | } 195 | if err == nil && !reflect.DeepEqual(tc.expectedImage, d.Spec.Template.Spec.Containers[0].Image) { 196 | t.Errorf("expected: %v, got: %v", tc.expectedImage, d.Spec.Template.Spec.Containers[0].Image) 197 | } 198 | if err == nil && !reflect.DeepEqual(tc.expectedArgs, d.Spec.Template.Spec.Containers[0].Args) { 199 | t.Errorf("i= %d expected: %v, got: %v", i, tc.expectedArgs, d.Spec.Template.Spec.Containers[0].Args) 200 | } 201 | for k, v := range tc.expectedLabels { 202 | if v != d.Spec.Template.Labels[k] { 203 | t.Errorf("expected: %v, got: %v", v, d.Labels[k]) 204 | } 205 | } 206 | for k, v := range tc.expectedAnnotations { 207 | if v != d.Spec.Template.Annotations[k] { 208 | t.Errorf("expected: %v, got: %v", v, d.Annotations[k]) 209 | } 210 | } 211 | } 212 | } 213 | 214 | func TestLable(t *testing.T) { 215 | type test struct { 216 | gen *generator 217 | expected string 218 | } 219 | 220 | tests := []test{ 221 | {gen: &generator{nb: notebookWithTemplate}, expected: JupyterNotebookName}, 222 | } 223 | 224 | for _, tc := range tests { 225 | mp := tc.gen.labels() 226 | if !reflect.DeepEqual(tc.expected, mp["notebook"]) { 227 | t.Errorf("expected: %v, got: %v", tc.expected, mp["notebook"]) 228 | } 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /pkg/notebook/reconcile.go: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making TKEStack 2 | // available. 3 | // 4 | // Copyright (C) 2012-2020 Tencent. All Rights Reserved. 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use 7 | // this file except in compliance with the License. You may obtain a copy of the 8 | // License at 9 | // 10 | // https://opensource.org/licenses/Apache-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | // WARRANTIES OF ANY KIND, either express or implied. See the License for the 15 | // specific language governing permissions and limitations under the License. 16 | 17 | package notebook 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | appsv1 "k8s.io/api/apps/v1" 24 | "k8s.io/apimachinery/pkg/api/equality" 25 | "k8s.io/apimachinery/pkg/api/errors" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/apimachinery/pkg/types" 28 | "k8s.io/client-go/tools/record" 29 | "sigs.k8s.io/controller-runtime/pkg/client" 30 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 31 | 32 | "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 33 | ) 34 | 35 | type Reconciler struct { 36 | cli client.Client 37 | log logr.Logger 38 | recorder record.EventRecorder 39 | scheme *runtime.Scheme 40 | 41 | instance *v1alpha1.JupyterNotebook 42 | gen *generator 43 | } 44 | 45 | func NewReconciler(cli client.Client, 46 | l logr.Logger, 47 | r record.EventRecorder, s *runtime.Scheme, 48 | i *v1alpha1.JupyterNotebook) (*Reconciler, error) { 49 | g, err := newGenerator(i) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return &Reconciler{ 54 | cli: cli, 55 | log: l, 56 | recorder: r, 57 | scheme: s, 58 | instance: i, 59 | gen: g, 60 | }, nil 61 | } 62 | 63 | func (r Reconciler) Reconcile() error { 64 | if err := r.reconcileDeployment(); err != nil { 65 | return err 66 | } 67 | return nil 68 | } 69 | 70 | func (r Reconciler) reconcileDeployment() error { 71 | desired, err := r.gen.DesiredDeploymentWithoutOwner() 72 | if err != nil { 73 | return err 74 | } 75 | 76 | if err := controllerutil.SetControllerReference( 77 | r.instance, desired, r.scheme); err != nil { 78 | r.log.Error(err, 79 | "Set controller reference error, requeuing the request") 80 | return err 81 | } 82 | 83 | actual := &appsv1.Deployment{} 84 | err = r.cli.Get(context.TODO(), 85 | types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()}, actual) 86 | 87 | // Create deployment if not found 88 | if err != nil && errors.IsNotFound(err) { 89 | r.log.Info("Creating deployment", "namespace", desired.Namespace, "name", desired.Name) 90 | if err := r.cli.Create(context.TODO(), desired); err != nil { 91 | r.log.Error(err, "Failed to create the deployment", 92 | "deployment", desired.Name) 93 | return err 94 | } 95 | } else if err != nil { 96 | r.log.Error(err, "failed to get the expected deployment", 97 | "deployment", desired.Name) 98 | return err 99 | } 100 | 101 | // Update deployment from desired to actural 102 | if !equality.Semantic.DeepEqual(desired.Spec, actual.Spec) { 103 | if err := r.cli.Update(context.TODO(), desired); err != nil { 104 | r.log.Error(err, "Failed to update deployment") 105 | return err 106 | } else { 107 | r.log.Info("deployment updated") 108 | } 109 | } 110 | 111 | return nil 112 | } 113 | -------------------------------------------------------------------------------- /pkg/notebook/reconcile_test.go: -------------------------------------------------------------------------------- 1 | package notebook 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "testing" 7 | "time" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | "k8s.io/apimachinery/pkg/runtime" 12 | "k8s.io/apimachinery/pkg/types" 13 | "k8s.io/client-go/kubernetes/scheme" 14 | "k8s.io/client-go/rest" 15 | "k8s.io/client-go/tools/record" 16 | ctrl "sigs.k8s.io/controller-runtime" 17 | "sigs.k8s.io/controller-runtime/pkg/client" 18 | "sigs.k8s.io/controller-runtime/pkg/envtest" 19 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 20 | logf "sigs.k8s.io/controller-runtime/pkg/log" 21 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 22 | "sigs.k8s.io/controller-runtime/pkg/manager" 23 | 24 | kubeflowtkestackiov1alpha1 "github.com/tkestack/elastic-jupyter-operator/api/v1alpha1" 25 | // +kubebuilder:scaffold:imports 26 | ) 27 | 28 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 29 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 30 | var ( 31 | cfg *rest.Config 32 | k8sClient client.Client 33 | k8sManager manager.Manager 34 | testEnv *envtest.Environment 35 | s *runtime.Scheme 36 | 37 | log = ctrl.Log.WithName("controllers").WithName("JupyterNotebook") 38 | rec = record.NewFakeRecorder(1024 * 1024) 39 | ) 40 | 41 | const ( 42 | timeout = time.Second * 10 43 | duration = time.Second * 10 44 | interval = time.Millisecond * 250 45 | ) 46 | 47 | func TestAPIs(t *testing.T) { 48 | RegisterFailHandler(Fail) 49 | 50 | RunSpecsWithDefaultAndCustomReporters(t, 51 | "Noteboook Suite", 52 | []Reporter{printer.NewlineReporter{}}) 53 | } 54 | 55 | var _ = BeforeSuite(func(done Done) { 56 | logf.SetLogger(zap.New(zap.UseDevMode(true))) 57 | 58 | By("bootstrapping test environment") 59 | testEnv = &envtest.Environment{ 60 | CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, 61 | } 62 | 63 | var err error 64 | cfg, err = testEnv.Start() 65 | Expect(err).ToNot(HaveOccurred()) 66 | Expect(cfg).ToNot(BeNil()) 67 | 68 | err = kubeflowtkestackiov1alpha1.AddToScheme(scheme.Scheme) 69 | Expect(err).NotTo(HaveOccurred()) 70 | 71 | // +kubebuilder:scaffold:scheme 72 | 73 | k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ 74 | Scheme: scheme.Scheme, 75 | }) 76 | Expect(err).ToNot(HaveOccurred()) 77 | 78 | go func() { 79 | err = k8sManager.Start(ctrl.SetupSignalHandler()) 80 | Expect(err).ToNot(HaveOccurred()) 81 | }() 82 | 83 | k8sClient = k8sManager.GetClient() 84 | Expect(k8sClient).ToNot(BeNil()) 85 | s = k8sManager.GetScheme() 86 | Expect(s).ToNot(BeNil()) 87 | 88 | close(done) 89 | }, 60) 90 | 91 | var _ = Describe("JupyterNotebook controller", func() { 92 | 93 | Context("Nil JupyterNotebook", func() { 94 | It("Should fail to NewReconciler", func() { 95 | _, err := NewReconciler(k8sClient, log, rec, s, nil) 96 | Expect(err).To(HaveOccurred()) 97 | }) 98 | }) 99 | 100 | Context("JupyterNotebook without template and notebook", func() { 101 | It("Should fail to reconcileDeployment", func() { 102 | var r *Reconciler 103 | var err error 104 | r, err = NewReconciler(k8sClient, log, rec, s, emptyNotebook) 105 | Expect(err).ToNot(HaveOccurred()) 106 | Expect(r).ToNot(BeNil()) 107 | err = r.reconcileDeployment() 108 | Expect(err).To(HaveOccurred()) 109 | }) 110 | }) 111 | 112 | Context("JupyterNotebook only have template", func() { 113 | It("Should reconcile deployment as desired", func() { 114 | var r *Reconciler 115 | var err error 116 | r, err = NewReconciler(k8sClient, log, rec, s, notebookWithTemplate) 117 | Expect(err).ToNot(HaveOccurred()) 118 | Expect(r).ToNot(BeNil()) 119 | 120 | err = r.cli.Create(context.TODO(), notebookWithTemplate) 121 | Expect(err).ToNot(HaveOccurred()) 122 | 123 | err = r.reconcileDeployment() 124 | Expect(err).ToNot(HaveOccurred()) 125 | 126 | By("Expecting template name") 127 | Eventually(func() string { 128 | actual := &kubeflowtkestackiov1alpha1.JupyterNotebook{} 129 | if err := k8sClient.Get(context.Background(), 130 | types.NamespacedName{Name: notebookWithTemplate.GetName(), Namespace: notebookWithTemplate.GetNamespace()}, actual); err == nil { 131 | return actual.Spec.Template.Spec.Containers[0].Name 132 | } 133 | return "" 134 | }, timeout, interval).Should(Equal(notebookWithTemplate.Spec.Template.Spec.Containers[0].Name)) 135 | 136 | err = r.Reconcile() 137 | Expect(err).ToNot(HaveOccurred()) 138 | }) 139 | }) 140 | }) 141 | 142 | var _ = AfterSuite(func() { 143 | By("tearing down the test environment") 144 | err := testEnv.Stop() 145 | Expect(err).ToNot(HaveOccurred()) 146 | }) 147 | --------------------------------------------------------------------------------