├── .dockerignore ├── .github ├── CODEOWNERS ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .gitmodules ├── .mvn └── wrapper │ ├── MavenWrapperDownloader.java │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .travis.yml ├── .travis ├── .travis.prepare.minikube.sh ├── .travis.prepare.openshift.sh ├── .travis.release.images.sh ├── .travis.test-common.sh ├── .travis.test-cross-ns.sh ├── .travis.test-oc-and-k8s.sh ├── .travis.test-restarts.sh ├── cache │ ├── .travis.before_cache.sh │ ├── .travis.before_install.sh │ ├── container-images-common.txt │ ├── container-images-k8s.txt │ └── container-images-oc.txt └── settings.xml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile.alpine ├── Dockerfile.ubi ├── LICENSE ├── Makefile ├── README.md ├── abstract-operator ├── pom.xml └── src │ └── main │ ├── java │ └── io │ │ └── radanalytics │ │ └── operator │ │ ├── SDKEntrypoint.java │ │ ├── common │ │ ├── AbstractOperator.java │ │ ├── AbstractWatcher.java │ │ ├── AnsiColors.java │ │ ├── ConfigMapWatcher.java │ │ ├── CustomResourceWatcher.java │ │ ├── EntityInfo.java │ │ ├── JSONSchemaReader.java │ │ ├── LogProducer.java │ │ ├── Operator.java │ │ ├── OperatorConfig.java │ │ ├── ProcessRunner.java │ │ └── crd │ │ │ ├── CrdDeployer.java │ │ │ ├── InfoClass.java │ │ │ ├── InfoList.java │ │ │ └── InfoStatus.java │ │ └── resource │ │ ├── HasDataHelper.java │ │ ├── LabelsHelper.java │ │ └── ResourceHelper.java │ ├── javadoc │ └── overview.html │ └── resources │ ├── META-INF │ └── beans.xml │ ├── log4j.properties │ └── schema │ └── test.js ├── annotator ├── pom.xml └── src │ └── main │ └── java │ └── io │ └── radanalytics │ └── operator │ └── annotator │ └── RegisterForReflectionAnnotator.java ├── docs ├── ascii.gif └── standardized-UML-diagram.png ├── examples ├── app.yaml ├── apps │ └── pyspark-ntlk.yaml ├── cluster-cm.yaml ├── cluster-with-config.yaml ├── cluster.yaml ├── spark-defaults.conf ├── test │ ├── cluster-1.yaml │ ├── cluster-2.yaml │ ├── cluster-cm-with-labels.yaml │ ├── cluster-cpumem.yaml │ ├── cluster-limits.yaml │ ├── cluster-limreq.yaml │ ├── cluster-node-tolerations.yaml │ ├── cluster-overwritelim.yaml │ ├── cluster-overwritereq.yaml │ ├── cluster-requests.yaml │ ├── cluster-with-config-1.yaml │ ├── cluster-with-config-2.yaml │ ├── cluster-with-config-3.yaml │ ├── cluster-with-labels.yaml │ ├── cm │ │ ├── app.yaml │ │ ├── cluster-1.yaml │ │ ├── cluster-2.yaml │ │ ├── cluster-cpumem.yaml │ │ ├── cluster-limits.yaml │ │ ├── cluster-limreq.yaml │ │ ├── cluster-node-tolerations.yaml │ │ ├── cluster-overwritelim.yaml │ │ ├── cluster-overwritereq.yaml │ │ ├── cluster-requests.yaml │ │ ├── cluster-with-config-1.yaml │ │ ├── cluster-with-config-2.yaml │ │ └── cluster-with-config-3.yaml │ └── history-server │ │ ├── Dockerfile │ │ ├── HistoryServer.md │ │ ├── externalStorage │ │ ├── ceph-nano.yaml │ │ ├── cluster-1.yaml │ │ ├── cluster-2.yaml │ │ └── history-server.yaml │ │ └── sharedVolume │ │ ├── cluster-1.yaml │ │ ├── cluster-2.yaml │ │ ├── history-server.yaml │ │ └── single-node-pvs.yaml └── with-prepared-data.yaml ├── helm └── spark-operator │ ├── .gitignore │ ├── .helmignore │ ├── Chart.yaml │ ├── Makefile │ ├── OWNERS │ ├── README.md │ ├── templates │ ├── NOTES.txt │ └── operator.yaml │ └── values.yaml ├── manifest ├── olm │ ├── configmap-based-all-in-one-csv.yaml │ └── crd │ │ ├── radanalytics-spark.package.yaml │ │ ├── sparkapplication.crd.yaml │ │ ├── sparkcluster.crd.yaml │ │ └── sparkclusteroperator.1.0.1.clusterserviceversion.yaml ├── operator-cm.yaml └── operator.yaml ├── monitoring ├── Monitoring.md ├── example-cluster-with-monitoring.yaml └── prometheus-operator.yaml ├── mvnw ├── mvnw.cmd ├── pom.xml ├── spark-operator ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── radanalytics │ │ │ └── operator │ │ │ ├── Constants.java │ │ │ ├── SparkOperatorEntrypoint.java │ │ │ ├── app │ │ │ ├── AppOperator.java │ │ │ └── KubernetesAppDeployer.java │ │ │ ├── cluster │ │ │ ├── InitContainersHelper.java │ │ │ ├── KubernetesSparkClusterDeployer.java │ │ │ ├── MetricsHelper.java │ │ │ ├── RunningClusters.java │ │ │ └── SparkClusterOperator.java │ │ │ └── historyServer │ │ │ ├── HistoryServerHelper.java │ │ │ ├── HistoryServerOperator.java │ │ │ └── KubernetesHistoryServerDeployer.java │ └── resources │ │ └── schema │ │ ├── sparkApplication.json │ │ ├── sparkCluster.json │ │ └── sparkHistoryServer.json │ └── test │ └── java │ └── io │ └── radanalytics │ └── operator │ └── resource │ └── YamlProcessingTest.java └── version-bump.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | # ignore all by default 2 | * 3 | 4 | # except these: 5 | !/spark-operator/ 6 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Each line is a file pattern followed by one or more owners. 2 | # These owners will be the default owners for everything in 3 | # the repo. Unless a later match takes precedence. 4 | # 5 | # examples: 6 | # * @global-owner 7 | # *.js @js-owner 8 | # /docs/ @doctocat 9 | # *.go @org/team-name 10 | 11 | * @Jiri-Kremser 12 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's: 3 | 4 | * Reporting a bug 5 | * Discussing the current state of the code 6 | * Submitting a fix 7 | * Proposing new features 8 | * Becoming a maintainer 9 | 10 | 11 | ## How to Contribute Code 12 | Pull requests are the best way to propose changes to the codebase. 13 | 14 | 1. Fork the repo and create your branch from master. 15 | 1. If you've added code that should be tested, add tests. 16 | 1. If you've changed APIs, update the documentation. 17 | 1. Ensure the test suite passes. 18 | 1. Make sure your code lints. 19 | 1. Issue that pull request! 20 | 21 | ## License 22 | By contributing, you agree that your contributions will be licensed under its Apache v2 License. 23 | 24 | Thank you! 25 | :octocat: 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### Description: 4 | 5 | 6 | #### Steps to reproduce: 7 | 8 | 1. 9 | 1. 10 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### Description 4 | 5 | 6 | #### Related Issue 7 | 8 | 9 | 10 | #### Types of changes 11 | 12 | 13 | :bulb: Updated docs / Refactor code / Added a tests case / Automation (non-breaking change) 14 | :bug: Bug fix (non-breaking change which fixes an issue) 15 | :sparkles: New feature (non-breaking change which adds functionality) 16 | :warning: Breaking change (fix or feature that would cause existing functionality to change) 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ###################### 2 | # Eclipse 3 | ###################### 4 | *.pydevproject 5 | .project 6 | .metadata 7 | tmp/ 8 | tmp/**/* 9 | *.tmp 10 | *.bak 11 | *.swp 12 | *~.nib 13 | local.properties 14 | .classpath 15 | .settings/ 16 | .loadpath 17 | .factorypath 18 | /src/main/resources/rebel.xml 19 | 20 | # External tool builders 21 | .externalToolBuilders/** 22 | 23 | # Locally stored "Eclipse launch configurations" 24 | *.launch 25 | 26 | # CDT-specific 27 | .cproject 28 | 29 | # PDT-specific 30 | .buildpath 31 | 32 | ###################### 33 | # Intellij 34 | ###################### 35 | .idea/ 36 | *.iml 37 | *.iws 38 | *.ipr 39 | *.ids 40 | *.orig 41 | classes/ 42 | 43 | ###################### 44 | # Visual Studio Code 45 | ###################### 46 | .vscode/ 47 | 48 | ###################### 49 | # Maven 50 | ###################### 51 | /log/ 52 | /annotator/target/ 53 | /abstract-operator/target/ 54 | /spark-operator/target/ 55 | pom.xml.versionsBackup 56 | 57 | ###################### 58 | # Gradle 59 | ###################### 60 | .gradle/ 61 | /build/ 62 | 63 | ###################### 64 | # Package Files 65 | ###################### 66 | *.jar 67 | *.war 68 | *.ear 69 | *.db 70 | 71 | ###################### 72 | # Windows 73 | ###################### 74 | # Windows image file caches 75 | Thumbs.db 76 | 77 | # Folder config file 78 | Desktop.ini 79 | 80 | ###################### 81 | # Mac OSX 82 | ###################### 83 | .DS_Store 84 | .svn 85 | 86 | # Thumbnails 87 | ._* 88 | 89 | # Files that might appear on external disk 90 | .Spotlight-V100 91 | .Trashes 92 | 93 | ###################### 94 | # Others 95 | ###################### 96 | *.class 97 | *.*~ 98 | *~ 99 | .merge_file* 100 | 101 | ###################### 102 | # Gradle Wrapper 103 | ###################### 104 | !gradle/wrapper/gradle-wrapper.jar 105 | 106 | ###################### 107 | # Maven Wrapper 108 | ###################### 109 | !.mvn/wrapper/maven-wrapper.jar 110 | 111 | ###################### 112 | # ESLint 113 | ###################### 114 | .eslintcache 115 | 116 | 117 | ########################## 118 | # Custom/Project Specific 119 | ########################## 120 | 121 | # generated manifests 122 | /k8s-spark-operator.yaml 123 | /openshift-spark-operator.yaml 124 | /manifest/operator-devel.yaml 125 | /manifest/operator-test.yaml 126 | 127 | # oc cluster up 128 | openshift.local.clusterup 129 | 130 | # spark shell 131 | metastore_db/ 132 | derby.log 133 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test"] 2 | path = test 3 | url = https://github.com/radanalyticsio/openshift-test-kit 4 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | 20 | import java.net.*; 21 | import java.io.*; 22 | import java.nio.channels.*; 23 | import java.util.Properties; 24 | 25 | public class MavenWrapperDownloader { 26 | 27 | /** 28 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 29 | */ 30 | private static final String DEFAULT_DOWNLOAD_URL = 31 | "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar"; 32 | 33 | /** 34 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 35 | * use instead of the default one. 36 | */ 37 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 38 | ".mvn/wrapper/maven-wrapper.properties"; 39 | 40 | /** 41 | * Path where the maven-wrapper.jar will be saved to. 42 | */ 43 | private static final String MAVEN_WRAPPER_JAR_PATH = 44 | ".mvn/wrapper/maven-wrapper.jar"; 45 | 46 | /** 47 | * Name of the property which should be used to override the default download url for the wrapper. 48 | */ 49 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 50 | 51 | public static void main(String args[]) { 52 | System.out.println("- Downloader started"); 53 | File baseDirectory = new File(args[0]); 54 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 55 | 56 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 57 | // wrapperUrl parameter. 58 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 59 | String url = DEFAULT_DOWNLOAD_URL; 60 | if(mavenWrapperPropertyFile.exists()) { 61 | FileInputStream mavenWrapperPropertyFileInputStream = null; 62 | try { 63 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 64 | Properties mavenWrapperProperties = new Properties(); 65 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 66 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 67 | } catch (IOException e) { 68 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 69 | } finally { 70 | try { 71 | if(mavenWrapperPropertyFileInputStream != null) { 72 | mavenWrapperPropertyFileInputStream.close(); 73 | } 74 | } catch (IOException e) { 75 | // Ignore ... 76 | } 77 | } 78 | } 79 | System.out.println("- Downloading from: : " + url); 80 | 81 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 82 | if(!outputFile.getParentFile().exists()) { 83 | if(!outputFile.getParentFile().mkdirs()) { 84 | System.out.println( 85 | "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 86 | } 87 | } 88 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 89 | try { 90 | downloadFileFromURL(url, outputFile); 91 | System.out.println("Done"); 92 | System.exit(0); 93 | } catch (Throwable e) { 94 | System.out.println("- Error downloading"); 95 | e.printStackTrace(); 96 | System.exit(1); 97 | } 98 | } 99 | 100 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 101 | URL website = new URL(urlString); 102 | ReadableByteChannel rbc; 103 | rbc = Channels.newChannel(website.openStream()); 104 | FileOutputStream fos = new FileOutputStream(destination); 105 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 106 | fos.close(); 107 | rbc.close(); 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/radanalyticsio/spark-operator/0fe4f9f97f6c82b9d94bd0189c01256d4d8b1662/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.1/apache-maven-3.6.1-bin.zip 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | services: 4 | - docker 5 | 6 | # skip install step 7 | install: true 8 | 9 | before_cache: 10 | - ./.travis/cache/.travis.before_cache.sh 11 | 12 | before_install: 13 | - ./.travis/cache/.travis.before_install.sh 14 | 15 | 16 | cache: 17 | timeout: 120 18 | directories: 19 | - $HOME/maven 20 | - $HOME/docker 21 | 22 | stages: 23 | - test 24 | - Kubernetes and Openshift tests 25 | - deploy 26 | 27 | jobs: 28 | include: 29 | - stage: test 30 | name: "Maven & cont. image build" 31 | language: java 32 | script: 33 | # run the tests 34 | - make test 35 | 36 | - stage: Kubernetes and Openshift tests 37 | name: "[oc • CMs] Full specs" 38 | env: BIN=oc VERSION=v3.9.0 CRD=0 39 | script: &oc-script-defaults 40 | - make build-travis 41 | - ./.travis/.travis.prepare.openshift.sh 42 | - ./.travis/.travis.test-oc-and-k8s.sh 43 | 44 | - stage: 45 | name: "[oc • CRs] Full specs" 46 | env: BIN=oc VERSION=v3.9.0 CRD=1 47 | script: *oc-script-defaults 48 | 49 | #- stage: 50 | # env: BIN=oc VERSION=v3.10.0 CRD=0 51 | # script: *oc-script-defaults 52 | 53 | - stage: 54 | name: "[K8s • CMs] Full specs" 55 | env: BIN=kubectl VERSION=v1.9.0 CRD=0 MINIKUBE_VERSION=v0.25.2 56 | script: &kc-script-defaults 57 | - make build-travis 58 | - ./.travis/.travis.prepare.minikube.sh 59 | - ./.travis/.travis.test-oc-and-k8s.sh 60 | 61 | - stage: 62 | name: "[K8s • CRs] Full specs" 63 | env: BIN=kubectl VERSION=v1.9.0 CRD=1 MINIKUBE_VERSION=v0.25.2 64 | script: *kc-script-defaults 65 | 66 | - stage: 67 | name: "[oc • CMs] Random restarts" 68 | env: BIN=oc VERSION=v3.9.0 CRD=0 69 | script: 70 | - make build-travis 71 | - ./.travis/.travis.prepare.openshift.sh 72 | - ./.travis/.travis.test-restarts.sh 73 | 74 | - stage: 75 | name: "[K8s • CRs] Random restarts" 76 | env: BIN=kubectl VERSION=v1.9.0 CRD=1 MINIKUBE_VERSION=v0.25.2 77 | script: 78 | - make build-travis 79 | - ./.travis/.travis.prepare.minikube.sh 80 | - ./.travis/.travis.test-restarts.sh 81 | 82 | - stage: 83 | name: "[oc • CMs] Cross namespaces (WATCH_NAMESPACE=*)" 84 | env: BIN=oc VERSION=v3.9.0 CRD=0 85 | script: &cross-ns-defaults 86 | - make build-travis 87 | - ./.travis/.travis.prepare.openshift.sh 88 | - ./.travis/.travis.test-cross-ns.sh 89 | 90 | - stage: 91 | name: "[oc • CRs] Cross namespaces (WATCH_NAMESPACE=*)" 92 | env: BIN=oc VERSION=v3.9.0 CRD=1 93 | script: *cross-ns-defaults 94 | 95 | - stage: deploy 96 | name: "Push container images" 97 | script: 98 | # release x.y.z or x.y.z-ubi if there is a release 99 | # or release the latest image if building the master branch 100 | - ./.travis/.travis.release.images.sh 101 | # release maven artifacts 102 | #- 'if [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $TRAVIS_BRANCH == "master" ]] ; then make build-travis && ./mvnw -s ./.travis/settings.xml clean deploy; fi' 103 | -------------------------------------------------------------------------------- /.travis/.travis.prepare.minikube.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | download_kubectl() { 6 | echo "Downloading kubectl binary for VERSION=${VERSION}" 7 | curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${VERSION}/bin/linux/amd64/kubectl && \ 8 | chmod +x kubectl && \ 9 | sudo mv kubectl /usr/local/bin/ && \ 10 | kubectl version || true 11 | } 12 | 13 | download_minikube() { 14 | echo "Downloading minikube binary for MINIKUBE_VERSION=${MINIKUBE_VERSION}" 15 | curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 && \ 16 | chmod +x minikube && \ 17 | sudo mv minikube /usr/local/bin/ && \ 18 | minikube version 19 | } 20 | 21 | setup_manifest() { 22 | sed -i'' 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator.yaml 23 | sed -i'' 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator-cm.yaml 24 | sed -i'' 's;imagePullPolicy: .*;imagePullPolicy: Never;g' manifest/operator.yaml 25 | sed -i'' 's;imagePullPolicy: .*;imagePullPolicy: Never;g' manifest/operator-cm.yaml 26 | [ "$CRD" = "0" ] && FOO="-cm" || FOO="" 27 | echo -e "'\nmanifest${FOO}:\n-----------\n" 28 | cat manifest/operator${FOO}.yaml 29 | } 30 | 31 | main() { 32 | echo -e "travis_fold:start:k8s\033[33;1mPrepare Minikube\033[0m" 33 | download_kubectl 34 | download_minikube 35 | setup_manifest 36 | echo -e "\ntravis_fold:end:k8s\r" 37 | } 38 | 39 | main 40 | -------------------------------------------------------------------------------- /.travis/.travis.prepare.openshift.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | RUN=${RUN:-"1"} 6 | 7 | download_openshift() { 8 | echo "Downloading oc binary for VERSION=${VERSION}" 9 | sudo docker cp $(docker create docker.io/openshift/origin:$VERSION):/bin/oc /usr/local/bin/oc 10 | oc version 11 | } 12 | 13 | setup_insecure_registry() { 14 | #sudo cat /etc/default/docker 15 | sudo cat /etc/docker/daemon.json 16 | sudo service docker stop 17 | cat << EOF | sudo tee /etc/docker/daemon.json 18 | { 19 | "insecure-registries": ["172.30.0.0/16"] 20 | } 21 | EOF 22 | #sudo sed -i -e 's/sock/sock --insecure-registry 172.30.0.0\/16/' /etc/default/docker 23 | #sudo cat /etc/default/docker 24 | sudo cat /etc/docker/daemon.json 25 | sudo service docker start 26 | sudo service docker status 27 | sudo mount --make-rshared / 28 | } 29 | 30 | setup_manifest() { 31 | sed -i'' 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator.yaml 32 | sed -i'' 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator-cm.yaml 33 | sed -i'' 's;imagePullPolicy: .*;imagePullPolicy: Never;g' manifest/operator.yaml 34 | sed -i'' 's;imagePullPolicy: .*;imagePullPolicy: Never;g' manifest/operator-cm.yaml 35 | [ "$CRD" = "0" ] && FOO="-cm" || FOO="" 36 | echo -e "'\nmanifest${FOO}:\n-----------\n" 37 | cat manifest/operator${FOO}.yaml 38 | } 39 | 40 | main() { 41 | echo -e "travis_fold:start:oc\033[33;1mPrepare OpenShift\033[0m" 42 | download_openshift 43 | setup_insecure_registry 44 | setup_manifest 45 | echo -e "\ntravis_fold:end:oc\r" 46 | } 47 | 48 | [ "$RUN" = "1" ] && main || echo "$0 sourced" 49 | -------------------------------------------------------------------------------- /.travis/.travis.release.images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -xe 4 | 5 | OWNER="${OWNER:-radanalyticsio}" 6 | IMAGE="${IMAGE:-spark-operator}" 7 | [ "$TRAVIS_BRANCH" = "master" -a "$TRAVIS_PULL_REQUEST" = "false" ] && LATEST=1 8 | 9 | main() { 10 | if [[ "$LATEST" = "1" ]]; then 11 | echo "Pushing the :latest and :latest-ubi images to docker.io and quay.io" 12 | loginDockerIo 13 | pushLatestImagesDockerIo 14 | loginQuayIo 15 | pushLatestImagesQuayIo 16 | elif [[ "${TRAVIS_TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 17 | echo "Pushing the '${TRAVIS_TAG}' and :latest-released images to docker.io and quay.io" 18 | buildReleaseImages 19 | loginDockerIo 20 | pushReleaseImages "docker.io" 21 | loginQuayIo 22 | pushReleaseImages "quay.io" 23 | else 24 | echo "Not doing the docker push, because the tag '${TRAVIS_TAG}' is not of form x.y.z" 25 | echo "and also it's not a build of the master branch" 26 | fi 27 | } 28 | 29 | loginDockerIo() { 30 | set +x 31 | docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" 32 | set -x 33 | } 34 | 35 | loginQuayIo() { 36 | set +x 37 | docker login -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io 38 | set -x 39 | } 40 | 41 | pushLatestImagesDockerIo() { 42 | make image-publish-all 43 | docker logout 44 | } 45 | 46 | pushLatestImagesQuayIo() { 47 | docker tag $OWNER/$IMAGE:latest "quay.io/"$OWNER/$IMAGE:latest 48 | docker tag $OWNER/$IMAGE:latest-alpine "quay.io/"$OWNER/$IMAGE:latest-alpine 49 | docker push "quay.io/"$OWNER/$IMAGE:latest 50 | docker push "quay.io/"$OWNER/$IMAGE:latest-alpine 51 | } 52 | 53 | buildReleaseImages() { 54 | # build ubi image 55 | make build-travis image-build-all 56 | } 57 | 58 | pushReleaseImages() { 59 | if [[ $# != 1 ]] && [[ $# != 2 ]]; then 60 | echo "Usage: pushReleaseImages image_repo" && exit 61 | fi 62 | REPO="$1" 63 | 64 | docker tag $OWNER/$IMAGE:ubi $REPO/$OWNER/$IMAGE:${TRAVIS_TAG} 65 | docker tag $OWNER/$IMAGE:ubi $REPO/$OWNER/$IMAGE:latest-released 66 | docker tag $OWNER/$IMAGE:alpine $REPO/$OWNER/$IMAGE:${TRAVIS_TAG}-alpine 67 | docker tag $OWNER/$IMAGE:alpine $REPO/$OWNER/$IMAGE:latest-released-alpine 68 | 69 | # push the latest-released and ${TRAVIS_TAG} images (and also -alpine images) 70 | docker push $REPO/$OWNER/$IMAGE:${TRAVIS_TAG} 71 | docker push $REPO/$OWNER/$IMAGE:latest-released 72 | docker push $REPO/$OWNER/$IMAGE:${TRAVIS_TAG}-alpine 73 | docker push $REPO/$OWNER/$IMAGE:latest-released-alpine 74 | docker logout 75 | } 76 | 77 | main 78 | -------------------------------------------------------------------------------- /.travis/.travis.test-cross-ns.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="${DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )}" 4 | BIN=${BIN:-oc} 5 | MANIFEST_SUFIX=${MANIFEST_SUFIX:-""} 6 | if [ "$CRD" = "1" ]; then 7 | CM="" 8 | KIND="SparkCluster" 9 | else 10 | CM="cm/" 11 | KIND="cm" 12 | fi 13 | 14 | source "${DIR}/.travis.test-common.sh" 15 | 16 | 17 | testCreateClusterInNamespace() { 18 | info 19 | echo -e "\n\n namespace:\n" 20 | oc project 21 | [ "$CRD" = "0" ] && FOO="-cm" || FOO="" 22 | os::cmd::expect_success_and_text "${BIN} create -f $DIR/../examples/cluster$FOO.yaml" '"?my-spark-cluster"? created' && \ 23 | os::cmd::try_until_text "${BIN} get pod -l radanalytics.io/deployment=my-spark-cluster-w -o yaml" 'ready: true' && \ 24 | os::cmd::try_until_text "${BIN} get pod -l radanalytics.io/deployment=my-spark-cluster-m -o yaml" 'ready: true' 25 | } 26 | 27 | testDeleteClusterInNamespace() { 28 | info 29 | os::cmd::expect_success_and_text '${BIN} delete ${KIND} my-spark-cluster' '"my-spark-cluster" deleted' 30 | sleep 5 31 | } 32 | 33 | 34 | run_tests() { 35 | testEditOperator 'WATCH_NAMESPACE="*"' || errorLogs 36 | sleep 15 # wait long enough, because the full reconciliation is not supported for this use-case (*) 37 | 38 | os::cmd::expect_success_and_text '${BIN} new-project foo' 'Now using project' || errorLogs 39 | testCreateClusterInNamespace || errorLogs 40 | testDeleteClusterInNamespace || errorLogs 41 | os::cmd::expect_success_and_text '${BIN} new-project bar' 'Now using project' || errorLogs 42 | testCreateClusterInNamespace || errorLogs 43 | testDeleteClusterInNamespace || errorLogs 44 | 45 | sleep 5 46 | logs 47 | } 48 | 49 | main() { 50 | export total=6 51 | export testIndex=0 52 | tear_down 53 | setup_testing_framework 54 | os::test::junit::declare_suite_start "operator/tests-cross-ns" 55 | cluster_up 56 | # cluster role is required for the cross namespace watching 57 | oc login -u system:admin 58 | oc adm policy add-cluster-role-to-user cluster-admin -z spark-operator 59 | testCreateOperator || { ${BIN} get events; ${BIN} get pods; exit 1; } 60 | if [ "$#" -gt 0 ]; then 61 | # run single test that is passed as arg 62 | $1 63 | else 64 | run_tests 65 | fi 66 | os::test::junit::declare_suite_end 67 | tear_down 68 | } 69 | 70 | main $@ 71 | -------------------------------------------------------------------------------- /.travis/.travis.test-oc-and-k8s.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="${DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )}" 4 | BIN=${BIN:-oc} 5 | MANIFEST_SUFIX=${MANIFEST_SUFIX:-""} 6 | if [ "$CRD" = "1" ]; then 7 | CR="" 8 | KIND="SparkCluster" 9 | else 10 | CR="cm/" 11 | KIND="cm" 12 | fi 13 | 14 | source "${DIR}/.travis.test-common.sh" 15 | 16 | run_custom_test() { 17 | testCustomCluster1 || errorLogs 18 | testCustomCluster2 || errorLogs 19 | testCustomCluster3 || errorLogs 20 | testCustomCluster4 || errorLogs 21 | testCustomCluster5 || errorLogs 22 | } 23 | 24 | run_limit_request_tests() { 25 | testNoLimits || errorLogs 26 | testCpuMem || errorLogs 27 | testJustLimits || errorLogs 28 | testJustRequests || errorLogs 29 | testRequestsAndLimits || errorLogs 30 | testOverwriteRequests || errorLogs 31 | testOverwriteLimits || errorLogs 32 | } 33 | 34 | run_tests() { 35 | testCreateCluster1 || errorLogs 36 | testScaleCluster || errorLogs 37 | sleep 45 38 | testNoPodRestartsOccurred "my-spark-cluster" || errorLogs 39 | testDeleteCluster || errorLogs 40 | sleep 5 41 | 42 | testCreateCluster2 || errorLogs 43 | testDownloadedData || errorLogs 44 | sleep 5 45 | 46 | testFullConfigCluster || errorLogs 47 | sleep 5 48 | 49 | run_custom_test || errorLogs 50 | sleep 5 51 | 52 | run_limit_request_tests || errorLogs 53 | sleep 5 54 | 55 | testNodeTolerations || errorLogs 56 | 57 | sleep 10 58 | testApp || appErrorLogs 59 | testAppResult || appErrorLogs 60 | testDeleteApp || appErrorLogs 61 | 62 | sleep 5 63 | testPythonApp || appErrorLogs 64 | testPythonAppResult || appErrorLogs 65 | 66 | testMetricServer || errorLogs 67 | logs 68 | } 69 | 70 | main() { 71 | export total=20 72 | export testIndex=0 73 | tear_down 74 | setup_testing_framework 75 | os::test::junit::declare_suite_start "operator/tests" 76 | cluster_up 77 | testCreateOperator || { ${BIN} get events; ${BIN} get pods; exit 1; } 78 | export operator_pod=`${BIN} get pod -l app.kubernetes.io/name=spark-operator -o='jsonpath="{.items[0].metadata.name}"' | sed 's/"//g'` 79 | if [ "$#" -gt 0 ]; then 80 | # run single test that is passed as arg 81 | $1 82 | else 83 | run_tests 84 | fi 85 | os::test::junit::declare_suite_end 86 | tear_down 87 | } 88 | 89 | main $@ 90 | -------------------------------------------------------------------------------- /.travis/.travis.test-restarts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="${DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )}" 4 | BIN=${BIN:-oc} 5 | MANIFEST_SUFIX=${MANIFEST_SUFIX:-""} 6 | if [ "$CRD" = "1" ]; then 7 | CR="" 8 | KIND="SparkCluster" 9 | else 10 | CR="cm/" 11 | KIND="cm" 12 | fi 13 | 14 | source "${DIR}/.travis.test-common.sh" 15 | 16 | run_tests() { 17 | testKillOperator || errorLogs 18 | testCreateCluster1 || errorLogs 19 | testKillOperator || errorLogs 20 | testScaleCluster || errorLogs 21 | testKillOperator || errorLogs 22 | testDeleteCluster || errorLogs 23 | testKillOperator || errorLogs 24 | 25 | sleep 10 26 | testApp || appErrorLogs 27 | testKillOperator || errorLogs 28 | testAppResult || appErrorLogs 29 | logs 30 | } 31 | 32 | main() { 33 | export total=11 34 | export testIndex=0 35 | tear_down 36 | setup_testing_framework 37 | os::test::junit::declare_suite_start "operator/tests-restarts" 38 | cluster_up 39 | testCreateOperator || { ${BIN} get events; ${BIN} get pods; exit 1; } 40 | if [ "$#" -gt 0 ]; then 41 | # run single test that is passed as arg 42 | $1 43 | else 44 | run_tests 45 | fi 46 | os::test::junit::declare_suite_end 47 | tear_down 48 | } 49 | 50 | main $@ 51 | -------------------------------------------------------------------------------- /.travis/cache/.travis.before_cache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | DIR="${DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )}" 6 | 7 | main() { 8 | docker_cache 9 | maven_cache 10 | } 11 | 12 | docker_cache(){ 13 | if [[ "$TRAVIS_JOB_NUMBER" == *.1 ]] || [[ "$TRAVIS_JOB_NUMBER" == *.10 ]]; then 14 | echo "Skipping docker cache for .1 and .10 jobs" 15 | exit 0 16 | else 17 | if [[ "$BIN" = "oc" ]]; then 18 | specific="${DIR}/container-images-oc.txt" 19 | elif [[ "$BIN" = "kubectl" ]]; then 20 | specific="${DIR}/container-images-k8s.txt" 21 | else 22 | echo "Unknown or empty \$BIN variable, skipping before-cache script.." 23 | exit 1 24 | fi 25 | fi 26 | 27 | mkdir -p $HOME/docker 28 | docker images -a --filter='dangling=false' --format '{{.Repository}}:{{.Tag}} {{.ID}}' > $HOME/docker/${BIN}-list.txt 29 | cat "${DIR}/container-images-common.txt ${specific}" | while read c 30 | do 31 | cat $HOME/docker/${BIN}-list.txt | grep "$c" | xargs -n 2 -t sh -c 'test -e $HOME/docker/$1.tar.gz || docker save $0 | gzip -2 > $HOME/docker/$1.tar.gz' 32 | done 33 | } 34 | 35 | maven_cache(){ 36 | mkdir -p $HOME/maven 37 | cd $HOME/.m2/repository 38 | tar cf - --exclude=io/radanalytics . | (cd $HOME/maven && tar xf - ) 39 | } 40 | 41 | main 42 | -------------------------------------------------------------------------------- /.travis/cache/.travis.before_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | DIR="${DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )}" 6 | 7 | main() { 8 | docker_cache 9 | maven_cache 10 | } 11 | 12 | docker_cache(){ 13 | if [[ "$TRAVIS_JOB_NUMBER" == *.1 ]] || [[ "$TRAVIS_JOB_NUMBER" == *.10 ]]; then 14 | images="${DIR}/container-images-common.txt" 15 | else 16 | if [[ "$BIN" = "oc" ]]; then 17 | images="${DIR}/container-images-common.txt ${DIR}/container-images-oc.txt" 18 | elif [[ "$BIN" = "kubectl" ]]; then 19 | images="${DIR}/container-images-common.txt ${DIR}/container-images-k8s.txt" 20 | else 21 | echo "Unknown or empty \$BIN variable, skipping before-cache script.." 22 | exit 1 23 | fi 24 | fi 25 | 26 | if [[ -d $HOME/docker ]] && [[ -e $HOME/docker/${BIN:-oc}-list.txt ]]; then 27 | cat ${images} | while read c 28 | do 29 | cat $HOME/docker/${BIN:-oc}-list.txt | grep "$c" | xargs -n 2 sh -c 'test -e $HOME/docker/$1.tar.gz && (zcat $HOME/docker/$1.tar.gz | docker load) || true' 30 | 31 | # make sure it's there, this should be a cheap operation if the previous command was successful 32 | docker pull ${c} 33 | done 34 | fi 35 | } 36 | 37 | maven_cache(){ 38 | mkdir -p $HOME/.m2/repository/ 39 | cp -r $HOME/maven $HOME/.m2/repository/ 40 | } 41 | 42 | main 43 | -------------------------------------------------------------------------------- /.travis/cache/container-images-common.txt: -------------------------------------------------------------------------------- 1 | fabric8/java-centos-openjdk8-jdk:1.5.1 2 | quay.io/radanalyticsio/ubi-jre-1.8.0-minimal:1.0 3 | quay.io/radanalyticsio/openshift-spark:2.4-latest 4 | quay.io/radanalyticsio/openshift-spark:2.4.5-2 5 | jkremser/spark-operator:2.4.0-ntlk 6 | busybox:latest 7 | -------------------------------------------------------------------------------- /.travis/cache/container-images-k8s.txt: -------------------------------------------------------------------------------- 1 | k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.1 2 | gcr.io/google-containers/kube-addon-manager:v6.5 3 | gcr.io/k8s-minikube/storage-provisioner:v1.8.0 4 | k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.5 5 | k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.5 6 | k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.5 7 | k8s.gcr.io/pause-amd64:3.0 -------------------------------------------------------------------------------- /.travis/cache/container-images-oc.txt: -------------------------------------------------------------------------------- 1 | openshift/origin-web-console:v3.9.0 2 | openshift/origin-docker-registry:v3.9.0 3 | openshift/origin-haproxy-router:v3.9.0 4 | openshift/origin-deployer:v3.9.0 5 | openshift/origin:v3.9.0 6 | openshift/origin-pod:v3.9.0 -------------------------------------------------------------------------------- /.travis/settings.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | ossrh 8 | ${env.SONATYPE_USER} 9 | ${env.SONATYPE_PASSWORD} 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 2020-09-2020 4 | 5 | - Change log added 6 | - Top-level *pom.xml* copied from https://github.com/jvm-operators/operator-parent-pom and turned into 7 | a local, unpublished artifact named spark-operator-parent 8 | - Subdirectory *annotator* copied from https://github.com/jvm-operators/abstract-operator and turned into 9 | a local, unpublished artifact named spark-operator-annotator 10 | - Subdirectory *abstract-operator* copied from https://github.com/jvm-operators/abstract-operator and turned into 11 | a local, unpublished artifact named spark-abstract-operator 12 | - Source code for the spark operator moved to *spark-operator* subdirectory and modified to reference 13 | spark-operator-parent, spark-operator-annotator, and spark-abstract-operator as dependencies instead 14 | of their precursors published in maven repositories. 15 | - Dockerfiles and Makefile modified to build with the new setup. 16 | - Version of the fabric8 kubernetes client changed to 4.10.3 17 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## spark-operator Community Code of Conduct v1.0 2 | 3 | ### Contributor Code of Conduct 4 | 5 | As contributors and maintainers of this project, and in the interest of fostering 6 | an open and welcoming community, we pledge to respect all people who contribute 7 | through reporting issues, posting feature requests, updating documentation, 8 | submitting pull requests or patches, and other activities. 9 | 10 | We are committed to making participation in this project a harassment-free experience for 11 | everyone, regardless of level of experience, gender, gender identity and expression, 12 | sexual orientation, disability, personal appearance, body size, race, ethnicity, age, 13 | religion, or nationality. 14 | 15 | Examples of unacceptable behavior by participants include: 16 | 17 | * The use of sexualized language or imagery 18 | * Personal attacks 19 | * Trolling or insulting/derogatory comments 20 | * Public or private harassment 21 | * Publishing other's private information, such as physical or electronic addresses, without explicit permission 22 | * Other unethical or unprofessional conduct. 23 | 24 | Project maintainers have the right and responsibility to remove, edit, or reject 25 | comments, commits, code, wiki edits, issues, and other contributions that are not 26 | aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers 27 | commit themselves to fairly and consistently applying these principles to every aspect 28 | of managing this project. Project maintainers who do not follow or enforce the Code of 29 | Conduct may be permanently removed from the project team. 30 | 31 | This code of conduct applies both within project spaces and in public spaces 32 | when an individual is representing the project or its community. 33 | 34 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 35 | contacting a spark-operator project maintainer Jiri Kremser . 36 | 37 | 38 | This Code of Conduct is adapted from the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md) which is 39 | adapted from the Contributor Covenant 40 | (http://contributor-covenant.org), version 1.2.0, available at 41 | http://contributor-covenant.org/version/1/2/0/ 42 | 43 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Maven Project Structure 2 | 3 | ## Version 1.0 branch 4 | 5 | In the 1.0 branch, the spark-operator depends on the [abstract-operator library](https://github.com/jvm-operators/abstract-operator) 6 | and the [operator-parent-pom](https://github.com/jvm-operators/operator-parent-pom). The top-level (only) pom.xml file in the 7 | 1.0 branch is for the spark-operator itself. 8 | 9 | Any work on a version of the spark-operator that maintains these dependencies should happen on the 1.0 branch. 10 | New versions/tags should use a 1.0.x versioning scheme. 11 | The 1.0 branch was created on 9/28/2020. 12 | 13 | ## Master branch 14 | 15 | After the creation of the 1.0 branch, the master branch of spark-operator was changed to eliminate the 16 | external dependencies on [abstract-operator](https://github.com/jvm-operators/abstract-operator) and [operator-parent-pom](https://github.com/jvm-operators/operator-parent-pom). 17 | 18 | Copies of the abstract-operator source code and the operator-parent-pom were added to the spark-operator repository, and their artifact 19 | ids were changed to *spark-abstractor-operator* and *spark-operator-parent*. The intention is to **not** publish these jars to 20 | a maven repository, but only reference them locally in building the spark-operator image. 21 | 22 | The top-level pom.xml file in the spark-operator repository is now the parent pom, and the following subdirectories 23 | contain all necessary source code for the spark-operator: 24 | 25 | * annotator 26 | * abstract-operator 27 | * spark-operator 28 | 29 | This change simplifies the build and release process and makes it easier to make changes in the abstract-operator code for the 30 | benefit of the spark-operator. 31 | 32 | Going forward, changes to the spark-operator source code itself that are independent of changes to the abstract-operator can 33 | be merged from the master branch into the 1.0 branch if applicable. Changes that are dependent on additional changes to the 34 | abstract-operator however cannot be merged into the 1.0 branch. 35 | 36 | ### Note on building in the master branch 37 | 38 | The *make package* command will use *mvn clean install* to build the parent pom, annotator, and abstract-operator and install them 39 | in the local .m2 repository before building the spark-operator. These components are not treated as modules, just co-located projects. 40 | The build is done this way to preserve spark-operator as an uberjar (making all of the components modules in the parent pom breaks this). 41 | 42 | ## CHANGELOG.md 43 | 44 | The changelog should be updated when changes are made to the repository. In particular, changes to the top-level pom file and/or 45 | the abstract-operator should be noted to comply with the Apache license in the original repositories for operator-parent-pom 46 | and abstract-operator. 47 | -------------------------------------------------------------------------------- /Dockerfile.alpine: -------------------------------------------------------------------------------- 1 | FROM jkremser/mini-jre:8.1 2 | 3 | ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm" 4 | 5 | LABEL BASE_IMAGE="jkremser/mini-jre:8" 6 | 7 | ADD spark-operator/target/spark-operator-*.jar /spark-operator.jar 8 | 9 | CMD ["/usr/bin/java", "-jar", "/spark-operator.jar"] 10 | -------------------------------------------------------------------------------- /Dockerfile.ubi: -------------------------------------------------------------------------------- 1 | FROM quay.io/radanalyticsio/ubi-jre-1.8.0-minimal:1.0 2 | 3 | ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm" 4 | 5 | LABEL BASE_IMAGE_2="quay.io/radanalyticsio/ubi-jre-1.8.0-minimal:1.0" 6 | 7 | ADD spark-operator/target/spark-operator-*.jar /spark-operator.jar 8 | 9 | CMD ["/usr/bin/java", "-jar", "/spark-operator.jar"] 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | IMAGE?=radanalyticsio/spark-operator 2 | 3 | .PHONY: build 4 | build: package image-build 5 | 6 | .PHONY: build-travis 7 | build-travis: 8 | echo -e "travis_fold:start:mvn\033[33;1mMaven and container build\033[0m" 9 | $(MAKE) build 10 | echo -e "\ntravis_fold:end:mvn\r" 11 | 12 | .PHONY: package 13 | package: 14 | # install parent pom in m2 cache 15 | MAVEN_OPTS="-Djansi.passthrough=true -Dplexus.logger.type=ansi $(MAVEN_OPTS)" ./mvnw clean install -DskipTests 16 | # install annotator in m2 cache 17 | MAVEN_OPTS="-Djansi.passthrough=true -Dplexus.logger.type=ansi $(MAVEN_OPTS)" ./mvnw clean install -f annotator/pom.xml -DskipTests 18 | # install abstract-operator in m2 cache 19 | MAVEN_OPTS="-Djansi.passthrough=true -Dplexus.logger.type=ansi $(MAVEN_OPTS)" ./mvnw clean install -f abstract-operator/pom.xml -DskipTests 20 | # build uberjar for spark-operator 21 | MAVEN_OPTS="-Djansi.passthrough=true -Dplexus.logger.type=ansi $(MAVEN_OPTS)" ./mvnw clean package -f spark-operator/pom.xml -DskipTests 22 | 23 | .PHONY: test 24 | test: 25 | MAVEN_OPTS="-Djansi.passthrough=true -Dplexus.logger.type=ansi $(MAVEN_OPTS)" ./mvnw clean test 26 | 27 | .PHONY: image-build 28 | image-build: 29 | docker build -t $(IMAGE):ubi -f Dockerfile.ubi . 30 | docker tag $(IMAGE):ubi $(IMAGE):latest 31 | 32 | .PHONY: image-build-alpine 33 | image-build-alpine: 34 | docker build -t $(IMAGE):alpine -f Dockerfile.alpine . 35 | 36 | .PHONY: image-build-all 37 | image-build-all: image-build image-build-alpine 38 | 39 | .PHONY: image-publish-alpine 40 | image-publish-alpine: image-build-alpine 41 | docker tag $(IMAGE):alpine $(IMAGE):alpine-`git rev-parse --short=8 HEAD` 42 | docker tag $(IMAGE):alpine $(IMAGE):latest-alpine 43 | docker push $(IMAGE):latest-alpine 44 | 45 | .PHONY: image-publish 46 | image-publish: image-build 47 | docker tag $(IMAGE):ubi $(IMAGE):`git rev-parse --short=8 HEAD`-ubi 48 | docker tag $(IMAGE):ubi $(IMAGE):latest-ubi 49 | docker push $(IMAGE):latest 50 | 51 | .PHONY: image-publish-all 52 | image-publish-all: build-travis image-build-all image-publish image-publish-alpine 53 | 54 | .PHONY: devel 55 | devel: build 56 | -docker kill `docker ps -q` || true 57 | oc cluster up ; oc login -u system:admin ; oc project default 58 | sed 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator.yaml > manifest/operator-devel.yaml && oc create -f manifest/operator-devel.yaml ; rm manifest/operator-devel.yaml || true 59 | until [ "true" = "`oc get pod -l app.kubernetes.io/name=spark-operator -o json 2> /dev/null | grep \"\\\"ready\\\": \" | sed -e 's;.*\(true\|false\),;\1;'`" ]; do printf "."; sleep 1; done 60 | oc logs -f `oc get pods --no-headers -l app.kubernetes.io/name=spark-operator | cut -f1 -d' '` 61 | 62 | .PHONY: devel-kubernetes 63 | devel-kubernetes: 64 | -minikube delete 65 | minikube start --vm-driver kvm2 66 | eval `minikube docker-env` && $(MAKE) build 67 | sed 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator.yaml > manifest/operator-devel.yaml && kubectl create -f manifest/operator.yaml ; rm manifest/operator-devel.yaml || true 68 | until [ "true" = "`kubectl get pod -l app.kubernetes.io/name=spark-operator -o json 2> /dev/null | grep \"\\\"ready\\\": \" | sed -e 's;.*\(true\|false\),;\1;'`" ]; do printf "."; sleep 1; done 69 | kubectl logs -f `kubectl get pods --no-headers -l app.kubernetes.io/name=spark-operator | cut -f1 -d' '` 70 | 71 | .PHONY: local-travis-tests 72 | local-travis-tests: build 73 | -docker kill `docker ps -q` || true 74 | sed 's;quay.io/radanalyticsio/spark-operator:latest-released;radanalyticsio/spark-operator:latest;g' manifest/operator.yaml > manifest/operator-test.yaml 75 | -BIN=oc CRD=0 MANIFEST_SUFIX="-test" .travis/.travis.test-oc-and-k8s.sh || true 76 | -BIN=oc CRD=0 MANIFEST_SUFIX="-test" .travis/.travis.test-restarts.sh || true 77 | -BIN=oc CRD=0 MANIFEST_SUFIX="-test" .travis/.travis.test-cross-ns.sh || true 78 | -rm manifest/operator-test.yaml || true 79 | -------------------------------------------------------------------------------- /abstract-operator/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.radanalytics 7 | spark-operator-parent 8 | 1.0.0 9 | 10 | io.radanalytics 11 | spark-abstract-operator 12 | 1.0.0 13 | 14 | scm:git:git@github.com:jvm-operators/abstract-operator.git 15 | scm:git:git@github.com:jvm-operators/abstract-operator.git 16 | https://github.com/jvm-operators/abstract-operator 17 | 18 | 19 | UTF-8 20 | 21 | 22 | 23 | io.quarkus 24 | quarkus-kubernetes-client 25 | 26 | 27 | io.quarkus 28 | quarkus-arc 29 | 30 | 31 | io.prometheus 32 | simpleclient 33 | 34 | 35 | io.prometheus 36 | simpleclient_httpserver 37 | 38 | 39 | io.prometheus 40 | simpleclient_hotspot 41 | 42 | 43 | io.prometheus 44 | simpleclient_log4j 45 | 46 | 47 | org.slf4j 48 | slf4j-api 49 | 50 | 51 | org.slf4j 52 | slf4j-log4j12 53 | 54 | 55 | org.yaml 56 | snakeyaml 57 | 58 | 59 | com.jcabi 60 | jcabi-manifests 61 | 62 | 63 | commons-collections 64 | commons-collections 65 | 66 | 67 | commons-lang 68 | commons-lang 69 | 70 | 71 | com.google.guava 72 | guava 73 | 74 | 75 | junit 76 | junit 77 | test 78 | 79 | 80 | 81 | 82 | sonatype-releases 83 | https://oss.sonatype.org/content/repositories/releases 84 | 85 | 86 | 87 | 88 | 89 | org.codehaus.mojo 90 | buildnumber-maven-plugin 91 | 92 | 93 | org.jsonschema2pojo 94 | jsonschema2pojo-maven-plugin 95 | 96 | 97 | io.quarkus 98 | quarkus-maven-plugin 99 | 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/AnsiColors.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | public class AnsiColors { 4 | 5 | // these shouldn't be used directly 6 | private static final String ANSI_R = "\u001B[31m"; 7 | private static final String ANSI_G = "\u001B[32m"; 8 | private static final String ANSI_Y = "\u001B[33m"; 9 | private static final String ANSI_RESET = "\u001B[0m"; 10 | 11 | // if empty, it's true 12 | public static final boolean COLORS = !"false".equals(System.getenv("COLORS")); 13 | 14 | public static String re() { 15 | return COLORS ? ANSI_R : ""; 16 | } 17 | 18 | public static String gr() { 19 | return COLORS ? ANSI_G : ""; 20 | } 21 | 22 | public static String ye() { 23 | return COLORS ? ANSI_Y : ""; 24 | } 25 | 26 | public static String xx() { 27 | return COLORS ? ANSI_RESET : ""; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/ConfigMapWatcher.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import io.fabric8.kubernetes.api.model.ConfigMap; 4 | import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; 5 | import io.fabric8.kubernetes.client.KubernetesClient; 6 | import io.radanalytics.operator.resource.HasDataHelper; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.CompletableFuture; 10 | import java.util.function.BiConsumer; 11 | import java.util.function.Function; 12 | import java.util.function.Predicate; 13 | 14 | import static io.radanalytics.operator.common.OperatorConfig.ALL_NAMESPACES; 15 | 16 | public class ConfigMapWatcher extends AbstractWatcher { 17 | 18 | // use via builder 19 | private ConfigMapWatcher(String namespace, 20 | String entityName, 21 | KubernetesClient client, 22 | Map selector, 23 | BiConsumer onAdd, 24 | BiConsumer onDelete, 25 | BiConsumer onModify, 26 | Predicate predicate, 27 | Function convert) { 28 | super(true, namespace, entityName, client, null, selector, onAdd, onDelete, onModify, predicate, convert, null); 29 | } 30 | 31 | public static class Builder { 32 | private boolean registered = false; 33 | private String namespace = ALL_NAMESPACES; 34 | private String entityName; 35 | private KubernetesClient client; 36 | private Map selector; 37 | 38 | private BiConsumer onAdd; 39 | private BiConsumer onDelete; 40 | private BiConsumer onModify; 41 | private Predicate predicate; 42 | private Function convert; 43 | 44 | public Builder withNamespace(String namespace) { 45 | this.namespace = namespace; 46 | return this; 47 | } 48 | 49 | public Builder withEntityName(String entityName) { 50 | this.entityName = entityName; 51 | return this; 52 | } 53 | 54 | public Builder withClient(KubernetesClient client) { 55 | this.client = client; 56 | return this; 57 | } 58 | 59 | public Builder withSelector(Map selector) { 60 | this.selector = selector; 61 | return this; 62 | } 63 | 64 | public Builder withOnAdd(BiConsumer onAdd) { 65 | this.onAdd = onAdd; 66 | return this; 67 | } 68 | 69 | public Builder withOnDelete(BiConsumer onDelete) { 70 | this.onDelete = onDelete; 71 | return this; 72 | } 73 | 74 | public Builder withOnModify(BiConsumer onModify) { 75 | this.onModify = onModify; 76 | return this; 77 | } 78 | 79 | public Builder withPredicate(Predicate predicate) { 80 | this.predicate = predicate; 81 | return this; 82 | } 83 | 84 | public Builder withConvert(Function convert) { 85 | this.convert = convert; 86 | return this; 87 | } 88 | 89 | public ConfigMapWatcher build() { 90 | if (!registered) { 91 | io.fabric8.kubernetes.internal.KubernetesDeserializer.registerCustomKind("v1#ConfigMap", ConfigMap.class); 92 | registered = true; 93 | } 94 | return new ConfigMapWatcher(namespace, entityName, client, selector, onAdd, onDelete, onModify, predicate, convert); 95 | } 96 | } 97 | 98 | public static T defaultConvert(Class clazz, ConfigMap cm) { 99 | return HasDataHelper.parseCM(clazz, cm); 100 | } 101 | 102 | @Override 103 | public CompletableFuture> watch() { 104 | return createConfigMapWatch().thenApply(watch -> this); 105 | } 106 | } 107 | 108 | 109 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/CustomResourceWatcher.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; 5 | import io.fabric8.kubernetes.client.KubernetesClient; 6 | import io.radanalytics.operator.common.crd.InfoClass; 7 | 8 | import java.util.concurrent.CompletableFuture; 9 | import java.util.function.BiConsumer; 10 | import java.util.function.Function; 11 | 12 | import static io.radanalytics.operator.common.OperatorConfig.ALL_NAMESPACES; 13 | 14 | public class CustomResourceWatcher extends AbstractWatcher { 15 | 16 | // use via builder 17 | private CustomResourceWatcher(String namespace, 18 | String entityName, 19 | KubernetesClient client, 20 | CustomResourceDefinition crd, 21 | BiConsumer onAdd, 22 | BiConsumer onDelete, 23 | BiConsumer onModify, 24 | Function convert) { 25 | super(true, namespace, entityName, client, crd, null, onAdd, onDelete, onModify, null, null, convert); 26 | } 27 | 28 | public static class Builder { 29 | private String namespace = ALL_NAMESPACES; 30 | private String entityName; 31 | private KubernetesClient client; 32 | private CustomResourceDefinition crd; 33 | 34 | private BiConsumer onAdd; 35 | private BiConsumer onDelete; 36 | private BiConsumer onModify; 37 | private Function convert; 38 | 39 | public Builder withNamespace(String namespace) { 40 | this.namespace = namespace; 41 | return this; 42 | } 43 | 44 | public Builder withEntityName(String entityName) { 45 | this.entityName = entityName; 46 | return this; 47 | } 48 | 49 | public Builder withClient(KubernetesClient client) { 50 | this.client = client; 51 | return this; 52 | } 53 | 54 | public Builder withCrd(CustomResourceDefinition crd) { 55 | this.crd = crd; 56 | return this; 57 | } 58 | 59 | public Builder withOnAdd(BiConsumer onAdd) { 60 | this.onAdd = onAdd; 61 | return this; 62 | } 63 | 64 | public Builder withOnDelete(BiConsumer onDelete) { 65 | this.onDelete = onDelete; 66 | return this; 67 | } 68 | 69 | public Builder withOnModify(BiConsumer onModify) { 70 | this.onModify = onModify; 71 | return this; 72 | } 73 | 74 | public Builder withConvert(Function convert) { 75 | this.convert = convert; 76 | return this; 77 | } 78 | 79 | public CustomResourceWatcher build() { 80 | return new CustomResourceWatcher(namespace, entityName, client, crd, onAdd, onDelete, onModify, convert); 81 | } 82 | } 83 | 84 | public static T defaultConvert(Class clazz, InfoClass info) { 85 | String name = info.getMetadata().getName(); 86 | String namespace = info.getMetadata().getNamespace(); 87 | ObjectMapper mapper = new ObjectMapper(); 88 | T infoSpec = mapper.convertValue(info.getSpec(), clazz); 89 | if (infoSpec == null) { // empty spec 90 | try { 91 | infoSpec = clazz.newInstance(); 92 | } catch (InstantiationException e) { 93 | e.printStackTrace(); 94 | } catch (IllegalAccessException e) { 95 | e.printStackTrace(); 96 | } 97 | } 98 | if (infoSpec.getName() == null) { 99 | infoSpec.setName(name); 100 | } 101 | if (infoSpec.getNamespace() == null) { 102 | infoSpec.setNamespace(namespace); 103 | } 104 | return infoSpec; 105 | } 106 | 107 | @Override 108 | public CompletableFuture> watch() { 109 | return createCustomResourceWatch().thenApply(watch -> this); 110 | } 111 | } 112 | 113 | 114 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/EntityInfo.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * Simple abstract class that captures the information about the object we are interested in in the Kubernetes cluster. 7 | * Field called 'name' is the only compulsory information and it represents the name of the configmap. 8 | * 9 | * By extending this class and adding new fields to it, you can create a rich configuration object. The structure 10 | * of this object will be expected in the watched config maps and there are also some prepared method for YAML -> 11 | * 'T extends EntityInfo' conversions prepared in 12 | * {@link io.radanalytics.operator.resource.HasDataHelper#parseCM(Class, io.fabric8.kubernetes.api.model.ConfigMap)}. 13 | */ 14 | public abstract class EntityInfo { 15 | protected String name; 16 | protected String namespace; 17 | 18 | public void setName(String name) { 19 | this.name = name; 20 | } 21 | 22 | public String getName() { 23 | return name; 24 | } 25 | 26 | public void setNamespace(String namespace) { 27 | this.namespace = namespace; 28 | } 29 | 30 | public String getNamespace() { 31 | return namespace; 32 | } 33 | 34 | @Override 35 | public boolean equals(Object o) { 36 | if (this == o) return true; 37 | if (o == null || getClass() != o.getClass()) return false; 38 | EntityInfo that = (EntityInfo) o; 39 | return Objects.equals(name, that.name) && Objects.equals(namespace, that.namespace); 40 | } 41 | 42 | @Override 43 | public int hashCode() { 44 | return Objects.hash(name, namespace); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/JSONSchemaReader.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import com.fasterxml.jackson.databind.DeserializationFeature; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import io.fabric8.kubernetes.api.model.apiextensions.v1.JSONSchemaProps; 6 | 7 | import java.io.IOException; 8 | import java.net.URL; 9 | 10 | public class JSONSchemaReader { 11 | 12 | public static JSONSchemaProps readSchema(Class infoClass) { 13 | ObjectMapper mapper = new ObjectMapper(); 14 | mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 15 | char[] chars = infoClass.getSimpleName().toCharArray(); 16 | chars[0] = Character.toLowerCase(chars[0]); 17 | String urlJson = "/schema/" + new String(chars) + ".json"; 18 | String urlJS = "/schema/" + new String(chars) + ".js"; 19 | URL in = infoClass.getResource(urlJson); 20 | if (null == in) { 21 | // try also if .js file exists 22 | in = infoClass.getResource(urlJS); 23 | } 24 | if (null == in) { 25 | return null; 26 | } 27 | try { 28 | return mapper.readValue(in, JSONSchemaProps.class); 29 | } catch (IOException e) { 30 | e.printStackTrace(); 31 | return null; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/LogProducer.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import javax.enterprise.context.Dependent; 7 | import javax.enterprise.inject.Produces; 8 | import javax.enterprise.inject.spi.InjectionPoint; 9 | 10 | @Dependent 11 | class LogProducer { 12 | 13 | @Produces 14 | Logger createLogger(InjectionPoint injectionPoint) { 15 | return LoggerFactory.getLogger(injectionPoint.getMember().getDeclaringClass().getName()); 16 | } 17 | } -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/Operator.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import java.lang.annotation.ElementType; 4 | import java.lang.annotation.Retention; 5 | import java.lang.annotation.RetentionPolicy; 6 | import java.lang.annotation.Target; 7 | 8 | @Target(ElementType.TYPE) 9 | @Retention(RetentionPolicy.RUNTIME) 10 | public @interface Operator { 11 | Class forKind(); 12 | String named() default ""; 13 | String prefix() default ""; 14 | String[] shortNames() default {}; 15 | String pluralName() default ""; 16 | boolean enabled() default true; 17 | boolean crd() default true; 18 | String[] additionalPrinterColumnNames() default {}; 19 | String[] additionalPrinterColumnPaths() default {}; 20 | String[] additionalPrinterColumnTypes() default {}; 21 | } 22 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/OperatorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017-2018 3 | * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). 4 | */ 5 | package io.radanalytics.operator.common; 6 | 7 | import java.util.Collections; 8 | import java.util.HashSet; 9 | import java.util.Map; 10 | import java.util.Set; 11 | import java.util.stream.Collectors; 12 | 13 | import static java.util.Arrays.asList; 14 | 15 | /** 16 | * Operator configuration 17 | */ 18 | public class OperatorConfig { 19 | 20 | public static final String WATCH_NAMESPACE = "WATCH_NAMESPACE"; 21 | public static final String SAME_NAMESPACE = "~"; 22 | public static final String ALL_NAMESPACES = "*"; 23 | public static final String METRICS = "METRICS"; 24 | public static final String METRICS_JVM = "METRICS_JVM"; 25 | public static final String METRICS_PORT = "METRICS_PORT"; 26 | public static final String FULL_RECONCILIATION_INTERVAL_S = "FULL_RECONCILIATION_INTERVAL_S"; 27 | public static final String OPERATOR_OPERATION_TIMEOUT_MS = "OPERATOR_OPERATION_TIMEOUT_MS"; 28 | 29 | public static final boolean DEFAULT_METRICS = true; 30 | public static final boolean DEFAULT_METRICS_JVM = false; 31 | public static final int DEFAULT_METRICS_PORT = 8080; 32 | public static final long DEFAULT_FULL_RECONCILIATION_INTERVAL_S = 180; 33 | public static final long DEFAULT_OPERATION_TIMEOUT_MS = 60_000; 34 | 35 | private final Set namespaces; 36 | private final boolean metrics; 37 | private final boolean metricsJvm; 38 | private final int metricsPort; 39 | private final long reconciliationIntervalS; 40 | private final long operationTimeoutMs; 41 | 42 | /** 43 | * Constructor 44 | * 45 | * @param namespaces namespace in which the operator will run and create resources 46 | * @param metrics whether the metrics server for prometheus should be started 47 | * @param metricsJvm whether to expose the internal JVM metrics, like heap, # of threads, etc. 48 | * @param metricsPort on which port the metrics server should be listening 49 | * @param reconciliationIntervalS specify every how many milliseconds the reconciliation runs 50 | * @param operationTimeoutMs timeout for internal operations specified in milliseconds 51 | */ 52 | public OperatorConfig(Set namespaces, boolean metrics, boolean metricsJvm, int metricsPort, 53 | long reconciliationIntervalS, long operationTimeoutMs) { 54 | this.namespaces = namespaces; 55 | this.reconciliationIntervalS = reconciliationIntervalS; 56 | this.operationTimeoutMs = operationTimeoutMs; 57 | this.metrics = metrics; 58 | this.metricsJvm = metricsJvm; 59 | this.metricsPort = metricsPort; 60 | } 61 | 62 | /** 63 | * Loads configuration parameters from a related map 64 | * 65 | * @param map map from which loading configuration parameters 66 | * @return Cluster Operator configuration instance 67 | */ 68 | public static OperatorConfig fromMap(Map map) { 69 | 70 | String namespacesList = map.get(WATCH_NAMESPACE); 71 | 72 | Set namespaces; 73 | if (namespacesList == null || namespacesList.isEmpty()) { 74 | // empty WATCH_NAMESPACE means we will be watching all the namespaces 75 | namespaces = Collections.singleton(ALL_NAMESPACES); 76 | } else { 77 | namespaces = new HashSet<>(asList(namespacesList.trim().split("\\s*,+\\s*"))); 78 | namespaces = namespaces.stream().map( 79 | ns -> ns.startsWith("\"") && ns.endsWith("\"") ? ns.substring(1, ns.length() - 1) : ns) 80 | .collect(Collectors.toSet()); 81 | } 82 | 83 | boolean metricsAux = DEFAULT_METRICS; 84 | String metricsEnvVar = map.get(METRICS); 85 | if (metricsEnvVar != null) { 86 | metricsAux = !"false".equals(metricsEnvVar.trim().toLowerCase()); 87 | } 88 | 89 | boolean metricsJvmAux = DEFAULT_METRICS_JVM; 90 | int metricsPortAux = DEFAULT_METRICS_PORT; 91 | if (metricsAux) { 92 | String metricsJvmEnvVar = map.get(METRICS_JVM); 93 | if (metricsJvmEnvVar != null) { 94 | metricsJvmAux = "true".equals(metricsJvmEnvVar.trim().toLowerCase()); 95 | } 96 | String metricsPortEnvVar = map.get(METRICS_PORT); 97 | if (metricsPortEnvVar != null) { 98 | metricsPortAux = Integer.parseInt(metricsPortEnvVar.trim().toLowerCase()); 99 | } 100 | } 101 | 102 | long reconciliationInterval = DEFAULT_FULL_RECONCILIATION_INTERVAL_S; 103 | String reconciliationIntervalEnvVar = map.get(FULL_RECONCILIATION_INTERVAL_S); 104 | if (reconciliationIntervalEnvVar != null) { 105 | reconciliationInterval = Long.parseLong(reconciliationIntervalEnvVar); 106 | } 107 | 108 | long operationTimeout = DEFAULT_OPERATION_TIMEOUT_MS; 109 | String operationTimeoutEnvVar = map.get(OPERATOR_OPERATION_TIMEOUT_MS); 110 | if (operationTimeoutEnvVar != null) { 111 | operationTimeout = Long.parseLong(operationTimeoutEnvVar); 112 | } 113 | 114 | return new OperatorConfig(namespaces, metricsAux, metricsJvmAux, metricsPortAux, reconciliationInterval, 115 | operationTimeout); 116 | } 117 | 118 | 119 | /** 120 | * @return namespaces in which the operator runs and creates resources 121 | */ 122 | public Set getNamespaces() { 123 | return namespaces; 124 | } 125 | 126 | /** 127 | * @return how many seconds among the reconciliation runs 128 | */ 129 | public long getReconciliationIntervalS() { 130 | return reconciliationIntervalS; 131 | } 132 | 133 | /** 134 | * @return how many milliseconds should we wait for Kubernetes operations 135 | */ 136 | public long getOperationTimeoutMs() { 137 | return operationTimeoutMs; 138 | } 139 | 140 | public boolean isMetrics() { 141 | return metrics; 142 | } 143 | 144 | public boolean isMetricsJvm() { 145 | return metricsJvm; 146 | } 147 | 148 | public int getMetricsPort() { 149 | return metricsPort; 150 | } 151 | 152 | @Override 153 | public String toString() { 154 | return "OperatorConfig{" + 155 | "namespaces=" + namespaces + 156 | ", metrics=" + metrics + 157 | ", metricsJvm=" + metricsJvm + 158 | ", metricsPort=" + metricsPort + 159 | ", reconciliationIntervalS=" + reconciliationIntervalS + 160 | ", operationTimeoutMs=" + operationTimeoutMs + 161 | '}'; 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/ProcessRunner.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.IOException; 8 | import java.io.InputStreamReader; 9 | import java.util.Arrays; 10 | 11 | import static io.radanalytics.operator.common.AnsiColors.*; 12 | 13 | /** 14 | * Helper class that can be used from the concrete operators as the glue code for running the OS process. 15 | */ 16 | public class ProcessRunner { 17 | private static final Logger log = LoggerFactory.getLogger(AbstractOperator.class.getName()); 18 | 19 | public static void runPythonScript(String path) { 20 | runCommand("python3 " + path); 21 | } 22 | 23 | public static void runShellScript(String path) { 24 | runCommand(path); 25 | } 26 | 27 | 28 | public static void runCommand(String command) { 29 | try { 30 | String[] commands = new String[] {"sh", "-c", "\"" + command + "\""}; 31 | log.info("running: {}", Arrays.toString(commands)); 32 | Process p = Runtime.getRuntime().exec(commands); 33 | BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); 34 | BufferedReader err = new BufferedReader(new InputStreamReader(p.getErrorStream())); 35 | String line; 36 | StringBuilder sb = new StringBuilder(); 37 | while ((line = in.readLine()) != null) { 38 | sb.append(line + "\n"); 39 | } 40 | String stdOutput = sb.toString(); 41 | if (!stdOutput.isEmpty()) { 42 | log.info("{}{}{}", gr(), stdOutput, xx()); 43 | } 44 | in.close(); 45 | 46 | sb = new StringBuilder(); 47 | while ((line = err.readLine()) != null) { 48 | sb.append(line + "\n"); 49 | } 50 | String errOutput = sb.toString(); 51 | if (!errOutput.isEmpty()) { 52 | log.error("{}{}{}", re(), stdOutput, xx()); 53 | } 54 | err.close(); 55 | } catch (IOException e) { 56 | log.error("Running '{}' failed with: {}", command, e.getMessage()); 57 | e.printStackTrace(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/crd/CrdDeployer.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common.crd; 2 | 3 | import com.fasterxml.jackson.databind.DeserializationFeature; 4 | import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; 5 | import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionBuilder; 6 | import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionFluent; 7 | import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceSubresourceStatus; 8 | import io.fabric8.kubernetes.api.model.apiextensions.v1.JSONSchemaProps; 9 | import io.fabric8.kubernetes.client.CustomResourceList; 10 | import io.fabric8.kubernetes.client.KubernetesClient; 11 | import io.fabric8.kubernetes.client.KubernetesClientException; 12 | import io.fabric8.kubernetes.client.utils.Serialization; 13 | import io.radanalytics.operator.common.EntityInfo; 14 | import io.radanalytics.operator.common.JSONSchemaReader; 15 | import org.slf4j.Logger; 16 | 17 | import javax.inject.Inject; 18 | import javax.inject.Singleton; 19 | import java.util.Arrays; 20 | import java.util.List; 21 | import java.util.stream.Collectors; 22 | 23 | @Singleton 24 | public class CrdDeployer { 25 | 26 | @Inject 27 | protected Logger log; 28 | 29 | public CustomResourceDefinition initCrds(KubernetesClient client, 30 | String prefix, 31 | String entityName, 32 | String[] shortNames, 33 | String pluralName, 34 | String[] additionalPrinterColumnNames, 35 | String[] additionalPrinterColumnPaths, 36 | String[] additionalPrinterColumnTypes, 37 | Class infoClass, 38 | boolean isOpenshift) { 39 | final String newPrefix = prefix.substring(0, prefix.length() - 1); 40 | CustomResourceDefinition crdToReturn; 41 | 42 | Serialization.jsonMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 43 | List crds = client.apiextensions().v1().customResourceDefinitions() 44 | .list() 45 | .getItems() 46 | .stream() 47 | .filter(p -> entityName.equals(p.getSpec().getNames().getKind()) && newPrefix.equals(p.getSpec().getGroup())) 48 | .collect(Collectors.toList()); 49 | if (!crds.isEmpty()) { 50 | crdToReturn = crds.get(0); 51 | log.info("CustomResourceDefinition for {} has been found in the K8s, so we are skipping the creation.", entityName); 52 | } else { 53 | log.info("Creating CustomResourceDefinition for {}.", entityName); 54 | JSONSchemaProps schema = JSONSchemaReader.readSchema(infoClass); 55 | CustomResourceDefinitionFluent.SpecNested builder; 56 | 57 | if (schema != null) { 58 | removeDefaultValues(schema); 59 | } 60 | builder = getCRDBuilder(newPrefix, 61 | entityName, 62 | shortNames, 63 | pluralName); 64 | crdToReturn = builder.endSpec().build(); 65 | try { 66 | client.apiextensions().v1().customResourceDefinitions().createOrReplace(crdToReturn); 67 | } catch (KubernetesClientException e) { 68 | // old version of K8s/openshift -> don't use schema validation 69 | log.warn("Consider upgrading the {}. Your version doesn't support schema validation for custom resources." 70 | , isOpenshift ? "OpenShift" : "Kubernetes"); 71 | crdToReturn = getCRDBuilder(newPrefix, 72 | entityName, 73 | shortNames, 74 | pluralName) 75 | .endSpec() 76 | .build(); 77 | client.apiextensions().v1().customResourceDefinitions().createOrReplace(crdToReturn); 78 | } 79 | } 80 | 81 | // register the new crd for json serialization 82 | io.fabric8.kubernetes.internal.KubernetesDeserializer.registerCustomKind(newPrefix + "/" + crdToReturn.getSpec().getVersions().get(0) + "#" + entityName, InfoClass.class); 83 | io.fabric8.kubernetes.internal.KubernetesDeserializer.registerCustomKind(newPrefix + "/" + crdToReturn.getSpec().getVersions().get(0) + "#" + entityName + "List", CustomResourceList.class); 84 | 85 | return crdToReturn; 86 | } 87 | 88 | private void removeDefaultValues(JSONSchemaProps schema) { 89 | if (null == schema) { 90 | return; 91 | } 92 | schema.setDefault(null); 93 | if (null != schema.getProperties()) { 94 | for (JSONSchemaProps prop : schema.getProperties().values()) { 95 | removeDefaultValues(prop); 96 | } 97 | } 98 | } 99 | 100 | private CustomResourceDefinitionFluent.SpecNested getCRDBuilder(String prefix, 101 | String entityName, 102 | String[] shortNames, 103 | String pluralName) { 104 | // if no plural name is specified, try to make one by adding "s" 105 | // also, plural names must be all lowercase 106 | String plural = pluralName; 107 | if (plural.isEmpty()) { 108 | plural = (entityName + "s"); 109 | } 110 | plural = plural.toLowerCase(); 111 | 112 | // short names must be all lowercase 113 | String[] shortNamesLower = Arrays.stream(shortNames) 114 | .map(sn -> sn.toLowerCase()) 115 | .toArray(String[]::new); 116 | 117 | return new CustomResourceDefinitionBuilder() 118 | .withApiVersion("apiextensions.k8s.io/v1") 119 | .withNewMetadata().withName(plural + "." + prefix) 120 | .endMetadata() 121 | .withNewSpec() 122 | .withNewNames() 123 | .withKind(entityName) 124 | .withPlural(plural) 125 | .withShortNames(Arrays.asList(shortNamesLower)).endNames() 126 | .withGroup(prefix) 127 | .withScope("Namespaced"); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/crd/InfoClass.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common.crd; 2 | 3 | import io.fabric8.kubernetes.client.CustomResource; 4 | 5 | public class InfoClass extends CustomResource { 6 | private U spec; 7 | private InfoStatus status; 8 | 9 | public InfoClass() { 10 | this.status = new InfoStatus(); 11 | } 12 | 13 | public InfoStatus getStatus() { 14 | return this.status; 15 | } 16 | 17 | public void setStatus(InfoStatus status) { 18 | this.status = status; 19 | } 20 | 21 | public U getSpec() { 22 | return spec; 23 | } 24 | 25 | // public void setSpec(U spec) { 26 | // this.spec = spec; 27 | // } 28 | } 29 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/crd/InfoList.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common.crd; 2 | 3 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 4 | import io.fabric8.kubernetes.client.CustomResourceList; 5 | import io.fabric8.kubernetes.internal.KubernetesDeserializer; 6 | 7 | @JsonDeserialize(using = KubernetesDeserializer.class) 8 | public class InfoList extends CustomResourceList> { 9 | } -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/common/crd/InfoStatus.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.common.crd; 2 | import io.fabric8.kubernetes.api.model.KubernetesResource; 3 | 4 | 5 | import java.text.DateFormat; 6 | import java.text.SimpleDateFormat; 7 | import java.util.Date; 8 | import java.util.TimeZone; 9 | 10 | public class InfoStatus implements KubernetesResource { 11 | 12 | private String state; 13 | private String lastTransitionTime; 14 | 15 | private static String toDateString(Date date) { 16 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'kk:mm:ss'Z'"); 17 | df.setTimeZone(TimeZone.getTimeZone( "GMT" )); 18 | return df.format(date); 19 | } 20 | 21 | public InfoStatus() { 22 | super(); 23 | this.state = "initial"; 24 | this.lastTransitionTime = toDateString(new Date()); 25 | } 26 | 27 | public InfoStatus(String state, Date dt) { 28 | super(); 29 | this.state = state; 30 | this.lastTransitionTime = toDateString(dt); 31 | } 32 | 33 | public void setState(String s) { 34 | this.state = s; 35 | } 36 | 37 | public String getState() { 38 | return this.state; 39 | } 40 | 41 | public void setLastTransitionTime(Date dt) { 42 | this.lastTransitionTime = toDateString(dt); 43 | } 44 | 45 | public String getLastTransitionTime() { 46 | return this.lastTransitionTime; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "InfoStatus{" + 52 | " state=" + state + 53 | " lastTransitionTime=" + lastTransitionTime + 54 | "}"; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/resource/HasDataHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 3 | * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). 4 | */ 5 | package io.radanalytics.operator.resource; 6 | 7 | import io.fabric8.kubernetes.api.model.ConfigMap; 8 | import io.radanalytics.operator.common.EntityInfo; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.yaml.snakeyaml.LoaderOptions; 12 | import org.yaml.snakeyaml.Yaml; 13 | import org.yaml.snakeyaml.constructor.Constructor; 14 | import org.yaml.snakeyaml.error.YAMLException; 15 | 16 | /** 17 | * A helper for parsing the data section inside the K8s resource (ConfigMap). 18 | * Type parameter T represents the concrete EntityInfo that captures the configuration obout the 19 | * objects in the clusters we are interested in, be it spark clusters, http servers, certificates, etc. 20 | * 21 | * One can create arbitrarily deep configurations by nesting the types in Class<T> and using 22 | * the Snake yaml or other library as for conversions between YAML and Java objects. 23 | */ 24 | public class HasDataHelper { 25 | private static final Logger log = LoggerFactory.getLogger(HasDataHelper.class.getName()); 26 | 27 | public static T parseYaml(Class clazz, String yamlDoc, String name) { 28 | 29 | LoaderOptions options = new LoaderOptions(); 30 | Yaml snake = new Yaml(new Constructor(clazz)); 31 | T entity = null; 32 | try { 33 | entity = snake.load(yamlDoc); 34 | } catch (YAMLException ex) { 35 | String msg = "Unable to parse yaml definition of configmap, check if you don't have typo: \n'\n" + 36 | yamlDoc + "\n'\n"; 37 | log.error(msg); 38 | throw new IllegalStateException(ex); 39 | } 40 | if (entity == null) { 41 | String msg = "Unable to parse yaml definition of configmap, check if you don't have typo: \n'\n" + 42 | yamlDoc + "\n'\n"; 43 | log.error(msg); 44 | try { 45 | entity = clazz.newInstance(); 46 | } catch (InstantiationException e) { 47 | e.printStackTrace(); 48 | } catch (IllegalAccessException e) { 49 | e.printStackTrace(); 50 | } 51 | } 52 | if (entity != null && entity.getName() == null) { 53 | entity.setName(name); 54 | } 55 | return entity; 56 | } 57 | 58 | /** 59 | * 60 | * @param clazz concrete class of type T that extends the EntityInfo. 61 | * This is the resulting type, we are convertion into. 62 | * @param cm input config map that will be converted into T. 63 | * We assume there is a multi-line section in the config map called config and it 64 | * contains a YAML structure that represents the object of type T. In other words the 65 | * keys in the yaml should be the same as the field names in the class T and the name of the 66 | * configmap will be assigned to the name of the object T. One can create arbitrarily deep 67 | * configuration by nesting the types in T and using the Snake yaml as the conversion library. 68 | * @param type parameter (T must extend {@link io.radanalytics.operator.common.EntityInfo}) 69 | * @return Java object of type T 70 | */ 71 | public static T parseCM(Class clazz, ConfigMap cm) { 72 | String yaml = cm.getData().get("config"); 73 | T entity = parseYaml(clazz, yaml, cm.getMetadata().getName()); 74 | return entity; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/resource/LabelsHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 3 | * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). 4 | */ 5 | package io.radanalytics.operator.resource; 6 | 7 | import io.fabric8.kubernetes.api.model.HasMetadata; 8 | 9 | import java.util.Collections; 10 | import java.util.Map; 11 | import java.util.Optional; 12 | 13 | /** 14 | * A helper for parsing the {@code metadata.labels} section inside the K8s resource 15 | */ 16 | public class LabelsHelper { 17 | 18 | /** 19 | * The kind of a ConfigMap: 20 | *
    21 | *
  • {@code radanalytics.io/kind=cluster} 22 | * identifies a ConfigMap that is intended to be consumed by 23 | * the cluster operator.
  • 24 | *
  • {@code radanalytics.io/kind=app} 25 | * identifies a ConfigMap that is intended to be consumed 26 | * by the app operator.
  • 27 | *
  • {@code radanalytics.io/kind=notebook} 28 | * identifies a ConfigMap that is intended to be consumed 29 | * by the notebook operator.
  • 30 | *
31 | */ 32 | public static final String OPERATOR_KIND_LABEL = "kind"; 33 | 34 | public static final String OPERATOR_SEVICE_TYPE_LABEL = "service"; 35 | public static final String OPERATOR_RC_TYPE_LABEL = "rcType"; 36 | public static final String OPERATOR_POD_TYPE_LABEL = "podType"; 37 | public static final String OPERATOR_DEPLOYMENT_LABEL = "deployment"; 38 | 39 | public static final Optional getKind(HasMetadata resource, String prefix) { 40 | return Optional.ofNullable(resource) 41 | .map(r -> r.getMetadata()) 42 | .map(m -> m.getLabels()) 43 | .map(l -> l.get(prefix + OPERATOR_KIND_LABEL)); 44 | } 45 | 46 | public static Map forKind(String kind, String prefix) { 47 | return Collections.singletonMap(prefix + OPERATOR_KIND_LABEL, kind); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /abstract-operator/src/main/java/io/radanalytics/operator/resource/ResourceHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 3 | * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html). 4 | */ 5 | package io.radanalytics.operator.resource; 6 | 7 | import io.fabric8.kubernetes.api.model.ConfigMap; 8 | 9 | import java.util.Optional; 10 | 11 | /** 12 | * A helper for parsing the top-lvl section inside the K8s resource 13 | */ 14 | public class ResourceHelper { 15 | 16 | /** 17 | * Returns the value of the {@code metadata.name} of the given {@code cm}. 18 | * 19 | * @param cm config map object 20 | * @return the name 21 | */ 22 | public static String name(ConfigMap cm) { 23 | return cm.getMetadata().getName(); 24 | } 25 | 26 | public static boolean isAKind(ConfigMap cm, String kind, String prefix) { 27 | return LabelsHelper.getKind(cm, prefix).map(k -> kind.equals(k)).orElse(false); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /abstract-operator/src/main/javadoc/overview.html: -------------------------------------------------------------------------------- 1 | 2 |
3 |

abstract-operator 9 |

10 |

Build status 15 | License

19 |

{ConfigMap|CRD}-based approach for lyfecycle management of various resources in Kubernetes and 20 | OpenShift. Using the Operator pattern, you can leverage the Kubernetes control loop and react on various events 21 | in the cluster.

22 |

Example Implementations 28 |

29 | 40 |

Code 46 |

47 |

This library can be simply used by adding it to classpath; creating a new class that extends AbstractOperator. 48 | This 'concrete operator' class needs to also have the @Operator annotation on it. For capturing the 49 | information about the monitored resources one has to also create a class that extends EntityInfo 50 | and have arbitrary fields on it with getters and setters.

51 |

This class can be also generated from the JSON schema. To do that add jsonschema2pojo 53 | plugin to the pom.xml and json schema to resources (example). 55 |

56 |

This is a no-op operator in Scala that simply logs into console when config map with label radanalytics.io/kind 57 | = foo is created.

58 |
@Operator(forKind = "foo", prefix = "radanalytics.io", infoClass = classOf[FooInfo])
 63 | class FooOperator extends AbstractOperator[FooInfo] {
 65 |   val log: Logger = LoggerFactory.getLogger(classOf[FooInfo].getName)
 68 | 
 69 |   @Override
 70 |   def onAdd(foo: FooInfo) = {
 72 |     log.info(s"created foo with name ${foo.name} and someParameter = ${foo.someParameter}")
 74 |   }
 75 | 
 76 |   @Override
 77 |   def onDelete(foo: FooInfo) = {
 79 |     log.info(s"deleted foo with name ${foo.name} and someParameter = ${foo.someParameter}")
 81 |   }
 82 | }
83 |
84 |

CRDs 90 |

91 |

By default the operator is based on CustomResourceDefinitions, if you want to create ConfigMap-based 92 | operator, add crd=false parameter in the @Operator annotation. Custom Resource operation mode will try to create 93 | the custom resource definition from the infoClass if it's not already there and then it listens on 94 | the newly created, deleted or modified custom resources (CR) of the given type.

95 |

For the CRDs the:

96 |
    97 |
  • forKind field represent the name of the CRD ('s' at the end for plular)
  • 98 |
  • pluralName explicitly se the plural name of the CR
  • 99 |
  • prefix field in the annotation represents the group
  • 100 |
  • shortNames short names that can be used instead of the long version (service -> svc, namespace -> ns, etc.)
  • 101 |
  • enabled by default it's enabled, but one may want to temporarily disable/silence the operator
  • 102 |
  • as for the version, currently the v1 is created automatically, but one can also create the 103 | CRD on his own before running the operator and providing the forKind and prefix 104 | matches, operator will use the existing CRD
  • 105 |
106 |

Documentation 112 |

113 |
114 | 115 | -------------------------------------------------------------------------------- /abstract-operator/src/main/resources/META-INF/beans.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | -------------------------------------------------------------------------------- /abstract-operator/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | 10 | log4j.logger.org.reflections=FATAL -------------------------------------------------------------------------------- /abstract-operator/src/main/resources/schema/test.js: -------------------------------------------------------------------------------- 1 | { 2 | "type":"object", 3 | "properties": { 4 | "foo": { 5 | "type": "string" 6 | }, 7 | "bar": { 8 | "type": "integer" 9 | }, 10 | "baz": { 11 | "type": "boolean" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /annotator/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.radanalytics 7 | spark-operator-parent 8 | 1.0.0 9 | 10 | io.radanalytics 11 | spark-operator-annotator 12 | 1.0.0 13 | 14 | scm:git:git@github.com:jvm-operators/abstract-operator.git 15 | scm:git:git@github.com:jvm-operators/abstract-operator.git 16 | https://github.com/jvm-operators/abstract-operator 17 | 18 | 19 | 20 | UTF-8 21 | 22 | 23 | 24 | org.jsonschema2pojo 25 | jsonschema2pojo-core 26 | 0.4.0 27 | 28 | 29 | io.quarkus 30 | quarkus-core 31 | 32 | 33 | 34 | 35 | sonatype-releases 36 | https://oss.sonatype.org/content/repositories/releases 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /annotator/src/main/java/io/radanalytics/operator/annotator/RegisterForReflectionAnnotator.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.annotator; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.sun.codemodel.JDefinedClass; 5 | import org.jsonschema2pojo.AbstractAnnotator; 6 | import io.quarkus.runtime.annotations.RegisterForReflection; 7 | 8 | public class RegisterForReflectionAnnotator extends AbstractAnnotator { 9 | 10 | @Override 11 | public void propertyOrder(JDefinedClass clazz, JsonNode propertiesNode) { 12 | super.propertyOrder(clazz, propertiesNode); 13 | clazz.annotate(RegisterForReflection.class); 14 | } 15 | } -------------------------------------------------------------------------------- /docs/ascii.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/radanalyticsio/spark-operator/0fe4f9f97f6c82b9d94bd0189c01256d4d8b1662/docs/ascii.gif -------------------------------------------------------------------------------- /docs/standardized-UML-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/radanalyticsio/spark-operator/0fe4f9f97f6c82b9d94bd0189c01256d4d8b1662/docs/standardized-UML-diagram.png -------------------------------------------------------------------------------- /examples/app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkApplication 3 | metadata: 4 | name: my-spark-app 5 | spec: 6 | image: quay.io/radanalyticsio/openshift-spark:2.4.5-2 7 | mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.4.5.jar 8 | mainClass: org.apache.spark.examples.SparkPi 9 | sleep: 300 # repeat each 5 minutes 10 | driver: 11 | cores: 0.2 12 | coreLimit: 200m 13 | executor: 14 | instances: 2 15 | cores: 1 16 | coreLimit: 400m 17 | labels: 18 | foo: bar 19 | -------------------------------------------------------------------------------- /examples/apps/pyspark-ntlk.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkApplication 3 | metadata: 4 | name: ntlk-example 5 | spec: 6 | image: jkremser/spark-operator:2.4.0-ntlk 7 | type: Python 8 | mainApplicationFile: local:///app.py 9 | deps: 10 | pyFiles: 11 | - local:///deps.zip 12 | driver: 13 | cores: 0.4 14 | coreLimit: 400m 15 | memory: 600m 16 | -------------------------------------------------------------------------------- /examples/cluster-cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "2" 11 | master: 12 | instances: "1" 13 | -------------------------------------------------------------------------------- /examples/cluster-with-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: sparky-cluster # compulsory 5 | spec: 6 | mavenDependencies: # optional: array of Maven resources identified by GAVs (groupId:artifactId:version) 7 | - com.amazonaws:aws-java-sdk-pom:1.10.34 8 | - org.apache.hadoop:hadoop-aws:2.7.3 9 | worker: 10 | instances: "2" # optional defaults to 1 11 | memory: "1Gi" # optional no defaults 12 | cpu: 0.2 # optional no defaults 13 | master: 14 | instances: "1" # optional defaults to 1 15 | memory: "1Gi" # optional no defaults 16 | cpu: 0.2 # optional no defaults 17 | customImage: quay.io/radanalyticsio/openshift-spark:2.4-latest # optional defaults to quay.io/radanalyticsio/openshift-spark:2.4-latest 18 | metrics: false # on each pod expose the metrics endpoint on port 7777 for prometheus, defautls to false 19 | sparkWebUI: true # create a service with the Spark UI, defaults to true 20 | env: # optional 21 | - name: SPARK_WORKER_CORES 22 | value: 2 23 | - name: FOO 24 | value: bar 25 | sparkConfigurationMap: my-config # optional defaults to ${name}-config 26 | # kubectl create configmap my-config --from-file=example/config.conf 27 | sparkConfiguration: # optional, it overrides the config defined above 28 | - name: spark.executor.memory 29 | value: 500m 30 | - name: spark.sql.conf.autoBroadcastJoinThreshold 31 | value: 20971520 32 | downloadData: # optional, it downloads these files to each node 33 | - url: https://raw.githubusercontent.com/radanalyticsio/spark-operator/master/README.md 34 | to: /tmp/ 35 | -------------------------------------------------------------------------------- /examples/cluster.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster 5 | spec: 6 | worker: 7 | instances: "2" 8 | master: 9 | instances: "1" 10 | -------------------------------------------------------------------------------- /examples/spark-defaults.conf: -------------------------------------------------------------------------------- 1 | spark.eventLog.enabled true 2 | spark.eventLog.dir file:/tmp/spark-events 3 | spark.executor.extraJavaOptions -XX:+PrintGCDetails 4 | spark.history.fs.logDirectory file:/tmp/spark-events 5 | spark.history.retainedApplications 100 6 | -------------------------------------------------------------------------------- /examples/test/cluster-1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-1 5 | spec: 6 | # this should fail 7 | w0rker: 8 | instances: "1" 9 | master: 10 | instances: "1" 11 | -------------------------------------------------------------------------------- /examples/test/cluster-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-2 5 | spec: 6 | worker: 7 | instances: "2" 8 | master: 9 | instances: "1" 10 | -------------------------------------------------------------------------------- /examples/test/cluster-cm-with-labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: spark-cluster-with-labels 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "2" 11 | labels: 12 | example-label-for-all-workers/bar: foo 13 | common-label-to-be-replaced-on-some-resources: worker-value 14 | master: 15 | instances: "1" 16 | labels: 17 | example-label-for-master/foo: bar 18 | common-label-to-be-replaced-on-some-resources: master-value 19 | labels: 20 | common-label-for-all-the-resources-operator-deploys/deployed-by: john 21 | common-label-to-be-replaced-on-some-resources: global-value 22 | -------------------------------------------------------------------------------- /examples/test/cluster-cpumem.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-cpumem 5 | spec: 6 | worker: 7 | instances: "1" 8 | cpu: 100m 9 | memory: 200m 10 | master: 11 | instances: "1" 12 | -------------------------------------------------------------------------------- /examples/test/cluster-limits.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-limits 5 | spec: 6 | worker: 7 | instances: "1" 8 | cpuLimit: 100m 9 | memoryLimit: 200m 10 | master: 11 | instances: "1" 12 | -------------------------------------------------------------------------------- /examples/test/cluster-limreq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-limreq 5 | spec: 6 | worker: 7 | instances: "1" 8 | cpuLimit: 125m 9 | memoryLimit: 225m 10 | cpuRequest: 100m 11 | memoryRequest: 200m 12 | master: 13 | instances: "1" 14 | -------------------------------------------------------------------------------- /examples/test/cluster-node-tolerations.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: spark-cluster-with-tolerations 5 | spec: 6 | nodeTolerations: 7 | - key: foo_key 8 | operator: Equal 9 | value: foo_value 10 | effect: NoExecute 11 | tolerationSeconds: 60 12 | - key: bar_key 13 | operator: Equal 14 | value: bar_value 15 | effect: NoSchedule 16 | worker: 17 | instances: "1" 18 | master: 19 | instances: "1" 20 | -------------------------------------------------------------------------------- /examples/test/cluster-overwritelim.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-overwritelim 5 | spec: 6 | worker: 7 | instances: "1" 8 | cpu: 150m 9 | memory: 250m 10 | cpuLimit: 175m 11 | memoryLimit: 275m 12 | master: 13 | instances: "1" 14 | -------------------------------------------------------------------------------- /examples/test/cluster-overwritereq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-overwritereq 5 | spec: 6 | worker: 7 | instances: "1" 8 | cpu: 150m 9 | memory: 250m 10 | cpuRequest: 100m 11 | memoryRequest: 200m 12 | master: 13 | instances: "1" 14 | -------------------------------------------------------------------------------- /examples/test/cluster-requests.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-requests 5 | spec: 6 | worker: 7 | instances: "1" 8 | cpuRequest: 150m 9 | memoryRequest: 250m 10 | master: 11 | instances: "1" 12 | -------------------------------------------------------------------------------- /examples/test/cluster-with-config-1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: sparky-cluster-1 5 | spec: 6 | sparkConfigurationMap: non-existent 7 | sparkConfiguration: 8 | - name: spark.executor.memory 9 | value: 1g 10 | downloadData: 11 | - url: https://raw.githubusercontent.com/radanalyticsio/spark-operator/master/README.md 12 | to: /tmp/ 13 | -------------------------------------------------------------------------------- /examples/test/cluster-with-config-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: sparky-cluster-2 5 | spec: 6 | sparkConfiguration: 7 | - name: spark.executor.memory 8 | value: 3g -------------------------------------------------------------------------------- /examples/test/cluster-with-config-3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: sparky-cluster-3 5 | spec: 6 | -------------------------------------------------------------------------------- /examples/test/cluster-with-labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: spark-cluster-with-labels 5 | spec: 6 | worker: 7 | instances: "2" 8 | labels: 9 | example-label-for-all-workers/bar: foo 10 | common-label-to-be-replaced-on-some-resources: worker-value 11 | master: 12 | instances: "1" 13 | labels: 14 | example-label-for-master/foo: bar 15 | common-label-to-be-replaced-on-some-resources: master-value 16 | labels: 17 | common-label-for-all-the-resources-operator-deploys/deployed-by: john 18 | common-label-to-be-replaced-on-some-resources: global-value 19 | -------------------------------------------------------------------------------- /examples/test/cm/app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-app 5 | labels: 6 | radanalytics.io/kind: SparkApplication 7 | data: 8 | config: |- 9 | image: quay.io/radanalyticsio/openshift-spark:2.4.5-2 10 | mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.4.5.jar 11 | mainClass: org.apache.spark.examples.SparkPi 12 | sleep: 300 # repeat each 5 minutes 13 | driver: 14 | cores: 0.2 15 | coreLimit: 200m 16 | executor: 17 | instances: 2 18 | cores: 1 19 | coreLimit: 400m 20 | labels: 21 | foo: bar 22 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-1 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | # this should fail 9 | config: |- 10 | w0rker: 11 | instances: "1" 12 | master: 13 | instances: "1" 14 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-2 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "2" -------------------------------------------------------------------------------- /examples/test/cm/cluster-cpumem.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-cpumem 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "1" 11 | cpu: 100m 12 | memory: 200m 13 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-limits.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-limits 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "1" 11 | cpuLimit: 100m 12 | memoryLimit: 200m 13 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-limreq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-limreq 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "1" 11 | cpuLimit: 125m 12 | memoryLimit: 225m 13 | cpuRequest: 100m 14 | memoryRequest: 200m 15 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-node-tolerations.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: spark-cluster-with-tolerations 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | nodeTolerations: 10 | - key: foo_key 11 | operator: Equal 12 | value: foo_value 13 | effect: NoExecute 14 | tolerationSeconds: 60 15 | - key: bar_key 16 | operator: Equal 17 | value: bar_value 18 | effect: NoSchedule 19 | worker: 20 | instances: "1" 21 | master: 22 | instances: "1" 23 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-overwritelim.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-overwritelim 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "1" 11 | cpu: 150m 12 | memory: 250m 13 | cpuLimit: 175m 14 | memoryLimit: 275m 15 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-overwritereq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-overwritereq 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "1" 11 | cpu: 150m 12 | memory: 250m 13 | cpuRequest: 100m 14 | memoryRequest: 200m 15 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-requests.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: my-spark-cluster-requests 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | worker: 10 | instances: "1" 11 | cpuRequest: 150m 12 | memoryRequest: 250m 13 | 14 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-with-config-1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: sparky-cluster-1 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | sparkConfigurationMap: non-existent 10 | sparkConfiguration: 11 | - name: spark.executor.memory 12 | value: 1g 13 | downloadData: 14 | - url: https://raw.githubusercontent.com/radanalyticsio/spark-operator/master/README.md 15 | to: /tmp/ 16 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-with-config-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: sparky-cluster-2 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | sparkConfiguration: 10 | - name: spark.executor.memory 11 | value: 3g 12 | -------------------------------------------------------------------------------- /examples/test/cm/cluster-with-config-3.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: sparky-cluster-3 5 | labels: 6 | radanalytics.io/kind: SparkCluster 7 | data: 8 | config: |- 9 | -------------------------------------------------------------------------------- /examples/test/history-server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/jkremser/openshift-spark:2.4.0 2 | ADD com.amazonaws_aws-java-sdk-1.7.4.jar /opt/spark/jars/ 3 | ADD org.apache.hadoop_hadoop-aws-2.7.3.jar /opt/spark/jars/ -------------------------------------------------------------------------------- /examples/test/history-server/HistoryServer.md: -------------------------------------------------------------------------------- 1 | # Using shared volume 2 | 3 | Assumptions: 4 | * Currently it's openshift only 5 | * Make sure the `/tmp/spark-events` on host is writable and readable by 'others', or use different directory in the PVs. 6 | * Spark 2.4 is installed locally and `spark-submit` is on `$PATH` 7 | * Spark operator is up and running 8 | 9 | ``` 10 | oc apply -f examples/test/history-server/sharedVolume/ 11 | ``` 12 | 13 | ``` 14 | oc get route 15 | ``` 16 | 17 | Open `http://my-history-server-default.127.0.0.1.nip.io/` or similar url in browser. 18 | 19 | ``` 20 | oc get pods -lradanalytics.io/podType=master -owide 21 | ``` 22 | 23 | Instead of `172.17.0.2`, use the correct ip from the command above 24 | ``` 25 | _jar_path=`type -P spark-submit | xargs dirname`/../examples/jars/spark-examples_*.jar 26 | spark-submit --master spark://172.17.0.2:7077 \ 27 | --conf spark.eventLog.enabled=true \ 28 | --conf spark.eventLog.dir=/tmp/spark-events/ \ 29 | --class org.apache.spark.examples.SparkPi \ 30 | --executor-memory 1G \ 31 | $_jar_path 42 32 | ``` 33 | 34 | 35 | # Using external object storage 36 | 37 | Assumptions: 38 | * Openshift is up and running (currently it's openshift only) 39 | * `aws` client is installed and configured on `$PATH` 40 | * Spark 2.4 is installed locally and `spark-submit` is on `$PATH` 41 | 42 | 43 | Deploy the operator (and wait for it to start): 44 | 45 | ``` 46 | oc login -u system:admin ; oc project default ; oc apply -f manifest/operator.yaml 47 | ``` 48 | 49 | Deploy ceph-nano, Minio or use S3 from Amazon directly: 50 | 51 | ``` 52 | oc --as system:admin adm policy add-scc-to-user anyuid system:serviceaccount:default:default 53 | oc apply -f examples/test/history-server/externalStorage/ 54 | ``` 55 | 56 | Configure the aws client: 57 | 58 | ``` 59 | aws configure 60 | AWS Access Key ID = foo 61 | AWS Secret Access Key = bar 62 | ``` 63 | 64 | Create new emtpy bucket for the event log called `my-history-server`: 65 | 66 | ``` 67 | oc expose pod ceph-nano-0 --type=NodePort 68 | _ceph=http://`oc get svc ceph-nano-0 --template={{.spec.clusterIP}}`:8000 69 | aws s3api create-bucket --bucket my-history-server --endpoint-url=$_ceph 70 | ``` 71 | 72 | Create some dummy file in the bucket (sparks needs the bucket to be non-empty from some reason): 73 | 74 | ``` 75 | aws s3 cp README.md s3://my-history-server/ --endpoint-url=$_ceph 76 | ``` 77 | 78 | 79 | ``` 80 | oc get pods -lradanalytics.io/podType=master -owide 81 | ``` 82 | 83 | Instead of `172.17.0.2`, use the correct ip from the command above 84 | 85 | ``` 86 | _jar_path=`type -P spark-submit | xargs dirname`/../examples/jars/spark-examples_*.jar 87 | spark-submit --master spark://172.17.0.4:7077 \ 88 | --packages com.amazonaws:aws-java-sdk-pom:1.10.34,org.apache.hadoop:hadoop-aws:2.7.3 \ 89 | --conf spark.eventLog.enabled=true \ 90 | --conf spark.eventLog.dir=s3a://my-history-server/ \ 91 | --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \ 92 | --conf spark.hadoop.fs.s3a.access.key=foo \ 93 | --conf spark.hadoop.fs.s3a.secret.key=bar \ 94 | --conf spark.hadoop.fs.s3a.endpoint=$_ceph \ 95 | --conf spark.driver.extraJavaOptions=-Dcom.amazonaws.services.s3.enableV4=true \ 96 | --class org.apache.spark.examples.SparkPi \ 97 | --executor-memory 1G \ 98 | $_jar_path 42 99 | ``` 100 | 101 | 102 | 103 | Delete the dummy file in the bucket (todo ugly): 104 | 105 | ``` 106 | aws s3 rm s3://my-history-server/README.md --endpoint-url=$_ceph 107 | ``` 108 | 109 | Check if the event has been written to the bucket: 110 | 111 | ``` 112 | aws s3 ls s3://my-history-server/ --endpoint-url=$_ceph 113 | ``` 114 | 115 | Check the history server on: 116 | 117 | ``` 118 | oc get route 119 | ``` 120 | 121 | Open `http://my-history-server-default.127.0.0.1.nip.io/` or similar url in browser. -------------------------------------------------------------------------------- /examples/test/history-server/externalStorage/ceph-nano.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: ceph-nano-services 7 | labels: 8 | app: ceph 9 | daemon: nano 10 | spec: 11 | ports: 12 | - name: cn-s3 13 | port: 80 14 | protocol: TCP 15 | targetPort: 8000 16 | type: LoadBalancer 17 | selector: 18 | app: ceph 19 | daemon: demo 20 | --- 21 | apiVersion: apps/v1 22 | kind: StatefulSet 23 | metadata: 24 | labels: 25 | app: ceph 26 | daemon: nano 27 | name: ceph-nano 28 | spec: 29 | replicas: 1 30 | serviceName: ceph-nano 31 | selector: 32 | matchLabels: 33 | app: ceph 34 | template: 35 | metadata: 36 | name: ceph-nano 37 | labels: 38 | app: ceph 39 | daemon: nano 40 | spec: 41 | # volumes: 42 | # - name: foo 43 | # emptyDir: {} 44 | # - name: bar 45 | # emptyDir: {} 46 | containers: 47 | - image: ceph/daemon 48 | imagePullPolicy: Always 49 | name: ceph-nano 50 | # volumeMounts: 51 | # - name: foo 52 | # mountPath: /var/lib/ceph 53 | # - name: bar 54 | # mountPath: /var/run/ceph/ 55 | ports: 56 | - containerPort: 8000 57 | name: cn-s3 58 | protocol: TCP 59 | resources: 60 | limits: 61 | cpu: "1" 62 | memory: 512M 63 | requests: 64 | cpu: "1" 65 | memory: 512M 66 | env: 67 | - name: NETWORK_AUTO_DETECT 68 | value: "4" 69 | - name: RGW_CIVETWEB_PORT 70 | value: "8000" 71 | - name: SREE_PORT 72 | value: "5001" 73 | - name: CEPH_DEMO_UID 74 | value: "nano" 75 | - name: CEPH_DAEMON 76 | value: "demo" 77 | - name: DEBUG 78 | value: "verbose" 79 | - name: CEPH_DEMO_ACCESS_KEY 80 | valueFrom: 81 | secretKeyRef: 82 | name: ceph-rgw-keys 83 | key: rgw_user_user_key 84 | - name: CEPH_DEMO_SECRET_KEY 85 | valueFrom: 86 | secretKeyRef: 87 | name: ceph-rgw-keys 88 | key: rgw_user_secret_key 89 | --- 90 | apiVersion: v1 91 | kind: Secret 92 | metadata: 93 | name: ceph-rgw-keys 94 | type: Opaque 95 | data: 96 | rgw_user_user_key: Zm9v 97 | rgw_user_secret_key: YmFy 98 | -------------------------------------------------------------------------------- /examples/test/history-server/externalStorage/cluster-1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-with-history-1 5 | spec: 6 | historyServer: 7 | type: remoteStorage 8 | name: my-history-server 9 | 10 | sparkConfiguration: 11 | - name: spark.hadoop.fs.s3a.impl 12 | value: org.apache.hadoop.fs.s3a.S3AFileSystem 13 | - name: spark.hadoop.fs.s3a.access.key 14 | value: foo 15 | - name: spark.hadoop.fs.s3a.secret.key 16 | value: bar 17 | 18 | worker: 19 | instances: "1" 20 | -------------------------------------------------------------------------------- /examples/test/history-server/externalStorage/cluster-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-with-history-2 5 | spec: 6 | historyServer: 7 | type: remoteStorage 8 | name: my-history-server 9 | 10 | sparkConfiguration: 11 | - name: spark.hadoop.fs.s3a.impl 12 | value: org.apache.hadoop.fs.s3a.S3AFileSystem 13 | - name: spark.hadoop.fs.s3a.access.key 14 | value: foo 15 | - name: spark.hadoop.fs.s3a.secret.key 16 | value: bar 17 | 18 | worker: 19 | instances: "1" 20 | -------------------------------------------------------------------------------- /examples/test/history-server/externalStorage/history-server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkHistoryServer 3 | metadata: 4 | name: my-history-server 5 | spec: 6 | type: remoteStorage 7 | expose: true 8 | logDirectory: s3a://my-history-server/ 9 | updateInterval: 10 10 | retainedApplications: 50 11 | customImage: quay.io/jkremser/openshift-spark:2.4.0-aws 12 | sparkConfiguration: 13 | - name: spark.hadoop.fs.s3a.impl 14 | value: org.apache.hadoop.fs.s3a.S3AFileSystem 15 | - name: spark.hadoop.fs.s3a.access.key 16 | value: foo 17 | - name: spark.hadoop.fs.s3a.secret.key 18 | value: bar 19 | - name: spark.hadoop.fs.s3a.endpoint 20 | value: http://ceph-nano-0:8000 -------------------------------------------------------------------------------- /examples/test/history-server/sharedVolume/cluster-1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-with-history-1 5 | spec: 6 | historyServer: 7 | name: my-history-server 8 | sharedVolume: 9 | size: 0.2Gi 10 | mountPath: /history/spark-events 11 | matchLabels: 12 | sparkClusterName: my-spark-cluster-with-history-1 13 | worker: 14 | instances: "1" 15 | -------------------------------------------------------------------------------- /examples/test/history-server/sharedVolume/cluster-2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: my-spark-cluster-with-history-2 5 | spec: 6 | historyServer: 7 | name: my-history-server 8 | worker: 9 | instances: "1" 10 | -------------------------------------------------------------------------------- /examples/test/history-server/sharedVolume/history-server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkHistoryServer 3 | metadata: 4 | name: my-history-server 5 | spec: 6 | type: sharedVolume 7 | sharedVolume: 8 | size: 0.3Gi 9 | mountPath: /history/spark-events 10 | matchLabels: 11 | myCustomLabel: foobar 12 | 13 | expose: true 14 | logDirectory: /history/spark-events 15 | updateInterval: 10 16 | retainedApplications: 50 17 | 18 | cleaner: 19 | enabled: false 20 | # interval: 10 21 | -------------------------------------------------------------------------------- /examples/test/history-server/sharedVolume/single-node-pvs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: PersistentVolume 3 | apiVersion: v1 4 | metadata: 5 | name: spark-history-server-pv 6 | labels: 7 | myCustomLabel: foobar 8 | spec: 9 | capacity: 10 | storage: 0.3Gi 11 | accessModes: 12 | - ReadWriteMany 13 | hostPath: 14 | path: "/tmp/spark-events" 15 | --- 16 | kind: PersistentVolume 17 | apiVersion: v1 18 | metadata: 19 | name: spark-master-1-pv 20 | labels: 21 | sparkClusterName: my-spark-cluster-with-history-1 22 | spec: 23 | capacity: 24 | storage: 0.3Gi 25 | accessModes: 26 | - ReadWriteMany 27 | hostPath: 28 | path: "/tmp/spark-events" 29 | --- 30 | kind: PersistentVolume 31 | apiVersion: v1 32 | metadata: 33 | name: spark-master-2-pv 34 | labels: 35 | radanalytics.io/SparkCluster: my-spark-cluster-with-history-2 36 | spec: 37 | capacity: 38 | storage: 0.3Gi 39 | accessModes: 40 | - ReadWriteMany 41 | hostPath: 42 | path: "/tmp/spark-events" 43 | --- 44 | #kind: PersistentVolume 45 | #apiVersion: v1 46 | #metadata: 47 | # name: example-nfs-pv 48 | #spec: 49 | # capacity: 50 | # storage: 0.3Gi 51 | # accessModes: 52 | # - ReadWriteMany 53 | # nfs: 54 | # server: "http://example.com" 55 | # path: "/tmp/spark-events" -------------------------------------------------------------------------------- /examples/with-prepared-data.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: radanalytics.io/v1 2 | kind: SparkCluster 3 | metadata: 4 | name: spark-cluster-with-data 5 | spec: 6 | worker: 7 | instances: "1" 8 | downloadData: 9 | - url: https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv 10 | to: /tmp/ 11 | - url: https://data.lacity.org/api/views/nxs9-385f/rows.csv 12 | to: /tmp/LA.csv 13 | -------------------------------------------------------------------------------- /helm/spark-operator/.gitignore: -------------------------------------------------------------------------------- 1 | template.yaml 2 | -------------------------------------------------------------------------------- /helm/spark-operator/.helmignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | # Common VCS dirs 3 | .git/ 4 | .gitignore 5 | .bzr/ 6 | .bzrignore 7 | .hg/ 8 | .hgignore 9 | .svn/ 10 | # Common backup files 11 | *.swp 12 | *.bak 13 | *.tmp 14 | *~ 15 | # Various IDEs 16 | .project 17 | .idea/ 18 | *.tmproj 19 | 20 | OWNERS 21 | -------------------------------------------------------------------------------- /helm/spark-operator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: spark-cluster-operator 3 | description: Operator for managing the Spark clusters and apps in Kubernetes and OpenShift 4 | version: 0.0.1 5 | appVersion: 1.0.1 6 | icon: https://upload.wikimedia.org/wikipedia/commons/thumb/f/f3/Apache_Spark_logo.svg/388px-Apache_Spark_logo.svg.png 7 | home: https://github.com/radanalyticsio/spark-operator 8 | sources: 9 | - https://github.com/radanalyticsio/spark-operator 10 | maintainers: 11 | - name: Jiri-Kremser 12 | email: jkremser@redhat.com 13 | keywords: 14 | - Apache Spark 15 | - Operator 16 | -------------------------------------------------------------------------------- /helm/spark-operator/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: template 2 | template: 3 | helm template . > template.yaml 4 | cat ./template.yaml 5 | 6 | template-crd: 7 | helm template --set env.crd=true . > template.yaml 8 | cat ./template.yaml 9 | -------------------------------------------------------------------------------- /helm/spark-operator/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - Jiri-Kremser 3 | reviewers: 4 | - Jiri-Kremser 5 | -------------------------------------------------------------------------------- /helm/spark-operator/README.md: -------------------------------------------------------------------------------- 1 | # spark-operator 2 | ConfigMap-based approach for managing the Spark clusters and apps in Kubernetes and OpenShift. 3 | 4 | # Installation 5 | ``` 6 | helm install incubator/spark-cluster-operator 7 | ``` 8 | 9 | or 10 | 11 | ``` 12 | helm install --set env.crd=true incubator/spark-cluster-operator 13 | ``` 14 | 15 | The operator needs to create Service Account, Role and Role Binding. If running in Minikube, you may need to 16 | start it this way: 17 | 18 | ``` 19 | minikube start --vm-driver kvm2 --bootstrapper kubeadm --kubernetes-version v1.7.10 20 | kubectl -n kube-system create sa tiller 21 | kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller 22 | helm init --service-account tiller 23 | ``` 24 | 25 | # Usage 26 | Create Apache Spark Cluster: 27 | 28 | ``` 29 | cat <&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar" 124 | FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( 125 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 126 | ) 127 | 128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 130 | if exist %WRAPPER_JAR% ( 131 | echo Found %WRAPPER_JAR% 132 | ) else ( 133 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 134 | echo Downloading from: %DOWNLOAD_URL% 135 | powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" 136 | echo Finished downloading %WRAPPER_JAR% 137 | ) 138 | @REM End of extension 139 | 140 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 141 | if ERRORLEVEL 1 goto error 142 | goto end 143 | 144 | :error 145 | set ERROR_CODE=1 146 | 147 | :end 148 | @endlocal & set ERROR_CODE=%ERROR_CODE% 149 | 150 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 151 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 152 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 153 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 154 | :skipRcPost 155 | 156 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 157 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 158 | 159 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 160 | 161 | exit /B %ERROR_CODE% 162 | -------------------------------------------------------------------------------- /spark-operator/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | io.radanalytics 5 | spark-operator-parent 6 | 1.0.0 7 | 8 | 4.0.0 9 | io.radanalytics 10 | spark-operator 11 | 1.0.11-SNAPSHOT 12 | 13 | 14 | https://github.com/radanalyticsio/spark-operator 15 | scm:git:git@github.com:radanalyticsio/spark-operator.git 16 | scm:git:git@github.com:radanalyticsio/spark-operator.git 17 | HEAD 18 | 19 | 20 | 21 | UTF-8 22 | 23 | 24 | 25 | 26 | io.radanalytics 27 | spark-abstract-operator 28 | 1.0.0 29 | 30 | 31 | junit 32 | junit 33 | test 34 | 35 | 36 | io.prometheus 37 | simpleclient 38 | 39 | 40 | io.fabric8 41 | openshift-client 42 | 43 | 44 | io.quarkus 45 | quarkus-core 46 | 47 | 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-jar-plugin 53 | 54 | 55 | org.codehaus.mojo 56 | buildnumber-maven-plugin 57 | 58 | 59 | org.jsonschema2pojo 60 | jsonschema2pojo-maven-plugin 61 | 62 | 63 | io.quarkus 64 | quarkus-maven-plugin 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/Constants.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator; 2 | 3 | public class Constants { 4 | 5 | public static String DEFAULT_SPARK_IMAGE = "quay.io/radanalyticsio/openshift-spark:2.4-latest"; 6 | public static String DEFAULT_SPARK_APP_IMAGE = "quay.io/radanalyticsio/openshift-spark:2.4-latest"; 7 | public static final String OPERATOR_TYPE_UI_LABEL = "ui"; 8 | public static final String OPERATOR_TYPE_MASTER_LABEL = "master"; 9 | public static final String OPERATOR_TYPE_WORKER_LABEL = "worker"; 10 | 11 | public static String getDefaultSparkImage() { 12 | String ret = DEFAULT_SPARK_IMAGE; 13 | if (System.getenv("DEFAULT_SPARK_CLUSTER_IMAGE") != null) { 14 | ret = System.getenv("DEFAULT_SPARK_CLUSTER_IMAGE"); 15 | } 16 | return ret; 17 | } 18 | 19 | public static String getDefaultSparkAppImage() { 20 | String ret = DEFAULT_SPARK_APP_IMAGE; 21 | if (System.getenv("DEFAULT_SPARK_APP_IMAGE") != null) { 22 | ret = System.getenv("DEFAULT_SPARK_APP_IMAGE"); 23 | } 24 | return ret; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/SparkOperatorEntrypoint.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator; 2 | 3 | import io.fabric8.kubernetes.api.model.*; 4 | import io.fabric8.kubernetes.api.model.extensions.HTTPIngressPathBuilder; 5 | import io.fabric8.kubernetes.api.model.extensions.Ingress; 6 | import io.fabric8.kubernetes.api.model.extensions.IngressBuilder; 7 | import io.fabric8.kubernetes.api.model.extensions.IngressRuleBuilder; 8 | import io.fabric8.kubernetes.client.KubernetesClient; 9 | import io.fabric8.openshift.api.model.Route; 10 | import io.fabric8.openshift.api.model.RouteBuilder; 11 | import io.fabric8.openshift.client.DefaultOpenShiftClient; 12 | import io.quarkus.runtime.ShutdownEvent; 13 | import io.quarkus.runtime.StartupEvent; 14 | import io.radanalytics.operator.common.AnsiColors; 15 | import org.slf4j.Logger; 16 | 17 | import javax.enterprise.context.ApplicationScoped; 18 | import javax.enterprise.event.Observes; 19 | import javax.inject.Inject; 20 | import java.util.ArrayList; 21 | import java.util.Collections; 22 | import java.util.List; 23 | import java.util.concurrent.Executors; 24 | import java.util.concurrent.ScheduledExecutorService; 25 | import java.util.concurrent.ScheduledFuture; 26 | 27 | import static java.util.concurrent.TimeUnit.SECONDS; 28 | 29 | @ApplicationScoped 30 | public class SparkOperatorEntrypoint{ 31 | @Inject 32 | private Logger log; 33 | 34 | @Inject 35 | private SDKEntrypoint entrypoint; 36 | 37 | public void onStart(@Observes StartupEvent event) { 38 | log.info("onStart.."); 39 | try { 40 | exposeMetrics(); 41 | } catch (Exception e) { 42 | // ignore, not critical (service may have been already exposed) 43 | log.warn("Unable to expose the metrics, cause: {}", e.getMessage()); 44 | } 45 | } 46 | 47 | void onStop(@Observes ShutdownEvent event) { 48 | // nothing special 49 | } 50 | 51 | private void exposeMetrics() { 52 | if (entrypoint.getConfig() != null && entrypoint.getConfig().isMetrics()) { 53 | List resources = new ArrayList<>(); 54 | KubernetesClient client = entrypoint.getClient(); 55 | Service svc = createServiceForMetrics(); 56 | resources.add(svc); 57 | if (entrypoint.isOpenShift()) { 58 | client = new DefaultOpenShiftClient(); 59 | Route route = createRouteForMetrics(); 60 | resources.add(route); 61 | } else { 62 | Ingress ingress = createIngressForMetrics(); 63 | resources.add(ingress); 64 | } 65 | KubernetesList k8sResources = new KubernetesListBuilder().withItems(resources).build(); 66 | client.resourceList(k8sResources).inNamespace(client.getNamespace()).createOrReplace(); 67 | 68 | if (entrypoint.isOpenShift()) { 69 | ScheduledExecutorService s = Executors.newScheduledThreadPool(1); 70 | int delay = 6; 71 | ScheduledFuture future = 72 | s.schedule(() -> { 73 | try { 74 | List routes = new DefaultOpenShiftClient().routes().withLabels(Collections.singletonMap("type", "operator-metrics")).list().getItems(); 75 | if (!routes.isEmpty()) { 76 | Route metrics = routes.iterator().next(); 77 | String host = metrics.getSpec().getHost(); 78 | log.info("Metrics for the Spark Operator are available at {} http://{} {}", AnsiColors.ye(), host, AnsiColors.xx()); 79 | } 80 | } catch (Throwable t) { 81 | log.warn("error during route retrieval: {}", t.getMessage()); 82 | t.printStackTrace(); 83 | } 84 | }, delay, SECONDS); 85 | } 86 | } 87 | 88 | } 89 | 90 | private Ingress createIngressForMetrics() { 91 | Ingress ingress = new IngressBuilder().withNewMetadata().withName("spark-operator-metrics") 92 | .withLabels(Collections.singletonMap("type", "operator-metrics")).endMetadata() 93 | .withNewSpec().withRules(new IngressRuleBuilder().withNewHttp() 94 | .withPaths(new HTTPIngressPathBuilder().withNewBackend().withServiceName("spark-operator-metrics") 95 | .withNewServicePort(entrypoint.getConfig().getMetricsPort()).endBackend().build()).endHttp().build()) 96 | .endSpec().build(); 97 | return ingress; 98 | } 99 | 100 | 101 | private Route createRouteForMetrics() { 102 | Route route = new RouteBuilder().withNewMetadata().withName("spark-operator-metrics") 103 | .withLabels(Collections.singletonMap("type", "operator-metrics")).endMetadata() 104 | .withNewSpec() 105 | .withNewTo("Service", "spark-operator-metrics", 100) 106 | .endSpec().build(); 107 | return route; 108 | } 109 | 110 | private Service createServiceForMetrics() { 111 | Service svc = new ServiceBuilder().withNewMetadata().withName("spark-operator-metrics") 112 | .withLabels(Collections.singletonMap("type", "operator-metrics")) 113 | .endMetadata().withNewSpec() 114 | .withSelector(Collections.singletonMap("app.kubernetes.io/name", "spark-operator")) 115 | .withPorts(new ServicePortBuilder().withPort(entrypoint.getConfig().getMetricsPort()) 116 | .withNewTargetPort().withIntVal(entrypoint.getConfig().getMetricsPort()).endTargetPort() 117 | .withProtocol("TCP").build()) 118 | .endSpec().build(); 119 | return svc; 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/app/AppOperator.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.app; 2 | 3 | import io.fabric8.kubernetes.api.model.KubernetesResourceList; 4 | import io.radanalytics.operator.common.AbstractOperator; 5 | import io.radanalytics.operator.common.Operator; 6 | import io.radanalytics.types.SparkApplication; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Map; 11 | import java.util.HashMap; 12 | import javax.inject.Inject; 13 | import javax.inject.Singleton; 14 | 15 | @Singleton 16 | @Operator(forKind = SparkApplication.class, prefix = "radanalytics.io") 17 | public class AppOperator extends AbstractOperator { 18 | 19 | @Inject 20 | private Logger log; 21 | private KubernetesAppDeployer deployer; 22 | private Map apps; 23 | 24 | public AppOperator(){ 25 | this.apps = new HashMap<>(); 26 | } 27 | 28 | private void put(SparkApplication app) { 29 | apps.put(app.getName(), app); 30 | } 31 | 32 | private void delete(String name) { 33 | if (apps.containsKey(name)) { 34 | apps.remove(name); 35 | } 36 | } 37 | 38 | private SparkApplication getApp(String name) { 39 | return this.apps.get(name); 40 | } 41 | 42 | private void updateStatus(SparkApplication app, String state) { 43 | for (int i=0; i<3; i++) { 44 | try { 45 | setCRStatus(state, app.getNamespace(), app.getName() ); 46 | break; 47 | } 48 | catch(Exception e) { 49 | try {Thread.sleep(500);} catch(Exception t) {} 50 | } 51 | } 52 | } 53 | 54 | @Override 55 | protected void onInit() { 56 | this.deployer = new KubernetesAppDeployer(entityName, prefix); 57 | } 58 | 59 | @Override 60 | protected void onAdd(SparkApplication app) { 61 | KubernetesResourceList list = deployer.getResourceList(app, namespace); 62 | client.resourceList(list).inNamespace(namespace).createOrReplace(); 63 | updateStatus(app, "ready" ); 64 | put(app); 65 | } 66 | 67 | @Override 68 | protected void onModify(SparkApplication newApp) { 69 | 70 | // TODO This comparison works to rule out a change in status because 71 | // we added the status block in the AbstractOperator universally, 72 | // ie it is not actually included in the SparkApplication type 73 | // definition generated from json. If that ever changes, then 74 | // this comparison will have to be a little smarter. 75 | SparkApplication existingApp = getApp(newApp.getName()); 76 | if (null == existingApp || !newApp.equals(existingApp)) { 77 | super.onModify(newApp); 78 | } 79 | } 80 | 81 | @Override 82 | protected void onDelete(SparkApplication app) { 83 | String name = app.getName(); 84 | updateStatus(app, "deleted"); 85 | delete(name); 86 | client.services().inNamespace(namespace).withLabels(deployer.getLabelsForDeletion(name)).delete(); 87 | client.replicationControllers().inNamespace(namespace).withLabels(deployer.getLabelsForDeletion(name)).delete(); 88 | client.pods().inNamespace(namespace).withLabels(deployer.getLabelsForDeletion(name)).delete(); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/app/KubernetesAppDeployer.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.app; 2 | 3 | import io.fabric8.kubernetes.api.model.*; 4 | import io.radanalytics.types.Deps; 5 | import io.radanalytics.types.Executor; 6 | import io.radanalytics.types.Driver; 7 | import io.radanalytics.types.SparkApplication; 8 | 9 | import java.util.*; 10 | import java.util.stream.Collectors; 11 | 12 | import static io.radanalytics.operator.Constants.getDefaultSparkAppImage; 13 | import static io.radanalytics.operator.cluster.KubernetesSparkClusterDeployer.env; 14 | import static io.radanalytics.operator.resource.LabelsHelper.OPERATOR_KIND_LABEL; 15 | 16 | public class KubernetesAppDeployer { 17 | 18 | private String entityName; 19 | private String prefix; 20 | 21 | KubernetesAppDeployer(String entityName, String prefix) { 22 | this.entityName = entityName; 23 | this.prefix = prefix; 24 | } 25 | 26 | public KubernetesResourceList getResourceList(SparkApplication app, String namespace) { 27 | checkForInjectionVulnerabilities(app, namespace); 28 | ReplicationController submitter = getSubmitterRc(app, namespace); 29 | KubernetesList resources = new KubernetesListBuilder().withItems(submitter).build(); 30 | return resources; 31 | } 32 | 33 | private ReplicationController getSubmitterRc(SparkApplication app, String namespace) { 34 | final String name = app.getName(); 35 | 36 | List envVars = new ArrayList<>(); 37 | envVars.add(env("APPLICATION_NAME", name)); 38 | app.getEnv().forEach(kv -> envVars.add(env(kv.getName(), kv.getValue()))); 39 | 40 | final Driver driver = Optional.ofNullable(app.getDriver()).orElse(new Driver()); 41 | final Executor executor = Optional.ofNullable(app.getExecutor()).orElse(new Executor()); 42 | 43 | String imageRef = getDefaultSparkAppImage(); // from Constants 44 | if (app.getImage() != null) { 45 | imageRef = app.getImage(); 46 | } 47 | 48 | StringBuilder command = new StringBuilder(); 49 | command.append("$SPARK_HOME/bin/spark-submit"); 50 | if (app.getMainClass() != null) { 51 | command.append(" --class ").append(app.getMainClass()); 52 | } 53 | command.append(" --master k8s://https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT"); 54 | command.append(" --conf spark.kubernetes.namespace=").append(namespace); 55 | command.append(" --deploy-mode ").append(app.getMode()); 56 | command.append(" --conf spark.app.name=").append(name); 57 | command.append(" --conf spark.kubernetes.container.image=").append(imageRef); 58 | command.append(" --conf spark.kubernetes.submission.waitAppCompletion=false"); 59 | command.append(" --conf spark.driver.cores=").append(driver.getCores()); 60 | command.append(" --conf spark.kubernetes.driver.limit.cores=").append(driver.getCoreLimit()); 61 | command.append(" --conf spark.driver.memory=").append(driver.getMemory()); 62 | if (driver.getMemoryOverhead() != null) { 63 | command.append(" --conf spark.driver.memoryOverhead=").append(driver.getMemoryOverhead()); 64 | } 65 | command.append(" --conf spark.kubernetes.authenticate.driver.serviceAccountName=").append(driver.getServiceAccount()); 66 | command.append(" --conf spark.kubernetes.driver.label.version=2.3.0"); 67 | 68 | // common labels 69 | final Map labels = getLabelsForDeletion(name); 70 | labels.put(prefix + entityName, name); 71 | if (app.getLabels() != null) labels.putAll(app.getLabels()); 72 | labels.forEach((k, v) -> { 73 | command.append(" --conf spark.kubernetes.driver.label.").append(k).append("=").append(v); 74 | command.append(" --conf spark.kubernetes.executor.label.").append(k).append("=").append(v); 75 | }); 76 | // driver labels 77 | if (driver.getLabels() != null) { 78 | driver.getLabels().forEach((k, v) -> 79 | command.append(" --conf spark.kubernetes.driver.label.").append(k).append("=").append(v)); 80 | } 81 | // executor labels 82 | if (executor.getLabels() != null) { 83 | executor.getLabels().forEach((k, v) -> 84 | command.append(" --conf spark.kubernetes.executor.label.").append(k).append("=").append(v)); 85 | } 86 | 87 | // env 88 | envVars.forEach(e -> { 89 | command.append(" --conf spark.kubernetes.driverEnv.").append(e.getName()).append("=").append(e.getValue()); 90 | command.append(" --conf spark.executorEnv.").append(e.getName()).append("=").append(e.getValue()); 91 | }); 92 | 93 | command.append(" --conf spark.executor.instances=").append(executor.getInstances()); 94 | command.append(" --conf spark.executor.cores=").append(executor.getCores()); 95 | command.append(" --conf spark.executor.memory=").append(executor.getMemory()); 96 | if (executor.getMemoryOverhead() != null) { 97 | command.append(" --conf spark.executor.memoryOverhead=").append(executor.getMemoryOverhead()); 98 | } 99 | 100 | // deps 101 | if (app.getDeps() != null) { 102 | Deps deps = app.getDeps(); 103 | if (deps.getPyFiles() != null && !deps.getPyFiles().isEmpty()) { 104 | command.append(" --py-files ").append(deps.getPyFiles().stream().collect(Collectors.joining(","))); 105 | } 106 | if (deps.getJars() != null && !deps.getJars().isEmpty()) { 107 | command.append(" --jars ").append(deps.getJars().stream().collect(Collectors.joining(","))); 108 | } 109 | if (deps.getFiles() != null && !deps.getFiles().isEmpty()) { 110 | command.append(" --files ").append(deps.getFiles().stream().collect(Collectors.joining(","))); 111 | } 112 | } 113 | 114 | command.append(" --conf spark.jars.ivy=/tmp/.ivy2"); 115 | // todo: check all the prerequisites 116 | if (app.getMainApplicationFile() == null) { 117 | throw new IllegalStateException("mainApplicationFile must be specified"); 118 | } 119 | command.append(" ").append(app.getMainApplicationFile()); 120 | 121 | if (app.getArguments() != null && !app.getArguments().trim().isEmpty()) { 122 | command.append(" ").append(app.getArguments()); 123 | } 124 | 125 | if (app.getSleep() > 0) { 126 | command.append(" && echo -e '\\n\\ntask/pod will be rescheduled in ").append(app.getSleep()).append(" seconds..'"); 127 | command.append(" && sleep ").append(app.getSleep()); 128 | } 129 | 130 | final String cmd = "echo -e '\\ncmd:\\n" + command.toString().replaceAll("'", "").replaceAll("--", "\\\\n--") + "\\n\\n' && " + command.toString(); 131 | ContainerBuilder containerBuilder = new ContainerBuilder() 132 | .withEnv(envVars) 133 | .withImage(imageRef) 134 | .withImagePullPolicy(app.getImagePullPolicy().value()) 135 | .withName(name + "-submitter") 136 | .withTerminationMessagePath("/dev/termination-log") 137 | .withTerminationMessagePolicy("File") 138 | .withCommand("/bin/sh", "-c") 139 | .withArgs(cmd); 140 | 141 | ReplicationController rc = new ReplicationControllerBuilder().withNewMetadata() 142 | .withName(name + "-submitter").withLabels(getDefaultLabels(name)) 143 | .endMetadata() 144 | .withNewSpec().withReplicas(1) 145 | .withSelector(getDefaultLabels(name)) 146 | .withNewTemplate().withNewMetadata().withLabels(getDefaultLabels(name)).withName(name + "-submitter") 147 | .endMetadata() 148 | .withNewSpec() 149 | .withContainers(containerBuilder.build()) 150 | .withServiceAccountName(driver.getServiceAccount()) 151 | .endSpec().endTemplate().endSpec().build(); 152 | 153 | return rc; 154 | } 155 | 156 | public Map getDefaultLabels(String name) { 157 | Map map = new HashMap<>(3); 158 | map.put(prefix + OPERATOR_KIND_LABEL, entityName); 159 | map.put(prefix + entityName, name); 160 | return map; 161 | } 162 | 163 | public Map getLabelsForDeletion(String name) { 164 | Map map = new HashMap<>(2); 165 | map.put(prefix + entityName, name); 166 | return map; 167 | } 168 | 169 | private void checkForInjectionVulnerabilities(SparkApplication app, String namespace) { 170 | //todo: this 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/cluster/MetricsHelper.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.cluster; 2 | 3 | import io.prometheus.client.Counter; 4 | import io.prometheus.client.Gauge; 5 | 6 | public class MetricsHelper { 7 | private static final String PREFIX = "operator_"; 8 | 9 | public static final Counter reconciliationsTotal = Counter.build() 10 | .name(PREFIX + "full_reconciliations_total") 11 | .help("How many times the full reconciliation has been run.") 12 | .labelNames("ns") 13 | .register(); 14 | 15 | public static final Gauge runningClusters = Gauge.build() 16 | .name(PREFIX + "running_clusters") 17 | .help("Spark clusters that are currently running.") 18 | .labelNames("ns") 19 | .register(); 20 | 21 | public static final Gauge workers = Gauge.build() 22 | .name(PREFIX + "running_workers") 23 | .help("Number of workers per cluster name.") 24 | .labelNames("cluster", "ns") 25 | .register(); 26 | 27 | public static final Gauge startedTotal = Gauge.build() 28 | .name(PREFIX + "started_clusters_total") 29 | .help("Spark clusters has been started by operator.") 30 | .labelNames("ns") 31 | .register(); 32 | } 33 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/cluster/RunningClusters.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.cluster; 2 | 3 | import io.radanalytics.types.SparkCluster; 4 | import io.radanalytics.types.Worker; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | import java.util.Optional; 9 | 10 | public class RunningClusters { 11 | 12 | private final Map clusters; 13 | private final String namespace; 14 | public RunningClusters(String namespace) { 15 | clusters = new HashMap<>(); 16 | this.namespace = namespace; 17 | MetricsHelper.runningClusters.labels(namespace).set(0); 18 | } 19 | 20 | public void put(SparkCluster ci) { 21 | MetricsHelper.runningClusters.labels(namespace).inc(); 22 | MetricsHelper.startedTotal.labels(namespace).inc(); 23 | MetricsHelper.workers.labels(ci.getName(), namespace).set(Optional.ofNullable(ci.getWorker()).orElse(new Worker()).getInstances()); 24 | clusters.put(ci.getName(), ci); 25 | } 26 | 27 | public void delete(String name) { 28 | if (clusters.containsKey(name)) { 29 | MetricsHelper.runningClusters.labels(namespace).dec(); 30 | MetricsHelper.workers.labels(name, namespace).set(0); 31 | clusters.remove(name); 32 | } 33 | } 34 | 35 | public SparkCluster getCluster(String name) { 36 | return this.clusters.get(name); 37 | } 38 | 39 | public void resetMetrics() { 40 | MetricsHelper.startedTotal.labels(namespace).set(0); 41 | clusters.forEach((c, foo) -> MetricsHelper.workers.labels(c, namespace).set(0)); 42 | MetricsHelper.startedTotal.labels(namespace).set(0); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/historyServer/HistoryServerHelper.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.historyServer; 2 | 3 | import io.radanalytics.types.HistoryServer; 4 | import io.radanalytics.types.SparkCluster; 5 | import io.radanalytics.types.SparkHistoryServer; 6 | 7 | public class HistoryServerHelper { 8 | 9 | public static boolean needsVolume(SparkHistoryServer hs) { 10 | return HistoryServer.Type.sharedVolume.value().equals(hs.getType().value()); 11 | } 12 | 13 | public static boolean needsVolume(SparkCluster cluster) { 14 | return null != cluster.getHistoryServer() && HistoryServer.Type.sharedVolume.value().equals(cluster.getHistoryServer().getType().value()); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /spark-operator/src/main/java/io/radanalytics/operator/historyServer/HistoryServerOperator.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.historyServer; 2 | 3 | import io.fabric8.kubernetes.api.model.KubernetesResourceList; 4 | import io.fabric8.openshift.client.DefaultOpenShiftClient; 5 | import io.radanalytics.operator.common.AbstractOperator; 6 | import io.radanalytics.operator.common.Operator; 7 | import io.radanalytics.types.SparkHistoryServer; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import javax.inject.Inject; 12 | import javax.inject.Singleton; 13 | 14 | import java.lang.Thread; 15 | import java.util.Map; 16 | import java.util.HashMap; 17 | import java.util.Optional; 18 | import java.util.WeakHashMap; 19 | 20 | @Singleton 21 | @Operator(forKind = SparkHistoryServer.class, prefix = "radanalytics.io") 22 | public class HistoryServerOperator extends AbstractOperator { 23 | 24 | @Inject 25 | private Logger log; 26 | private KubernetesHistoryServerDeployer deployer; 27 | private boolean osClient = false; 28 | private Map cache = new WeakHashMap<>(); 29 | private Map hss; 30 | 31 | public HistoryServerOperator() { 32 | this.hss = new HashMap<>(); 33 | } 34 | 35 | private void put(SparkHistoryServer hs) { 36 | hss.put(hs.getName(), hs); 37 | } 38 | 39 | private void delete(String name) { 40 | if (hss.containsKey(name)) { 41 | hss.remove(name); 42 | } 43 | } 44 | 45 | private SparkHistoryServer getHS(String name) { 46 | return this.hss.get(name); 47 | } 48 | 49 | private void updateStatus(SparkHistoryServer hs, String state) { 50 | for (int i=0; i<3; i++) { 51 | try { 52 | setCRStatus(state, hs.getNamespace(), hs.getName() ); 53 | break; 54 | } 55 | catch(Exception e) { 56 | log.warn("failed to update status {} for {} in {}", state, hs.getName(), hs.getNamespace()); 57 | try {Thread.sleep(500);} catch(Exception t) {} 58 | } 59 | } 60 | } 61 | 62 | @Override 63 | protected void onInit() { 64 | this.deployer = new KubernetesHistoryServerDeployer(entityName, prefix); 65 | } 66 | 67 | @Override 68 | protected void onAdd(SparkHistoryServer hs) { 69 | log.info("Spark history server added"); 70 | 71 | KubernetesResourceList list = deployer.getResourceList(hs, namespace, isOpenshift); 72 | if (isOpenshift && hs.getExpose() && !osClient) { 73 | 74 | // we will create openshift specific resource (Route) 75 | this.client = new DefaultOpenShiftClient(); 76 | osClient = true; 77 | } 78 | client.resourceList(list).inNamespace(namespace).createOrReplace(); 79 | cache.put(hs.getName(), list); 80 | updateStatus(hs, "ready"); 81 | put(hs); 82 | } 83 | 84 | @Override 85 | protected void onModify(SparkHistoryServer newHs) { 86 | 87 | // TODO This comparison works to rule out a change in status because 88 | // we added the status block in the AbstractOperator universally, 89 | // ie it is not actually included in the SparkHistoryServer type 90 | // definition generated from json. If that ever changes, then 91 | // this comparison will have to be a little smarter. 92 | SparkHistoryServer existingHs = getHS(newHs.getName()); 93 | if (null == existingHs || !newHs.equals(existingHs)) { 94 | super.onModify(newHs); 95 | } 96 | } 97 | 98 | @Override 99 | protected void onDelete(SparkHistoryServer hs) { 100 | log.info("Spark history server removed"); 101 | String name = hs.getName(); 102 | updateStatus(hs, "deleted"); 103 | delete(name); 104 | KubernetesResourceList list = Optional.ofNullable(cache.get(name)).orElse(deployer.getResourceList(hs, namespace, isOpenshift)); 105 | client.resourceList(list).inNamespace(namespace).delete(); 106 | cache.remove(name); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /spark-operator/src/main/resources/schema/sparkApplication.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "description": "A Spark application configuration", 4 | "type": "object", 5 | "extends": { 6 | "type": "object", 7 | "existingJavaType": "io.radanalytics.operator.common.EntityInfo" 8 | }, 9 | "properties": { 10 | "deps": { 11 | "type": "object", 12 | "properties": { 13 | "jars": { 14 | "type": "array", 15 | "items": { 16 | "type": "string" 17 | } 18 | }, 19 | "files": { 20 | "type": "array", 21 | "items": { 22 | "type": "string" 23 | } 24 | }, 25 | "pyFiles": { 26 | "type": "array", 27 | "items": { 28 | "type": "string" 29 | } 30 | }, 31 | "jarsDownloadDir": { 32 | "type": "string" 33 | }, 34 | "filesDownloadDir": { 35 | "type": "string" 36 | }, 37 | "downloadTimeout": { 38 | "type": "integer", 39 | "default": "60", 40 | "minimum": "0" 41 | }, 42 | "maxSimultaneousDownloads": { 43 | "type": "integer", 44 | "default": "5", 45 | "minimum": "0" 46 | } 47 | } 48 | }, 49 | "historyServer": { 50 | "type": "string" 51 | }, 52 | "driver": { 53 | "type": "object", 54 | "properties": { 55 | "memory": { 56 | "type": "string", 57 | "default": "512m" 58 | }, 59 | "memoryOverhead": { 60 | "type": "string" 61 | }, 62 | "image": { 63 | "type": "string" 64 | }, 65 | "labels": { 66 | "type": "string", 67 | "existingJavaType": "java.util.Map", 68 | "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]" 69 | }, 70 | "cores": { 71 | "type": "string", 72 | "default": "0.5" 73 | }, 74 | "coreLimit": { 75 | "type": "string", 76 | "default": "500m" 77 | }, 78 | "serviceAccount": { 79 | "type": "string", 80 | "default": "spark-operator" 81 | } 82 | } 83 | }, 84 | "executor": { 85 | "type": "object", 86 | "properties": { 87 | "memory": { 88 | "type": "string", 89 | "default": "512m" 90 | }, 91 | "memoryOverhead": { 92 | "type": "string" 93 | }, 94 | "image": { 95 | "type": "string" 96 | }, 97 | "labels": { 98 | "type": "string", 99 | "existingJavaType": "java.util.Map", 100 | "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]" 101 | }, 102 | "instances": { 103 | "type": "integer", 104 | "default": "1", 105 | "minimum": "0" 106 | }, 107 | "cores": { 108 | "type": "string", 109 | "default": "1", 110 | "description": "Must be a positive integer" 111 | }, 112 | "coreLimit": { 113 | "type": "string", 114 | "default": "1000m" 115 | } 116 | } 117 | }, 118 | "image": { 119 | "type": "string" 120 | }, 121 | "mainApplicationFile": { 122 | "type": "string" 123 | }, 124 | "mainClass": { 125 | "type": "string" 126 | }, 127 | "arguments": { 128 | "type": "string" 129 | }, 130 | "mode": { 131 | "type": "string", 132 | "default": "cluster", 133 | "enum": [ 134 | "client", 135 | "cluster" 136 | ], 137 | "javaEnumNames": [ 138 | "client", 139 | "cluster" 140 | ] 141 | }, 142 | "restartPolicy": { 143 | "type": "string", 144 | "default": "Always", 145 | "enum": [ 146 | "Always", 147 | "OnFailure", 148 | "Never" 149 | ], 150 | "javaEnumNames": [ 151 | "Always", 152 | "OnFailure", 153 | "Never" 154 | ] 155 | }, 156 | "imagePullPolicy": { 157 | "type": "string", 158 | "default": "IfNotPresent", 159 | "enum": [ 160 | "IfNotPresent", 161 | "Always", 162 | "Never" 163 | ], 164 | "javaEnumNames": [ 165 | "IfNotPresent", 166 | "Always", 167 | "Never" 168 | ] 169 | }, 170 | "type": { 171 | "type": "string", 172 | "default": "Java", 173 | "enum": [ 174 | "Java", 175 | "Scala", 176 | "Python", 177 | "R" 178 | ], 179 | "javaEnumNames": [ 180 | "Java", 181 | "Scala", 182 | "Python", 183 | "R" 184 | ] 185 | }, 186 | "sleep": { 187 | "type": "integer", 188 | "default": "31536000", 189 | "minimum": "0" 190 | }, 191 | "labels": { 192 | "type": "string", 193 | "existingJavaType": "java.util.Map", 194 | "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]" 195 | }, 196 | "env": { 197 | "type": "array", 198 | "items": { 199 | "type": "object", 200 | "properties": { 201 | "name": { 202 | "type": "string" 203 | }, 204 | "value": { 205 | "type": "string" 206 | } 207 | }, 208 | "required": [ 209 | "name", 210 | "value" 211 | ] 212 | } 213 | }, 214 | "sparkConfigMap": { 215 | "type": "string" 216 | } 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /spark-operator/src/main/resources/schema/sparkCluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "description": "A Spark cluster configuration", 4 | "dependencies": null, 5 | "type": "object", 6 | "extends": { 7 | "type": "object", 8 | "existingJavaType": "io.radanalytics.operator.common.EntityInfo" 9 | }, 10 | "properties": { 11 | "master": { 12 | "type": "object", 13 | "properties": { 14 | "instances": { 15 | "type": "integer", 16 | "default": "1", 17 | "minimum": "1" 18 | }, 19 | "memory": { 20 | "type": "string" 21 | }, 22 | "memoryRequest": { 23 | "type": "string" 24 | }, 25 | "memoryLimit": { 26 | "type": "string" 27 | }, 28 | "cpu": { 29 | "type": "string" 30 | }, 31 | "cpuRequest": { 32 | "type": "string" 33 | }, 34 | "cpuLimit": { 35 | "type": "string" 36 | }, 37 | "labels": { 38 | "existingJavaType": "java.util.Map", 39 | "type": "string", 40 | "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]" 41 | }, 42 | "command": { 43 | "type": "array", 44 | "items": { 45 | "type": "string" 46 | } 47 | }, 48 | "commandArgs": { 49 | "type": "array", 50 | "items": { 51 | "type": "string" 52 | } 53 | } 54 | } 55 | }, 56 | "worker": { 57 | "type": "object", 58 | "properties": { 59 | "instances": { 60 | "type": "integer", 61 | "default": "1", 62 | "minimum": "0" 63 | }, 64 | "memory": { 65 | "type": "string" 66 | }, 67 | "memoryRequest": { 68 | "type": "string" 69 | }, 70 | "memoryLimit": { 71 | "type": "string" 72 | }, 73 | "cpu": { 74 | "type": "string" 75 | }, 76 | "cpuRequest": { 77 | "type": "string" 78 | }, 79 | "cpuLimit": { 80 | "type": "string" 81 | }, 82 | "labels": { 83 | "existingJavaType": "java.util.Map", 84 | "type": "string", 85 | "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]" 86 | }, 87 | "command": { 88 | "type": "array", 89 | "items": { 90 | "type": "string" 91 | } 92 | }, 93 | "commandArgs": { 94 | "type": "array", 95 | "items": { 96 | "type": "string" 97 | } 98 | } 99 | } 100 | }, 101 | "nodeTolerations": { 102 | "type": "array", 103 | "items": { 104 | "type": "object", 105 | "properties": { 106 | "key": { 107 | "type": "string" 108 | }, 109 | "operator": { 110 | "type": "string" 111 | }, 112 | "value": { 113 | "type": "string" 114 | }, 115 | "effect": { 116 | "type": "string" 117 | }, 118 | "tolerationSeconds": { 119 | "type": "integer", 120 | "default": null 121 | } 122 | }, 123 | "required": [ 124 | "key", 125 | "operator", 126 | "value", 127 | "effect" 128 | ] 129 | } 130 | }, 131 | "mavenDependencies": { 132 | "type": "array", 133 | "items": { 134 | "type": "string" 135 | } 136 | }, 137 | "mavenRepositories": { 138 | "type": "array", 139 | "items": { 140 | "type": "string" 141 | } 142 | }, 143 | "customImage": { 144 | "type": "string" 145 | }, 146 | "customInitContainerImage": { 147 | "type": "string" 148 | }, 149 | "metrics": { 150 | "type": "boolean", 151 | "default": "false" 152 | }, 153 | "sparkWebUI": { 154 | "type": "boolean", 155 | "default": "true" 156 | }, 157 | "sparkConfigurationMap": { 158 | "type": "string" 159 | }, 160 | "env": { 161 | "type": "array", 162 | "items": { 163 | "type": "object", 164 | "javaType": "io.radanalytics.types.Env", 165 | "properties": { 166 | "name": { "type": "string" }, 167 | "value": { "type": "string" } 168 | }, 169 | "required": ["name", "value"] 170 | } 171 | }, 172 | "sparkConfiguration": { 173 | "type": "array", 174 | "items": { 175 | "type": "object", 176 | "properties": { 177 | "name": { "type": "string" }, 178 | "value": { "type": "string" } 179 | }, 180 | "required": ["name", "value"] 181 | } 182 | }, 183 | "labels": { 184 | "type": "object", 185 | "existingJavaType": "java.util.Map" 186 | }, 187 | "historyServer": { 188 | "type": "object", 189 | "properties": { 190 | "name": { 191 | "type": "string" 192 | }, 193 | "type": { 194 | "type": "string", 195 | "default": "sharedVolume", 196 | "enum": [ 197 | "sharedVolume", 198 | "remoteStorage" 199 | ], 200 | "javaEnumNames": [ 201 | "sharedVolume", 202 | "remoteStorage" 203 | ] 204 | }, 205 | 206 | "sharedVolume": { 207 | "type": "object", 208 | "properties": { 209 | "size": { 210 | "type": "string", 211 | "default": "0.3Gi" 212 | }, 213 | "mountPath": { 214 | "type": "string", 215 | "default": "/history/spark-events" 216 | }, 217 | "matchLabels": { 218 | "type": "object", 219 | "existingJavaType": "java.util.Map" 220 | } 221 | } 222 | }, 223 | "remoteURI": { 224 | "type": "string", 225 | "description": "s3 bucket or hdfs path" 226 | } 227 | } 228 | }, 229 | "downloadData": { 230 | "type": "array", 231 | "items": { 232 | "type": "object", 233 | "properties": { 234 | "url": { 235 | "type": "string" 236 | }, 237 | "to": { 238 | "type": "string" 239 | } 240 | }, 241 | "required": [ 242 | "url", 243 | "to" 244 | ] 245 | } 246 | } 247 | }, 248 | "required": [] 249 | } 250 | -------------------------------------------------------------------------------- /spark-operator/src/main/resources/schema/sparkHistoryServer.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "description": "A Spark history server configuration", 4 | "type": "object", 5 | "extends": { 6 | "type": "object", 7 | "existingJavaType": "io.radanalytics.operator.common.EntityInfo" 8 | }, 9 | "properties": { 10 | "type": { 11 | "type": "string", 12 | "default": "sharedVolume", 13 | "enum": ["sharedVolume", "remoteStorage"], 14 | "javaEnumNames" : ["sharedVolume","remoteStorage"] 15 | }, 16 | "sharedVolume": { 17 | "type": "object", 18 | "existingJavaType": "io.radanalytics.types.SharedVolume", 19 | "properties": { 20 | "size": { 21 | "type": "string", 22 | "default": "0.3Gi" 23 | }, 24 | "mountPath": { 25 | "type": "string", 26 | "default": "/history/spark-events" 27 | }, 28 | "matchLabels": { 29 | "type": "object", 30 | "existingJavaType": "java.util.Map" 31 | } 32 | } 33 | }, 34 | "sparkConfiguration": { 35 | "type": "array", 36 | "items": { 37 | "type": "object", 38 | "existingJavaType": "io.radanalytics.types.SparkConfiguration", 39 | "properties": { 40 | "name": { "type": "string" }, 41 | "value": { "type": "string" } 42 | }, 43 | "required": ["name", "value"] 44 | } 45 | }, 46 | "remoteURI": { 47 | "type": "string", 48 | "description": "s3 bucket or hdfs path" 49 | }, 50 | "expose": { 51 | "type": "boolean", 52 | "default": "false", 53 | "description": "Should the operator also expose the service? For OpenShift the route is created, while for Kubernetes the Ingress object is created." 54 | }, 55 | "host": { 56 | "type": "string", 57 | "default": "", 58 | "description": "Custom dns hostname under which the Spark History server will be exposed. If not specified it should be generated by OpenShift route, for K8s the Ingress resource is created and it's up to the Ingress controller." 59 | }, 60 | "customImage": { 61 | "type": "string", 62 | "description": "Container image that will be used for the spark history server. It assumes the standard Spark distribution under /opt/spark" 63 | }, 64 | "logDirectory": { 65 | "type": "string", 66 | "default": "file:/history/spark-events", 67 | "description": "For the filesystem history provider, the URL to the directory containing application event logs to load. This can be a local file:// path, an HDFS path hdfs://namenode/shared/spark-logs or that of an alternative filesystem supported by the Hadoop APIs." 68 | }, 69 | "updateInterval": { 70 | "type": "integer", 71 | "default": "10", 72 | "minimum": "1", 73 | "description": "The period (seconds) at which the filesystem history provider checks for new or updated logs in the log directory. A shorter interval detects new applications faster, at the expense of more server load re-reading updated applications. As soon as an update has completed, listings of the completed and incomplete applications will reflect the changes." 74 | }, 75 | "internalPort": { 76 | "type": "integer", 77 | "default": "18080", 78 | "minimum": "1025", 79 | "description": "The port on pod to which the web interface of the history server binds. If exposed via Route or Ingress, this internal port will probably map to some other port." 80 | }, 81 | "retainedApplications": { 82 | "type": "integer", 83 | "default": "50", 84 | "minimum": "1", 85 | "description": "The number of applications to retain UI data for in the cache. If this cap is exceeded, then the oldest applications will be removed from the cache. If an application is not in the cache, it will have to be loaded from disk if it is accessed from the UI." 86 | }, 87 | "maxApplications": { 88 | "type": "integer", 89 | "default": "999999", 90 | "minimum": "1", 91 | "description": "The number of applications to display on the history summary page. Application UIs are still available by accessing their URLs directly even if they are not displayed on the history summary page." 92 | }, 93 | "provider": { 94 | "type": "string", 95 | "default": "org.apache.spark.deploy.history.FsHistoryProvider", 96 | "description": "Name of the class implementing the application history backend. Currently there is only one implementation, provided by Spark, which looks for application logs stored in the file system." 97 | }, 98 | "kerberos": { 99 | "type": "object", 100 | "properties": { 101 | "enabled": { 102 | "type": "boolean", 103 | "default": "false", 104 | "description": "Indicates whether the history server should use kerberos to login. This is required if the history server is accessing HDFS files on a secure Hadoop cluster. If this is true, it uses the configs spark.history.kerberos.principal and spark.history.kerberos.keytab." 105 | }, 106 | "principal": { 107 | "type": "string", 108 | "description": "Kerberos principal name for the History Server." 109 | }, 110 | "keytab": { 111 | "type": "string", 112 | "description": "Location of the kerberos keytab file for the History Server." 113 | } 114 | } 115 | }, 116 | "cleaner": { 117 | "type": "object", 118 | "properties": { 119 | "enabled": { 120 | "type": "boolean", 121 | "default": "false", 122 | "description": "Specifies whether the History Server should periodically clean up event logs from storage." 123 | }, 124 | "interval": { 125 | "type": "integer", 126 | "default": "1", 127 | "minimum": "1", 128 | "description": "How often (days) the filesystem job history cleaner checks for files to delete. Files are only deleted if they are older than spark.history.fs.cleaner.maxAge" 129 | }, 130 | "maxAge": { 131 | "type": "integer", 132 | "default": "7", 133 | "minimum": "1", 134 | "description": "# of days, job history files older than this will be deleted when the filesystem history cleaner runs." 135 | } 136 | } 137 | }, 138 | "endEventReparseChunkSize": { 139 | "type": "integer", 140 | "default": "1", 141 | "minimum": "1", 142 | "description": "# of MB; How many bytes to parse at the end of log files looking for the end event. This is used to speed up generation of application listings by skipping unnecessary parts of event log files. It can be disabled by setting this config to 0." 143 | }, 144 | "inProgressOptimization": { 145 | "type": "boolean", 146 | "default": "true", 147 | "description": "Enable optimized handling of in-progress logs. This option may leave finished applications that fail to rename their event logs listed as in-progress." 148 | }, 149 | "numReplayThreads": { 150 | "type": "string", 151 | "description": "Number of threads that will be used by history server to process event logs. If empty, 25% of available cores will be used." 152 | }, 153 | "maxDiskUsage": { 154 | "type": "integer", 155 | "default": "10", 156 | "minimum": "1", 157 | "description": "# of GB; Maximum disk usage for the local directory where the cache application history information are stored." 158 | }, 159 | "persistentPath": { 160 | "type": "string", 161 | "description": "Local directory where to cache application history data. If set, the history server will store application data on disk instead of keeping it in memory. The data written to disk will be re-used in the event of a history server restart." 162 | } 163 | }, 164 | "required": [] 165 | } 166 | -------------------------------------------------------------------------------- /spark-operator/src/test/java/io/radanalytics/operator/resource/YamlProcessingTest.java: -------------------------------------------------------------------------------- 1 | package io.radanalytics.operator.resource; 2 | 3 | 4 | import io.fabric8.kubernetes.api.model.ConfigMap; 5 | import io.fabric8.kubernetes.client.DefaultKubernetesClient; 6 | import io.fabric8.kubernetes.client.KubernetesClient; 7 | import io.radanalytics.types.SparkApplication; 8 | import io.radanalytics.types.SparkCluster; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | 12 | import java.io.IOException; 13 | import java.nio.charset.StandardCharsets; 14 | import java.nio.file.Files; 15 | import java.nio.file.Paths; 16 | 17 | import static org.junit.Assert.*; 18 | 19 | public class YamlProcessingTest { 20 | 21 | private String path1 = "../examples/cluster-cm.yaml"; 22 | private String path2 = "../examples/test/cm/app.yaml"; 23 | private String path3 = "../examples/test/cm/cluster-with-config-1.yaml"; 24 | private String cluster1; 25 | private String application; 26 | private KubernetesClient client = new DefaultKubernetesClient(); 27 | 28 | @Before 29 | public void prepare() throws IOException { 30 | this.cluster1 = readFile(path1); 31 | this.application = readFile(path2); 32 | ConfigMap cm1 = client.configMaps().load(path1).get(); 33 | ConfigMap cm2 = client.configMaps().load(path2).get(); 34 | this.cluster1 = cm1.getData().get("config"); 35 | this.application = cm2.getData().get("config"); 36 | } 37 | 38 | @Test 39 | public void testParseCM1() { 40 | ConfigMap cm1 = client.configMaps().load(path1).get(); 41 | SparkCluster clusterInfo = HasDataHelper.parseCM(SparkCluster.class, cm1); 42 | 43 | assertEquals(clusterInfo.getName(), "my-spark-cluster"); 44 | assertEquals(clusterInfo.getWorker().getInstances().intValue(), 2); 45 | assertEquals(clusterInfo.getMaster().getInstances().intValue(), 1); 46 | } 47 | 48 | @Test 49 | public void testParseCM2() { 50 | ConfigMap cm1 = client.configMaps().load(path2).get(); 51 | SparkApplication sparkApplication = HasDataHelper.parseCM(SparkApplication.class, cm1); 52 | 53 | assertEquals(sparkApplication.getName(), "my-spark-app"); 54 | assertEquals(sparkApplication.getMainClass(), "org.apache.spark.examples.SparkPi"); 55 | assertEquals(sparkApplication.getExecutor().getInstances().intValue(), 2); 56 | } 57 | 58 | @Test 59 | public void testParseCM3() { 60 | ConfigMap cm3 = client.configMaps().load(path3).get(); 61 | SparkCluster clusterInfo = HasDataHelper.parseCM(SparkCluster.class, cm3); 62 | 63 | assertNull(clusterInfo.getMaster()); 64 | assertEquals(clusterInfo.getSparkConfiguration().size(), 1); 65 | assertEquals(clusterInfo.getSparkConfiguration().get(0).getName(), "spark.executor.memory"); 66 | 67 | assertEquals(clusterInfo.getDownloadData().size(), 1); 68 | assertEquals(clusterInfo.getDownloadData().get(0).getTo(), "/tmp/"); 69 | } 70 | 71 | @Test 72 | public void testParseYaml1() { 73 | SparkCluster clusterInfo = HasDataHelper.parseYaml(SparkCluster.class, cluster1, "foo"); 74 | 75 | assertEquals(clusterInfo.getName(), "foo"); 76 | assertEquals(clusterInfo.getWorker().getInstances().intValue(), 2); 77 | assertEquals(clusterInfo.getCustomImage(), null); 78 | } 79 | 80 | @Test 81 | public void testParseYaml2() { 82 | SparkApplication sparkApplication = HasDataHelper.parseYaml(SparkApplication.class, application, "bar"); 83 | 84 | assertEquals(sparkApplication.getName(), "bar"); 85 | assertNull(sparkApplication.getArguments()); 86 | assertEquals(sparkApplication.getSleep().intValue(), 300); 87 | } 88 | 89 | @Test 90 | public void testParseGeneral() { 91 | SparkCluster clusterInfo1 = HasDataHelper.parseYaml(SparkCluster.class, cluster1, "foobar"); 92 | ConfigMap cm1 = client.configMaps().load(path1).get(); 93 | 94 | SparkCluster clusterInfo2 = HasDataHelper.parseCM(SparkCluster.class, cm1); 95 | SparkCluster clusterInfo3 = HasDataHelper.parseYaml(SparkCluster.class, cluster1, "my-spark-cluster"); 96 | 97 | // different name 98 | assertNotEquals(clusterInfo1, clusterInfo2); 99 | 100 | assertEquals(clusterInfo2.getName(), clusterInfo3.getName()); 101 | } 102 | 103 | private String readFile(String path) throws IOException { 104 | byte[] encoded = Files.readAllBytes(Paths.get(path)); 105 | return new String(encoded, StandardCharsets.UTF_8); 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /version-bump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | checkParams() { 4 | [[ $# -lt 1 ]] && printUsage && exit 1 5 | if [[ "$1" != "micro" ]] && [[ "$1" != "minor" ]] && [[ "$1" != "major" ]]; then 6 | printUsage 7 | exit 1 8 | fi 9 | } 10 | 11 | checkUntracked() { 12 | [[ -z $(git status -s) ]] || { 13 | echo "there are untracked files, commit them first" 14 | exit 1 15 | } 16 | } 17 | 18 | printUsage() { 19 | echo "usage: version-bump.sh " 20 | } 21 | 22 | gitFu() { 23 | [[ $# -lt 2 ]] && "usage: gitFu x.y.z x'.y'.z' " && exit 1 24 | old=$1 25 | new=$2 26 | mvn -U versions:set -DnewVersion=$new 27 | git add pom.xml 28 | set -x 29 | git commit -m "$3 version bump from $old to $new" 30 | set +x 31 | } 32 | 33 | main() { 34 | checkUntracked 35 | 36 | checkParams $@ 37 | PARAM=$1 38 | 39 | CURRENT=`mvn -U org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep -Ev "(^\[|Download\w+:)"|grep -v "Download"` 40 | VERSION=`echo $CURRENT | sed 's/-SNAPSHOT//g'` 41 | 42 | maj=`echo $VERSION | sed 's/^\([0-9]\+\)\..*$/\1/g'` 43 | min=`echo $VERSION | sed 's/^[0-9]\+\.\([0-9]\+\)\..*$/\1/g'` 44 | mic=`echo $VERSION | sed 's/.*\.\([0-9]\+\)$/\1/g'` 45 | 46 | echo "Current version: $CURRENT" 47 | echo "version: $VERSION" 48 | echo "major: $maj" 49 | echo "minor: $min" 50 | echo "micro: $mic" 51 | 52 | if [[ "$PARAM" = "micro" ]]; then 53 | echo "Updating micro version" 54 | elif [[ "$PARAM" = "minor" ]]; then 55 | echo "Updating minor version" 56 | ((min++)) 57 | mic=0 58 | elif [[ "$PARAM" = "major" ]]; then 59 | echo "Updating major version" 60 | ((maj++)) 61 | min=0 62 | mic=0 63 | else 64 | echo "Unrecognized param: $PARAM" 65 | printUsage && exit 1 66 | fi 67 | 68 | DESIRED="$maj.$min.$mic" 69 | gitFu "$CURRENT" "$DESIRED" "$PARAM" 70 | git tag -d "$DESIRED" || true 71 | git tag "$DESIRED" 72 | 73 | ((mic++)) 74 | DESIRED_NEW="$maj.$min.$mic-SNAPSHOT" 75 | 76 | echo "Desired new version: $DESIRED_NEW" 77 | gitFu "$DESIRED" "$DESIRED_NEW" "$PARAM" 78 | 79 | echo -e "if everything is ok, you may want to continue with: \n\n git push personal master $DESIRED\n\n" 80 | } 81 | 82 | main $@ 83 | --------------------------------------------------------------------------------