├── .gitignore ├── Brewfile ├── Makefile ├── README.adoc ├── build.gradle.kts ├── common ├── build.gradle.kts └── src │ └── main │ └── kotlin │ └── skk │ ├── KafkaConfig.kt │ └── Question.kt ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── ksqldb-setup-deployment.yaml ├── ksqldb-setup ├── build.gradle.kts └── src │ └── main │ └── kotlin │ └── skk │ └── Setup.kt ├── scripts ├── ccloud │ ├── ccloud-generate-cp-config.sh │ ├── ccloud_key_password.sh │ ├── ccloud_library.sh │ ├── ccloud_stack_create.sh │ └── ccloud_stack_destroy.sh └── common │ ├── Makefile │ ├── colors.sh │ └── helper.sh ├── settings.gradle.kts ├── skaffold.yaml ├── web-ui ├── build.gradle.kts └── src │ ├── main │ ├── kotlin │ │ └── skk │ │ │ ├── Html.kt │ │ │ └── Main.kt │ └── resources │ │ └── META-INF │ │ └── resources │ │ └── assets │ │ ├── index.css │ │ ├── index.js │ │ └── lang.js │ └── test │ └── kotlin │ └── skk │ └── TestKafkaConfigFactory.kt ├── ws-to-kafka-app-deployment.yaml ├── ws-to-kafka-app-secret-template.yaml └── ws-to-kafka ├── build.gradle.kts └── src ├── main ├── kotlin │ └── skk │ │ └── Main.kt └── resources │ └── application.properties └── test └── kotlin └── skk └── TestKafkaProducerFactory.kt /.gitignore: -------------------------------------------------------------------------------- 1 | /.gradle/ 2 | /.idea/ 3 | build/ 4 | delta_configs/ 5 | stack-configs/ 6 | ws-to-kafka-app-secret.yaml 7 | -------------------------------------------------------------------------------- /Brewfile: -------------------------------------------------------------------------------- 1 | brew "kubernetes-cli" 2 | brew "jq" 3 | brew "skaffold" 4 | brew "k9s" 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | THIS_MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) 2 | THIS_MKFILE_DIR := $(dir $(THIS_MKFILE_PATH)) 3 | 4 | include $(THIS_MKFILE_DIR)scripts/common/Makefile 5 | 6 | GCP_PROJECT_ID ?= $(shell gcloud config list --format 'value(core.project)') 7 | GKE_BASE_MACHINE_TYPE ?= n1-highmem-2 8 | 9 | CLUSTER_NAME=portable-serverless-workshop 10 | CLUSTER_ZONE=us-central1-a 11 | 12 | gke-check-dependencies: check-dependencies 13 | @$(call check-var-defined,GCP_PROJECT_ID) 14 | @$(call check-dependency,gcloud) 15 | @$(call echo_pass,gke-base dependencies verified) 16 | 17 | gke-create-cluster: gke-check-dependencies 18 | @$(call print-header,"Creating a new cluster Creating GKE") 19 | @$(call print-prompt) 20 | gcloud --quiet container --project $(GCP_PROJECT_ID) clusters create ${CLUSTER_NAME} --num-nodes 2 --machine-type $(GKE_BASE_MACHINE_TYPE) --zone ${CLUSTER_ZONE} 21 | export PROJECT_ID=${GCP_PROJECT_ID} 22 | 23 | gke-enable-cloud-run: gke-check-dependencies 24 | @$(call print-header,"Enabling CloudRun APIs") 25 | @$(call print-prompt) 26 | gcloud services enable cloudapis.googleapis.com container.googleapis.com containerregistry.googleapis.com run.googleapis.com --project=${GCP_PROJECT_ID} 27 | 28 | gke-scale-cluster-%: gke-check-dependencies 29 | @$(call print-header,"Scaling my ${CLUSTER_NAME}") 30 | @$(call print-prompt) 31 | gcloud --quiet container clusters resize ${CLUSTER_NAME} --num-nodes=$* --zone ${CLUSTER_ZONE} 32 | 33 | gke-destroy-cluster: gke-check-dependencies 34 | @$(call print-header, "Delete GKE cluster") 35 | @$(call print-prompt) 36 | gcloud --quiet container --project $(GCP_PROJECT_ID) clusters delete ${CLUSTER_NAME} --zone ${CLUSTER_ZONE} 37 | @$(call echo_stdout_footer_pass,GKE Cluster Deleted) 38 | 39 | ccloud-cli: 40 | @echo "Installing ccloud" 41 | @curl -L -s --http1.1 https://cnfl.io/ccloud-cli | sh -s -- -b . 42 | @sudo install -m 755 ccloud ~/bin/ccloud 43 | @rm -f ccloud 44 | @$(caller echo_stdout_footer_pass, "ccloud cli installed") 45 | 46 | install-deps: ccloud 47 | @brew bundle 48 | @$(caller echo_stdout_footer_pass, "dependencies installed") 49 | 50 | ccloud-create-cluster: 51 | @$(call print-header,"☁️ Creating ccloud Cluster...") 52 | @$(call print-prompt) 53 | ./scripts/ccloud/ccloud_stack_create.sh 54 | 55 | ccloud-destroy-cluster: 56 | @$(call print-header,"🧨 Destroying ccloud Cluster...") 57 | @$(call print-prompt) 58 | ./scripts/ccloud/ccloud_stack_destroy.sh ${THIS_MKFILE_DIR}$(filter-out $@,$(MAKECMDGOALS)) 59 | 60 | ccloud-get-kafka-key-password: 61 | . ./scripts/ccloud/ccloud_key_password.sh 62 | @$(call ccloud-validate) 63 | 64 | ccloud-validate: 65 | @$(call print-header,"🌐 environment variables...") 66 | @$(call print-prompt) 67 | @echo "CLUSTER_ID: $(CLUSTER_ID)" 68 | @echo "KAFKA_BOOTSTRAP_SERVERS: $(KAFKA_BOOTSTRAP_SERVERS)" 69 | @echo "KAFKA_USERNAME: $(KAFKA_USERNAME)" 70 | @echo "KAFKA_PASSWORD: $(KAFKA_PASSWORD:0:6)..." 71 | @echo "SCHEMA_REGISTRY_URL: $(SCHEMA_REGISTRY_URL)" 72 | @echo "SCHEMA_REGISTRY_ID: $(SCHEMA_REGISTRY_ID)" 73 | @echo "SCHEMA_REGISTRY_KEY: $(SCHEMA_REGISTRY_KEY)" 74 | @echo "SCHEMA_REGISTRY_PASSWORD: $(SCHEMA_REGISTRY_PASSWORD:0:6)..." 75 | @echo "KSQLDB_ENDPOINT: $(KSQLDB_ENDPOINT)" 76 | @echo "KSQLDB_USERNAME: $(KSQLDB_USERNAME)" 77 | @echo "KSQLDB_PASSWORD: $(KSQLDB_PASSWORD:0:6)..." 78 | @echo "PROJECT_ID: $(PROJECT_ID)" 79 | 80 | kube-generate-secret: ccloud-validate 81 | envsubst < ws-to-kafka-app-secret-template.yaml > ws-to-kafka-app-secret.yaml 82 | 83 | kube-deploy-ws-to-kafka: 84 | skaffold dev 85 | 86 | ksqldb-deploy-streams: 87 | @$(call print-header,"🚀 deploying ksqlDB app...") 88 | @$(call print-prompt) 89 | ./gradlew :ksqldb-setup:run 90 | 91 | gke-deploy-ksql-app: 92 | ./gradlew :web-ui:bootBuildImage --imageName=gcr.io/${GCP_PROJECT_ID}/skk-web-ui 93 | docker push gcr.io/${GCP_PROJECT_ID}/skk-web-ui 94 | gcloud run deploy --image=gcr.io/${GCP_PROJECT_ID}/skk-web-ui --platform=managed --allow-unauthenticated --memory=512Mi --region=us-central1 --project=${GCP_PROJECT_ID} --set-env-vars=KSQLDB_ENDPOINT=${KSQLDB_ENDPOINT} --set-env-vars=KSQLDB_USERNAME=${KSQLDB_USERNAME} --set-env-vars=KSQLDB_PASSWORD=${KSQLDB_PASSWORD} skk-web-ui -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | == Serverless Kotlin Kafka 2 | 3 | == ws-to-kafka 4 | 5 | === Dev (TestContainer Kafka): 6 | 7 | [source,shell script] 8 | ---- 9 | ./gradlew :ws-to-kafka:bootRun 10 | ---- 11 | 12 | .consume messaged with schema 13 | [source,shell script] 14 | ---- 15 | docker exec -it schema-registry /usr/bin/kafka-json-schema-console-consumer --topic mytopic --bootstrap-server broker:9092 16 | ---- 17 | 18 | 19 | === Prod (External Kafka): 20 | 21 | [source,shell script] 22 | ---- 23 | export KAFKA_BOOTSTRAP_SERVERS=YOUR_BOOTSTRAP_SERVERS 24 | export KAFKA_USERNAME=YOUR_CCLOUD_KEY 25 | export KAFKA_PASSWORD=YOUR_CCLOUD_PASSWORD 26 | export SCHEMA_REGISTRY_URL=YOUR_CCLOUD_SCHEMA_REGISTRY_URL 27 | export SCHEMA_REGISTRY_KEY=YOUR_SCHEMA_REGISTRY_USERNAME 28 | export SCHEMA_REGISTRY_PASSWORD=YOUR_SCHEMA_REGISTRY_PASSWORD 29 | 30 | ./gradlew :ws-to-kafka:run 31 | ---- 32 | 33 | ==== run on Kubernetes 34 | 35 | * provision Kubernetes cluster on GCP 36 | + 37 | 38 | [source,bash] 39 | ---- 40 | make create-gke-cluster # <1> 41 | 42 | ./gradlew :ws-to-kafka:bootBuildImage --imageName=gcr.io/devx-testing/ws-to-kafka && docker push gcr.io/devx-testing/ws-to-kafka # <2> 43 | 44 | envsubst < ws-to-kafka-app-secret-template.yaml | kubectl apply -f - # <3> 45 | # or 46 | envsubst < ws-to-kafka-app-secret-template.yaml > ws-to-kafka-app-secret.yaml 47 | 48 | skaffold dev #<4> 49 | ---- 50 | <1> create Kubernetes cluster 51 | <2> build the image and push to Google Container Registry 52 | <3> create Kubernetes secret 53 | <4> deploy app using Skaffold 54 | 55 | .containerize & run 56 | [source,shell script] 57 | ---- 58 | ./gradlew :ws-to-kafka:bootBuildImage --imageName=skk-ws-to-kafka 59 | 60 | # using env vars from above 61 | docker run -it \ 62 | -eKAFKA_BOOTSTRAP_SERVERS=$KAFKA_BOOTSTRAP_SERVERS \ 63 | -eKAFKA_USERNAME=$KAFKA_USERNAME \ 64 | -eKAFKA_PASSWORD=$KAFKA_PASSWORD \ 65 | -eSCHEMA_REGISTRY_URL=$SCHEMA_REGISTRY_URL \ 66 | -eSCHEMA_REGISTRY_KEY=$SCHEMA_REGISTRY_KEY \ 67 | -eSCHEMA_REGISTRY_PASSWORD=$SCHEMA_REGISTRY_PASSWORD \ 68 | skk-ws-to-kafka 69 | ---- 70 | 71 | 72 | == ksqldb-setup 73 | 74 | === Prod 75 | 76 | [source,shell script] 77 | ---- 78 | export KSQLDB_ENDPOINT=YOUR_KSQLDB_ENDPOINT 79 | export KSQLDB_USERNAME=YOUR_KSQLDB_USERNAME 80 | export KSQLDB_PASSWORD=YOUR_KSQLDB_PASSWORD 81 | 82 | ./gradlew :ksqldb-setup:run 83 | ---- 84 | 85 | .containerize & run 86 | [source,shell script] 87 | ---- 88 | ./gradlew :ksqldb-setup:bootBuildImage --imageName=skk-ksqldb-setup 89 | 90 | # using env vars from above 91 | docker run -it \ 92 | -eKSQLDB_ENDPOINT=$KSQLDB_ENDPOINT \ 93 | -eKSQLDB_USERNAME=$KSQLDB_USERNAME \ 94 | -eKSQLDB_PASSWORD=$KSQLDB_PASSWORD \ 95 | skk-ksqldb-setup 96 | ---- 97 | 98 | 99 | == web-ui 100 | 101 | === Dev (TestContainer Kafka): 102 | 103 | [source,shell script] 104 | ---- 105 | ./gradlew :web-ui:bootRun 106 | ---- 107 | 108 | View the Web UI: http://localhost:8080 109 | 110 | === Prod (External Kafka): 111 | 112 | [source,shell script] 113 | ---- 114 | export KSQLDB_ENDPOINT=YOUR_KSQLDB_ENDPOINT 115 | export KSQLDB_USERNAME=YOUR_KSQLDB_USERNAME 116 | export KSQLDB_PASSWORD=YOUR_KSQLDB_PASSWORD 117 | 118 | ./gradlew :web-ui:run 119 | ---- 120 | 121 | .containerize & run 122 | [source,shell script] 123 | ---- 124 | ./gradlew :web-ui:bootBuildImage --imageName=skk-web-ui 125 | 126 | # using env vars from above 127 | docker run -it \ 128 | -eKSQLDB_ENDPOINT=$KSQLDB_ENDPOINT \ 129 | -eKSQLDB_USERNAME=$KSQLDB_USERNAME \ 130 | -eKSQLDB_PASSWORD=$KSQLDB_PASSWORD \ 131 | skk-web-ui 132 | ---- 133 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | kotlin("jvm") version "1.4.30" apply false 3 | kotlin("plugin.spring") version "1.4.30" apply false 4 | id("org.springframework.boot") version "2.4.2" apply false 5 | id("io.spring.dependency-management") version "1.0.11.RELEASE" apply false 6 | } 7 | 8 | allprojects { 9 | repositories { 10 | mavenCentral() 11 | jcenter() 12 | maven("https://packages.confluent.io/maven") 13 | maven("https://repository.mulesoft.org/nexus/content/repositories/public/") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /common/build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | `java-library` 3 | id("org.springframework.boot") 4 | id("io.spring.dependency-management") 5 | kotlin("jvm") 6 | kotlin("plugin.spring") 7 | } 8 | 9 | java { 10 | sourceCompatibility = JavaVersion.VERSION_1_8 11 | targetCompatibility = JavaVersion.VERSION_1_8 12 | } 13 | 14 | dependencies { 15 | implementation(kotlin("reflect")) 16 | implementation(kotlin("stdlib-jdk8")) 17 | 18 | api("org.springframework.boot:spring-boot-autoconfigure") 19 | api("com.fasterxml.jackson.module:jackson-module-kotlin") 20 | 21 | api("io.confluent.ksql:ksqldb-api-client:6.1.0") { 22 | exclude("org.slf4j", "slf4j-log4j12") 23 | } 24 | 25 | annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") 26 | } 27 | 28 | tasks.withType { 29 | kotlinOptions { 30 | freeCompilerArgs = listOf("-Xjsr305=strict") 31 | jvmTarget = JavaVersion.VERSION_1_8.toString() 32 | } 33 | } 34 | 35 | tasks.withType { 36 | enabled = false 37 | } 38 | 39 | tasks.withType { 40 | enabled = true 41 | } 42 | -------------------------------------------------------------------------------- /common/src/main/kotlin/skk/KafkaConfig.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import io.confluent.ksql.api.client.Client 4 | import io.confluent.ksql.api.client.ClientOptions 5 | import org.springframework.beans.factory.annotation.Value 6 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean 7 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty 8 | import org.springframework.context.annotation.Bean 9 | import org.springframework.context.annotation.Configuration 10 | import org.springframework.context.annotation.Lazy 11 | import java.net.URL 12 | 13 | data class KafkaTopicConfig(val replicas: Int, val partitions: Int, val name: String) 14 | 15 | data class KafkaConfig(val bootstrapServers: String, val username: String? = null, val password: String? = null) { 16 | 17 | val authConfig = if (username.isNullOrEmpty() && password.isNullOrEmpty()) { 18 | emptyMap() 19 | } 20 | else { 21 | mapOf( 22 | "ssl.endpoint.identification.algorithm" to "https", 23 | "sasl.mechanism" to "PLAIN", 24 | "sasl.jaas.config" to "org.apache.kafka.common.security.plain.PlainLoginModule required username='$username' password='$password';", 25 | "security.protocol" to "SASL_SSL", 26 | ) 27 | } 28 | 29 | val config = mapOf( 30 | "bootstrap.servers" to bootstrapServers, 31 | ) + authConfig 32 | 33 | } 34 | 35 | data class SchemaRegistryConfig(val url: String, val username: String? = null, val password: String? = null) { 36 | val authConfig = if (username.isNullOrEmpty() && password.isNullOrEmpty()) { 37 | emptyMap() 38 | } 39 | else { 40 | mapOf( 41 | "basic.auth.credentials.source" to "USER_INFO", 42 | "schema.registry.basic.auth.user.info" to "$username:$password", 43 | ) 44 | } 45 | 46 | val config = mapOf( 47 | "schema.registry.url" to url 48 | ) + authConfig 49 | } 50 | 51 | data class KsqldbConfig(val endpoint: URL, val maybeUsername: String? = null, val maybePassword: String? = null) 52 | 53 | @Configuration 54 | class KafkaConfigs { 55 | 56 | @Bean 57 | @ConditionalOnProperty(name = ["ksqldb.endpoint"]) 58 | fun ksqlClient( 59 | @Value("\${ksqldb.endpoint}") endpoint: URL, 60 | @Value("\${ksqldb.username:}") username: String, 61 | @Value("\${ksqldb.password:}") password: String, 62 | ): KsqldbConfig { 63 | val maybeUsername = if (username.isEmpty()) null else username 64 | val maybePassword = if (password.isEmpty()) null else password 65 | return KsqldbConfig(endpoint, maybeUsername, maybePassword) 66 | } 67 | 68 | @Bean 69 | @ConditionalOnProperty(name = ["serverless.kotlin.kafka.mytopic.name", "serverless.kotlin.kafka.mytopic.replicas", "serverless.kotlin.kafka.mytopic.partitions"]) 70 | @ConditionalOnMissingBean(KafkaTopicConfig::class) 71 | fun kafkaTopicConfig( 72 | @Value("\${serverless.kotlin.kafka.mytopic.name}") name: String, 73 | @Value("\${serverless.kotlin.kafka.mytopic.replicas}") replicas: Int, 74 | @Value("\${serverless.kotlin.kafka.mytopic.partitions}") partitions: Int, 75 | ): KafkaTopicConfig { 76 | return KafkaTopicConfig(replicas, partitions, name) 77 | } 78 | 79 | @Bean 80 | @ConditionalOnProperty(name = ["kafka.bootstrap.servers"]) 81 | fun kafkaConfig( 82 | @Value("\${kafka.bootstrap.servers}") bootstrapServers: String, 83 | @Value("\${kafka.username:}") username: String, 84 | @Value("\${kafka.password:}") password: String, 85 | ): KafkaConfig { 86 | return KafkaConfig(bootstrapServers, username, password) 87 | } 88 | 89 | @Bean 90 | @ConditionalOnProperty(name = ["schema.registry.url"]) 91 | fun schemaRegistryConfig( 92 | @Value("\${schema.registry.url}") url: String, 93 | @Value("\${schema.registry.key:}") key: String, 94 | @Value("\${schema.registry.password:}") password: String, 95 | ): SchemaRegistryConfig { 96 | return SchemaRegistryConfig(url, key, password) 97 | } 98 | 99 | @Bean 100 | @Lazy 101 | fun client(ksqldbConfig: KsqldbConfig): Client { 102 | val baseOptions = ClientOptions.create() 103 | .setHost(ksqldbConfig.endpoint.host) 104 | .setPort(ksqldbConfig.endpoint.port) 105 | 106 | val options = ksqldbConfig.maybeUsername?.let { username -> 107 | ksqldbConfig.maybePassword?.let { password -> 108 | baseOptions 109 | .setUseTls(true) 110 | .setUseAlpn(true) 111 | .setBasicAuthCredentials(username, password) 112 | } 113 | } ?: baseOptions 114 | 115 | return Client.create(options) 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /common/src/main/kotlin/skk/Question.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty 4 | import com.fasterxml.jackson.core.JsonParser 5 | import com.fasterxml.jackson.core.JsonProcessingException 6 | import com.fasterxml.jackson.databind.DeserializationContext 7 | import com.fasterxml.jackson.databind.JsonDeserializer 8 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize 9 | import java.io.IOException 10 | 11 | // from https://github.com/ktorio/ktor/blob/master/ktor-utils/common/src/io/ktor/util/Text.kt#L10 12 | fun String.escapeHTML(): String { 13 | val text = this@escapeHTML 14 | if (text.isEmpty()) return text 15 | 16 | return buildString(length) { 17 | for (element in text) { 18 | when (element) { 19 | '\'' -> append("'") 20 | '\"' -> append(""") 21 | '&' -> append("&") 22 | '<' -> append("<") 23 | '>' -> append(">") 24 | else -> append(element) 25 | } 26 | } 27 | } 28 | } 29 | 30 | class QuestionBodyDeserializer : JsonDeserializer() { 31 | @Throws(IOException::class, JsonProcessingException::class) 32 | override fun deserialize(parser: JsonParser, context: DeserializationContext): String { 33 | return parser.text.escapeHTML() 34 | } 35 | } 36 | 37 | class QuestionTagsDeserializer : JsonDeserializer>() { 38 | @Throws(IOException::class, JsonProcessingException::class) 39 | @Suppress("UNCHECKED_CAST") 40 | override fun deserialize(parser: JsonParser, context: DeserializationContext): List { 41 | // data is [foo|bar] so we need to manually split it 42 | val deserializer: JsonDeserializer = context.findRootValueDeserializer(context.constructType(List::class.java)) 43 | val maybeList = deserializer.deserialize(parser, context) as? List 44 | return maybeList?.let { it.firstOrNull()?.split('|') } ?: emptyList() 45 | } 46 | } 47 | 48 | // JSON-Schema will generated from POKO in SR by Serializer 49 | data class Question( 50 | val url: String, 51 | val title: String, 52 | @JsonProperty("favorite_count") val favoriteCount: Int, 53 | @JsonProperty("view_count") val viewCount: Int, 54 | @JsonDeserialize(using = QuestionTagsDeserializer::class) val tags: List, 55 | @JsonDeserialize(using = QuestionBodyDeserializer::class) val body: String 56 | ) 57 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | kotlin.code.style=official 2 | org.gradle.parallel=true 3 | org.gradle.caching=true -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamesward/serverless-kotlin-kafka/7081110f01d6b8f62953aac9fa4a98f365abb900/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /ksqldb-setup-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: ksqldb-setup 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: ksqldb-setup 9 | ttlSecondsAfterFinished: 10 10 | template: 11 | metadata: 12 | labels: 13 | app: ksqldb-setup 14 | spec: 15 | containers: 16 | - name: ksqldb-setup 17 | image: gcr.io/devx-testing/ksqldb-setup 18 | env: 19 | - name: KSQLDB_ENDPOINT 20 | valueFrom: 21 | secretKeyRef: 22 | key: KSQLDB_ENDPOINT 23 | name: ccloud-secret 24 | - name: KSQLDB_USERNAME 25 | valueFrom: 26 | secretKeyRef: 27 | key: KSQLDB_USERNAME 28 | name: ccloud-secret 29 | - name: KSQLDB_PASSWORD 30 | valueFrom: 31 | secretKeyRef: 32 | key: KSQLDB_PASSWORD 33 | name: ccloud-secret 34 | resources: 35 | requests: 36 | memory: 512Mi # 768Mi 37 | cpu: 500m # 1000m 38 | -------------------------------------------------------------------------------- /ksqldb-setup/build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 2 | 3 | plugins { 4 | application 5 | id("org.springframework.boot") 6 | id("io.spring.dependency-management") 7 | kotlin("jvm") 8 | kotlin("plugin.spring") 9 | } 10 | 11 | java { 12 | sourceCompatibility = JavaVersion.VERSION_1_8 13 | targetCompatibility = JavaVersion.VERSION_1_8 14 | } 15 | 16 | dependencies { 17 | api(project(":common")) 18 | 19 | implementation(kotlin("reflect")) 20 | implementation(kotlin("stdlib-jdk8")) 21 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8") 22 | 23 | runtimeOnly("ch.qos.logback:logback-classic") 24 | } 25 | 26 | tasks.withType { 27 | kotlinOptions { 28 | freeCompilerArgs = listOf("-Xjsr305=strict") 29 | jvmTarget = JavaVersion.VERSION_1_8.toString() 30 | } 31 | } 32 | 33 | tasks.withType { 34 | dependsOn("testClasses") 35 | args("--spring.profiles.active=dev") 36 | classpath += sourceSets["test"].runtimeClasspath 37 | } 38 | 39 | application { 40 | mainClass.set("skk.SetupKt") 41 | } 42 | 43 | tasks.withType { 44 | enabled = false 45 | } 46 | 47 | tasks.withType { 48 | enabled = true 49 | } 50 | -------------------------------------------------------------------------------- /ksqldb-setup/src/main/kotlin/skk/Setup.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import io.confluent.ksql.api.client.Client 4 | import kotlinx.coroutines.future.await 5 | import kotlinx.coroutines.runBlocking 6 | import org.springframework.boot.CommandLineRunner 7 | import org.springframework.boot.autoconfigure.SpringBootApplication 8 | import org.springframework.boot.runApplication 9 | 10 | @SpringBootApplication 11 | class Setup(val client: Client) : CommandLineRunner { 12 | 13 | override fun run(vararg args: String?): Unit = runBlocking { 14 | val stackoverflowStream = """ 15 | CREATE STREAM IF NOT EXISTS STACKOVERFLOW WITH (KAFKA_TOPIC='mytopic', VALUE_FORMAT='JSON_SR'); 16 | """.trimIndent() 17 | client.executeStatement(stackoverflowStream).await() 18 | 19 | val stackoverflowAllStream = """ 20 | CREATE STREAM IF NOT EXISTS STACKOVERFLOW_ALL AS 21 | SELECT 22 | 1 AS ONE, FAVORITE_COUNT 23 | FROM 24 | STACKOVERFLOW; 25 | """.trimIndent() 26 | client.executeStatement(stackoverflowAllStream).await() 27 | 28 | val stackoverflowExplodedStream = """ 29 | CREATE STREAM IF NOT EXISTS TAGS AS 30 | SELECT 31 | TITLE, BODY, URL, VIEW_COUNT, FAVORITE_COUNT, EXPLODE(STACKOVERFLOW.TAGS) TAG 32 | FROM 33 | STACKOVERFLOW 34 | EMIT CHANGES; 35 | """.trimIndent() 36 | client.executeStatement(stackoverflowExplodedStream).await() 37 | 38 | val tagsQuestionsTable = """ 39 | CREATE TABLE IF NOT EXISTS TAGS_QUESTIONS AS 40 | SELECT 41 | TAG, 42 | COUNT(*) QUESTION_COUNT 43 | FROM 44 | TAGS 45 | GROUP BY TAG 46 | EMIT CHANGES; 47 | """.trimIndent() 48 | client.executeStatement(tagsQuestionsTable).await() 49 | 50 | val stackoverflowTotalsTable = """ 51 | CREATE TABLE IF NOT EXISTS TOTALS AS 52 | SELECT 53 | ONE, SUM(FAVORITE_COUNT) AS TOTAL 54 | FROM 55 | STACKOVERFLOW_ALL 56 | GROUP BY ONE 57 | EMIT CHANGES; 58 | """.trimIndent() 59 | client.executeStatement(stackoverflowTotalsTable, mapOf("auto.offset.reset" to "earliest")).await() 60 | } 61 | 62 | } 63 | 64 | fun main(args: Array) { 65 | runApplication(*args).close() 66 | } 67 | -------------------------------------------------------------------------------- /scripts/ccloud/ccloud-generate-cp-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2020 Confluent Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | ############################################################################### 19 | # Overview: 20 | # 21 | # This code reads a local Confluent Cloud configuration file 22 | # and writes delta configuration files into ./delta_configs for 23 | # Confluent Platform components and clients connecting to Confluent Cloud. 24 | # 25 | # Confluent Platform Components: 26 | # - Confluent Schema Registry 27 | # - KSQL Data Generator 28 | # - ksqlDB server 29 | # - Confluent Replicator (executable) 30 | # - Confluent Control Center 31 | # - Confluent Metrics Reporter 32 | # - Confluent REST Proxy 33 | # - Kafka Connect 34 | # - Kafka connector 35 | # - Kafka command line tools 36 | # 37 | # Kafka Clients: 38 | # - Java (Producer/Consumer) 39 | # - Java (Streams) 40 | # - Python 41 | # - .NET 42 | # - Go 43 | # - Node.js (https://github.com/Blizzard/node-rdkafka) 44 | # - C++ 45 | # 46 | # Documentation for using this script: 47 | # 48 | # https://docs.confluent.io/current/cloud/connect/auto-generate-configs.html 49 | # 50 | # Arguments: 51 | # 52 | # 1 (optional) - CONFIG_FILE, defaults to ~/.ccloud/config, (required if specifying SR_CONFIG_FILE) 53 | # 2 (optional) - SR_CONFIG_FILE, defaults to CONFIG_FILE 54 | # 55 | # Example CONFIG_FILE at ~/.ccloud/config 56 | # 57 | # $ cat $HOME/.ccloud/config 58 | # 59 | # bootstrap.servers= 60 | # ssl.endpoint.identification.algorithm=https 61 | # security.protocol=SASL_SSL 62 | # sasl.mechanism=PLAIN 63 | # sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username\="" password\=""; 64 | # 65 | # If you are using Confluent Cloud Schema Registry, add the following configuration parameters 66 | # either to file above (arg 1 CONFIG_FILE) or to a separate file (arg 2 SR_CONFIG_FILE) 67 | # 68 | # basic.auth.credentials.source=USER_INFO 69 | # schema.registry.basic.auth.user.info=: 70 | # schema.registry.url=https:// 71 | # 72 | # If you are using Confluent Cloud ksqlDB, add the following configuration parameters 73 | # to file above (arg 1 CONFIG_FILE) 74 | # 75 | # ksql.endpoint= 76 | # ksql.basic.auth.user.info=: 77 | # 78 | ################################################################################ 79 | CONFIG_FILE=$1 80 | if [[ -z "$CONFIG_FILE" ]]; then 81 | CONFIG_FILE=~/.ccloud/config 82 | fi 83 | if [[ ! -f "$CONFIG_FILE" ]]; then 84 | echo "File $CONFIG_FILE is not found. Please create this properties file to connect to your Confluent Cloud cluster and then try again" 85 | echo "See https://docs.confluent.io/current/cloud/connect/auto-generate-configs.html for more information" 86 | exit 1 87 | fi 88 | 89 | SR_CONFIG_FILE=$2 90 | if [[ -z "$SR_CONFIG_FILE" ]]; then 91 | SR_CONFIG_FILE=$CONFIG_FILE 92 | fi 93 | if [[ ! -f "$SR_CONFIG_FILE" ]]; then 94 | echo "File $SR_CONFIG_FILE is not found. Please create this properties file to connect to your Schema Registry and then try again" 95 | echo "See https://docs.confluent.io/current/cloud/connect/auto-generate-configs.html for more information" 96 | exit 1 97 | fi 98 | 99 | echo -e "\nGenerating component configurations from $CONFIG_FILE and Schema Registry configurations from $SR_CONFIG_FILE" 100 | echo -e "\n(If you want to run any of these components to talk to Confluent Cloud, these are the configurations to add to the properties file for each component)" 101 | 102 | # Set permissions 103 | PERM=600 104 | if ls --version 2>/dev/null | grep -q 'coreutils' ; then 105 | # GNU binutils 106 | PERM=$(stat -c "%a" $CONFIG_FILE) 107 | else 108 | # BSD 109 | PERM=$(stat -f "%OLp" $CONFIG_FILE) 110 | fi 111 | 112 | # Make destination 113 | DEST="delta_configs" 114 | mkdir -p $DEST 115 | 116 | ################################################################################ 117 | # Glean parameters from the Confluent Cloud configuration file 118 | ################################################################################ 119 | 120 | # Kafka cluster 121 | BOOTSTRAP_SERVERS=$( grep "^bootstrap.server" $CONFIG_FILE | awk -F'=' '{print $2;}' ) 122 | BOOTSTRAP_SERVERS=${BOOTSTRAP_SERVERS/\\/} 123 | SASL_JAAS_CONFIG=$( grep "^sasl.jaas.config" $CONFIG_FILE | cut -d'=' -f2- ) 124 | SASL_JAAS_CONFIG_PROPERTY_FORMAT=${SASL_JAAS_CONFIG/username\\=/username=} 125 | SASL_JAAS_CONFIG_PROPERTY_FORMAT=${SASL_JAAS_CONFIG_PROPERTY_FORMAT/password\\=/password=} 126 | CLOUD_KEY=$( echo $SASL_JAAS_CONFIG | awk '{print $3}' | awk -F'"' '$0=$2' ) 127 | CLOUD_SECRET=$( echo $SASL_JAAS_CONFIG | awk '{print $4}' | awk -F'"' '$0=$2' ) 128 | 129 | # Schema Registry 130 | BASIC_AUTH_CREDENTIALS_SOURCE=$( grep "^basic.auth.credentials.source" $SR_CONFIG_FILE | awk -F'=' '{print $2;}' ) 131 | SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO=$( grep "^schema.registry.basic.auth.user.info" $SR_CONFIG_FILE | awk -F'=' '{print $2;}' ) 132 | SCHEMA_REGISTRY_URL=$( grep "^schema.registry.url" $SR_CONFIG_FILE | awk -F'=' '{print $2;}' ) 133 | 134 | # ksqlDB 135 | KSQLDB_ENDPOINT=$( grep "^ksql.endpoint" $CONFIG_FILE | awk -F'=' '{print $2;}' ) 136 | KSQLDB_BASIC_AUTH_USER_INFO=$( grep "^ksql.basic.auth.user.info" $CONFIG_FILE | awk -F'=' '{print $2;}' ) 137 | 138 | ################################################################################ 139 | # Build configuration file with CCloud connection parameters and 140 | # Confluent Monitoring Interceptors for Streams Monitoring in Confluent Control Center 141 | ################################################################################ 142 | INTERCEPTORS_CONFIG_FILE=$DEST/interceptors-ccloud.config 143 | rm -f $INTERCEPTORS_CONFIG_FILE 144 | echo "# Configuration derived from $CONFIG_FILE" > $INTERCEPTORS_CONFIG_FILE 145 | while read -r line 146 | do 147 | # Skip lines that are commented out 148 | if [[ ! -z $line && ${line:0:1} == '#' ]]; then 149 | continue 150 | fi 151 | # Skip lines that contain just whitespace 152 | if [[ -z "${line// }" ]]; then 153 | continue 154 | fi 155 | if [[ ${line:0:9} == 'bootstrap' ]]; then 156 | line=${line/\\/} 157 | fi 158 | echo $line >> $INTERCEPTORS_CONFIG_FILE 159 | done < "$CONFIG_FILE" 160 | echo -e "\n# Confluent Monitoring Interceptor specific configuration" >> $INTERCEPTORS_CONFIG_FILE 161 | while read -r line 162 | do 163 | # Skip lines that are commented out 164 | if [[ ! -z $line && ${line:0:1} == '#' ]]; then 165 | continue 166 | fi 167 | # Skip lines that contain just whitespace 168 | if [[ -z "${line// }" ]]; then 169 | continue 170 | fi 171 | if [[ ${line:0:9} == 'bootstrap' ]]; then 172 | line=${line/\\/} 173 | fi 174 | if [[ ${line:0:4} == 'sasl' || 175 | ${line:0:3} == 'ssl' || 176 | ${line:0:8} == 'security' || 177 | ${line:0:9} == 'bootstrap' ]]; then 178 | echo "confluent.monitoring.interceptor.$line" >> $INTERCEPTORS_CONFIG_FILE 179 | fi 180 | done < "$CONFIG_FILE" 181 | chmod $PERM $INTERCEPTORS_CONFIG_FILE 182 | 183 | echo -e "\nConfluent Platform Components:" 184 | 185 | ################################################################################ 186 | # Confluent Schema Registry instance (local) for Confluent Cloud 187 | ################################################################################ 188 | SR_CONFIG_DELTA=$DEST/schema-registry-ccloud.delta 189 | echo "$SR_CONFIG_DELTA" 190 | rm -f $SR_CONFIG_DELTA 191 | while read -r line 192 | do 193 | if [[ ! -z $line && ${line:0:1} != '#' ]]; then 194 | if [[ ${line:0:29} != 'basic.auth.credentials.source' && ${line:0:15} != 'schema.registry' ]]; then 195 | echo "kafkastore.$line" >> $SR_CONFIG_DELTA 196 | fi 197 | fi 198 | done < "$CONFIG_FILE" 199 | chmod $PERM $SR_CONFIG_DELTA 200 | 201 | ################################################################################ 202 | # Confluent Replicator (executable) for Confluent Cloud 203 | ################################################################################ 204 | REPLICATOR_PRODUCER_DELTA=$DEST/replicator-to-ccloud-producer.delta 205 | echo "$REPLICATOR_PRODUCER_DELTA" 206 | rm -f $REPLICATOR_PRODUCER_DELTA 207 | cp $INTERCEPTORS_CONFIG_FILE $REPLICATOR_PRODUCER_DELTA 208 | echo -e "\n# Confluent Replicator (executable) specific configuration" >> $REPLICATOR_PRODUCER_DELTA 209 | echo "interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" >> $REPLICATOR_PRODUCER_DELTA 210 | REPLICATOR_SASL_JAAS_CONFIG=$SASL_JAAS_CONFIG 211 | REPLICATOR_SASL_JAAS_CONFIG=${REPLICATOR_SASL_JAAS_CONFIG//\\=/=} 212 | REPLICATOR_SASL_JAAS_CONFIG=${REPLICATOR_SASL_JAAS_CONFIG//\"/\\\"} 213 | chmod $PERM $REPLICATOR_PRODUCER_DELTA 214 | 215 | ################################################################################ 216 | # ksqlDB Server runs locally and connects to Confluent Cloud 217 | ################################################################################ 218 | KSQLDB_SERVER_DELTA=$DEST/ksqldb-server-ccloud.delta 219 | echo "$KSQLDB_SERVER_DELTA" 220 | cp $INTERCEPTORS_CONFIG_FILE $KSQLDB_SERVER_DELTA 221 | echo -e "\n# ksqlDB Server specific configuration" >> $KSQLDB_SERVER_DELTA 222 | echo "producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" >> $KSQLDB_SERVER_DELTA 223 | echo "consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" >> $KSQLDB_SERVER_DELTA 224 | echo "ksql.streams.producer.retries=2147483647" >> $KSQLDB_SERVER_DELTA 225 | echo "ksql.streams.producer.confluent.batch.expiry.ms=9223372036854775807" >> $KSQLDB_SERVER_DELTA 226 | echo "ksql.streams.producer.request.timeout.ms=300000" >> $KSQLDB_SERVER_DELTA 227 | echo "ksql.streams.producer.max.block.ms=9223372036854775807" >> $KSQLDB_SERVER_DELTA 228 | echo "ksql.streams.replication.factor=3" >> $KSQLDB_SERVER_DELTA 229 | echo "ksql.internal.topic.replicas=3" >> $KSQLDB_SERVER_DELTA 230 | echo "ksql.sink.replicas=3" >> $KSQLDB_SERVER_DELTA 231 | echo -e "\n# Confluent Schema Registry configuration for ksqlDB Server" >> $KSQLDB_SERVER_DELTA 232 | while read -r line 233 | do 234 | if [[ ${line:0:29} == 'basic.auth.credentials.source' ]]; then 235 | echo "ksql.schema.registry.$line" >> $KSQLDB_SERVER_DELTA 236 | elif [[ ${line:0:15} == 'schema.registry' ]]; then 237 | echo "ksql.$line" >> $KSQLDB_SERVER_DELTA 238 | fi 239 | done < $SR_CONFIG_FILE 240 | chmod $PERM $KSQLDB_SERVER_DELTA 241 | 242 | ################################################################################ 243 | # KSQL DataGen for Confluent Cloud 244 | ################################################################################ 245 | KSQL_DATAGEN_DELTA=$DEST/ksql-datagen.delta 246 | echo "$KSQL_DATAGEN_DELTA" 247 | rm -f $KSQL_DATAGEN_DELTA 248 | cp $INTERCEPTORS_CONFIG_FILE $KSQL_DATAGEN_DELTA 249 | echo -e "\n# KSQL DataGen specific configuration" >> $KSQL_DATAGEN_DELTA 250 | echo "interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" >> $KSQL_DATAGEN_DELTA 251 | echo -e "\n# Confluent Schema Registry configuration for KSQL DataGen" >> $KSQL_DATAGEN_DELTA 252 | while read -r line 253 | do 254 | if [[ ${line:0:29} == 'basic.auth.credentials.source' ]]; then 255 | echo "ksql.schema.registry.$line" >> $KSQL_DATAGEN_DELTA 256 | elif [[ ${line:0:15} == 'schema.registry' ]]; then 257 | echo "ksql.$line" >> $KSQL_DATAGEN_DELTA 258 | fi 259 | done < $SR_CONFIG_FILE 260 | chmod $PERM $KSQL_DATAGEN_DELTA 261 | 262 | ################################################################################ 263 | # Confluent Control Center runs locally, monitors Confluent Cloud, and uses Confluent Cloud cluster as the backstore 264 | ################################################################################ 265 | C3_DELTA=$DEST/control-center-ccloud.delta 266 | echo "$C3_DELTA" 267 | rm -f $C3_DELTA 268 | echo -e "\n# Confluent Control Center specific configuration" >> $C3_DELTA 269 | while read -r line 270 | do 271 | if [[ ! -z $line && ${line:0:1} != '#' ]]; then 272 | if [[ ${line:0:9} == 'bootstrap' ]]; then 273 | line=${line/\\/} 274 | echo "$line" >> $C3_DELTA 275 | fi 276 | if [[ ${line:0:4} == 'sasl' || ${line:0:3} == 'ssl' || ${line:0:8} == 'security' ]]; then 277 | echo "confluent.controlcenter.streams.$line" >> $C3_DELTA 278 | fi 279 | fi 280 | done < "$CONFIG_FILE" 281 | # max.message.bytes is enforced to 8MB in Confluent Cloud 282 | echo "confluent.metrics.topic.max.message.bytes=8388608" >> $C3_DELTA 283 | echo -e "\n# Confluent Schema Registry configuration for Confluent Control Center" >> $C3_DELTA 284 | while read -r line 285 | do 286 | if [[ ${line:0:29} == 'basic.auth.credentials.source' ]]; then 287 | echo "confluent.controlcenter.schema.registry.$line" >> $C3_DELTA 288 | elif [[ ${line:0:15} == 'schema.registry' ]]; then 289 | echo "confluent.controlcenter.$line" >> $C3_DELTA 290 | fi 291 | done < $SR_CONFIG_FILE 292 | chmod $PERM $C3_DELTA 293 | 294 | ################################################################################ 295 | # Confluent Metrics Reporter to Confluent Cloud 296 | ################################################################################ 297 | METRICS_REPORTER_DELTA=$DEST/metrics-reporter.delta 298 | echo "$METRICS_REPORTER_DELTA" 299 | rm -f $METRICS_REPORTER_DELTA 300 | echo "metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter" >> $METRICS_REPORTER_DELTA 301 | echo "confluent.metrics.reporter.topic.replicas=3" >> $METRICS_REPORTER_DELTA 302 | while read -r line 303 | do 304 | if [[ ! -z $line && ${line:0:1} != '#' ]]; then 305 | if [[ ${line:0:9} == 'bootstrap' || ${line:0:4} == 'sasl' || ${line:0:3} == 'ssl' || ${line:0:8} == 'security' ]]; then 306 | echo "confluent.metrics.reporter.$line" >> $METRICS_REPORTER_DELTA 307 | fi 308 | fi 309 | done < "$CONFIG_FILE" 310 | chmod $PERM $METRICS_REPORTER_DELTA 311 | 312 | ################################################################################ 313 | # Confluent REST Proxy to Confluent Cloud 314 | ################################################################################ 315 | REST_PROXY_DELTA=$DEST/rest-proxy.delta 316 | echo "$REST_PROXY_DELTA" 317 | rm -f $REST_PROXY_DELTA 318 | while read -r line 319 | do 320 | if [[ ! -z $line && ${line:0:1} != '#' ]]; then 321 | if [[ ${line:0:9} == 'bootstrap' || ${line:0:4} == 'sasl' || ${line:0:3} == 'ssl' || ${line:0:8} == 'security' ]]; then 322 | echo "$line" >> $REST_PROXY_DELTA 323 | echo "client.$line" >> $REST_PROXY_DELTA 324 | fi 325 | fi 326 | done < "$CONFIG_FILE" 327 | echo -e "\n# Confluent Schema Registry configuration for REST Proxy" >> $REST_PROXY_DELTA 328 | while read -r line 329 | do 330 | if [[ ${line:0:29} == 'basic.auth.credentials.source' || ${line:0:36} == 'schema.registry.basic.auth.user.info' ]]; then 331 | echo "client.$line" >> $REST_PROXY_DELTA 332 | elif [[ ${line:0:19} == 'schema.registry.url' ]]; then 333 | echo "$line" >> $REST_PROXY_DELTA 334 | fi 335 | done < $SR_CONFIG_FILE 336 | chmod $PERM $REST_PROXY_DELTA 337 | 338 | ################################################################################ 339 | # Kafka Connect runs locally and connects to Confluent Cloud 340 | ################################################################################ 341 | CONNECT_DELTA=$DEST/connect-ccloud.delta 342 | echo "$CONNECT_DELTA" 343 | rm -f $CONNECT_DELTA 344 | cat < $CONNECT_DELTA 345 | # Configuration for embedded admin client 346 | replication.factor=3 347 | config.storage.replication.factor=3 348 | offset.storage.replication.factor=3 349 | status.storage.replication.factor=3 350 | 351 | EOF 352 | while read -r line 353 | do 354 | if [[ ! -z $line && ${line:0:1} != '#' ]]; then 355 | if [[ ${line:0:9} == 'bootstrap' ]]; then 356 | line=${line/\\/} 357 | echo "$line" >> $CONNECT_DELTA 358 | fi 359 | if [[ ${line:0:4} == 'sasl' || ${line:0:3} == 'ssl' || ${line:0:8} == 'security' ]]; then 360 | echo "$line" >> $CONNECT_DELTA 361 | fi 362 | fi 363 | done < "$CONFIG_FILE" 364 | 365 | for prefix in "producer" "consumer" "producer.confluent.monitoring.interceptor" "consumer.confluent.monitoring.interceptor" ; do 366 | 367 | echo -e "\n# Configuration for embedded $prefix" >> $CONNECT_DELTA 368 | while read -r line 369 | do 370 | if [[ ! -z $line && ${line:0:1} != '#' ]]; then 371 | if [[ ${line:0:9} == 'bootstrap' ]]; then 372 | line=${line/\\/} 373 | fi 374 | if [[ ${line:0:4} == 'sasl' || ${line:0:3} == 'ssl' || ${line:0:8} == 'security' ]]; then 375 | echo "${prefix}.$line" >> $CONNECT_DELTA 376 | fi 377 | fi 378 | done < "$CONFIG_FILE" 379 | 380 | done 381 | 382 | 383 | cat <> $CONNECT_DELTA 384 | 385 | # Confluent Schema Registry for Kafka Connect 386 | value.converter=io.confluent.connect.avro.AvroConverter 387 | value.converter.basic.auth.credentials.source=$BASIC_AUTH_CREDENTIALS_SOURCE 388 | value.converter.schema.registry.basic.auth.user.info=$SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO 389 | value.converter.schema.registry.url=$SCHEMA_REGISTRY_URL 390 | EOF 391 | chmod $PERM $CONNECT_DELTA 392 | 393 | ################################################################################ 394 | # Kafka connector 395 | ################################################################################ 396 | CONNECTOR_DELTA=$DEST/connector-ccloud.delta 397 | echo "$CONNECTOR_DELTA" 398 | rm -f $CONNECTOR_DELTA 399 | cat <> $CONNECTOR_DELTA 400 | // Confluent Schema Registry for Kafka connectors 401 | value.converter=io.confluent.connect.avro.AvroConverter 402 | value.converter.basic.auth.credentials.source=$BASIC_AUTH_CREDENTIALS_SOURCE 403 | value.converter.schema.registry.basic.auth.user.info=$SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO 404 | value.converter.schema.registry.url=$SCHEMA_REGISTRY_URL 405 | EOF 406 | chmod $PERM $CONNECTOR_DELTA 407 | 408 | ################################################################################ 409 | # AK command line tools 410 | ################################################################################ 411 | AK_TOOLS_DELTA=$DEST/ak-tools-ccloud.delta 412 | echo "$AK_TOOLS_DELTA" 413 | rm -f $AK_TOOLS_DELTA 414 | cp $CONFIG_FILE $AK_TOOLS_DELTA 415 | chmod $PERM $AK_TOOLS_DELTA 416 | 417 | ################################################################################ 418 | # ENV 419 | ################################################################################ 420 | ENV_CONFIG=$DEST/env.delta 421 | echo "$ENV_CONFIG" 422 | rm -f $ENV_CONFIG 423 | 424 | cat <> $ENV_CONFIG 425 | export BOOTSTRAP_SERVERS=$BOOTSTRAP_SERVERS 426 | export SASL_JAAS_CONFIG='$SASL_JAAS_CONFIG_PROPERTY_FORMAT' 427 | export SASL_JAAS_CONFIG_PROPERTY_FORMAT='$SASL_JAAS_CONFIG_PROPERTY_FORMAT' 428 | export REPLICATOR_SASL_JAAS_CONFIG='$REPLICATOR_SASL_JAAS_CONFIG' 429 | export BASIC_AUTH_CREDENTIALS_SOURCE=$BASIC_AUTH_CREDENTIALS_SOURCE 430 | export SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO=$SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO 431 | export SCHEMA_REGISTRY_URL=$SCHEMA_REGISTRY_URL 432 | export CLOUD_KEY=$CLOUD_KEY 433 | export CLOUD_SECRET=$CLOUD_SECRET 434 | export KSQLDB_ENDPOINT=$KSQLDB_ENDPOINT 435 | export KSQLDB_BASIC_AUTH_USER_INFO=$KSQLDB_BASIC_AUTH_USER_INFO 436 | EOF 437 | chmod $PERM $ENV_CONFIG -------------------------------------------------------------------------------- /scripts/ccloud/ccloud_key_password.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Kafka 4 | CLUSTER_ID=$(ccloud kafka cluster list -o json | jq -r '.[0].id') 5 | export CLUSTER_ID 6 | 7 | ccloud kafka cluster use "$CLUSTER_ID" 8 | 9 | KAFKA_USER_PASS=$(ccloud api-key create --resource "${CLUSTER_ID}" -ojson) 10 | KAFKA_USERNAME=$(echo "$KAFKA_USER_PASS" | jq -r '.key') 11 | KAFKA_PASSWORD=$(echo "$KAFKA_USER_PASS" | jq -r '.secret') 12 | 13 | export KAFKA_USERNAME KAFKA_PASSWORD 14 | ccloud api-key use "$KAFKA_USERNAME" --resource "$CLUSTER_ID" 15 | 16 | KAFKA_BOOTSTRAP_SERVERS=$(ccloud kafka cluster describe "$CLUSTER_ID" -ojson | jq -r '.endpoint') 17 | export KAFKA_BOOTSTRAP_SERVERS 18 | 19 | # Schema Registry 20 | SR_CLUSTER=$(ccloud schema-registry cluster enable --cloud gcp --geo us -ojson) 21 | SCHEMA_REGISTRY_ID=$(echo "$SR_CLUSTER" | jq -r '.id') 22 | SCHEMA_REGISTRY_URL=$(echo "$SR_CLUSTER" | jq -r '.endpoint_url') 23 | 24 | export SCHEMA_REGISTRY_ID SCHEMA_REGISTRY_URL 25 | 26 | SR_USER_PASS=$(ccloud api-key create --resource "$SCHEMA_REGISTRY_ID" -ojson) 27 | SCHEMA_REGISTRY_KEY=$(echo "$SR_USER_PASS" | jq -r '.key') 28 | SCHEMA_REGISTRY_PASSWORD=$(echo "$SR_USER_PASS" | jq -r '.secret') 29 | 30 | export SCHEMA_REGISTRY_KEY SCHEMA_REGISTRY_PASSWORD 31 | 32 | # ksqDB 33 | KSQLDB_APPS=$(ccloud ksql app list -ojson ) 34 | KSQLDB_ID=$(echo "$KSQLDB_APPS" | jq -r '.[0].id') 35 | KSQLDB_ENDPOINT=$(echo "$KSQLDB_APPS" | jq -r '.[0].endpoint') 36 | 37 | export KSQLDB_ID KSQLDB_ENDPOINT 38 | 39 | ccloud ksql app configure-acls "$KSQLDB_ID" mytopic 40 | 41 | KSQLDB_KEY_PASS=$(ccloud api-key create --resource "$KSQLDB_ID" -o json) 42 | KSQLDB_USERNAME=$(echo "$KSQLDB_KEY_PASS" | jq -r '.key') 43 | KSQLDB_PASSWORD=$(echo "$KSQLDB_KEY_PASS" | jq -r '.secret') 44 | 45 | export KSQLDB_USERNAME KSQLDB_PASSWORD 46 | 47 | -------------------------------------------------------------------------------- /scripts/ccloud/ccloud_library.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ################################################################ 4 | # ccloud_library.sh 5 | # -------------------------------------------------------------- 6 | # This library of functions automates common tasks with Confluent Cloud https://confluent.cloud/ 7 | # 8 | # Example usage in https://github.com/confluentinc/examples 9 | # 10 | # Get the library: 11 | # 12 | # wget -O ccloud_library.sh https://raw.githubusercontent.com/confluentinc/examples/latest/utils/ccloud_library.sh 13 | # 14 | # Use the library from your script: 15 | # 16 | # source ./ccloud_library.sh 17 | # 18 | # Support: 19 | # 20 | # 1. Community support via https://github.com/confluentinc/examples/issues 21 | # 2. There are no guarantees for backwards compatibility 22 | # 3. PRs welcome ;) 23 | ################################################################ 24 | 25 | RED='\033[0;31m' 26 | NC='\033[0m' # No Color 27 | GREEN='\033[0;32m' 28 | BLUE='\033[0;34m' 29 | YELLOW='\033[1;33m' 30 | BOLD='\033[1m' 31 | 32 | # -------------------------------------------------------------- 33 | # Initialize 34 | # -------------------------------------------------------------- 35 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 36 | 37 | # -------------------------------------------------------------- 38 | # Library 39 | # -------------------------------------------------------------- 40 | 41 | function ccloud::prompt_continue_ccloud_demo() { 42 | echo -e "${BLUE}This demo uses real Confluent Cloud ☁️ resources." 43 | echo -e "${RED} ${BOLD}💰 To avoid unexpected charges, carefully evaluate the cost of resources before launching the script and ensure all resources are destroyed after you are done running it.${NC}" 44 | read -p "Do you still want to run this script? [y/n] " -n 1 -r 45 | echo 46 | if [[ ! $REPLY =~ ^[Yy]$ ]] 47 | then 48 | exit 1 49 | fi 50 | 51 | return 0 52 | } 53 | function ccloud::validate_expect_installed() { 54 | if [[ $(type expect 2>&1) =~ "not found" ]]; then 55 | echo "'expect' is not found. Install 'expect' and try again" 56 | exit 1 57 | fi 58 | 59 | return 0 60 | } 61 | function ccloud::validate_ccloud_cli_installed() { 62 | if [[ $(type ccloud 2>&1) =~ "not found" ]]; then 63 | echo "'ccloud' is not found. Install Confluent Cloud CLI (https://docs.confluent.io/current/quickstart/cloud-quickstart/index.html#step-2-install-the-ccloud-cli) and try again" 64 | exit 1 65 | fi 66 | } 67 | 68 | function ccloud::validate_ccloud_cli_v2() { 69 | ccloud::validate_ccloud_cli_installed || exit 1 70 | 71 | if [[ -z $(ccloud version 2>&1 | grep "Go") ]]; then 72 | echo "This demo requires the new Confluent Cloud CLI. Please update your version and try again." 73 | exit 1 74 | fi 75 | 76 | return 0 77 | } 78 | 79 | function ccloud::validate_logged_in_ccloud_cli() { 80 | ccloud::validate_ccloud_cli_v2 || exit 1 81 | 82 | if [[ "$(ccloud kafka cluster list 2>&1)" == "Error: You must log in to run that command." ]]; then 83 | echo "ERROR: Log into Confluent Cloud with the command 'ccloud login --save' before running the demo. The '--save' argument saves your Confluent Cloud user login credentials or refresh token (in the case of SSO) to the local netrc file." 84 | exit 1 85 | fi 86 | 87 | return 0 88 | } 89 | 90 | function ccloud::get_version_ccloud_cli() { 91 | ccloud version | grep "^Version:" | cut -d':' -f2 | cut -d'v' -f2 92 | } 93 | 94 | function ccloud::validate_version_ccloud_cli() { 95 | 96 | ccloud::validate_ccloud_cli_installed || exit 1 97 | 98 | REQUIRED_CCLOUD_VER=${1:-"1.7.0"} 99 | CCLOUD_VER=$(ccloud::get_version_ccloud_cli) 100 | 101 | if ccloud::version_gt $REQUIRED_CCLOUD_VER $CCLOUD_VER; then 102 | echo "ccloud version ${REQUIRED_CCLOUD_VER} or greater is required. Current reported version: ${CCLOUD_VER}" 103 | echo 'To update run: ccloud update' 104 | exit 1 105 | fi 106 | } 107 | 108 | function ccloud::validate_psql_installed() { 109 | if [[ $(type psql 2>&1) =~ "not found" ]]; then 110 | echo "psql is not found. Install psql and try again" 111 | exit 1 112 | fi 113 | 114 | return 0 115 | } 116 | 117 | function ccloud::validate_aws_cli_installed() { 118 | if [[ $(type aws 2>&1) =~ "not found" ]]; then 119 | echo "AWS CLI is not found. Install AWS CLI and try again" 120 | exit 1 121 | fi 122 | 123 | return 0 124 | } 125 | 126 | function ccloud::get_version_aws_cli() { 127 | version_major=$(aws --version 2>&1 | awk -F/ '{print $2;}' | head -c 1) 128 | if [[ "$version_major" -eq 2 ]]; then 129 | echo "2" 130 | else 131 | echo "1" 132 | fi 133 | return 0 134 | } 135 | 136 | function ccloud::validate_gsutil_installed() { 137 | if [[ $(type gsutil 2>&1) =~ "not found" ]]; then 138 | echo "Google Cloud gsutil is not found. Install Google Cloud gsutil and try again" 139 | exit 1 140 | fi 141 | 142 | return 0 143 | } 144 | 145 | function ccloud::validate_az_installed() { 146 | if [[ $(type az 2>&1) =~ "not found" ]]; then 147 | echo "Azure CLI is not found. Install Azure CLI and try again" 148 | exit 1 149 | fi 150 | 151 | return 0 152 | } 153 | 154 | function ccloud::validate_cloud_source() { 155 | config=$1 156 | 157 | source $config 158 | 159 | if [[ "$DATA_SOURCE" == "kinesis" ]]; then 160 | ccloud::validate_aws_cli_installed || exit 1 161 | if [[ -z "$KINESIS_REGION" || -z "$AWS_PROFILE" ]]; then 162 | echo "ERROR: DATA_SOURCE=kinesis, but KINESIS_REGION or AWS_PROFILE is not set. Please set these parameters in config/demo.cfg and try again." 163 | exit 1 164 | fi 165 | aws kinesis list-streams --profile $AWS_PROFILE --region $KINESIS_REGION > /dev/null \ 166 | || { echo "Could not run 'aws kinesis list-streams'. Check credentials and run again." ; exit 1; } 167 | elif [[ "$DATA_SOURCE" == "rds" ]]; then 168 | ccloud::validate_aws_cli_installed || exit 1 169 | if [[ -z "$RDS_REGION" || -z "$AWS_PROFILE" ]]; then 170 | echo "ERROR: DATA_SOURCE=rds, but RDS_REGION or AWS_PROFILE is not set. Please set these parameters in config/demo.cfg and try again." 171 | exit 1 172 | fi 173 | aws rds describe-db-instances --profile $AWS_PROFILE --region $RDS_REGION > /dev/null \ 174 | || { echo "Could not run 'aws rds describe-db-instances'. Check credentials and run again." ; exit 1; } 175 | else 176 | echo "Cloud source $cloudsource is not valid. Must be one of [kinesis|rds]." 177 | exit 1 178 | fi 179 | 180 | return 0 181 | } 182 | 183 | function ccloud::validate_cloud_storage() { 184 | config=$1 185 | 186 | source $config 187 | storage=$DESTINATION_STORAGE 188 | 189 | if [[ "$storage" == "s3" ]]; then 190 | ccloud::validate_aws_cli_installed || exit 1 191 | ccloud::validate_credentials_s3 $S3_PROFILE $S3_BUCKET || exit 1 192 | aws s3api list-buckets --profile $S3_PROFILE --region $STORAGE_REGION > /dev/null \ 193 | || { echo "Could not run 'aws s3api list-buckets'. Check credentials and run again." ; exit 1; } 194 | elif [[ "$storage" == "gcs" ]]; then 195 | ccloud::validate_gsutil_installed || exit 1 196 | ccloud::validate_credentials_gcp $GCS_CREDENTIALS_FILE $GCS_BUCKET || exit 1 197 | elif [[ "$storage" == "az" ]]; then 198 | ccloud::validate_az_installed || exit 1 199 | ccloud::validate_credentials_az $AZBLOB_STORAGE_ACCOUNT $AZBLOB_CONTAINER || exit 1 200 | else 201 | echo "Storage destination $storage is not valid. Must be one of [s3|gcs|az]." 202 | exit 1 203 | fi 204 | 205 | return 0 206 | } 207 | 208 | function ccloud::validate_credentials_gcp() { 209 | GCS_CREDENTIALS_FILE=$1 210 | GCS_BUCKET=$2 211 | 212 | if [[ -z "$GCS_CREDENTIALS_FILE" || -z "$GCS_BUCKET" ]]; then 213 | echo "ERROR: DESTINATION_STORAGE=gcs, but GCS_CREDENTIALS_FILE or GCS_BUCKET is not set. Please set these parameters in config/demo.cfg and try again." 214 | exit 1 215 | fi 216 | 217 | gcloud auth activate-service-account --key-file $GCS_CREDENTIALS_FILE || { 218 | echo "ERROR: Cannot activate service account with key file $GCS_CREDENTIALS_FILE. Verify your credentials and try again." 219 | exit 1 220 | } 221 | 222 | # Create JSON-formatted string of the GCS credentials 223 | export GCS_CREDENTIALS=$(python ./stringify-gcp-credentials.py $GCS_CREDENTIALS_FILE) 224 | # Remove leading and trailing double quotes, otherwise connector creation from CLI fails 225 | GCS_CREDENTIALS=$(echo "${GCS_CREDENTIALS:1:${#GCS_CREDENTIALS}-2}") 226 | 227 | return 0 228 | } 229 | 230 | function ccloud::validate_credentials_az() { 231 | AZBLOB_STORAGE_ACCOUNT=$1 232 | AZBLOB_CONTAINER=$2 233 | 234 | if [[ -z "$AZBLOB_STORAGE_ACCOUNT" || -z "$AZBLOB_CONTAINER" ]]; then 235 | echo "ERROR: DESTINATION_STORAGE=az, but AZBLOB_STORAGE_ACCOUNT or AZBLOB_CONTAINER is not set. Please set these parameters in config/demo.cfg and try again." 236 | exit 1 237 | fi 238 | 239 | if [[ "$AZBLOB_STORAGE_ACCOUNT" == "default" ]]; then 240 | echo "ERROR: Azure Blob storage account name cannot be 'default'. Verify the value of the storage account name (did you create one?) in config/demo.cfg, as specified by the parameter AZBLOB_STORAGE_ACCOUNT, and try again." 241 | exit 1 242 | fi 243 | 244 | exists=$(az storage account check-name --name $AZBLOB_STORAGE_ACCOUNT | jq -r .reason) 245 | if [[ "$exists" != "AlreadyExists" ]]; then 246 | echo "ERROR: Azure Blob storage account name $AZBLOB_STORAGE_ACCOUNT does not exist. Check the value of AZBLOB_STORAGE_ACCOUNT in config/demo.cfg and try again." 247 | exit 1 248 | fi 249 | export AZBLOB_ACCOUNT_KEY=$(az storage account keys list --account-name $AZBLOB_STORAGE_ACCOUNT | jq -r '.[0].value') 250 | if [[ "$AZBLOB_ACCOUNT_KEY" == "" ]]; then 251 | echo "ERROR: Cannot get the key for Azure Blob storage account name $AZBLOB_STORAGE_ACCOUNT. Check the value of AZBLOB_STORAGE_ACCOUNT in config/demo.cfg, and your key, and try again." 252 | exit 1 253 | fi 254 | 255 | return 0 256 | } 257 | 258 | function ccloud::validate_credentials_s3() { 259 | S3_PROFILE=$1 260 | S3_BUCKET=$2 261 | 262 | if [[ -z "$S3_PROFILE" || -z "$S3_BUCKET" ]]; then 263 | echo "ERROR: DESTINATION_STORAGE=s3, but S3_PROFILE or S3_BUCKET is not set. Please set these parameters in config/demo.cfg and try again." 264 | exit 1 265 | fi 266 | 267 | aws configure get aws_access_key_id --profile $S3_PROFILE 1>/dev/null || { 268 | echo "ERROR: Cannot determine aws_access_key_id from S3_PROFILE=$S3_PROFILE. Verify your credentials and try again." 269 | exit 1 270 | } 271 | aws configure get aws_secret_access_key --profile $S3_PROFILE 1>/dev/null || { 272 | echo "ERROR: Cannot determine aws_secret_access_key from S3_PROFILE=$S3_PROFILE. Verify your credentials and try again." 273 | exit 1 274 | } 275 | return 0 276 | } 277 | 278 | function ccloud::validate_schema_registry_up() { 279 | auth=$1 280 | sr_endpoint=$2 281 | 282 | curl --silent -u $auth $sr_endpoint > /dev/null || { 283 | echo "ERROR: Could not validate credentials to Confluent Cloud Schema Registry. Please troubleshoot" 284 | exit 1 285 | } 286 | 287 | echo "Validated credentials to Confluent Cloud Schema Registry at $sr_endpoint" 288 | return 0 289 | } 290 | 291 | 292 | function ccloud::create_and_use_environment() { 293 | ENVIRONMENT_NAME=$1 294 | 295 | OUTPUT=$(ccloud environment create $ENVIRONMENT_NAME -o json) 296 | if [[ $? != 0 ]]; then 297 | echo "ERROR: Failed to create environment $ENVIRONMENT_NAME. Please troubleshoot (maybe run ./clean.sh) and run again" 298 | exit 1 299 | fi 300 | ENVIRONMENT=$(echo "$OUTPUT" | jq -r ".id") 301 | ccloud environment use $ENVIRONMENT &>/dev/null 302 | 303 | echo $ENVIRONMENT 304 | 305 | return 0 306 | } 307 | 308 | function ccloud::find_cluster() { 309 | CLUSTER_NAME=$1 310 | CLUSTER_CLOUD=$2 311 | CLUSTER_REGION=$3 312 | 313 | local FOUND_CLUSTER=$(ccloud kafka cluster list -o json | jq -c -r '.[] | select((.name == "'"$CLUSTER_NAME"'") and (.provider == "'"$CLUSTER_CLOUD"'") and (.region == "'"$CLUSTER_REGION"'"))') 314 | [[ ! -z "$FOUND_CLUSTER" ]] && { 315 | echo "$FOUND_CLUSTER" | jq -r .id 316 | return 0 317 | } || { 318 | return 1 319 | } 320 | } 321 | 322 | function ccloud::create_and_use_cluster() { 323 | CLUSTER_NAME=$1 324 | CLUSTER_CLOUD=$2 325 | CLUSTER_REGION=$3 326 | 327 | OUTPUT=$(ccloud kafka cluster create "$CLUSTER_NAME" --cloud $CLUSTER_CLOUD --region $CLUSTER_REGION 2>&1) 328 | if [ $? -eq 0 ]; then 329 | CLUSTER=$(echo "$OUTPUT" | grep '| Id' | awk '{print $4;}') 330 | ccloud kafka cluster use $CLUSTER 331 | echo $CLUSTER 332 | else 333 | echo "Error creating cluster: $OUTPUT. Troubleshoot and try again" 334 | exit 1 335 | fi 336 | 337 | return 0 338 | } 339 | 340 | function ccloud::maybe_create_and_use_cluster() { 341 | CLUSTER_NAME=$1 342 | CLUSTER_CLOUD=$2 343 | CLUSTER_REGION=$3 344 | CLUSTER_ID=$(ccloud::find_cluster $CLUSTER_NAME $CLUSTER_CLOUD $CLUSTER_REGION) 345 | if [ $? -eq 0 ] 346 | then 347 | ccloud kafka cluster use $CLUSTER_ID 348 | echo $CLUSTER_ID 349 | else 350 | ccloud::create_and_use_cluster "$CLUSTER_NAME" "$CLUSTER_CLOUD" "$CLUSTER_REGION" 351 | fi 352 | 353 | return 0 354 | } 355 | 356 | function ccloud::create_service_account() { 357 | SERVICE_NAME=$1 358 | 359 | OUTPUT=$(ccloud service-account create $SERVICE_NAME --description $SERVICE_NAME -o json) 360 | SERVICE_ACCOUNT_ID=$(echo "$OUTPUT" | jq -r ".id") 361 | 362 | echo $SERVICE_ACCOUNT_ID 363 | 364 | return 0 365 | } 366 | 367 | function ccloud::enable_schema_registry() { 368 | SCHEMA_REGISTRY_CLOUD=$1 369 | SCHEMA_REGISTRY_GEO=$2 370 | 371 | OUTPUT=$(ccloud schema-registry cluster enable --cloud $SCHEMA_REGISTRY_CLOUD --geo $SCHEMA_REGISTRY_GEO -o json) 372 | SCHEMA_REGISTRY=$(echo "$OUTPUT" | jq -r ".id") 373 | 374 | echo $SCHEMA_REGISTRY 375 | 376 | return 0 377 | } 378 | 379 | function ccloud::find_credentials_resource() { 380 | SERVICE_ACCOUNT_ID=$1 381 | RESOURCE=$2 382 | local FOUND_CRED=$(ccloud api-key list -o json | jq -c -r 'map(select((.resource_id == "'"$RESOURCE"'") and (.owner = "'"$SERVICE_ACCOUNT_ID"'")))') 383 | local FOUND_COUNT=$(echo "$FOUND_CRED" | jq 'length') 384 | [[ $FOUND_COUNT -ne 0 ]] && { 385 | echo "$FOUND_CRED" | jq -r '.[0].key' 386 | return 0 387 | } || { 388 | return 1 389 | } 390 | } 391 | function ccloud::create_credentials_resource() { 392 | SERVICE_ACCOUNT_ID=$1 393 | RESOURCE=$2 394 | 395 | OUTPUT=$(ccloud api-key create --service-account $SERVICE_ACCOUNT_ID --resource $RESOURCE -o json) 396 | API_KEY_SA=$(echo "$OUTPUT" | jq -r ".key") 397 | API_SECRET_SA=$(echo "$OUTPUT" | jq -r ".secret") 398 | 399 | echo "${API_KEY_SA}:${API_SECRET_SA}" 400 | 401 | return 0 402 | } 403 | ##################################################################### 404 | # The return from this function will be a colon ':' deliminted 405 | # list, if the api-key is created the second element of the 406 | # list will the secret. If the api-key is being reused 407 | # the second element of the list will be empty 408 | ##################################################################### 409 | function ccloud::maybe_create_credentials_resource() { 410 | SERVICE_ACCOUNT_ID=$1 411 | RESOURCE=$2 412 | 413 | local KEY=$(ccloud::find_credentials_resource $SERVICE_ACCOUNT_ID $RESOURCE) 414 | [[ -z $KEY ]] && { 415 | ccloud::create_credentials_resource $SERVICE_ACCOUNT_ID $RESOURCE 416 | } || { 417 | echo "$KEY:"; # the secret cannot be retrieved from a found key, caller needs to handle this 418 | return 0 419 | } 420 | } 421 | 422 | function ccloud::find_ksqldb_app() { 423 | KSQLDB_NAME=$1 424 | CLUSTER=$2 425 | 426 | local FOUND_APP=$(ccloud ksql app list -o json | jq -c -r 'map(select((.name == "'"$KSQLDB_NAME"'") and (.kafka == "'"$CLUSTER"'")))') 427 | local FOUND_COUNT=$(echo "$FOUND_APP" | jq 'length') 428 | [[ $FOUND_COUNT -ne 0 ]] && { 429 | echo "$FOUND_APP" | jq -r '.[].id' 430 | return 0 431 | } || { 432 | return 1 433 | } 434 | } 435 | 436 | function ccloud::create_ksqldb_app() { 437 | KSQLDB_NAME=$1 438 | CLUSTER=$2 439 | 440 | KSQLDB=$(ccloud ksql app create --cluster $CLUSTER -o json "$KSQLDB_NAME" | jq -r ".id") 441 | echo $KSQLDB 442 | 443 | return 0 444 | } 445 | function ccloud::maybe_create_ksqldb_app() { 446 | KSQLDB_NAME=$1 447 | CLUSTER=$2 448 | 449 | APP_ID=$(ccloud::find_ksqldb_app $KSQLDB_NAME $CLUSTER) 450 | if [ $? -eq 0 ] 451 | then 452 | echo $APP_ID 453 | else 454 | ccloud::create_ksqldb_app "$KSQLDB_NAME" "$CLUSTER" 455 | fi 456 | 457 | return 0 458 | } 459 | 460 | function ccloud::create_acls_all_resources_full_access() { 461 | SERVICE_ACCOUNT_ID=$1 462 | [[ $QUIET == "true" ]] && 463 | local REDIRECT_TO="/dev/null" || 464 | local REDIRECT_TO="/dev/stdout" 465 | 466 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation CREATE --topic '*' &>"$REDIRECT_TO" 467 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --topic '*' &>"$REDIRECT_TO" 468 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation READ --topic '*' &>"$REDIRECT_TO" 469 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation DESCRIBE --topic '*' &>"$REDIRECT_TO" 470 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation DESCRIBE_CONFIGS --topic '*' &>"$REDIRECT_TO" 471 | 472 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation READ --consumer-group '*' &>"$REDIRECT_TO" 473 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --consumer-group '*' &>"$REDIRECT_TO" 474 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation CREATE --consumer-group '*' &>"$REDIRECT_TO" 475 | 476 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation DESCRIBE --transactional-id '*' &>"$REDIRECT_TO" 477 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --transactional-id '*' &>"$REDIRECT_TO" 478 | 479 | ccloud kafka acl create --allow --service-account $SERVICE_ACCOUNT_ID --operation IDEMPOTENT-WRITE --cluster-scope &>"$REDIRECT_TO" 480 | 481 | return 0 482 | } 483 | 484 | function ccloud::delete_acls_ccloud_stack() { 485 | SERVICE_ACCOUNT_ID=$1 486 | 487 | [[ $QUIET == "true" ]] && 488 | local REDIRECT_TO="/dev/null" || 489 | local REDIRECT_TO="/dev/stdout" 490 | 491 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation CREATE --topic '*' &>"$REDIRECT_TO" 492 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --topic '*' &>"$REDIRECT_TO" 493 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation READ --topic '*' &>"$REDIRECT_TO" 494 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation DESCRIBE --topic '*' &>"$REDIRECT_TO" 495 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation DESCRIBE_CONFIGS --topic '*' &>"$REDIRECT_TO" 496 | 497 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation READ --consumer-group '*' &>"$REDIRECT_TO" 498 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --consumer-group '*' &>"$REDIRECT_TO" 499 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation CREATE --consumer-group '*' &>"$REDIRECT_TO" 500 | 501 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation DESCRIBE --transactional-id '*' &>"$REDIRECT_TO" 502 | ccloud kafka acl delete --allow --service-account $SERVICE_ACCOUNT_ID --operation WRITE --transactional-id '*' &>"$REDIRECT_TO" 503 | 504 | return 0 505 | } 506 | 507 | function ccloud::validate_ccloud_config() { 508 | expected_configfile=$1 509 | 510 | if [[ ! -f "$expected_configfile" ]]; then 511 | echo "Confluent Cloud configuration file does not exist at $expected_configfile. Please create the configuration file with properties set to your Confluent Cloud cluster and try again." 512 | exit 1 513 | else 514 | cat "$CONFIG_FILE" | jq . &> /dev/null 515 | status=$? 516 | if [[ $status == 0 ]]; then 517 | echo "ERROR: File $CONFIG_FILE is not properly formatted as key=value pairs (did you accidentally point to the Confluent Cloud CLI 'config.json' file?--this will not work). Manually create the required properties file to connect to your Confluent Cloud cluster and then try again." 518 | echo "See https://docs.confluent.io/current/cloud/connect/auto-generate-configs.html for more information" 519 | exit 1 520 | elif ! [[ $(grep "^\s*bootstrap.server" $expected_configfile) ]]; then 521 | echo "Missing 'bootstrap.server' in $expected_configfile. Please modify the configuration file with properties set to your Confluent Cloud cluster and try again." 522 | exit 1 523 | fi 524 | fi 525 | 526 | return 0 527 | } 528 | 529 | function ccloud::validate_ksqldb_up() { 530 | ksqldb_endpoint=$1 531 | ccloud_config_file=$2 532 | credentials=$3 533 | 534 | ccloud::validate_logged_in_ccloud_cli || exit 1 535 | 536 | if [[ "$ksqldb_endpoint" == "" ]]; then 537 | echo "ERROR: Provision a ksqlDB cluster via the Confluent Cloud UI and add the configuration parameter ksql.endpoint and ksql.basic.auth.user.info into your Confluent Cloud configuration file at $ccloud_config_file and try again." 538 | exit 1 539 | fi 540 | ksqlDBAppId=$(ccloud ksql app list | grep "$ksqldb_endpoint" | awk '{print $1}') 541 | if [[ "$ksqlDBAppId" == "" ]]; then 542 | echo "ERROR: Confluent Cloud ksqlDB endpoint $ksqldb_endpoint is not found. Provision a ksqlDB cluster via the Confluent Cloud UI and add the configuration parameter ksql.endpoint and ksql.basic.auth.user.info into your Confluent Cloud configuration file at $ccloud_config_file and try again." 543 | exit 1 544 | fi 545 | STATUS=$(ccloud ksql app describe $ksqlDBAppId | grep "Status" | grep UP) 546 | if [[ "$STATUS" == "" ]]; then 547 | echo "ERROR: Confluent Cloud ksqlDB endpoint $ksqldb_endpoint with id $ksqlDBAppId is not in UP state. Troubleshoot and try again." 548 | exit 1 549 | fi 550 | 551 | ccloud::validate_credentials_ksqldb "$ksqldb_endpoint" "$ccloud_config_file" "$credentials" || exit 1 552 | 553 | return 0 554 | } 555 | 556 | function ccloud::validate_azure_account() { 557 | AZBLOB_STORAGE_ACCOUNT=$1 558 | 559 | if [[ "$AZBLOB_STORAGE_ACCOUNT" == "default" ]]; then 560 | echo "ERROR: Azure Blob storage account name cannot be 'default'. Verify the value of the storage account name (did you create one?) in config/demo.cfg, as specified by the parameter AZBLOB_STORAGE_ACCOUNT, and try again." 561 | exit 1 562 | fi 563 | 564 | exists=$(az storage account check-name --name $AZBLOB_STORAGE_ACCOUNT | jq -r .reason) 565 | if [[ "$exists" != "AlreadyExists" ]]; then 566 | echo "ERROR: Azure Blob storage account name $AZBLOB_STORAGE_ACCOUNT does not exist. Check the value of STORAGE_PROFILE in config/demo.cfg and try again." 567 | exit 1 568 | fi 569 | export AZBLOB_ACCOUNT_KEY=$(az storage account keys list --account-name $AZBLOB_STORAGE_ACCOUNT | jq -r '.[0].value') 570 | if [[ "$AZBLOB_ACCOUNT_KEY" == "" ]]; then 571 | echo "ERROR: Cannot get the key for Azure Blob storage account name $AZBLOB_STORAGE_ACCOUNT. Check the value of STORAGE_PROFILE in config/demo.cfg, and your key, and try again." 572 | exit 1 573 | fi 574 | 575 | return 0 576 | } 577 | 578 | function ccloud::validate_credentials_ksqldb() { 579 | ksqldb_endpoint=$1 580 | ccloud_config_file=$2 581 | credentials=$3 582 | 583 | response=$(curl ${ksqldb_endpoint}/info \ 584 | -H "Content-Type: application/vnd.ksql.v1+json; charset=utf-8" \ 585 | --silent \ 586 | -u $credentials) 587 | if [[ "$response" =~ "Unauthorized" ]]; then 588 | echo "ERROR: Authorization failed to the ksqlDB cluster. Check your ksqlDB credentials set in the configuration parameter ksql.basic.auth.user.info in your Confluent Cloud configuration file at $ccloud_config_file and try again." 589 | exit 1 590 | fi 591 | 592 | echo "Validated credentials to Confluent Cloud ksqlDB at $ksqldb_endpoint" 593 | return 0 594 | } 595 | 596 | function ccloud::create_connector() { 597 | file=$1 598 | 599 | echo -e "\nCreating connector from $file\n" 600 | 601 | # About the Confluent Cloud CLI command 'ccloud connector create': 602 | # - Typical usage of this CLI would be 'ccloud connector create --config ' 603 | # - However, in this demo, the connector's configuration file contains parameters that need to be first substituted 604 | # so the CLI command includes eval and heredoc. 605 | # - The '-vvv' is added for verbose output 606 | ccloud connector create -vvv --config <(eval "cat </dev/null 658 | return $? 659 | } 660 | 661 | function ccloud::validate_topic_exists() { 662 | topic=$1 663 | 664 | ccloud kafka topic describe $topic &>/dev/null 665 | return $? 666 | } 667 | 668 | function ccloud::validate_subject_exists() { 669 | subject=$1 670 | sr_url=$2 671 | sr_credentials=$3 672 | 673 | curl --silent -u $sr_credentials $sr_url/subjects/$subject/versions/latest | jq -r ".subject" | grep $subject > /dev/null 674 | return $? 675 | } 676 | 677 | function ccloud::login_ccloud_cli(){ 678 | 679 | URL=$1 680 | EMAIL=$2 681 | PASSWORD=$3 682 | 683 | ccloud::validate_expect_installed 684 | 685 | echo -e "\n# Login" 686 | OUTPUT=$( 687 | expect </dev/null 796 | ccloud kafka acl create --allow --service-account $serviceAccount --operation WRITE --topic $TOPIC --prefix 797 | ccloud kafka acl create --allow --service-account $serviceAccount --operation READ --topic $TOPIC --prefix 798 | done 799 | 800 | ccloud kafka acl create --allow --service-account $serviceAccount --operation READ --consumer-group connect-cloud 801 | 802 | echo "Connectors: creating topics and ACLs for service account $serviceAccount" 803 | ccloud kafka acl create --allow --service-account $serviceAccount --operation READ --consumer-group connect-replicator 804 | ccloud kafka acl create --allow --service-account $serviceAccount --operation describe --cluster-scope 805 | 806 | return 0 807 | } 808 | 809 | function ccloud::validate_ccloud_stack_up() { 810 | CLOUD_KEY=$1 811 | CONFIG_FILE=$2 812 | enable_ksqldb=$3 813 | 814 | if [ -z "$enable_ksqldb" ]; then 815 | enable_ksqldb=true 816 | fi 817 | 818 | ccloud::validate_environment_set || exit 1 819 | ccloud::set_kafka_cluster_use "$CLOUD_KEY" "$CONFIG_FILE" || exit 1 820 | ccloud::validate_schema_registry_up "$SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO" "$SCHEMA_REGISTRY_URL" || exit 1 821 | if $enable_ksqldb ; then 822 | ccloud::validate_ksqldb_up "$KSQLDB_ENDPOINT" "$CONFIG_FILE" "$KSQLDB_BASIC_AUTH_USER_INFO" || exit 1 823 | fi 824 | } 825 | 826 | function ccloud::validate_environment_set() { 827 | ccloud environment list | grep '*' &>/dev/null || { 828 | echo "ERROR: could not determine if environment is set. Run 'ccloud environment list' and set 'ccloud environment use' and try again" 829 | exit 1 830 | } 831 | 832 | return 0 833 | 834 | } 835 | 836 | function ccloud::set_kafka_cluster_use() { 837 | CLOUD_KEY=$1 838 | CONFIG_FILE=$2 839 | 840 | if [[ "$CLOUD_KEY" == "" ]]; then 841 | echo "ERROR: could not parse the broker credentials from $CONFIG_FILE. Verify your credentials and try again." 842 | exit 1 843 | fi 844 | kafkaCluster=$(ccloud api-key list | grep "$CLOUD_KEY" | awk '{print $8;}') 845 | if [[ "$kafkaCluster" == "" ]]; then 846 | echo "ERROR: Could not associate key $CLOUD_KEY to a Confluent Cloud Kafka cluster. Verify your credentials, ensure the API key has a set resource type, and try again." 847 | exit 1 848 | fi 849 | ccloud kafka cluster use $kafkaCluster 850 | endpoint=$(ccloud kafka cluster describe $kafkaCluster -o json | jq -r ".endpoint" | cut -c 12-) 851 | echo -e "\nAssociated key $CLOUD_KEY to Confluent Cloud Kafka cluster $kafkaCluster at $endpoint" 852 | 853 | return 0 854 | } 855 | 856 | function ccloud::create_ccloud_stack() { 857 | QUIET="${QUIET:-true}" 858 | REPLICATION_FACTOR=${REPLICATION_FACTOR:-1} 859 | enable_ksqldb=$1 860 | 861 | if [[ -z "$SERVICE_ACCOUNT_ID" ]]; then 862 | # Service Account is not received so it will be created 863 | local RANDOM_NUM=$((1 + RANDOM % 1000000)) 864 | SERVICE_NAME=${SERVICE_NAME:-"demo-app-$RANDOM_NUM"} 865 | SERVICE_ACCOUNT_ID=$(ccloud::create_service_account $SERVICE_NAME) 866 | fi 867 | 868 | if [[ "$SERVICE_NAME" == "" ]]; then 869 | echo "ERROR: SERVICE_NAME is not defined. If you are providing the SERVICE_ACCOUNT_ID to this function please also provide the SERVICE_NAME" 870 | exit 1 871 | fi 872 | 873 | echo -e "☁️ Creating Confluent Cloud stack for service account ${GREEN}${SERVICE_NAME}${NC}, ID: ${GREEN}${SERVICE_ACCOUNT_ID}${NC}." 874 | 875 | if [[ -z "$ENVIRONMENT" ]]; 876 | then 877 | # Environment is not received so it will be created 878 | ENVIRONMENT_NAME=${ENVIRONMENT_NAME:-"demo-env-$SERVICE_ACCOUNT_ID"} 879 | ENVIRONMENT=$(ccloud::create_and_use_environment $ENVIRONMENT_NAME) 880 | else 881 | ccloud environment use $ENVIRONMENT &>/dev/null 882 | fi 883 | 884 | CLUSTER_NAME=${CLUSTER_NAME:-"demo-kafka-cluster-$SERVICE_ACCOUNT_ID"} 885 | CLUSTER_CLOUD="${CLUSTER_CLOUD:-aws}" 886 | CLUSTER_REGION="${CLUSTER_REGION:-us-west-2}" 887 | CLUSTER=$(ccloud::maybe_create_and_use_cluster "$CLUSTER_NAME" $CLUSTER_CLOUD $CLUSTER_REGION) 888 | if [[ "$CLUSTER" == "" ]] ; then 889 | echo "Kafka cluster id is empty" 890 | echo "ERROR: Could not create cluster. Please troubleshoot" 891 | exit 1 892 | fi 893 | BOOTSTRAP_SERVERS=$(ccloud kafka cluster describe $CLUSTER -o json | jq -r ".endpoint" | cut -c 12-) 894 | CLUSTER_CREDS=$(ccloud::maybe_create_credentials_resource $SERVICE_ACCOUNT_ID $CLUSTER) 895 | 896 | MAX_WAIT=720 897 | echo "" 898 | echo "Waiting up to $MAX_WAIT seconds for Confluent Cloud cluster to be ready and for credentials to propagate" 899 | ccloud::retry $MAX_WAIT ccloud::validate_ccloud_cluster_ready || exit 1 900 | 901 | # Estimating another 80s wait still sometimes required 902 | WARMUP_TIME=${WARMUP_TIME:-80} 903 | echo "Sleeping an additional ${WARMUP_TIME} seconds to ensure propagation of all metadata" 904 | sleep $WARMUP_TIME 905 | 906 | SCHEMA_REGISTRY_GEO="${SCHEMA_REGISTRY_GEO:-us}" 907 | SCHEMA_REGISTRY=$(ccloud::enable_schema_registry $CLUSTER_CLOUD $SCHEMA_REGISTRY_GEO) 908 | SCHEMA_REGISTRY_ENDPOINT=$(ccloud schema-registry cluster describe -o json | jq -r ".endpoint_url") 909 | SCHEMA_REGISTRY_CREDS=$(ccloud::maybe_create_credentials_resource $SERVICE_ACCOUNT_ID $SCHEMA_REGISTRY) 910 | 911 | if $enable_ksqldb ; then 912 | KSQLDB_NAME=${KSQLDB_NAME:-"demo-ksqldb-$SERVICE_ACCOUNT_ID"} 913 | KSQLDB=$(ccloud::maybe_create_ksqldb_app "$KSQLDB_NAME" $CLUSTER) 914 | KSQLDB_ENDPOINT=$(ccloud ksql app describe $KSQLDB -o json | jq -r ".endpoint") 915 | KSQLDB_CREDS=$(ccloud::maybe_create_credentials_resource $SERVICE_ACCOUNT_ID $KSQLDB) 916 | KSQLDB_SERVICE_ACCOUNT_ID=$(ccloud service-account list -o json 2>/dev/null | jq -r "map(select(.name == \"KSQL.$KSQLDB\")) | .[0].id") 917 | ccloud ksql app configure-acls $KSQLDB 918 | fi 919 | 920 | ccloud::create_acls_all_resources_full_access $SERVICE_ACCOUNT_ID 921 | 922 | CLOUD_API_KEY=`echo $CLUSTER_CREDS | awk -F: '{print $1}'` 923 | CLOUD_API_SECRET=`echo $CLUSTER_CREDS | awk -F: '{print $2}'` 924 | ccloud api-key use $CLOUD_API_KEY --resource ${CLUSTER} 925 | 926 | if [[ -z "$SKIP_CONFIG_FILE_WRITE" ]]; then 927 | if [[ -z "$CLIENT_CONFIG" ]]; then 928 | mkdir -p stack-configs 929 | CLIENT_CONFIG="stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config" 930 | fi 931 | 932 | cat < $CLIENT_CONFIG 933 | # -------------------------------------- 934 | # Confluent Cloud connection information 935 | # -------------------------------------- 936 | # ENVIRONMENT ID: ${ENVIRONMENT} 937 | # SERVICE ACCOUNT ID: ${SERVICE_ACCOUNT_ID} 938 | # KAFKA CLUSTER ID: ${CLUSTER} 939 | # SCHEMA REGISTRY CLUSTER ID: ${SCHEMA_REGISTRY} 940 | EOF 941 | if $enable_ksqldb ; then 942 | cat <> $CLIENT_CONFIG 943 | # KSQLDB APP ID: ${KSQLDB} 944 | EOF 945 | fi 946 | cat <> $CLIENT_CONFIG 947 | # -------------------------------------- 948 | ssl.endpoint.identification.algorithm=https 949 | sasl.mechanism=PLAIN 950 | security.protocol=SASL_SSL 951 | bootstrap.servers=${BOOTSTRAP_SERVERS} 952 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username\="${CLOUD_API_KEY}" password\="${CLOUD_API_SECRET}"; 953 | basic.auth.credentials.source=USER_INFO 954 | schema.registry.url=${SCHEMA_REGISTRY_ENDPOINT} 955 | schema.registry.basic.auth.user.info=`echo $SCHEMA_REGISTRY_CREDS | awk -F: '{print $1}'`:`echo $SCHEMA_REGISTRY_CREDS | awk -F: '{print $2}'` 956 | replication.factor=${REPLICATION_FACTOR} 957 | EOF 958 | if $enable_ksqldb ; then 959 | cat <> $CLIENT_CONFIG 960 | ksql.endpoint=${KSQLDB_ENDPOINT} 961 | ksql.basic.auth.user.info=`echo $KSQLDB_CREDS | awk -F: '{print $1}'`:`echo $KSQLDB_CREDS | awk -F: '{print $2}'` 962 | EOF 963 | fi 964 | 965 | echo 966 | echo -e "${GREEN}Client configuration file saved to: ${BLUE}$CLIENT_CONFIG${NC}" 967 | fi 968 | 969 | return 0 970 | } 971 | 972 | function ccloud::destroy_ccloud_stack() { 973 | SERVICE_ACCOUNT_ID=$1 974 | 975 | ENVIRONMENT_NAME=${ENVIRONMENT_NAME:-"demo-env-$SERVICE_ACCOUNT_ID"} 976 | CLUSTER_NAME=${CLUSTER_NAME:-"demo-kafka-cluster-$SERVICE_ACCOUNT_ID"} 977 | CLIENT_CONFIG=${CLIENT_CONFIG:-"stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config"} 978 | KSQLDB_NAME=${KSQLDB_NAME:-"demo-ksqldb-$SERVICE_ACCOUNT_ID"} 979 | 980 | QUIET="${QUIET:-true}" 981 | [[ $QUIET == "true" ]] && 982 | local REDIRECT_TO="/dev/null" || 983 | local REDIRECT_TO="/dev/stdout" 984 | 985 | echo "Destroying Confluent Cloud stack associated to service account id $SERVICE_ACCOUNT_ID" 986 | 987 | if [[ $KSQLDB_ENDPOINT != "" ]]; then 988 | KSQLDB=$(ccloud ksql app list | grep $KSQLDB_NAME | awk '{print $1;}') 989 | echo "KSQLDB: $KSQLDB" 990 | ccloud ksql app delete $KSQLDB &>"$REDIRECT_TO" 991 | fi 992 | 993 | ccloud::delete_acls_ccloud_stack $SERVICE_ACCOUNT_ID 994 | ccloud service-account delete $SERVICE_ACCOUNT_ID &>"$REDIRECT_TO" 995 | 996 | CLUSTER=$(ccloud kafka cluster list | grep $CLUSTER_NAME | tr -d '\*' | awk '{print $1;}') 997 | echo "CLUSTER: $CLUSTER" 998 | ccloud kafka cluster delete $CLUSTER &> "$REDIRECT_TO" 999 | 1000 | ENVIRONMENT=$(ccloud environment list | grep $ENVIRONMENT_NAME | tr -d '\*' | awk '{print $1;}') 1001 | echo "ENVIRONMENT: $ENVIRONMENT" 1002 | ccloud environment delete $ENVIRONMENT &> "$REDIRECT_TO" 1003 | 1004 | rm -f $CLIENT_CONFIG 1005 | 1006 | return 0 1007 | } 1008 | 1009 | ############################################## 1010 | # These are some duplicate functions from 1011 | # helper.sh to decouple the script files. In 1012 | # the future we can work to remove this 1013 | # duplication if necessary 1014 | ############################################## 1015 | function ccloud::retry() { 1016 | local -r -i max_wait="$1"; shift 1017 | local -r cmd="$@" 1018 | 1019 | local -i sleep_interval=5 1020 | local -i curr_wait=0 1021 | 1022 | until $cmd 1023 | do 1024 | if (( curr_wait >= max_wait )) 1025 | then 1026 | echo "ERROR: Failed after $curr_wait seconds. Please troubleshoot and run again." 1027 | return 1 1028 | else 1029 | printf "." 1030 | curr_wait=$((curr_wait+sleep_interval)) 1031 | sleep $sleep_interval 1032 | fi 1033 | done 1034 | printf "\n" 1035 | } 1036 | function ccloud::version_gt() { 1037 | test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; 1038 | } 1039 | -------------------------------------------------------------------------------- /scripts/ccloud/ccloud_stack_create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ######################################### 4 | # This script uses real Confluent Cloud resources. 5 | # To avoid unexpected charges, carefully evaluate the cost of resources before launching the script and ensure all resources are destroyed after you are done running it. 6 | ######################################### 7 | 8 | export CLUSTER_CLOUD=gcp 9 | # Use to find the closest http://www.gcping.com/ 10 | export CLUSTER_REGION=us-central1 11 | #CLUSTER_CLOUD=aws 12 | 13 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 14 | 15 | # Source library 16 | source $DIR/../common/colors.sh 17 | source $DIR/../common/helper.sh 18 | source $DIR/ccloud_library.sh 19 | 20 | ccloud::validate_version_ccloud_cli 1.7.0 || exit 1 21 | check_jq || exit 1 22 | ccloud::validate_logged_in_ccloud_cli || exit 1 23 | 24 | ccloud::prompt_continue_ccloud_demo || exit 1 25 | 26 | enable_ksqldb=false 27 | echo -e "Do you also want to create a Confluent Cloud 🚀 ksqlDB app ${RED}(hourly charges may apply)${NC}? [y/n] " 28 | read -n 1 -r 29 | echo 30 | if [[ $REPLY =~ ^[Yy]$ ]] 31 | then 32 | enable_ksqldb=true 33 | fi 34 | 35 | echo 36 | ccloud::create_ccloud_stack $enable_ksqldb || exit 1 37 | 38 | echo 39 | echo -e "${BLUE}Validating...${NC}" 40 | SERVICE_ACCOUNT_ID=$(ccloud kafka cluster list -o json | jq -r '.[0].name' | awk -F'-' '{print $4;}') 41 | CONFIG_FILE=stack-configs/java-service-account-$SERVICE_ACCOUNT_ID.config 42 | ccloud::validate_ccloud_config $CONFIG_FILE || exit 1 43 | $DIR/ccloud-generate-cp-config.sh $CONFIG_FILE > /dev/null 44 | source delta_configs/env.delta 45 | 46 | if $enable_ksqldb ; then 47 | MAX_WAIT=500 48 | echo -e "${GREEN}Waiting up to $MAX_WAIT seconds for Confluent Cloud ksqlDB cluster to be UP${NC}" 49 | retry $MAX_WAIT ccloud::validate_ccloud_ksqldb_endpoint_ready $KSQLDB_ENDPOINT || exit 1 50 | fi 51 | 52 | ccloud::validate_ccloud_stack_up $CLOUD_KEY $CONFIG_FILE $enable_ksqldb || exit 1 53 | 54 | echo 55 | echo "ACLs in this cluster:" 56 | ccloud kafka acl list 57 | 58 | echo 59 | echo -e "${GREEN}Local client configuration file written to ${BOLD}${BLUE}${CONFIG_FILE}${NC}" 60 | echo 61 | 62 | echo 63 | echo -e "${YELLOW}To destroy this Confluent Cloud stack run ->${NC}" 64 | echo -e " ./ccloud_stack_destroy.sh $CONFIG_FILE" 65 | echo 66 | 67 | echo 68 | ENVIRONMENT=$(ccloud environment list | grep demo-env-$SERVICE_ACCOUNT_ID | tr -d '\*' | awk '{print $1;}') 69 | echo -e "${BLUE}Tip:${NC} 'ccloud' CLI has been set to the new environment ${GREEN}${ENVIRONMENT}${NC}" -------------------------------------------------------------------------------- /scripts/ccloud/ccloud_stack_destroy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 5 | 6 | # Source library 7 | source $DIR/../common/colors.sh 8 | source $DIR/../common/helper.sh 9 | source $DIR/ccloud_library.sh 10 | 11 | ccloud::validate_version_ccloud_cli 1.7.0 || exit 1 12 | ccloud::validate_logged_in_ccloud_cli || exit 1 13 | check_jq || exit 1 14 | 15 | if [ -z "$1" ]; then 16 | echo -e "${RED}ERROR: Must supply argument that is the client configuration file created from './ccloud_stack_create.sh'. (Is it in stack-configs/ folder?) ${NC}" 17 | exit 1 18 | else 19 | CONFIG_FILE=$1 20 | fi 21 | 22 | read -p "This script will destroy the entire environment specified in $CONFIG_FILE. Do you want to proceed? [y/n] " -n 1 -r 23 | echo 24 | if [[ ! $REPLY =~ ^[Yy]$ ]] 25 | then 26 | exit 1 27 | fi 28 | 29 | ccloud::validate_ccloud_config $CONFIG_FILE || exit 1 30 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 31 | $DIR/ccloud-generate-cp-config.sh $CONFIG_FILE > /dev/null 32 | source delta_configs/env.delta 33 | SERVICE_ACCOUNT_ID=$(ccloud::get_service_account $CLOUD_KEY $CONFIG_FILE) || exit 1 34 | 35 | echo 36 | ccloud::destroy_ccloud_stack $SERVICE_ACCOUNT_ID 37 | 38 | echo 39 | echo -e "${BOLD}${BLUE}Tip:${NC} 'ccloud' CLI currently has no environment set" 40 | -------------------------------------------------------------------------------- /scripts/common/Makefile: -------------------------------------------------------------------------------- 1 | echo_fail = printf "\e[31m✘ \033\e[0m$(1)\n" 2 | echo_pass = printf "\e[32m✔ \033\e[0m$(1)\n" 3 | echo_stdout_header = printf "\n+++++++++++++ $(1)\n" 4 | echo_stdout_footer = printf "+++++++++++++ $(1)\n" 5 | echo_stdout_footer_pass = printf "\e[32m✔ \033\e[0m ========= $(1) \n" 6 | 7 | check-dependency = $(if $(shell command -v $(1)),$(call echo_pass,found $(1)),$(call echo_fail,$(1) not installed);exit 1) 8 | 9 | check-var-defined = $(if $(strip $($1)),,$(error "$1" is not defined)) 10 | 11 | define print-prompt = 12 | printf "\e[96m➜ \e[0m" 13 | endef 14 | 15 | define print-header = 16 | printf "\n%-50s\n" $1 | tr ' ~' '- ' 17 | endef 18 | 19 | check-dependencies: 20 | @$(call check-dependency,curl) 21 | @$(call check-dependency,kubectl) 22 | @$(call check-dependency,jq) 23 | @$(call check-dependency,envsubst) 24 | -------------------------------------------------------------------------------- /scripts/common/colors.sh: -------------------------------------------------------------------------------- 1 | # ---------------------------------- 2 | # Colors 3 | # ---------------------------------- 4 | NOCOLOR='\033[0m' 5 | NC='\033[0m' 6 | RED='\033[0;31m' 7 | GREEN='\033[0;32m' 8 | ORANGE='\033[0;33m' 9 | BLUE='\033[0;34m' 10 | PURPLE='\033[0;35m' 11 | CYAN='\033[0;36m' 12 | LIGHTGRAY='\033[0;37m' 13 | DARKGRAY='\033[1;30m' 14 | LIGHTRED='\033[1;31m' 15 | LIGHTGREEN='\033[1;32m' 16 | YELLOW='\033[1;33m' 17 | LIGHTBLUE='\033[1;34m' 18 | LIGHTPURPLE='\033[1;35m' 19 | LIGHTCYAN='\033[1;36m' 20 | WHITE='\033[1;37m' 21 | BOLD='\033[1m' 22 | -------------------------------------------------------------------------------- /scripts/common/helper.sh: -------------------------------------------------------------------------------- 1 | function check_jq() { 2 | if [[ $(type jq 2>&1) =~ "not found" ]]; then 3 | echo "'jq' is not found. Install 'jq' and try again" 4 | exit 1 5 | fi 6 | 7 | return 0 8 | } 9 | 10 | function check_ccloud_config() { 11 | expected_configfile=$1 12 | 13 | if [[ ! -f "$expected_configfile" ]]; then 14 | echo "Confluent Cloud configuration file does not exist at $expected_configfile. Please create the configuration file with properties set to your Confluent Cloud cluster and try again." 15 | exit 1 16 | elif ! [[ $(grep "^\s*bootstrap.server" $expected_configfile) ]]; then 17 | echo "Missing 'bootstrap.server' in $expected_configfile. Please modify the configuration file with properties set to your Confluent Cloud cluster and try again." 18 | exit 1 19 | fi 20 | 21 | return 0 22 | } 23 | 24 | function validate_confluent_cloud_schema_registry() { 25 | auth=$1 26 | sr_endpoint=$2 27 | 28 | curl --silent -u $auth $sr_endpoint 29 | if [[ "$?" -ne 0 ]]; then 30 | echo "ERROR: Could not validate credentials to Confluent Cloud Schema Registry. Please troubleshoot" 31 | exit 1 32 | fi 33 | return 0 34 | } 35 | 36 | function check_docker() { 37 | if ! docker ps -q &>/dev/null; then 38 | echo "This demo requires Docker but it doesn't appear to be running. Please start Docker and try again." 39 | exit 1 40 | fi 41 | 42 | return 0 43 | } 44 | 45 | 46 | retry() { 47 | local -r -i max_wait="$1"; shift 48 | local -r cmd="$@" 49 | 50 | local -i sleep_interval=5 51 | local -i curr_wait=0 52 | 53 | until $cmd 54 | do 55 | if (( curr_wait >= max_wait )) 56 | then 57 | echo "ERROR: Failed after $curr_wait seconds. Please troubleshoot and run again." 58 | return 1 59 | else 60 | printf "." 61 | curr_wait=$((curr_wait+sleep_interval)) 62 | sleep $sleep_interval 63 | fi 64 | done 65 | printf "\n" 66 | } -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "serverless_kotlin_kafka" 2 | 3 | include("common", "ksqldb-setup", "ws-to-kafka", "web-ui") 4 | -------------------------------------------------------------------------------- /skaffold.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: skaffold/v2beta10 2 | kind: Config 3 | metadata: 4 | name: portable-serverless 5 | deploy: 6 | kubectl: 7 | manifests: 8 | - ws-to-kafka-app-secret.yaml 9 | - ws-to-kafka-app-deployment.yaml 10 | - ksqldb-setup-deployment.yaml -------------------------------------------------------------------------------- /web-ui/build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 2 | 3 | plugins { 4 | application 5 | id("org.springframework.boot") 6 | id("io.spring.dependency-management") 7 | kotlin("jvm") 8 | kotlin("plugin.spring") 9 | } 10 | 11 | java { 12 | sourceCompatibility = JavaVersion.VERSION_1_8 13 | targetCompatibility = JavaVersion.VERSION_1_8 14 | } 15 | 16 | dependencies { 17 | implementation(project(":common")) 18 | implementation(kotlin("reflect")) 19 | implementation(kotlin("stdlib-jdk8")) 20 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8") 21 | implementation("io.projectreactor.kotlin:reactor-kotlin-extensions") 22 | runtimeOnly("org.jetbrains.kotlinx:kotlinx-coroutines-reactor") 23 | 24 | implementation("org.springframework.boot:spring-boot-starter-webflux") 25 | 26 | implementation("org.webjars:bootstrap:4.5.3") 27 | implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.7.2") 28 | 29 | testImplementation("org.testcontainers:kafka:1.15.2") 30 | 31 | developmentOnly("org.springframework.boot:spring-boot-devtools") 32 | } 33 | 34 | tasks.withType { 35 | kotlinOptions { 36 | freeCompilerArgs = listOf("-Xjsr305=strict") 37 | jvmTarget = JavaVersion.VERSION_1_8.toString() 38 | } 39 | } 40 | 41 | tasks.withType { 42 | dependsOn("testClasses") 43 | dependsOn(":ws-to-kafka:bootBuildImage") 44 | dependsOn(":ksqldb-setup:bootBuildImage") 45 | args("--spring.profiles.active=dev") 46 | classpath += sourceSets["test"].runtimeClasspath 47 | } 48 | 49 | application { 50 | mainClass.set("skk.MainKt") 51 | } 52 | -------------------------------------------------------------------------------- /web-ui/src/main/kotlin/skk/Html.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import kotlinx.html.* 4 | import kotlinx.html.dom.createHTMLDocument 5 | import org.w3c.dom.Document 6 | 7 | 8 | object Html { 9 | 10 | class TEMPLATE(consumer: TagConsumer<*>) : 11 | HTMLTag("template", consumer, emptyMap(), 12 | inlineTag = true, 13 | emptyTag = false), HtmlInlineTag 14 | 15 | fun FlowContent.template(block: TEMPLATE.() -> Unit = {}) { 16 | TEMPLATE(consumer).visit(block) 17 | } 18 | 19 | fun TEMPLATE.li(classes : String? = null, block : LI.() -> Unit = {}) { 20 | LI(attributesMapOf("class", classes), consumer).visit(block) 21 | } 22 | 23 | fun page(js: String, content: FlowContent.() -> Unit = {}): HTML.() -> Unit = { 24 | head { 25 | link("/webjars/bootstrap/4.5.3/css/bootstrap.min.css", LinkRel.stylesheet) 26 | link("/assets/index.css", LinkRel.stylesheet) 27 | script(ScriptType.textJavaScript) { 28 | src = "/assets/$js" 29 | } 30 | } 31 | body { 32 | nav("navbar fixed-top navbar-light bg-light") { 33 | a("/", classes = "navbar-brand") { 34 | +"Serverless Kotlin Kafka" 35 | } 36 | } 37 | 38 | div("container-fluid") { 39 | content() 40 | } 41 | } 42 | } 43 | 44 | val indexHTML = page("index.js") { 45 | template { 46 | id = "total-template" 47 | +"Total Favorites: {{total}}" 48 | } 49 | 50 | div { 51 | id = "total" 52 | } 53 | 54 | ul { 55 | id = "recent-questions" 56 | 57 | template { 58 | id = "recent-questions-template" 59 | 60 | li { 61 | id = "lang-{{lang}}" 62 | 63 | a("{{lang}}") { 64 | +"{{lang}} = {{num}}" 65 | } 66 | } 67 | } 68 | } 69 | } 70 | 71 | val index: Document = createHTMLDocument().html(block = indexHTML) 72 | 73 | fun langHTML(name: String) = page("lang.js") { 74 | +"Questions For `$name`" 75 | 76 | ul { 77 | id = "questions" 78 | 79 | template { 80 | id = "question-template" 81 | 82 | li { 83 | a("{{url}}") { 84 | +"{{title}}" 85 | } 86 | +" (favorites: {{favorite_count}}, views: {{view_count}})" 87 | } 88 | } 89 | } 90 | } 91 | 92 | fun lang(name: String): Document = createHTMLDocument().html(block = langHTML(name)) 93 | 94 | } 95 | -------------------------------------------------------------------------------- /web-ui/src/main/kotlin/skk/Main.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import io.confluent.ksql.api.client.Client 4 | import kotlinx.coroutines.future.await 5 | import kotlinx.html.dom.serialize 6 | import org.springframework.boot.autoconfigure.SpringBootApplication 7 | import org.springframework.boot.runApplication 8 | import org.springframework.context.annotation.Bean 9 | import org.springframework.context.annotation.Configuration 10 | import org.springframework.web.bind.annotation.GetMapping 11 | import org.springframework.web.bind.annotation.PathVariable 12 | import org.springframework.web.bind.annotation.RestController 13 | import org.springframework.web.reactive.handler.SimpleUrlHandlerMapping 14 | import org.springframework.web.reactive.socket.WebSocketHandler 15 | import org.springframework.web.reactive.socket.WebSocketSession 16 | import org.springframework.web.reactive.socket.server.support.WebSocketHandlerAdapter 17 | import reactor.kotlin.core.publisher.toFlux 18 | import reactor.kotlin.core.publisher.toMono 19 | 20 | 21 | @SpringBootApplication 22 | @RestController 23 | class WebApp(val client: Client) { 24 | 25 | @GetMapping("/") 26 | fun index(): String { 27 | return Html.index.serialize(true) 28 | } 29 | 30 | @GetMapping("/total") 31 | suspend fun total() = run { 32 | val rows = client.executeQuery("SELECT * FROM TOTALS WHERE ONE = 1;").await() 33 | rows?.firstOrNull()?.getLong("TOTAL") ?: 0L 34 | } 35 | 36 | @GetMapping("/{name}") 37 | fun lang(@PathVariable name: String): String { 38 | return Html.lang(name).serialize(true) 39 | } 40 | 41 | } 42 | 43 | 44 | @Configuration 45 | class WebSocketConfig { 46 | 47 | @Bean 48 | fun simpleUrlHandlerMapping(client: Client): SimpleUrlHandlerMapping { 49 | return SimpleUrlHandlerMapping(mapOf( 50 | "/langs" to langs(client), 51 | ), 0) 52 | } 53 | 54 | fun langs(client: Client): WebSocketHandler { 55 | return WebSocketHandler { session: WebSocketSession -> 56 | val query = "SELECT * FROM TAGS_QUESTIONS EMIT CHANGES;" 57 | 58 | client.streamQuery(query).toMono().flatMap { kafkaMessages -> 59 | val webSocketMessages = kafkaMessages.toFlux().map { message -> 60 | val lang = message.getString("TAG") 61 | val num = message.getInteger("QUESTION_COUNT") 62 | session.textMessage("$lang:$num") 63 | } 64 | session.send(webSocketMessages) 65 | } 66 | 67 | } 68 | } 69 | 70 | @Bean 71 | fun webSocketHandlerAdapter(): WebSocketHandlerAdapter { 72 | return WebSocketHandlerAdapter() 73 | } 74 | 75 | } 76 | 77 | fun main(args: Array) { 78 | runApplication(*args) 79 | } 80 | -------------------------------------------------------------------------------- /web-ui/src/main/resources/META-INF/resources/assets/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin-top: 75px; 3 | font-family: monospace; 4 | } 5 | -------------------------------------------------------------------------------- /web-ui/src/main/resources/META-INF/resources/assets/index.js: -------------------------------------------------------------------------------- 1 | const wsProto = (window.location.protocol === 'https:') ? 'wss:' : 'ws:'; 2 | const wsBase = `${wsProto}//${window.location.hostname}:${window.location.port}`; 3 | 4 | window.addEventListener('load', () => { 5 | fetch('/total') 6 | .then(response => response.text()) 7 | .then(body => { 8 | const template = document.getElementById('total-template'); 9 | const total = document.importNode(template.content, true); 10 | const content = total.firstChild.textContent.replace('{{total}}', body); 11 | document.getElementById('total').innerHTML = content; 12 | }); 13 | }); 14 | 15 | 16 | function langsWS() { 17 | let ws = new WebSocket(`${wsBase}/langs`); 18 | 19 | ws.onmessage = function(event) { 20 | const [lang, num] = event.data.split(':'); 21 | 22 | const template = document.getElementById('recent-questions-template'); 23 | const recentQuestion = document.importNode(template.content, true); 24 | 25 | function replace(e) { 26 | return e.replaceAll('{{lang}}', lang).replaceAll('{{num}}', num); 27 | } 28 | 29 | for (const element of recentQuestion.children) { 30 | Array.from(element.attributes).forEach(attr => attr.value = replace(attr.value)); 31 | element.innerHTML = replace(element.innerHTML); 32 | } 33 | 34 | const recentQuestions = document.getElementById('recent-questions'); 35 | 36 | const existingRecentQuestion = document.getElementById(`lang-${lang}`); 37 | 38 | if (existingRecentQuestion != null) { 39 | existingRecentQuestion.replaceWith(recentQuestion); 40 | } 41 | else { 42 | recentQuestions.appendChild(recentQuestion); 43 | } 44 | } 45 | 46 | ws.onclose = function() { 47 | window.setTimeout(() => { ws = langsWS() }, 500); 48 | } 49 | } 50 | 51 | langsWS(); 52 | -------------------------------------------------------------------------------- /web-ui/src/main/resources/META-INF/resources/assets/lang.js: -------------------------------------------------------------------------------- 1 | const lang = window.location.pathname.slice(1); 2 | 3 | const ws = new WebSocket('wss://stackoverflow-to-ws-x5ht4amjia-uc.a.run.app/questions'); 4 | 5 | ws.onmessage = function(event) { 6 | const data = JSON.parse(event.data); 7 | const tags = data.tags[0].split('|'); 8 | 9 | if (tags.includes(lang)) { 10 | const template = document.getElementById('question-template'); 11 | const question = document.importNode(template.content, true); 12 | 13 | for (const element of question.children) { 14 | for (const key in data) { 15 | element.innerHTML = element.innerHTML.replace(new RegExp('{{' + key + '}}'), data[key]); 16 | } 17 | } 18 | 19 | document.getElementById('questions').appendChild(question); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /web-ui/src/test/kotlin/skk/TestKafkaConfigFactory.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import com.github.dockerjava.api.async.ResultCallback 4 | import com.github.dockerjava.api.command.WaitContainerResultCallback 5 | import com.github.dockerjava.api.model.Frame 6 | import com.github.dockerjava.api.model.StreamType 7 | import org.springframework.context.annotation.Bean 8 | import org.springframework.context.annotation.Configuration 9 | import org.springframework.context.annotation.Primary 10 | import org.springframework.stereotype.Component 11 | import org.testcontainers.containers.GenericContainer 12 | import org.testcontainers.containers.KafkaContainer 13 | import org.testcontainers.containers.Network 14 | import org.testcontainers.containers.wait.strategy.Wait 15 | import org.testcontainers.utility.DockerImageName 16 | import java.net.URL 17 | import javax.annotation.PreDestroy 18 | 19 | 20 | @Component 21 | class TestZookeeperContainer : 22 | GenericContainer(DockerImageName.parse("confluentinc/cp-zookeeper:6.1.0")) { 23 | 24 | val testNetwork: Network = Network.newNetwork() 25 | 26 | val port = 2181 27 | 28 | init { 29 | withNetwork(testNetwork) 30 | withEnv("ZOOKEEPER_CLIENT_PORT", port.toString()) 31 | start() 32 | } 33 | 34 | @PreDestroy 35 | fun destroy() { 36 | stop() 37 | } 38 | 39 | } 40 | 41 | @Component 42 | class TestKafkaContainer(testZookeeperContainer: TestZookeeperContainer) : 43 | KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.1.0")) { 44 | 45 | val port = 9092 46 | 47 | init { 48 | withNetwork(testZookeeperContainer.network) 49 | withExternalZookeeper(testZookeeperContainer.networkAliases.first() + ":" + testZookeeperContainer.port) 50 | withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1") 51 | withEnv("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1") 52 | withEnv("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1") 53 | start() 54 | } 55 | 56 | @PreDestroy 57 | fun destroy() { 58 | stop() 59 | } 60 | 61 | } 62 | 63 | @Component 64 | class TestSchemaRegistryContainer(testKafkaContainer: TestKafkaContainer) : 65 | GenericContainer(DockerImageName.parse("confluentinc/cp-schema-registry:6.1.0")) { 66 | 67 | fun url() = "http://${networkAliases.first()}:${exposedPorts.first()}" 68 | 69 | init { 70 | withNetwork(testKafkaContainer.network) 71 | withExposedPorts(8081) 72 | withEnv("SCHEMA_REGISTRY_HOST_NAME", "schema-registry") 73 | withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "${testKafkaContainer.networkAliases.first()}:${testKafkaContainer.port}") 74 | start() 75 | } 76 | 77 | @PreDestroy 78 | fun destroy() { 79 | stop() 80 | } 81 | 82 | } 83 | 84 | @Component 85 | class TestWsToKafkaContainer(testKafkaContainer: TestKafkaContainer, testSchemaRegistryContainer: TestSchemaRegistryContainer) : 86 | GenericContainer(DockerImageName.parse("ws-to-kafka")) { 87 | 88 | init { 89 | // todo: logger 90 | withNetwork(testKafkaContainer.network) 91 | withEnv("KAFKA_BOOTSTRAP_SERVERS", "${testKafkaContainer.networkAliases.first()}:${testKafkaContainer.port}") 92 | withEnv("SCHEMA_REGISTRY_URL", testSchemaRegistryContainer.url()) 93 | withEnv("SERVERLESS_KOTLIN_KAFKA_MYTOPIC_REPLICAS", "1") 94 | withEnv("SERVERLESS_KOTLIN_KAFKA_MYTOPIC_PARTITIONS", "3") 95 | waitingFor(Wait.forLogMessage(".*Kafka startTimeMs.*", 1)) 96 | start() 97 | } 98 | 99 | @PreDestroy 100 | fun destroy() { 101 | stop() 102 | } 103 | 104 | } 105 | 106 | @Component 107 | class TestKsqlDbServerContainer(testKafkaContainer: TestKafkaContainer, testSchemaRegistryContainer: TestSchemaRegistryContainer, testWsToKafkaContainer: TestWsToKafkaContainer) : 108 | GenericContainer(DockerImageName.parse("confluentinc/ksqldb-server:0.15.0")) { 109 | 110 | init { 111 | withNetwork(testKafkaContainer.network) 112 | withExposedPorts(8088) 113 | withEnv("KSQL_BOOTSTRAP_SERVERS", "${testKafkaContainer.networkAliases.first()}:${testKafkaContainer.port}") 114 | withEnv("KSQL_OPTS", "-Dksql.schema.registry.url=${testSchemaRegistryContainer.url()}") 115 | waitingFor(Wait.forLogMessage(".*INFO Server up and running.*\\n", 1)) 116 | start() 117 | 118 | val ksqldbUrl = "http://${this.networkAliases.first()}:8088" 119 | 120 | val createKsqldbSetup = dockerClient 121 | .createContainerCmd("ksqldb-setup") 122 | .withNetworkMode(testKafkaContainer.network.id) 123 | .withEnv("KSQLDB_ENDPOINT=$ksqldbUrl") 124 | .exec() 125 | 126 | dockerClient.startContainerCmd(createKsqldbSetup.id).exec() 127 | 128 | val logger = object: ResultCallback.Adapter() { 129 | override fun onNext(frame: Frame?) { 130 | when(frame?.streamType) { 131 | // todo: logger? 132 | StreamType.STDOUT, StreamType.STDERR -> print(String(frame.payload)) 133 | } 134 | } 135 | } 136 | 137 | dockerClient.logContainerCmd(createKsqldbSetup.id).withStdErr(true).withStdOut(true).withFollowStream(true).withTailAll().exec(logger).awaitCompletion() 138 | 139 | val exit = dockerClient.waitContainerCmd(createKsqldbSetup.id).exec(WaitContainerResultCallback()).awaitStatusCode() 140 | 141 | if (exit > 0) { 142 | throw Exception("Could not run ksqldb-setup") 143 | } 144 | } 145 | 146 | @PreDestroy 147 | fun destroy() { 148 | stop() 149 | } 150 | 151 | } 152 | 153 | 154 | @Configuration 155 | class TestKafkaConfigFactory { 156 | 157 | @Bean 158 | fun ksqldbConfig(testKsqlDbServerContainer: TestKsqlDbServerContainer): KsqldbConfig { 159 | return KsqldbConfig(URL("http", testKsqlDbServerContainer.host, testKsqlDbServerContainer.firstMappedPort, "")) 160 | } 161 | 162 | @Bean 163 | @Primary 164 | fun kafkaTopicConfig(): KafkaTopicConfig { 165 | return KafkaTopicConfig(1, 3, "testtopic") 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /ws-to-kafka-app-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: ws-to-kafka 5 | spec: 6 | serviceName: ws-to-kafka 7 | podManagementPolicy: Parallel 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: ws-to-kafka 12 | template: 13 | metadata: 14 | labels: 15 | app: ws-to-kafka 16 | spec: 17 | containers: 18 | - name: ws-to-kafka 19 | image: gcr.io/devx-testing/ws-to-kafka 20 | env: 21 | - name: KAFKA_BOOTSTRAP_SERVERS 22 | valueFrom: 23 | secretKeyRef: 24 | key: KAFKA_BOOTSTRAP_SERVERS 25 | name: ccloud-secret 26 | - name: KAFKA_USERNAME 27 | valueFrom: 28 | secretKeyRef: 29 | key: KAFKA_USERNAME 30 | name: ccloud-secret 31 | - name: KAFKA_PASSWORD 32 | valueFrom: 33 | secretKeyRef: 34 | key: KAFKA_PASSWORD 35 | name: ccloud-secret 36 | - name: SCHEMA_REGISTRY_URL 37 | valueFrom: 38 | secretKeyRef: 39 | key: SCHEMA_REGISTRY_URL 40 | name: ccloud-secret 41 | - name: SCHEMA_REGISTRY_KEY 42 | valueFrom: 43 | secretKeyRef: 44 | key: SCHEMA_REGISTRY_KEY 45 | name: ccloud-secret 46 | - name: SCHEMA_REGISTRY_PASSWORD 47 | valueFrom: 48 | secretKeyRef: 49 | key: SCHEMA_REGISTRY_PASSWORD 50 | name: ccloud-secret 51 | - name: KSQLDB_ENDPOINT 52 | valueFrom: 53 | secretKeyRef: 54 | key: KSQLDB_ENDPOINT 55 | name: ccloud-secret 56 | - name: KSQLDB_USERNAME 57 | valueFrom: 58 | secretKeyRef: 59 | key: KSQLDB_USERNAME 60 | name: ccloud-secret 61 | - name: KSQLDB_PASSWORD 62 | valueFrom: 63 | secretKeyRef: 64 | key: KSQLDB_PASSWORD 65 | name: ccloud-secret 66 | resources: 67 | requests: 68 | memory: 512Mi # 768Mi 69 | cpu: 500m # 1000m 70 | --- 71 | apiVersion: v1 72 | kind: Service 73 | metadata: 74 | name: ws-to-kafka 75 | spec: 76 | clusterIP: None 77 | -------------------------------------------------------------------------------- /ws-to-kafka-app-secret-template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: ccloud-secret 5 | type: Opaque 6 | stringData: 7 | KAFKA_BOOTSTRAP_SERVERS: $KAFKA_BOOTSTRAP_SERVERS 8 | KAFKA_USERNAME: $KAFKA_USERNAME 9 | KAFKA_PASSWORD: $KAFKA_PASSWORD 10 | SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL 11 | SCHEMA_REGISTRY_KEY: $SCHEMA_REGISTRY_KEY 12 | SCHEMA_REGISTRY_PASSWORD: $SCHEMA_REGISTRY_PASSWORD 13 | KSQLDB_ENDPOINT: $KSQLDB_ENDPOINT 14 | KSQLDB_USERNAME: $KSQLDB_USERNAME 15 | KSQLDB_PASSWORD: $KSQLDB_PASSWORD 16 | -------------------------------------------------------------------------------- /ws-to-kafka/build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 2 | 3 | plugins { 4 | application 5 | id("org.springframework.boot") 6 | id("io.spring.dependency-management") 7 | kotlin("jvm") 8 | kotlin("plugin.spring") 9 | } 10 | 11 | java { 12 | sourceCompatibility = JavaVersion.VERSION_1_8 13 | targetCompatibility = JavaVersion.VERSION_1_8 14 | } 15 | 16 | dependencies { 17 | implementation(project(":common")) 18 | implementation(kotlin("reflect")) 19 | implementation(kotlin("stdlib-jdk8")) 20 | implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8") 21 | implementation("io.confluent:kafka-json-schema-serializer:6.0.0") 22 | 23 | implementation("org.springframework.boot:spring-boot-starter-webflux") 24 | implementation("org.springframework.kafka:spring-kafka") 25 | 26 | testImplementation("org.testcontainers:kafka:1.15.2") 27 | } 28 | 29 | tasks.withType { 30 | kotlinOptions { 31 | freeCompilerArgs = listOf("-Xjsr305=strict") 32 | jvmTarget = JavaVersion.VERSION_1_8.toString() 33 | useIR = true 34 | } 35 | } 36 | 37 | tasks.withType { 38 | dependsOn("testClasses") 39 | args("--spring.profiles.active=dev") 40 | classpath = sourceSets["test"].runtimeClasspath 41 | } 42 | 43 | application { 44 | mainClass.set("skk.MainKt") 45 | } 46 | -------------------------------------------------------------------------------- /ws-to-kafka/src/main/kotlin/skk/Main.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import com.fasterxml.jackson.databind.PropertyNamingStrategy 4 | import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper 5 | import com.fasterxml.jackson.module.kotlin.readValue 6 | import kotlinx.coroutines.runBlocking 7 | import org.apache.kafka.clients.admin.NewTopic 8 | import org.springframework.beans.factory.annotation.Value 9 | import org.springframework.boot.CommandLineRunner 10 | import org.springframework.boot.autoconfigure.SpringBootApplication 11 | import org.springframework.boot.autoconfigure.kafka.KafkaProperties 12 | import org.springframework.boot.runApplication 13 | import org.springframework.context.annotation.Bean 14 | import org.springframework.context.annotation.Configuration 15 | import org.springframework.kafka.config.TopicBuilder 16 | import org.springframework.kafka.core.DefaultKafkaProducerFactory 17 | import org.springframework.kafka.core.KafkaAdmin 18 | import org.springframework.kafka.core.KafkaTemplate 19 | import org.springframework.kafka.core.ProducerFactory 20 | import org.springframework.web.reactive.socket.WebSocketMessage 21 | import org.springframework.web.reactive.socket.client.ReactorNettyWebSocketClient 22 | import java.net.URI 23 | 24 | 25 | @SpringBootApplication 26 | class Main( 27 | @Value("\${serverless.kotlin.kafka.so-to-ws.url}") val soToWsUrl: String, 28 | val kafkaTopicConfig: KafkaTopicConfig, 29 | val kafkaTemplate: KafkaTemplate, 30 | ) : CommandLineRunner { 31 | 32 | val wsClient = ReactorNettyWebSocketClient() 33 | val mapper = jacksonObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) 34 | 35 | override fun run(vararg args: String?): Unit = runBlocking { 36 | val uri = URI(soToWsUrl) 37 | 38 | val send: (WebSocketMessage) -> Unit = { message -> 39 | 40 | // parsing string to Kotlin object 41 | val question = mapper.readValue(message.payloadAsText) 42 | println(question.url) 43 | 44 | //sending to Kafka topic 45 | kafkaTemplate.send(kafkaTopicConfig.name, question.url, question) 46 | } 47 | 48 | wsClient.execute(uri) { session -> 49 | session.receive().doOnNext(send).then() 50 | }.block() 51 | } 52 | 53 | } 54 | 55 | @Configuration 56 | class KafkaSetup { 57 | 58 | @Bean 59 | fun newTopic(kafkaTopicConfig: KafkaTopicConfig): NewTopic { 60 | return TopicBuilder.name(kafkaTopicConfig.name).partitions(kafkaTopicConfig.partitions).replicas(kafkaTopicConfig.replicas).build() 61 | } 62 | 63 | @Bean 64 | fun kafkaAdmin(kafkaConfig: KafkaConfig): KafkaAdmin { 65 | return KafkaAdmin(kafkaConfig.config) 66 | } 67 | 68 | @Bean 69 | fun producerFactory(kafkaProperties: KafkaProperties, kafkaConfig: KafkaConfig, schemaRegistryConfig: SchemaRegistryConfig): ProducerFactory { 70 | val config = kafkaProperties.buildProducerProperties() + kafkaConfig.config + schemaRegistryConfig.config 71 | return DefaultKafkaProducerFactory(config) 72 | } 73 | 74 | @Bean 75 | fun kafkaTemplate(producerFactory: ProducerFactory): KafkaTemplate { 76 | return KafkaTemplate(producerFactory) 77 | } 78 | 79 | } 80 | 81 | fun main(args: Array) { 82 | runApplication
(*args) 83 | } 84 | -------------------------------------------------------------------------------- /ws-to-kafka/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.main.web-application-type=none 2 | 3 | # kafka application config 4 | serverless.kotlin.kafka.so-to-ws.url=wss://stackoverflow-to-ws-x5ht4amjia-uc.a.run.app/questions 5 | 6 | serverless.kotlin.kafka.mytopic.name=mytopic 7 | serverless.kotlin.kafka.mytopic.replicas=3 8 | serverless.kotlin.kafka.mytopic.partitions=8 9 | 10 | # kafka producer config 11 | spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer 12 | spring.kafka.producer.value-serializer=io.confluent.kafka.serializers.json.KafkaJsonSchemaSerializer 13 | 14 | -------------------------------------------------------------------------------- /ws-to-kafka/src/test/kotlin/skk/TestKafkaProducerFactory.kt: -------------------------------------------------------------------------------- 1 | package skk 2 | 3 | import org.springframework.context.annotation.Bean 4 | import org.springframework.context.annotation.Configuration 5 | import org.springframework.context.annotation.Primary 6 | import org.springframework.stereotype.Component 7 | import org.testcontainers.containers.GenericContainer 8 | import org.testcontainers.containers.KafkaContainer 9 | import org.testcontainers.containers.Network 10 | import org.testcontainers.utility.DockerImageName 11 | import javax.annotation.PreDestroy 12 | 13 | 14 | @Component 15 | class TestZookeeperContainer : GenericContainer(DockerImageName.parse("confluentinc/cp-zookeeper:6.1.0")) { 16 | 17 | val testNetwork: Network = Network.newNetwork() 18 | 19 | init { 20 | withNetwork(testNetwork) 21 | withNetworkAliases("zookeeper") 22 | withEnv("ZOOKEEPER_CLIENT_PORT", "2181") 23 | start() 24 | } 25 | 26 | @PreDestroy 27 | fun destroy() { 28 | stop() 29 | } 30 | 31 | } 32 | 33 | @Component 34 | class TestKafkaContainer(testZookeeperContainer: TestZookeeperContainer) : KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.1.0")) { 35 | 36 | init { 37 | withNetwork(testZookeeperContainer.network) 38 | withNetworkAliases("broker") 39 | withExternalZookeeper("zookeeper:2181") 40 | start() 41 | } 42 | 43 | @PreDestroy 44 | fun destroy() { 45 | stop() 46 | } 47 | 48 | } 49 | 50 | @Component 51 | class TestSchemaRegistryContainer(testKafkaContainer: TestKafkaContainer) : GenericContainer(DockerImageName.parse("confluentinc/cp-schema-registry:6.1.0")) { 52 | 53 | init { 54 | withCreateContainerCmdModifier { it.withName("schema-registry") } 55 | withNetwork(testKafkaContainer.network) 56 | withExposedPorts(8081) 57 | withEnv("SCHEMA_REGISTRY_HOST_NAME", "schema-registry") 58 | withEnv("SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS", "broker:9092") 59 | start() 60 | } 61 | 62 | @PreDestroy 63 | fun destroy() { 64 | stop() 65 | } 66 | 67 | } 68 | 69 | @Configuration 70 | class TestKafkaProducerFactory { 71 | 72 | @Bean 73 | fun kafkaConfig(testKafkaContainer: TestKafkaContainer): KafkaConfig { 74 | return KafkaConfig(testKafkaContainer.bootstrapServers) 75 | } 76 | 77 | @Bean 78 | fun schemaRegistryConfig(testSchemaRegistryContainer: TestSchemaRegistryContainer): SchemaRegistryConfig { 79 | return SchemaRegistryConfig("http://${testSchemaRegistryContainer.host}:${testSchemaRegistryContainer.firstMappedPort}") 80 | } 81 | 82 | @Bean 83 | @Primary 84 | fun kafkaTopicConfig(): KafkaTopicConfig { 85 | return KafkaTopicConfig(1, 3, "mytopic") 86 | } 87 | 88 | } 89 | --------------------------------------------------------------------------------