├── .github ├── dependabot.yml └── workflows │ ├── codeql.yml │ ├── gradle.yml │ └── publish.yml ├── .gitignore ├── .idea └── .gitignore ├── LICENSE ├── README.md ├── build.gradle.kts ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle.kts └── src ├── main └── kotlin │ └── com │ └── lapanthere │ └── flink │ └── api │ └── kotlin │ ├── api │ └── OutputTag.kt │ └── typeutils │ ├── DataClassTypeComparator.kt │ ├── DataClassTypeInfoFactory.kt │ ├── DataClassTypeInformation.kt │ ├── DataClassTypeSerializer.kt │ ├── DataClassTypeSerializerSnapshot.kt │ └── TypeInformation.kt └── test └── kotlin └── com └── lapanthere └── flink └── api └── kotlin └── typeutils ├── Classes.kt ├── DataClassTypeComparatorTest.kt ├── DataClassTypeInfoFactoryTest.kt ├── DataClassTypeInformationTest.kt ├── DataClassTypeSerializerTest.kt ├── ExampleTest.kt └── TypeInformationTest.kt /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | on: 3 | push: 4 | branches: [ "main" ] 5 | pull_request: 6 | branches: [ "main" ] 7 | schedule: 8 | - cron: '41 12 * * 6' 9 | jobs: 10 | analyze: 11 | name: Analyze 12 | runs-on: ubuntu-latest 13 | permissions: 14 | actions: read 15 | contents: read 16 | security-events: write 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | language: [ 'java' ] 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v4 24 | - name: Initialize CodeQL 25 | uses: github/codeql-action/init@v3 26 | with: 27 | languages: ${{ matrix.language }} 28 | - name: Set up JDK 11 29 | uses: actions/setup-java@v4 30 | with: 31 | distribution: 'temurin' 32 | java-version: '11' 33 | cache: 'gradle' 34 | - name: Build with Gradle 35 | run: ./gradlew assemble 36 | - name: Perform CodeQL Analysis 37 | uses: github/codeql-action/analyze@v3 38 | with: 39 | category: "/language:${{matrix.language}}" 40 | -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [ push ] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - name: Set up JDK 11 9 | uses: actions/setup-java@v4 10 | with: 11 | distribution: 'temurin' 12 | java-version: '11' 13 | cache: 'gradle' 14 | - name: Build with Gradle 15 | run: ./gradlew build 16 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | on: 3 | release: 4 | types: [ published ] 5 | jobs: 6 | publish-release: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - name: Set up JDK 11 11 | uses: actions/setup-java@v4 12 | with: 13 | distribution: 'temurin' 14 | java-version: '11' 15 | cache: 'gradle' 16 | - name: Publish artifact 17 | env: 18 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 19 | GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }} 20 | GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} 21 | GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} 22 | # The GITHUB_REF tag comes in the format 'refs/tags/xxx'. 23 | # So if we split on '/' and take the 3rd value, we can get the release name. 24 | run: | 25 | NEW_VERSION=$(echo "${GITHUB_REF}" | cut -d "/" -f3) 26 | ./gradlew -Pversion=${NEW_VERSION} publish 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # Package Files 8 | *.jar 9 | *.war 10 | *.nar 11 | *.ear 12 | *.zip 13 | *.tar.gz 14 | *.rar 15 | 16 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 17 | hs_err_pid* 18 | 19 | # Gradle config 20 | .gradle 21 | /build/ 22 | 23 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 24 | !gradle-wrapper.jar 25 | 26 | # User-specific stuff 27 | .idea/**/workspace.xml 28 | .idea/**/kotlinc.xml 29 | .idea/**/tasks.xml 30 | .idea/**/usage.statistics.xml 31 | .idea/**/dictionaries 32 | .idea/**/shelf 33 | 34 | # Generated files 35 | .idea/**/contentModel.xml 36 | 37 | # Sensitive or high-churn files 38 | .idea/**/dataSources/ 39 | .idea/**/dataSources.ids 40 | .idea/**/dataSources.local.xml 41 | .idea/**/sqlDataSources.xml 42 | .idea/**/dynamic.xml 43 | .idea/**/uiDesigner.xml 44 | .idea/**/dbnavigator.xml 45 | 46 | # Gradle 47 | .idea/**/gradle.xml 48 | .idea/**/libraries 49 | 50 | # Gradle and Maven with auto-import 51 | .idea/modules.xml 52 | .idea/*.iml 53 | .idea/modules 54 | 55 | # File-based project format 56 | *.iws 57 | 58 | # IntelliJ 59 | out/ 60 | 61 | # JIRA plugin 62 | atlassian-ide-plugin.xml 63 | 64 | # Editor-based Rest Client 65 | .idea/httpRequests 66 | 67 | # Operate tooling 68 | package/tmp 69 | 70 | .DS_Store 71 | /.jqwik-database 72 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021-2022 Timothée Peignier 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kotlin support for Apache Flink 2 | 3 | This package provides type information and 4 | specialized [type serializers](https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/dev/datastream/fault-tolerance/serialization/types_serialization/) 5 | for kotlin data classes (including `Pair` and `Triple`), as well as for Kotlin `Map` and `Collection` types. 6 | 7 | ## Installation 8 | 9 | The package is available on [Maven Central](https://search.maven.org/search?q=g:com.lapanthere%20AND%20a:flink-kotlin) 10 | and [Github](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-gradle-registry#using-a-published-package), 11 | using Gradle: 12 | 13 | ```kotlin 14 | implementation("com.lapanthere:flink-kotlin:0.1.0") 15 | ``` 16 | 17 | ## Usage 18 | 19 | ### TypeInformation 20 | 21 | Using the `createTypeInformation` that will return a Kotlin friendly `TypeInformation` for data classes, collections, 22 | maps, etc: 23 | 24 | ```kotlin 25 | dataStream.process( 26 | processFunction(), 27 | createTypeInformation() 28 | ) 29 | ``` 30 | 31 | It also supports fields name in the definition of keys, i.e. you will be able to use name of fields directly: 32 | 33 | ```kotlin 34 | dataStream.join(another).where("name").equalTo("personName") 35 | ``` 36 | 37 | You can also annotate your data classes with the `@TypeInfo` annotation: 38 | 39 | ```kotlin 40 | @TypeInfo(DataClassTypeInfoFactory::class) 41 | data class Record( 42 | val name: String, 43 | val value: Long 44 | ) 45 | ``` 46 | 47 | ## Schema evolution 48 | 49 | Schema evolution for data classes follow this set of rules: 50 | 51 | 1. Fields can be removed. Once removed, the previous value for the removed field will be dropped in future checkpoints 52 | and savepoints. 53 | 2. New fields can be added. The new field will be initialized to the default value for its type, as defined by Java. 54 | 3. Declared fields types cannot change. 55 | 4. Class name of type cannot change, including the namespace of the class. 56 | 5. Null fields are not supported. 57 | 58 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | plugins { 2 | kotlin("jvm") version "2.1.21" 3 | `java-library` 4 | `maven-publish` 5 | signing 6 | 7 | id("org.jmailen.kotlinter") version "5.1.0" 8 | id("org.jetbrains.dokka") version "2.0.0" 9 | } 10 | 11 | group = "com.lapanthere" 12 | 13 | repositories { 14 | mavenCentral() 15 | } 16 | 17 | dependencies { 18 | implementation(kotlin("reflect")) 19 | 20 | val flinkVersion = "1.20.1" 21 | implementation("org.apache.flink:flink-java:$flinkVersion") 22 | 23 | testImplementation("org.apache.flink:flink-test-utils:$flinkVersion") 24 | testImplementation("org.apache.flink:flink-core:$flinkVersion:tests") 25 | testImplementation(platform("org.junit:junit-bom:5.11.4")) 26 | testImplementation("org.junit.jupiter:junit-jupiter") 27 | testImplementation("org.junit.vintage:junit-vintage-engine") 28 | } 29 | 30 | tasks.named("test") { 31 | useJUnitPlatform() 32 | 33 | maxHeapSize = "1G" 34 | 35 | testLogging { 36 | events("passed") 37 | } 38 | } 39 | 40 | tasks.register("sourcesJar") { 41 | archiveClassifier.set("sources") 42 | from(sourceSets.main.get().allSource) 43 | } 44 | 45 | tasks.register("javadocJar") { 46 | group = JavaBasePlugin.DOCUMENTATION_GROUP 47 | archiveClassifier.set("javadoc") 48 | from(tasks.named("dokkaHtml")) 49 | } 50 | 51 | dokka { 52 | moduleName.set("flink-kotlin") 53 | dokkaSourceSets.main { 54 | sourceLink { 55 | localDirectory.set(file("src/main/kotlin")) 56 | remoteUrl("https://github.com/cyberdelia/flink-kotlin") 57 | } 58 | } 59 | } 60 | 61 | kotlin { 62 | explicitApi() 63 | } 64 | 65 | publishing { 66 | repositories { 67 | maven { 68 | name = "Github" 69 | url = uri("https://maven.pkg.github.com/cyberdelia/flink-kotlin") 70 | credentials { 71 | username = System.getenv("GITHUB_ACTOR") 72 | password = System.getenv("GITHUB_TOKEN") 73 | } 74 | } 75 | maven { 76 | name = "OSSRH" 77 | url = uri("https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/") 78 | credentials { 79 | username = System.getenv("OSSRH_USERNAME") 80 | password = System.getenv("OSSRH_PASSWORD") 81 | } 82 | } 83 | } 84 | 85 | publications { 86 | create("default") { 87 | from(components["java"]) 88 | artifact(tasks["sourcesJar"]) 89 | artifact(tasks["javadocJar"]) 90 | pom { 91 | name.set("flink-kotlin") 92 | description.set("Kotlin extensions for Apache Flink") 93 | url.set("https://github.com/cyberdelia/flink-kotlin") 94 | scm { 95 | connection.set("scm:git:git://github.com/cyberdelia/flink-kotlin.git") 96 | developerConnection.set("scm:git:ssh://github.com/cyberdelia/flink-kotlin.git") 97 | url.set("https://github.com/cyberdelia/flink-kotlin") 98 | } 99 | licenses { 100 | license { 101 | name.set("MIT License") 102 | url.set("http://www.opensource.org/licenses/mit-license.php") 103 | } 104 | } 105 | developers { 106 | developer { 107 | id.set("cyberdelia") 108 | name.set("Timothée Peignier") 109 | email.set("tim@lapanthere.com") 110 | organization.set("La Panthère") 111 | organizationUrl.set("https://lapanthere.com") 112 | } 113 | } 114 | } 115 | } 116 | } 117 | } 118 | 119 | signing { 120 | useInMemoryPgpKeys( 121 | System.getenv("GPG_KEY_ID"), 122 | System.getenv("GPG_PRIVATE_KEY"), 123 | System.getenv("GPG_PASSPHRASE") 124 | ) 125 | sign(publishing.publications) 126 | } 127 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.jetbrains.dokka.experimental.gradle.pluginMode=V2EnabledWithHelpers 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cyberdelia/flink-kotlin/a990fe4c41b72077e78ed908a462ca0f6a9ada07/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.12.1-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # SPDX-License-Identifier: Apache-2.0 19 | # 20 | 21 | ############################################################################## 22 | # 23 | # Gradle start up script for POSIX generated by Gradle. 24 | # 25 | # Important for running: 26 | # 27 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 28 | # noncompliant, but you have some other compliant shell such as ksh or 29 | # bash, then to run this script, type that shell name before the whole 30 | # command line, like: 31 | # 32 | # ksh Gradle 33 | # 34 | # Busybox and similar reduced shells will NOT work, because this script 35 | # requires all of these POSIX shell features: 36 | # * functions; 37 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 38 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 39 | # * compound commands having a testable exit status, especially «case»; 40 | # * various built-in commands including «command», «set», and «ulimit». 41 | # 42 | # Important for patching: 43 | # 44 | # (2) This script targets any POSIX shell, so it avoids extensions provided 45 | # by Bash, Ksh, etc; in particular arrays are avoided. 46 | # 47 | # The "traditional" practice of packing multiple parameters into a 48 | # space-separated string is a well documented source of bugs and security 49 | # problems, so this is (mostly) avoided, by progressively accumulating 50 | # options in "$@", and eventually passing that to Java. 51 | # 52 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 53 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 54 | # see the in-line comments for details. 55 | # 56 | # There are tweaks for specific operating systems such as AIX, CygWin, 57 | # Darwin, MinGW, and NonStop. 58 | # 59 | # (3) This script is generated from the Groovy template 60 | # https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 61 | # within the Gradle project. 62 | # 63 | # You can find Gradle at https://github.com/gradle/gradle/. 64 | # 65 | ############################################################################## 66 | 67 | # Attempt to set APP_HOME 68 | 69 | # Resolve links: $0 may be a link 70 | app_path=$0 71 | 72 | # Need this for daisy-chained symlinks. 73 | while 74 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 75 | [ -h "$app_path" ] 76 | do 77 | ls=$( ls -ld "$app_path" ) 78 | link=${ls#*' -> '} 79 | case $link in #( 80 | /*) app_path=$link ;; #( 81 | *) app_path=$APP_HOME$link ;; 82 | esac 83 | done 84 | 85 | # This is normally unused 86 | # shellcheck disable=SC2034 87 | APP_BASE_NAME=${0##*/} 88 | # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) 89 | APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s 90 | ' "$PWD" ) || exit 91 | 92 | # Use the maximum available, or set MAX_FD != -1 to use that value. 93 | MAX_FD=maximum 94 | 95 | warn () { 96 | echo "$*" 97 | } >&2 98 | 99 | die () { 100 | echo 101 | echo "$*" 102 | echo 103 | exit 1 104 | } >&2 105 | 106 | # OS specific support (must be 'true' or 'false'). 107 | cygwin=false 108 | msys=false 109 | darwin=false 110 | nonstop=false 111 | case "$( uname )" in #( 112 | CYGWIN* ) cygwin=true ;; #( 113 | Darwin* ) darwin=true ;; #( 114 | MSYS* | MINGW* ) msys=true ;; #( 115 | NONSTOP* ) nonstop=true ;; 116 | esac 117 | 118 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 119 | 120 | 121 | # Determine the Java command to use to start the JVM. 122 | if [ -n "$JAVA_HOME" ] ; then 123 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 124 | # IBM's JDK on AIX uses strange locations for the executables 125 | JAVACMD=$JAVA_HOME/jre/sh/java 126 | else 127 | JAVACMD=$JAVA_HOME/bin/java 128 | fi 129 | if [ ! -x "$JAVACMD" ] ; then 130 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 131 | 132 | Please set the JAVA_HOME variable in your environment to match the 133 | location of your Java installation." 134 | fi 135 | else 136 | JAVACMD=java 137 | if ! command -v java >/dev/null 2>&1 138 | then 139 | die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 140 | 141 | Please set the JAVA_HOME variable in your environment to match the 142 | location of your Java installation." 143 | fi 144 | fi 145 | 146 | # Increase the maximum file descriptors if we can. 147 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 148 | case $MAX_FD in #( 149 | max*) 150 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 151 | # shellcheck disable=SC2039,SC3045 152 | MAX_FD=$( ulimit -H -n ) || 153 | warn "Could not query maximum file descriptor limit" 154 | esac 155 | case $MAX_FD in #( 156 | '' | soft) :;; #( 157 | *) 158 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 159 | # shellcheck disable=SC2039,SC3045 160 | ulimit -n "$MAX_FD" || 161 | warn "Could not set maximum file descriptor limit to $MAX_FD" 162 | esac 163 | fi 164 | 165 | # Collect all arguments for the java command, stacking in reverse order: 166 | # * args from the command line 167 | # * the main class name 168 | # * -classpath 169 | # * -D...appname settings 170 | # * --module-path (only if needed) 171 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 172 | 173 | # For Cygwin or MSYS, switch paths to Windows format before running java 174 | if "$cygwin" || "$msys" ; then 175 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 176 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 177 | 178 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 179 | 180 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 181 | for arg do 182 | if 183 | case $arg in #( 184 | -*) false ;; # don't mess with options #( 185 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 186 | [ -e "$t" ] ;; #( 187 | *) false ;; 188 | esac 189 | then 190 | arg=$( cygpath --path --ignore --mixed "$arg" ) 191 | fi 192 | # Roll the args list around exactly as many times as the number of 193 | # args, so each arg winds up back in the position where it started, but 194 | # possibly modified. 195 | # 196 | # NB: a `for` loop captures its iteration list before it begins, so 197 | # changing the positional parameters here affects neither the number of 198 | # iterations, nor the values presented in `arg`. 199 | shift # remove old arg 200 | set -- "$@" "$arg" # push replacement arg 201 | done 202 | fi 203 | 204 | 205 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 206 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 207 | 208 | # Collect all arguments for the java command: 209 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, 210 | # and any embedded shellness will be escaped. 211 | # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be 212 | # treated as '${Hostname}' itself on the command line. 213 | 214 | set -- \ 215 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 216 | -classpath "$CLASSPATH" \ 217 | org.gradle.wrapper.GradleWrapperMain \ 218 | "$@" 219 | 220 | # Stop when "xargs" is not available. 221 | if ! command -v xargs >/dev/null 2>&1 222 | then 223 | die "xargs is not available" 224 | fi 225 | 226 | # Use "xargs" to parse quoted args. 227 | # 228 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 229 | # 230 | # In Bash we could simply go: 231 | # 232 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 233 | # set -- "${ARGS[@]}" "$@" 234 | # 235 | # but POSIX shell has neither arrays nor command substitution, so instead we 236 | # post-process each arg (as a line of input to sed) to backslash-escape any 237 | # character that might be a shell metacharacter, then use eval to reverse 238 | # that process (while maintaining the separation between arguments), and wrap 239 | # the whole thing up as a single "set" statement. 240 | # 241 | # This will of course break if any of these variables contains a newline or 242 | # an unmatched quote. 243 | # 244 | 245 | eval "set -- $( 246 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 247 | xargs -n1 | 248 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 249 | tr '\n' ' ' 250 | )" '"$@"' 251 | 252 | exec "$JAVACMD" "$@" 253 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "flink-kotlin" 2 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/api/OutputTag.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.api 2 | 3 | import com.lapanthere.flink.api.kotlin.typeutils.createTypeInformation 4 | import org.apache.flink.util.OutputTag 5 | 6 | @Suppress("ktlint:standard:function-naming") 7 | public inline fun OutputTag(id: String): OutputTag = OutputTag(id, createTypeInformation()) 8 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeComparator.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeutils.TypeComparator 4 | import org.apache.flink.api.common.typeutils.TypeSerializer 5 | import org.apache.flink.api.java.typeutils.runtime.TupleComparatorBase 6 | import org.apache.flink.core.memory.MemorySegment 7 | import org.apache.flink.types.NullKeyFieldException 8 | 9 | internal class DataClassTypeComparator( 10 | keys: IntArray, 11 | comparators: Array>, 12 | serializers: Array>, 13 | ) : TupleComparatorBase(keys, comparators, serializers) { 14 | override fun hash(record: T?): Int { 15 | val code = comparators.first().hash(record?.component(0)) 16 | return comparators.drop(1).foldIndexed(code) { i, hash, comparator -> 17 | val j = i + 1 18 | try { 19 | (hash * HASH_SALT[j and 0x1f]) + comparator.hash(record?.component(j)) 20 | } catch (e: NullPointerException) { 21 | throw NullKeyFieldException(keyPositions[j]) 22 | } 23 | } 24 | } 25 | 26 | override fun setReference(toCompare: T?) { 27 | keyPositions.zip(comparators).forEach { (position, comparator) -> 28 | try { 29 | comparator.setReference(toCompare?.component(position)) 30 | } catch (e: NullPointerException) { 31 | throw NullKeyFieldException(position) 32 | } 33 | } 34 | } 35 | 36 | override fun equalToReference(candidate: T?): Boolean = 37 | keyPositions.zip(comparators).all { (position, comparator) -> 38 | try { 39 | comparator.equalToReference(candidate?.component(position)) 40 | } catch (e: NullPointerException) { 41 | throw NullKeyFieldException(position) 42 | } 43 | } 44 | 45 | override fun compare( 46 | first: T?, 47 | second: T?, 48 | ): Int { 49 | keyPositions.zip(comparators).forEach { (position, comparator) -> 50 | try { 51 | val cmp = comparator.compare(first?.component(position), second?.component(position)) 52 | if (cmp != 0) { 53 | return cmp 54 | } 55 | } catch (e: NullPointerException) { 56 | throw NullKeyFieldException(position) 57 | } 58 | } 59 | return 0 60 | } 61 | 62 | override fun putNormalizedKey( 63 | record: T?, 64 | target: MemorySegment, 65 | offset: Int, 66 | numBytes: Int, 67 | ) { 68 | var localNumBytes = numBytes 69 | var localOffset = offset 70 | var i = 0 71 | try { 72 | while (i < numLeadingNormalizableKeys && numBytes > 0) { 73 | var len: Int = normalizedKeyLengths[i] 74 | len = if (numBytes >= len) len else numBytes 75 | val comparator = comparators[i] 76 | comparator.putNormalizedKey(record?.component(keyPositions[i]), target, offset, len) 77 | localNumBytes -= len 78 | localOffset += len 79 | i += 1 80 | } 81 | } catch (e: NullPointerException) { 82 | throw NullKeyFieldException(keyPositions[i]) 83 | } 84 | } 85 | 86 | override fun duplicate(): TypeComparator { 87 | instantiateDeserializationUtils() 88 | val comparator = DataClassTypeComparator(keyPositions, comparators, serializers) 89 | comparator.privateDuplicate(this) 90 | return comparator 91 | } 92 | 93 | override fun extractKeys( 94 | record: Any?, 95 | target: Array, 96 | index: Int, 97 | ): Int { 98 | var localIndex = index 99 | keyPositions.zip(comparators).forEach { (position, comparator) -> 100 | localIndex += comparator.extractKeys(position, target, localIndex) 101 | } 102 | return localIndex - index 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeInfoFactory.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInfoFactory 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.java.typeutils.TypeExtractionUtils 6 | import java.lang.reflect.Type 7 | import kotlin.reflect.full.starProjectedType 8 | 9 | public class DataClassTypeInfoFactory : TypeInfoFactory() { 10 | @Suppress("UNCHECKED_CAST") 11 | override fun createTypeInfo( 12 | t: Type, 13 | genericParameters: Map>, 14 | ): TypeInformation? { 15 | val klass = TypeExtractionUtils.typeToClass(t).kotlin 16 | if (!klass.isData) { 17 | return null 18 | } 19 | return createTypeInformation(klass.starProjectedType) as TypeInformation 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeInformation.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.operators.Keys 5 | import org.apache.flink.api.common.serialization.SerializerConfig 6 | import org.apache.flink.api.common.typeinfo.TypeInformation 7 | import org.apache.flink.api.common.typeutils.CompositeType 8 | import org.apache.flink.api.common.typeutils.TypeComparator 9 | import org.apache.flink.api.common.typeutils.TypeSerializer 10 | import org.apache.flink.api.java.typeutils.TupleTypeInfoBase 11 | 12 | private const val REGEX_INT_FIELD = "[0-9]+" 13 | private const val REGEX_FIELD = "[\\p{L}_\\$][\\p{L}\\p{Digit}_\\$]*" 14 | private const val REGEX_NESTED_FIELDS = "($REGEX_FIELD)(\\.(.+))?" 15 | private const val REGEX_NESTED_FIELDS_WILDCARD = "$REGEX_NESTED_FIELDS|\\${Keys.ExpressionKeys.SELECT_ALL_CHAR}" 16 | 17 | private val PATTERN_NESTED_FIELDS = Regex(REGEX_NESTED_FIELDS) 18 | private val PATTERN_NESTED_FIELDS_WILDCARD = Regex(REGEX_NESTED_FIELDS_WILDCARD) 19 | private val PATTERN_INT_FIELD = Regex(REGEX_INT_FIELD) 20 | 21 | public class DataClassTypeInformation( 22 | private val klass: Class, 23 | private val typeParameters: Map>, 24 | kotlinFieldTypes: Array>, 25 | private val kotlinFieldNames: Array, 26 | ) : TupleTypeInfoBase(klass, *kotlinFieldTypes) { 27 | override fun toString(): String = 28 | buildString { 29 | append(klass.simpleName) 30 | if (types.isNotEmpty()) { 31 | append(types.joinToString(", ", "<", ">")) 32 | } 33 | } 34 | 35 | public override fun equals(other: Any?): Boolean = 36 | when (other) { 37 | is DataClassTypeInformation<*> -> 38 | other.canEqual(this) && 39 | super.equals(other) && 40 | genericParameters == other.genericParameters && 41 | fieldNames.contentEquals(other.fieldNames) 42 | else -> false 43 | } 44 | 45 | public override fun canEqual(obj: Any): Boolean = obj is DataClassTypeInformation<*> 46 | 47 | public override fun hashCode(): Int = 48 | 31 * (31 * super.hashCode() + fieldNames.contentHashCode()) + genericParameters.values.toTypedArray().contentHashCode() 49 | 50 | @Deprecated("Deprecated in Java") 51 | override fun createSerializer(config: ExecutionConfig): TypeSerializer = 52 | DataClassTypeSerializer( 53 | klass, 54 | types.take(arity).map { it.createSerializer(config) }.toTypedArray(), 55 | ) 56 | 57 | override fun createSerializer(config: SerializerConfig?): TypeSerializer = 58 | DataClassTypeSerializer(klass, types.take(arity).map { it.createSerializer(config) }.toTypedArray()) 59 | 60 | override fun createTypeComparatorBuilder(): TypeComparatorBuilder = DataClassTypeComparatorBuilder() 61 | 62 | override fun getFieldNames(): Array = kotlinFieldNames 63 | 64 | override fun getFieldIndex(fieldName: String): Int = kotlinFieldNames.indexOf(fieldName) 65 | 66 | override fun getFlatFields( 67 | fieldExpression: String, 68 | offset: Int, 69 | result: MutableList, 70 | ) { 71 | val match = 72 | PATTERN_NESTED_FIELDS_WILDCARD.matchEntire(fieldExpression) 73 | ?: throw InvalidFieldReferenceException("""Invalid tuple field reference "$fieldExpression".""") 74 | var field = match.groups[0]?.value!! 75 | if (field == Keys.ExpressionKeys.SELECT_ALL_CHAR) { 76 | var keyPosition = 0 77 | fieldTypes.forEach { fieldType -> 78 | when (fieldType) { 79 | is CompositeType<*> -> { 80 | fieldType.getFlatFields(Keys.ExpressionKeys.SELECT_ALL_CHAR, offset + keyPosition, result) 81 | keyPosition += fieldType.totalFields - 1 82 | } 83 | 84 | else -> result.add(FlatFieldDescriptor(offset + keyPosition, fieldType)) 85 | } 86 | } 87 | } else { 88 | field = match.groups[1]?.value!! 89 | val intFieldMatch = PATTERN_INT_FIELD.matchEntire(field) 90 | if (intFieldMatch != null) { 91 | field = "_" + (Integer.valueOf(field) + 1) 92 | } 93 | 94 | val tail = match.groups[3]?.value 95 | if (tail == null) { 96 | fun extractFlatFields( 97 | index: Int, 98 | pos: Int, 99 | ) { 100 | if (index >= fieldNames.size) { 101 | throw InvalidFieldReferenceException("""Unable to find field "$field" in type "$this".""") 102 | } else if (field == fieldNames[index]) { 103 | when (val fieldType = fieldTypes[index]) { 104 | is CompositeType<*> -> fieldType.getFlatFields("*", pos, result) 105 | else -> result.add(FlatFieldDescriptor(pos, fieldType)) 106 | } 107 | } else { 108 | extractFlatFields(index + 1, pos + fieldTypes[index].totalFields) 109 | } 110 | } 111 | extractFlatFields(0, offset) 112 | } else { 113 | fun extractFlatFields( 114 | index: Int, 115 | pos: Int, 116 | ) { 117 | if (index >= fieldNames.size) { 118 | throw InvalidFieldReferenceException("""Unable to find field "$field" in type "$this".""") 119 | } else if (field == fieldNames[index]) { 120 | when (val fieldType = fieldTypes[index]) { 121 | is CompositeType<*> -> fieldType.getFlatFields(tail, pos, result) 122 | else -> throw InvalidFieldReferenceException( 123 | """Nested field expression "$tail" not possible on atomic type "$fieldType".""", 124 | ) 125 | } 126 | } else { 127 | extractFlatFields(index + 1, pos + fieldTypes[index].totalFields) 128 | } 129 | } 130 | extractFlatFields(0, offset) 131 | } 132 | } 133 | } 134 | 135 | override fun getTypeAt(fieldExpression: String): TypeInformation { 136 | val match = 137 | PATTERN_NESTED_FIELDS.matchEntire(fieldExpression) 138 | ?: if (fieldExpression.startsWith(Keys.ExpressionKeys.SELECT_ALL_CHAR)) { 139 | throw InvalidFieldReferenceException("Wildcard expressions are not allowed here.") 140 | } else { 141 | throw InvalidFieldReferenceException("""Invalid format of data class field expression "$fieldExpression".""") 142 | } 143 | 144 | var field = match.groups[1]?.value!! 145 | val tail = match.groups[3]?.value 146 | val intFieldMatcher = PATTERN_INT_FIELD.matchEntire(field) 147 | if (intFieldMatcher != null) { 148 | field = "_" + (Integer.valueOf(field) + 1) 149 | } 150 | 151 | fieldNames.zip(fieldTypes).forEachIndexed { i, (fieldName, fieldType) -> 152 | if (fieldName == field) { 153 | return if (tail == null) { 154 | getTypeAt(i) 155 | } else { 156 | when (fieldType) { 157 | is CompositeType<*> -> fieldType.getTypeAt(i) 158 | else -> throw InvalidFieldReferenceException( 159 | """Nested field expression "$tail" not possible on atomic type "$fieldType".""", 160 | ) 161 | } 162 | } 163 | } 164 | } 165 | throw InvalidFieldReferenceException("""Unable to find field "$field" in type "$this".""") 166 | } 167 | 168 | override fun getGenericParameters(): Map> = typeParameters 169 | 170 | private inner class DataClassTypeComparatorBuilder : TypeComparatorBuilder { 171 | private val fieldComparators: MutableList> = mutableListOf() 172 | private val logicalKeyFields: MutableList = mutableListOf() 173 | 174 | override fun initializeTypeComparatorBuilder(size: Int) {} 175 | 176 | override fun addComparatorField( 177 | fieldId: Int, 178 | comparator: TypeComparator<*>, 179 | ) { 180 | fieldComparators += comparator 181 | logicalKeyFields += fieldId 182 | } 183 | 184 | override fun createTypeComparator(config: ExecutionConfig): TypeComparator = 185 | DataClassTypeComparator( 186 | logicalKeyFields.toIntArray(), 187 | fieldComparators.toTypedArray(), 188 | types.take(logicalKeyFields.max() + 1).map { it.createSerializer(config) }.toTypedArray(), 189 | ) 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeSerializer.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeutils.TypeSerializer 4 | import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot 5 | import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase 6 | import org.apache.flink.core.memory.DataInputView 7 | import org.apache.flink.core.memory.DataOutputView 8 | import kotlin.reflect.full.functions 9 | import kotlin.reflect.full.primaryConstructor 10 | 11 | public class DataClassTypeSerializer( 12 | type: Class?, 13 | fieldSerializers: Array>, 14 | ) : TupleSerializerBase(type, fieldSerializers) { 15 | override fun duplicate(): TypeSerializer = 16 | DataClassTypeSerializer( 17 | tupleClass, 18 | fieldSerializers.map { it.duplicate() }.toTypedArray(), 19 | ) 20 | 21 | override fun createInstance(fields: Array): T? = 22 | try { 23 | tupleClass.kotlin.primaryConstructor?.call(*fields) 24 | } catch (e: Throwable) { 25 | null 26 | } 27 | 28 | override fun createInstance(): T? = createInstance(fieldSerializers.map { it.createInstance() }.toTypedArray()) 29 | 30 | override fun deserialize(source: DataInputView): T? = createInstance(fieldSerializers.map { it.deserialize(source) }.toTypedArray()) 31 | 32 | override fun snapshotConfiguration(): TypeSerializerSnapshot = DataClassTypeSerializerSnapshot(this) 33 | 34 | override fun createOrReuseInstance( 35 | fields: Array, 36 | reuse: T, 37 | ): T? = createInstance(fields) 38 | 39 | override fun deserialize( 40 | reuse: T?, 41 | source: DataInputView, 42 | ): T? = deserialize(source) 43 | 44 | override fun serialize( 45 | record: T?, 46 | target: DataOutputView, 47 | ) { 48 | fieldSerializers.forEachIndexed { i, serializer -> 49 | serializer.serialize(record?.component(i), target) 50 | } 51 | } 52 | 53 | override fun copy( 54 | from: T?, 55 | reuse: T, 56 | ): T? = copy(from) 57 | 58 | override fun copy(from: T?): T? = 59 | if (from == null) { 60 | null 61 | } else { 62 | createInstance( 63 | fieldSerializers 64 | .mapIndexed { i, serializer -> 65 | serializer.copy(from.component(i)) 66 | }.toTypedArray(), 67 | ) 68 | } 69 | } 70 | 71 | internal fun T.component(i: Int): Any? = this::class.functions.first { it.name == "component${i + 1}" }.call(this) 72 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeSerializerSnapshot.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot 4 | import org.apache.flink.api.common.typeutils.TypeSerializer 5 | import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot 6 | import org.apache.flink.core.memory.DataInputView 7 | import org.apache.flink.core.memory.DataOutputView 8 | import org.apache.flink.util.InstantiationUtil 9 | 10 | public class DataClassTypeSerializerSnapshot : CompositeTypeSerializerSnapshot> { 11 | private var type: Class? = null 12 | 13 | public constructor() : super() 14 | 15 | public constructor(type: Class) : super() { 16 | this.type = type 17 | } 18 | 19 | public constructor(serializerInstance: DataClassTypeSerializer) : super(serializerInstance) { 20 | type = serializerInstance.tupleClass 21 | } 22 | 23 | override fun getCurrentOuterSnapshotVersion(): Int = 1 24 | 25 | override fun getNestedSerializers(outerSerializer: DataClassTypeSerializer): Array> = 26 | outerSerializer.fieldSerializers 27 | 28 | override fun createOuterSerializerWithNestedSerializers(nestedSerializers: Array>): DataClassTypeSerializer = 29 | DataClassTypeSerializer(type!!, nestedSerializers) 30 | 31 | override fun writeOuterSnapshot(out: DataOutputView) { 32 | out.writeUTF(type!!.name) 33 | } 34 | 35 | override fun readOuterSnapshot( 36 | readOuterSnapshotVersion: Int, 37 | `in`: DataInputView, 38 | userCodeClassLoader: ClassLoader, 39 | ) { 40 | type = InstantiationUtil.resolveClassByName(`in`, userCodeClassLoader) 41 | } 42 | 43 | @Deprecated("Deprecated in Java.") 44 | override fun resolveOuterSchemaCompatibility(newSerializer: DataClassTypeSerializer): OuterSchemaCompatibility = 45 | if (type == newSerializer.tupleClass) { 46 | OuterSchemaCompatibility.COMPATIBLE_AS_IS 47 | } else { 48 | OuterSchemaCompatibility.INCOMPATIBLE 49 | } 50 | 51 | override fun resolveOuterSchemaCompatibility(oldSerializerSnapshot: TypeSerializerSnapshot): OuterSchemaCompatibility = 52 | if (oldSerializerSnapshot is DataClassTypeSerializerSnapshot) { 53 | OuterSchemaCompatibility.COMPATIBLE_AS_IS 54 | } else { 55 | OuterSchemaCompatibility.INCOMPATIBLE 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/kotlin/com/lapanthere/flink/api/kotlin/typeutils/TypeInformation.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation 4 | import org.apache.flink.api.java.typeutils.ListTypeInfo 5 | import org.apache.flink.api.java.typeutils.MapTypeInfo 6 | import org.apache.flink.api.java.typeutils.TypeExtractor 7 | import kotlin.reflect.KType 8 | import kotlin.reflect.full.isSubclassOf 9 | import kotlin.reflect.full.primaryConstructor 10 | import kotlin.reflect.full.starProjectedType 11 | import kotlin.reflect.jvm.javaType 12 | import kotlin.reflect.jvm.jvmErasure 13 | import kotlin.reflect.typeOf 14 | 15 | public fun createTypeInformation(type: KType): TypeInformation<*> { 16 | val klass = type.jvmErasure 17 | return when { 18 | klass.isData -> { 19 | val generics = klass.typeParameters.map { it.starProjectedType } 20 | val projected = type.arguments.map { it.type ?: Any::class.starProjectedType } 21 | val mapping = generics.zip(projected).toMap() 22 | val fields = klass.primaryConstructor?.parameters ?: emptyList() 23 | val parameters = fields.map { mapping.getOrDefault(it.type, it.type) } 24 | DataClassTypeInformation( 25 | klass.java, 26 | mapping.map { (key, value) -> key.javaType.typeName to createTypeInformation(value) }.toMap(), 27 | parameters.map { createTypeInformation(it) }.toTypedArray(), 28 | fields.map { it.name!! }.toTypedArray(), 29 | ) 30 | } 31 | klass.isSubclassOf(Map::class) -> { 32 | val (key, value) = type.arguments.map { it.type ?: Any::class.starProjectedType } 33 | MapTypeInfo(createTypeInformation(key), createTypeInformation(value)) 34 | } 35 | klass.isSubclassOf(Collection::class) -> { 36 | ListTypeInfo(createTypeInformation(type.arguments.map { it.type ?: Any::class.starProjectedType }.first())) 37 | } 38 | else -> TypeExtractor.createTypeInfo(type.javaType) 39 | } 40 | } 41 | 42 | @Suppress("UNCHECKED_CAST") 43 | public inline fun createTypeInformation(): TypeInformation = createTypeInformation(typeOf()) as TypeInformation 44 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/Classes.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInfo 4 | import java.io.Serializable 5 | import java.util.Date 6 | 7 | @TypeInfo(DataClassTypeInfoFactory::class) 8 | data class Word( 9 | val ignoreStaticField: Int = 0, 10 | @Transient 11 | private val ignoreTransientField: Int = 0, 12 | val date: Date, 13 | val someFloat: Float = 0f, 14 | var nothing: Any, 15 | val collection: List = emptyList(), 16 | ) 17 | 18 | @TypeInfo(DataClassTypeInfoFactory::class) 19 | data class WordCount( 20 | val count: Int = 0, 21 | val word: Word, 22 | ) 23 | 24 | @TypeInfo(DataClassTypeInfoFactory::class) 25 | class NotADataClass 26 | 27 | @TypeInfo(DataClassTypeInfoFactory::class) 28 | data class Basic( 29 | val abc: String, 30 | val field: Int, 31 | ) 32 | 33 | @TypeInfo(DataClassTypeInfoFactory::class) 34 | data class ParameterizedClass( 35 | val name: String, 36 | val field: T, 37 | ) 38 | 39 | @TypeInfo(DataClassTypeInfoFactory::class) 40 | data class Nested( 41 | val string: String, 42 | ) : Serializable 43 | 44 | @TypeInfo(DataClassTypeInfoFactory::class) 45 | data class DataClass( 46 | val string: String, 47 | val long: Long, 48 | val nested: Nested, 49 | ) : Serializable 50 | 51 | data class Purchase( 52 | val amount: Double, 53 | ) 54 | 55 | data class Order( 56 | val purchases: List, 57 | ) { 58 | constructor(vararg purchases: Purchase) : this(purchases.toList()) 59 | } 60 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeComparatorTest.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.operators.Keys.ExpressionKeys 5 | import org.apache.flink.api.common.typeutils.ComparatorTestBase 6 | import org.apache.flink.api.common.typeutils.CompositeType 7 | import org.apache.flink.api.common.typeutils.TypeComparator 8 | import org.apache.flink.api.common.typeutils.TypeSerializer 9 | import org.apache.flink.api.java.typeutils.TypeExtractor 10 | import java.util.Arrays 11 | 12 | internal class DataClassTypeComparatorTest : ComparatorTestBase() { 13 | private val type = TypeExtractor.getForClass(DataClass::class.java) 14 | 15 | override fun createComparator(ascending: Boolean): TypeComparator { 16 | require(type is CompositeType) 17 | val keys = ExpressionKeys(arrayOf("*"), type) 18 | val orders = BooleanArray(keys.numberOfKeyFields) 19 | Arrays.fill(orders, ascending) 20 | return type.createComparator( 21 | keys.computeLogicalKeyPositions(), 22 | orders, 23 | 0, 24 | ExecutionConfig(), 25 | ) 26 | } 27 | 28 | override fun createSerializer(): TypeSerializer = type.createSerializer(ExecutionConfig()) 29 | 30 | override fun getSortedTestData(): Array = 31 | arrayOf( 32 | DataClass("abc", 1, Nested("abc")), 33 | DataClass("def", 2, Nested("def")), 34 | DataClass("xyz", 3, Nested("xyz")), 35 | ) 36 | } 37 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeInfoFactoryTest.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.functions.InvalidTypesException 4 | import org.apache.flink.api.java.typeutils.TypeExtractor 5 | import org.junit.jupiter.api.Assertions.assertEquals 6 | import org.junit.jupiter.api.Assertions.assertFalse 7 | import org.junit.jupiter.api.Assertions.assertTrue 8 | import org.junit.jupiter.api.Test 9 | import org.junit.jupiter.api.assertThrows 10 | import java.util.Date 11 | 12 | internal class DataClassTypeInfoFactoryTest { 13 | @Test 14 | fun `reject non data classes`() { 15 | assertThrows { TypeExtractor.createTypeInfo(NotADataClass::class.java) } 16 | } 17 | 18 | @Test 19 | fun `allows basic data classes`() { 20 | val typeInformation = TypeExtractor.createTypeInfo(Basic::class.java) 21 | assertTrue(typeInformation is DataClassTypeInformation) 22 | } 23 | 24 | @Test 25 | fun `allows data classes with type parameters`() { 26 | val typeInformation = TypeExtractor.createTypeInfo(ParameterizedClass::class.java) 27 | assertTrue(typeInformation is DataClassTypeInformation>) 28 | } 29 | 30 | @Test 31 | fun `detects all fields and nested classes`() { 32 | val typeInformation = TypeExtractor.createTypeInfo(WordCount::class.java) 33 | assertFalse(typeInformation.isBasicType) 34 | assertTrue(typeInformation.isTupleType) 35 | assertEquals(7, typeInformation.totalFields) 36 | 37 | assertTrue(typeInformation is DataClassTypeInformation) 38 | require(typeInformation is DataClassTypeInformation) 39 | 40 | val fields = arrayOf("count", "word.date", "word.someFloat", "word.collection", "word.nothing") 41 | val positions = arrayOf(0, 3, 4, 6, 5) 42 | assertEquals(fields.size, positions.size) 43 | fields.zip(positions).forEach { (field, position) -> 44 | val fieldDescriptors = typeInformation.getFlatFields(field) 45 | assertEquals(1, fieldDescriptors.size) 46 | assertEquals(position, fieldDescriptors.first().position) 47 | } 48 | 49 | val fieldDescriptors = typeInformation.getFlatFields("word.*") 50 | assertEquals(6, fieldDescriptors.size) 51 | fieldDescriptors.sortBy { it.position } 52 | assertEquals(Int::class.javaObjectType, fieldDescriptors[0].type.typeClass) 53 | assertEquals(Int::class.javaObjectType, fieldDescriptors[1].type.typeClass) 54 | assertEquals(Date::class.java, fieldDescriptors[2].type.typeClass) 55 | assertEquals(Float::class.javaObjectType, fieldDescriptors[3].type.typeClass) 56 | assertEquals(Any::class.java, fieldDescriptors[4].type.typeClass) 57 | assertEquals(List::class.java, fieldDescriptors[5].type.typeClass) 58 | 59 | val nestedTypeInformation = typeInformation.getTypeAt(1) 60 | assertEquals(6, nestedTypeInformation.arity) 61 | assertEquals(6, nestedTypeInformation.totalFields) 62 | assertTrue(nestedTypeInformation is DataClassTypeInformation) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeInformationTest.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeutils.TypeInformationTestBase 4 | 5 | internal class DataClassTypeInformationTest : TypeInformationTestBase>() { 6 | override fun getTestData(): Array> = 7 | arrayOf( 8 | createTypeInformation() as DataClassTypeInformation, 9 | createTypeInformation() as DataClassTypeInformation, 10 | createTypeInformation>() as DataClassTypeInformation>, 11 | createTypeInformation>() as DataClassTypeInformation>, 12 | createTypeInformation>() as DataClassTypeInformation>, 13 | ) 14 | } 15 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/DataClassTypeSerializerTest.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.SerializerTestBase 6 | import org.apache.flink.api.common.typeutils.TypeSerializer 7 | 8 | internal abstract class AbstractDataClassTypeSerializerTest : SerializerTestBase() { 9 | protected abstract val typeInformation: TypeInformation 10 | 11 | override fun createSerializer(): TypeSerializer = typeInformation.createSerializer(ExecutionConfig()) 12 | 13 | override fun getLength(): Int = -1 14 | 15 | override fun getTypeClass(): Class = typeInformation.typeClass 16 | } 17 | 18 | internal class DataClassTypeSerializerTest : AbstractDataClassTypeSerializerTest() { 19 | override val typeInformation: TypeInformation = createTypeInformation() 20 | 21 | override fun getTestData(): Array = 22 | arrayOf( 23 | DataClass("string", 1, Nested("string")), 24 | DataClass("string", 123, Nested("123")), 25 | ) 26 | } 27 | 28 | internal class ParameterizedTypeSerializerTest : AbstractDataClassTypeSerializerTest>() { 29 | override val typeInformation: TypeInformation> = createTypeInformation() 30 | 31 | override fun getTestData(): Array> = 32 | arrayOf( 33 | ParameterizedClass("string", 1), 34 | ParameterizedClass("string", 4), 35 | ) 36 | } 37 | 38 | internal class OrderTypeSerializerTest : AbstractDataClassTypeSerializerTest() { 39 | override val typeInformation: TypeInformation = createTypeInformation() 40 | 41 | override fun getTestData(): Array = 42 | arrayOf( 43 | Order(Purchase(2.0), Purchase(1.0)), 44 | Order(Purchase(20.0), Purchase(15.0)), 45 | ) 46 | } 47 | 48 | internal class PairTypeSerializerTest : AbstractDataClassTypeSerializerTest>() { 49 | override val typeInformation: TypeInformation> = createTypeInformation() 50 | 51 | override fun getTestData(): Array> = 52 | arrayOf( 53 | Pair("Hello", 1), 54 | Pair("World", 2), 55 | ) 56 | } 57 | 58 | internal class TripleTypeSerializerTest : AbstractDataClassTypeSerializerTest>() { 59 | override val typeInformation: TypeInformation> = createTypeInformation() 60 | 61 | override fun getTestData(): Array> = 62 | arrayOf( 63 | Triple("Hello", "World", 1), 64 | Triple("Super", "Mario", 2), 65 | ) 66 | } 67 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/ExampleTest.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration 4 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment 5 | import org.apache.flink.test.util.MiniClusterWithClientResource 6 | import org.junit.jupiter.api.AfterAll 7 | import org.junit.jupiter.api.Assertions.assertEquals 8 | import org.junit.jupiter.api.BeforeAll 9 | import org.junit.jupiter.api.Test 10 | 11 | internal class ExampleTest { 12 | companion object { 13 | @JvmStatic 14 | private val cluster = 15 | MiniClusterWithClientResource( 16 | MiniClusterResourceConfiguration 17 | .Builder() 18 | .setNumberSlotsPerTaskManager(2) 19 | .setNumberTaskManagers(1) 20 | .build(), 21 | ) 22 | 23 | @JvmStatic 24 | @BeforeAll 25 | fun beforeAll(): Unit = cluster.before() 26 | 27 | @JvmStatic 28 | @AfterAll 29 | fun afterAll(): Unit = cluster.after() 30 | } 31 | 32 | private val environment = 33 | StreamExecutionEnvironment.getExecutionEnvironment().apply { 34 | config.disableGenericTypes() 35 | } 36 | 37 | @Test 38 | fun `handles data classes`() { 39 | val purchases = 40 | environment 41 | .fromCollection(listOf(Purchase(2.0), Purchase(1.0)), createTypeInformation()) 42 | .executeAndCollect(10) 43 | assertEquals(2, purchases.size) 44 | } 45 | 46 | @Test 47 | fun `handles list`() { 48 | val orders = 49 | environment 50 | .fromCollection(listOf(Order(Purchase(2.0), Purchase(1.0))), createTypeInformation()) 51 | .executeAndCollect(10) 52 | assertEquals(1, orders.size) 53 | assertEquals(2, orders.first().purchases.size) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/kotlin/com/lapanthere/flink/api/kotlin/typeutils/TypeInformationTest.kt: -------------------------------------------------------------------------------- 1 | package com.lapanthere.flink.api.kotlin.typeutils 2 | 3 | import org.apache.flink.api.common.typeinfo.Types 4 | import org.apache.flink.api.java.typeutils.ListTypeInfo 5 | import org.apache.flink.api.java.typeutils.MapTypeInfo 6 | import org.junit.jupiter.api.Assertions.assertEquals 7 | import org.junit.jupiter.api.Assertions.assertTrue 8 | import org.junit.jupiter.api.Test 9 | 10 | internal class TypeInformationTest { 11 | @Test 12 | fun `returns Triple type information`() { 13 | val typeInformation = createTypeInformation>() 14 | assertTrue(typeInformation is DataClassTypeInformation>) 15 | assertEquals(Types.STRING, typeInformation.genericParameters["A"]) 16 | assertEquals(Types.STRING, typeInformation.genericParameters["B"]) 17 | assertEquals(Types.INT, typeInformation.genericParameters["C"]) 18 | } 19 | 20 | @Test 21 | fun `returns Pair type information`() { 22 | val typeInformation = createTypeInformation>() 23 | assertTrue(typeInformation is DataClassTypeInformation>) 24 | assertEquals(Types.STRING, typeInformation.genericParameters["A"]) 25 | assertEquals(Types.STRING, typeInformation.genericParameters["B"]) 26 | } 27 | 28 | @Test 29 | fun `returns data class type information`() { 30 | assertTrue(createTypeInformation() is DataClassTypeInformation) 31 | assertTrue(createTypeInformation>() is DataClassTypeInformation>) 32 | } 33 | 34 | @Test 35 | fun `returns generic parameters`() { 36 | val typeInformation = createTypeInformation>() 37 | assertEquals(Types.INT, typeInformation.genericParameters["T"]) 38 | } 39 | 40 | @Test 41 | fun `returns list type information`() { 42 | val typeInformation = createTypeInformation>() 43 | require(typeInformation is ListTypeInfo<*>) 44 | assertEquals(Types.STRING, typeInformation.elementTypeInfo) 45 | } 46 | 47 | @Test 48 | fun `returns map type information`() { 49 | val typeInformation = createTypeInformation>() 50 | require(typeInformation is MapTypeInfo<*, *>) 51 | assertEquals(Types.STRING, typeInformation.keyTypeInfo) 52 | assertEquals(Types.INT, typeInformation.valueTypeInfo) 53 | } 54 | } 55 | --------------------------------------------------------------------------------