├── .git-blame-ignore-revs ├── .gitattributes ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .scala-steward.conf ├── .scalafmt.conf ├── LICENSE.md ├── README.md ├── build.sbt ├── modules ├── examples │ ├── notebooks │ │ └── word_count_example.ipynb │ ├── scripts │ │ ├── PubSubConnectorWithJson.sc │ │ ├── debug-sql.sc │ │ ├── flink-amm.sc │ │ ├── flink-scala-cli.scala │ │ ├── gen-csv-file.sc │ │ ├── gen-kafka-data.sc │ │ ├── hybrid-source.sc │ │ └── logback.xml │ └── src │ │ ├── main │ │ ├── protobuf │ │ │ └── simple.proto │ │ ├── resources │ │ │ ├── cities.csv │ │ │ └── logback.xml │ │ └── scala │ │ │ └── org │ │ │ └── example │ │ │ ├── Job.scala │ │ │ ├── SocketTextStreamWordCount.scala │ │ │ ├── TransactonIOs.scala │ │ │ ├── connectedStreams.scala │ │ │ ├── fileFilter.scala │ │ │ ├── fraud │ │ │ ├── FraudDetectionJob.scala │ │ │ ├── FraudDetector.scala │ │ │ └── RunningAverage.scala │ │ │ ├── runningSum.scala │ │ │ ├── troubleshooting │ │ │ ├── fakeKafkaSource.scala │ │ │ ├── measurements.scala │ │ │ └── troubleshootingExample.scala │ │ │ └── wordCount.scala │ │ └── test │ │ └── scala │ │ └── org │ │ └── example │ │ ├── ConnectedStreamsTest.scala │ │ ├── CustomTriggerTests.scala │ │ ├── MyKeyedProcessFunctionTest.scala │ │ └── fraud │ │ ├── FakeRuntimeContext.scala │ │ └── FraudDetectorTest.scala ├── flink-1-api │ └── src │ │ ├── main │ │ ├── scala-2 │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flinkx │ │ │ │ └── api │ │ │ │ └── LowPrioImplicits.scala │ │ ├── scala-3 │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flinkx │ │ │ │ └── api │ │ │ │ └── LowPrioImplicits.scala │ │ └── scala │ │ │ └── org │ │ │ └── apache │ │ │ ├── flink │ │ │ └── streaming │ │ │ │ └── util │ │ │ │ └── typeutils │ │ │ │ └── DefaultScalaProductFieldAccessorFactory.scala │ │ │ └── flinkx │ │ │ └── api │ │ │ ├── AllWindowedStream.scala │ │ │ ├── AsyncDataStream.scala │ │ │ ├── BroadcastConnectedStream.scala │ │ │ ├── CloseableIterator.scala │ │ │ ├── ClosureCleaner.scala │ │ │ ├── CoGroupedStreams.scala │ │ │ ├── ConnectedStreams.scala │ │ │ ├── DataStream.scala │ │ │ ├── DataStreamUtils.scala │ │ │ ├── JoinedStreams.scala │ │ │ ├── KeyedStream.scala │ │ │ ├── OutputTag.scala │ │ │ ├── ScalaStreamOps.scala │ │ │ ├── StreamExecutionEnvironment.scala │ │ │ ├── WindowedStream.scala │ │ │ ├── async │ │ │ ├── AsyncFunction.scala │ │ │ ├── JavaResultFutureWrapper.scala │ │ │ ├── ResultFuture.scala │ │ │ ├── RichAsyncFunction.scala │ │ │ └── ScalaRichAsyncFunctionWrapper.scala │ │ │ ├── conv.scala │ │ │ ├── extensions │ │ │ ├── impl │ │ │ │ └── acceptPartialFunctions │ │ │ │ │ ├── OnConnectedStream.scala │ │ │ │ │ ├── OnDataStream.scala │ │ │ │ │ ├── OnJoinedStream.scala │ │ │ │ │ ├── OnKeyedStream.scala │ │ │ │ │ └── OnWindowedStream.scala │ │ │ └── ops.scala │ │ │ ├── function │ │ │ ├── AllWindowFunction.scala │ │ │ ├── ProcessAllWindowFunction.scala │ │ │ ├── ProcessWindowFunction.scala │ │ │ ├── RichAllWindowFunction.scala │ │ │ ├── RichWindowFunction.scala │ │ │ ├── StatefulFunction.scala │ │ │ ├── WindowFunction.scala │ │ │ └── util │ │ │ │ ├── ScalaAllWindowFunction.scala │ │ │ │ ├── ScalaAllWindowFunctionWrapper.scala │ │ │ │ ├── ScalaProcessWindowFunctionWrapper.scala │ │ │ │ ├── ScalaReduceFunction.scala │ │ │ │ ├── ScalaWindowFunction.scala │ │ │ │ └── ScalaWindowFunctionWrapper.scala │ │ │ ├── mapper │ │ │ ├── BigDecMapper.scala │ │ │ ├── BigIntMapper.scala │ │ │ └── UuidMapper.scala │ │ │ ├── serializer │ │ │ ├── ArraySerializer.scala │ │ │ ├── CollectionSerializerSnapshot.scala │ │ │ ├── CoproductSerializer.scala │ │ │ ├── ListCCSerializer.scala │ │ │ ├── ListSerializer.scala │ │ │ ├── MapSerializer.scala │ │ │ ├── MappedSerializer.scala │ │ │ ├── ScalaCaseObjectSerializer.scala │ │ │ ├── SeqSerializer.scala │ │ │ ├── SetSerializer.scala │ │ │ └── VectorSerializer.scala │ │ │ ├── serializers.scala │ │ │ └── typeinfo │ │ │ ├── CaseClassTypeInfo.scala │ │ │ ├── CollectionTypeInformation.scala │ │ │ ├── CoproductTypeInformation.scala │ │ │ ├── EitherTypeInfo.scala │ │ │ ├── MappedTypeInformation.scala │ │ │ ├── OptionTypeInfo.scala │ │ │ ├── ProductTypeInformation.scala │ │ │ ├── SimpleTypeInformation.scala │ │ │ └── UnitTypeInformation.scala │ │ └── test │ │ ├── resources │ │ ├── click.dat │ │ ├── logback.xml │ │ ├── old-serializer-snapshot.dat │ │ └── without-arity-test.dat │ │ ├── scala-2 │ │ └── org │ │ │ └── apache │ │ │ └── flinkx │ │ │ └── api │ │ │ └── GenericCaseClassScala2Test.scala │ │ ├── scala-3 │ │ └── org │ │ │ └── apache │ │ │ └── flinkx │ │ │ └── api │ │ │ ├── GenericCaseClassScala3Test.scala │ │ │ └── Scala3EnumTest.scala │ │ └── scala │ │ └── org │ │ └── apache │ │ └── flinkx │ │ └── api │ │ ├── AnyTest.scala │ │ ├── AsyncDataStreamTest.scala │ │ ├── CatsTest.scala │ │ ├── CoGroupedStreamsTest.scala │ │ ├── DataStreamTest.scala │ │ ├── ExampleTest.scala │ │ ├── IntegrationTest.scala │ │ ├── IntegrationTestSink.scala │ │ ├── JoinedStreamsTest.scala │ │ ├── MappedTypeInfoTest.scala │ │ ├── ProcessFunctionTest.scala │ │ ├── SchemaEvolutionTest.scala │ │ ├── SerializerSnapshotTest.scala │ │ ├── SerializerTest.scala │ │ ├── StreamExecutionEnvironmentTest.scala │ │ ├── TestUtils.scala │ │ ├── TypeInfoTest.scala │ │ └── serializer │ │ ├── ArraySerializerTest.scala │ │ └── CoproductSerializerTest.scala ├── flink-2-api │ └── src │ │ ├── main │ │ ├── scala-2 │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flinkx │ │ │ │ └── api │ │ │ │ └── LowPrioImplicits.scala │ │ ├── scala-3 │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── flinkx │ │ │ │ └── api │ │ │ │ └── LowPrioImplicits.scala │ │ └── scala │ │ │ └── org │ │ │ └── apache │ │ │ ├── flink │ │ │ └── streaming │ │ │ │ └── util │ │ │ │ └── typeutils │ │ │ │ └── DefaultScalaProductFieldAccessorFactory.scala │ │ │ └── flinkx │ │ │ └── api │ │ │ ├── AllWindowedStream.scala │ │ │ ├── AsyncDataStream.scala │ │ │ ├── BroadcastConnectedStream.scala │ │ │ ├── CloseableIterator.scala │ │ │ ├── ClosureCleaner.scala │ │ │ ├── CoGroupedStreams.scala │ │ │ ├── ConnectedStreams.scala │ │ │ ├── DataStream.scala │ │ │ ├── DataStreamUtils.scala │ │ │ ├── JoinedStreams.scala │ │ │ ├── KeyedStream.scala │ │ │ ├── OutputTag.scala │ │ │ ├── ScalaStreamOps.scala │ │ │ ├── StreamExecutionEnvironment.scala │ │ │ ├── WindowedStream.scala │ │ │ ├── async │ │ │ ├── AsyncFunction.scala │ │ │ ├── JavaResultFutureWrapper.scala │ │ │ ├── ResultFuture.scala │ │ │ ├── RichAsyncFunction.scala │ │ │ └── ScalaRichAsyncFunctionWrapper.scala │ │ │ ├── conv.scala │ │ │ ├── extensions │ │ │ ├── impl │ │ │ │ └── acceptPartialFunctions │ │ │ │ │ ├── OnConnectedStream.scala │ │ │ │ │ ├── OnDataStream.scala │ │ │ │ │ ├── OnJoinedStream.scala │ │ │ │ │ ├── OnKeyedStream.scala │ │ │ │ │ └── OnWindowedStream.scala │ │ │ └── ops.scala │ │ │ ├── function │ │ │ ├── AllWindowFunction.scala │ │ │ ├── ProcessAllWindowFunction.scala │ │ │ ├── ProcessWindowFunction.scala │ │ │ ├── RichAllWindowFunction.scala │ │ │ ├── RichWindowFunction.scala │ │ │ ├── StatefulFunction.scala │ │ │ ├── WindowFunction.scala │ │ │ └── util │ │ │ │ ├── ScalaAllWindowFunction.scala │ │ │ │ ├── ScalaAllWindowFunctionWrapper.scala │ │ │ │ ├── ScalaProcessWindowFunctionWrapper.scala │ │ │ │ ├── ScalaReduceFunction.scala │ │ │ │ ├── ScalaWindowFunction.scala │ │ │ │ └── ScalaWindowFunctionWrapper.scala │ │ │ ├── mapper │ │ │ ├── BigDecMapper.scala │ │ │ ├── BigIntMapper.scala │ │ │ └── UuidMapper.scala │ │ │ ├── serializer │ │ │ ├── ArraySerializer.scala │ │ │ ├── CollectionSerializerSnapshot.scala │ │ │ ├── CoproductSerializer.scala │ │ │ ├── ListCCSerializer.scala │ │ │ ├── ListSerializer.scala │ │ │ ├── MapSerializer.scala │ │ │ ├── MappedSerializer.scala │ │ │ ├── ScalaCaseObjectSerializer.scala │ │ │ ├── SeqSerializer.scala │ │ │ ├── SetSerializer.scala │ │ │ └── VectorSerializer.scala │ │ │ ├── serializers.scala │ │ │ └── typeinfo │ │ │ ├── CaseClassTypeInfo.scala │ │ │ ├── CollectionTypeInformation.scala │ │ │ ├── CoproductTypeInformation.scala │ │ │ ├── EitherTypeInfo.scala │ │ │ ├── MappedTypeInformation.scala │ │ │ ├── OptionTypeInfo.scala │ │ │ ├── ProductTypeInformation.scala │ │ │ ├── SimpleTypeInformation.scala │ │ │ └── UnitTypeInformation.scala │ │ └── test │ │ └── scala │ │ └── org │ │ └── apache │ │ └── flinkx │ │ └── api │ │ └── serializer │ │ ├── CollectionSerializerSnapshotTest.scala │ │ ├── CoproductSerializerSnapshotTest.scala │ │ ├── MapSerializerSnapshotTest.scala │ │ └── MappedSerializerSnapshotTest.scala └── flink-common-api │ └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── flinkx │ │ │ └── api │ │ │ └── serializer │ │ │ ├── ScalaCaseClassSerializerSnapshot.java │ │ │ ├── ScalaEitherSerializerSnapshot.java │ │ │ └── ScalaOptionSerializerSnapshot.java │ ├── scala-2 │ │ └── org │ │ │ └── apache │ │ │ └── flinkx │ │ │ └── api │ │ │ └── serializer │ │ │ └── ConstructorCompat.scala │ ├── scala-3 │ │ └── org │ │ │ └── apache │ │ │ └── flinkx │ │ │ └── api │ │ │ ├── TaggedDerivation.scala │ │ │ ├── TypeTag.scala │ │ │ ├── TypeTagMacro.scala │ │ │ └── serializer │ │ │ └── ConstructorCompat.scala │ └── scala │ │ └── org │ │ └── apache │ │ └── flinkx │ │ └── api │ │ ├── serializer │ │ ├── CaseClassSerializer.scala │ │ ├── EitherSerializer.scala │ │ ├── ImmutableSerializer.scala │ │ ├── MutableSerializer.scala │ │ ├── NothingSerializer.scala │ │ ├── OptionSerializer.scala │ │ └── UnitSerializer.scala │ │ ├── typeinfo │ │ ├── CaseClassComparator.scala │ │ ├── FailFastTypeInfoFactory.scala │ │ └── OptionTypeComparator.scala │ │ └── util │ │ └── ClassUtil.scala │ └── test │ └── scala │ └── org │ └── apache │ └── flinkx │ └── api │ ├── serializer │ └── CaseClassSerializerTest.scala │ └── util │ └── ClassUtilTest.scala ├── project ├── build.properties └── plugins.sbt ├── release.sh └── version.sbt /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Scala Steward: Reformat with scalafmt 3.5.9 2 | a19cdcb35801ca76ccc2d7263f5170309e2d266e 3 | 4 | # Scala Steward: Reformat with scalafmt 3.7.7 5 | a744067645fb512553ada2007190487c9019fb86 6 | 7 | # Scala Steward: Reformat with scalafmt 3.7.13 8 | 0c01ba437ef9e9590ffd52557c3e2242e5c7187c 9 | 10 | # Scala Steward: Reformat with scalafmt 3.7.15 11 | cd88dbd39d341e4d83dbaed57b1fe497159ba0be 12 | 13 | # Scala Steward: Reformat with scalafmt 3.8.0 14 | ac01058463c34eb216d3d3c0500dfce038fc4cff 15 | 16 | # Scala Steward: Reformat with scalafmt 3.8.1 17 | fade5fe3272078547e5ca2a7ef80a085a8d24b24 18 | 19 | # Scala Steward: Reformat with scalafmt 3.8.2 20 | 0931a26c8728f0f02202ce0a73085e6034110dd4 21 | 22 | # Scala Steward: Reformat with scalafmt 3.8.3 23 | 263d8f24393c0607dddc5a3b8cd2fbc1f0de0141 24 | 25 | # Scala Steward: Reformat with scalafmt 3.9.0 26 | 5428c1b0fd387b524dea6571e34ab17c5df39760 27 | 28 | # Scala Steward: Reformat with scalafmt 3.9.1 29 | ae9aadfa2b4779b5096a9004d6d7f5d9db2c2c7f 30 | 31 | # Scala Steward: Reformat with scalafmt 3.9.5 32 | 86e22a329b56041cf014ca16e78ab03e8fd7df00 33 | 34 | # Scala Steward: Reformat with scalafmt 3.9.6 35 | 9fd34d49283551ea5253cbe6177a10661be715d3 36 | 37 | # Scala Steward: Reformat with scalafmt 3.9.7 38 | a82026bb66f0e17cf76f2d91c6ea286431bbbd31 39 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.tsv filter=lfs diff=lfs merge=lfs -text 2 | *.gz filter=lfs diff=lfs merge=lfs -text 3 | *.json filter=lfs diff=lfs merge=lfs -text 4 | *.dat filter=lfs diff=lfs merge=lfs -text 5 | *.jpg filter=lfs diff=lfs merge=lfs -text 6 | *.svg filter=lfs diff=lfs merge=lfs -text 7 | *.png filter=lfs diff=lfs merge=lfs -text 8 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: CI 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-22.04 15 | strategy: 16 | matrix: 17 | java: [11] 18 | scala: [2.13.16, 3.3.6] 19 | flink: [1.18.1, 1.19.1] 20 | sbt-module: ['flink-1-api'] 21 | include: 22 | - scala: 3.3.6 23 | java: 17 24 | flink: 1.20.0 25 | sbt-module: 'flink-1-api' 26 | - scala: 3.3.6 27 | java: 17 28 | flink: 2.0.0 29 | sbt-module: 'flink-2-api' 30 | env: 31 | JAVA_OPTIONS: '--add-opens java.base/java.lang=ALL-UNNAMED' 32 | steps: 33 | - uses: actions/checkout@v3 34 | with: 35 | fetch-depth: 0 36 | - name: Set up JDK 37 | uses: actions/setup-java@v3 38 | with: 39 | distribution: temurin 40 | java-version: ${{ matrix.java }} 41 | cache: sbt 42 | - name: Compile Docs 43 | run: JAVA_OPTS=$JAVA_OPTIONS sbt "++ ${{ matrix.scala }} docs/mdoc" 44 | - name: Run tests on examples 45 | # always running on Scala 3.x version by default 46 | if: ${{ !startsWith(matrix.flink, '1.18') && !startsWith(matrix.flink, '2.') }} 47 | run: JAVA_OPTS=$JAVA_OPTIONS sbt -DflinkVersion1=${{ matrix.flink }} "project examples; test" 48 | - name: Run tests on Flink API 49 | run: JAVA_OPTS=$JAVA_OPTIONS sbt -DflinkVersion1=${{ matrix.flink }} -DflinkVersion2=${{ matrix.flink }} "++ ${{ matrix.scala }}; project ${{ matrix.sbt-module }}; test" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | project/project 3 | project/target 4 | target 5 | .DS_STORE 6 | .git 7 | .bsp 8 | .run 9 | .bloop 10 | .metals 11 | .vscode 12 | metals.sbt 13 | .scala-build 14 | .ammonite/* 15 | *.jar 16 | *checkpoint 17 | sink-table 18 | *.iml -------------------------------------------------------------------------------- /.scala-steward.conf: -------------------------------------------------------------------------------- 1 | updates.ignore = [ 2 | { groupId = "org.apache.flink" } 3 | ] -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | style = defaultWithAlign 2 | maxColumn = 120 3 | version = 3.9.7 4 | assumeStandardLibraryStripMargin = true 5 | align.stripMargin = true 6 | runner.dialect = scala3 -------------------------------------------------------------------------------- /modules/examples/scripts/debug-sql.sc: -------------------------------------------------------------------------------- 1 | import $ivy.`org.flinkextended::flink-scala-api:1.18.1_1.2.4` 2 | 3 | import $ivy.`org.apache.flink:flink-clients:1.18.1` 4 | 5 | import $ivy.`org.apache.flink:flink-streaming-scala_2.12:1.18.1` 6 | 7 | import $ivy.`org.apache.flink:flink-table-api-java:1.18.1` 8 | import $ivy.`org.apache.flink:flink-table-api-java-bridge:1.18.1` 9 | import $ivy.`org.apache.flink:flink-table-runtime:1.18.1` 10 | import $ivy.`org.apache.flink:flink-table-planner_2.12:1.18.1` 11 | 12 | import org.apache.flink.table.api._ 13 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment 14 | import org.apache.flink.connector.datagen.table.DataGenConnectorOptions 15 | 16 | import org.apache.flinkx.api._ 17 | import org.apache.flinkx.api.serializers._ 18 | 19 | import java.lang.{Long => JLong} 20 | 21 | val env = StreamExecutionEnvironment.getExecutionEnvironment 22 | val tEnv = StreamTableEnvironment.create(env.getJavaEnv) 23 | 24 | val settings = EnvironmentSettings.newInstance().inStreamingMode().build() 25 | 26 | val table = TableEnvironment.create(settings) 27 | 28 | table.createTemporaryTable( 29 | "SourceTable", 30 | TableDescriptor 31 | .forConnector("datagen") 32 | .schema( 33 | Schema.newBuilder 34 | .column("BookId", DataTypes.INT()) 35 | .build 36 | ) 37 | .option(DataGenConnectorOptions.ROWS_PER_SECOND, new JLong(1)) 38 | .build 39 | ) 40 | 41 | val tableDescriptor = TableDescriptor 42 | .forConnector("datagen") 43 | .schema( 44 | Schema.newBuilder 45 | .column("id", DataTypes.INT.notNull) 46 | .column("a", DataTypes.ROW(DataTypes.FIELD("np", DataTypes.INT.notNull())).notNull()) 47 | .build 48 | ) 49 | .build 50 | table.createTemporaryTable("t1", tableDescriptor) 51 | table.createTemporaryTable("t2", tableDescriptor) 52 | // table.dropTemporaryTable("t1") 53 | // table.dropTemporaryTable("t2") 54 | 55 | val res = table.executeSql( 56 | "EXPLAIN SELECT a.id, COALESCE(a.a.np, b.a.np) c1, IFNULL(a.a.np, b.a.np) c2 FROM t1 a left JOIN t2 b ON a.id=b.id where a.a is null or a.a.np is null" 57 | ) 58 | res.print 59 | -------------------------------------------------------------------------------- /modules/examples/scripts/flink-amm.sc: -------------------------------------------------------------------------------- 1 | import $cp.lib.`flink-faker-0.4.0.jar` 2 | 3 | import $ivy.`org.flinkextended::flink-scala-api:1.15.4_1.0.0` 4 | import $ivy.`org.apache.flink:flink-clients:1.15.2` 5 | import $ivy.`org.apache.flink:flink-csv:1.15.2` 6 | import $ivy.`org.apache.flink:flink-table-api-java:1.15.2` 7 | import $ivy.`org.apache.flink:flink-table-api-java-bridge:1.15.2` 8 | import $ivy.`org.apache.flink:flink-table-runtime:1.15.2` 9 | import $ivy.`org.apache.flink:flink-table-planner-loader:1.15.2` 10 | 11 | import org.apache.flink.table.api._ 12 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment 13 | import org.apache.flink.connector.datagen.table.DataGenConnectorOptions 14 | 15 | import org.apache.flink.api._ 16 | import org.apache.flink.api.serializers._ 17 | 18 | import _root_.java.lang.{Long => JLong} 19 | 20 | val env = StreamExecutionEnvironment.getExecutionEnvironment 21 | val tEnv = StreamTableEnvironment.create(env.getJavaEnv) 22 | val settings = EnvironmentSettings.newInstance().inStreamingMode().build() 23 | val table = TableEnvironment.create(settings) 24 | 25 | val tableDescriptor = TableDescriptor 26 | .forConnector("faker") 27 | .schema( 28 | Schema.newBuilder 29 | .column( 30 | "id", 31 | DataTypes.INT // .notNull 32 | ) 33 | .column( 34 | "a", 35 | DataTypes.ROW(DataTypes.FIELD("np", DataTypes.INT)) 36 | ) 37 | .build 38 | ) 39 | .option("fields.id.expression", "#{number.numberBetween '0','10'}") 40 | .option("fields.a.np.expression", "#{number.numberBetween '20','30'}") 41 | // .option("fields.a.np.null-rate", "0.5") 42 | .option("fields.a.null-rate", "0.5") 43 | .option("rows-per-second", "50") 44 | .build 45 | table.createTemporaryTable("t1", tableDescriptor) 46 | // table.dropTemporaryTable("t1") 47 | 48 | val res = table.executeSql( 49 | "SELECT a.id, COALESCE(a.a.np, a.id) c1, IFNULL(a.a.np, a.id) c2, a.a.np FROM t1 a" 50 | // "show create table t1" 51 | ) 52 | res.print 53 | -------------------------------------------------------------------------------- /modules/examples/scripts/flink-scala-cli.scala: -------------------------------------------------------------------------------- 1 | //> using dep "org.flinkextended::flink-scala-api:1.18.1_1.2.4" 2 | //> using dep "org.apache.flink:flink-clients:1.18.1" 3 | 4 | import org.apache.flinkx.api.* 5 | import org.apache.flinkx.api.serializers.* 6 | import org.slf4j.LoggerFactory 7 | import java.io.File 8 | 9 | @main def wordCountExample = 10 | val logger = LoggerFactory.getLogger(this.getClass()) 11 | val files = File(".").listFiles ++ Option(File("/flink/lib/").listFiles) 12 | .getOrElse(Array.empty[File]) 13 | val elems = files.filter(_.isFile).map(_.getAbsolutePath()) 14 | 15 | val env = StreamExecutionEnvironment.getExecutionEnvironment 16 | val text = env.fromElements(elems*) 17 | 18 | text.addSink(logger.info(_)) 19 | 20 | env.execute("wordCount") 21 | -------------------------------------------------------------------------------- /modules/examples/scripts/gen-csv-file.sc: -------------------------------------------------------------------------------- 1 | //> using dep "org.flinkextended::flink-scala-api-1:1.2.7" 2 | //> using dep "org.apache.flink:flink-clients:1.20.1" 3 | //> using dep "org.apache.flink:flink-csv:1.20.1" 4 | //> using dep "org.apache.flink:flink-connector-files:1.20.1" 5 | //> using dep "org.apache.flink:flink-table-runtime:1.20.1" 6 | //> using dep "org.apache.flink:flink-table-planner-loader:1.20.1" 7 | 8 | import org.apache.flink.table.api._ 9 | import org.apache.flink.connector.datagen.table.DataGenConnectorOptions 10 | import org.apache.flinkx.api._ 11 | import org.apache.flinkx.api.serializers._ 12 | 13 | import java.lang.{Long => JLong} 14 | 15 | val env = StreamExecutionEnvironment.getExecutionEnvironment 16 | val settings = EnvironmentSettings.newInstance.inStreamingMode.build 17 | val table = TableEnvironment.create(settings) 18 | val schema = Schema.newBuilder 19 | .column("id", DataTypes.INT()) 20 | .column("bid_price", DataTypes.DOUBLE()) 21 | .column("order_time", DataTypes.TIMESTAMP(2)) 22 | .build 23 | 24 | table.createTemporaryTable( 25 | "SourceTable", 26 | TableDescriptor 27 | .forConnector("datagen") 28 | .schema(schema) 29 | .option(DataGenConnectorOptions.NUMBER_OF_ROWS, JLong(1000)) 30 | .option("fields.id.kind", "sequence") 31 | .option("fields.id.start", "1") 32 | .option("fields.id.end", "10000") 33 | .build 34 | ) 35 | 36 | val currentDirectory = java.io.File(".").getCanonicalPath 37 | 38 | table.createTemporaryTable( 39 | "SinkTable", 40 | TableDescriptor 41 | .forConnector("filesystem") 42 | .schema(schema) 43 | .option("format", "csv") 44 | .option("sink.rolling-policy.file-size", "124 kb") 45 | .option("path", s"file://$currentDirectory/sink-table") 46 | .build 47 | ) 48 | 49 | table.executeSql("insert into SinkTable select * from SourceTable").print 50 | -------------------------------------------------------------------------------- /modules/examples/scripts/gen-kafka-data.sc: -------------------------------------------------------------------------------- 1 | //> using dep "org.flinkextended::flink-scala-api:1.18.1_1.2.4" 2 | //> using dep "org.apache.flink:flink-clients:1.18.1" 3 | //> using dep "org.apache.flink:flink-csv:1.18.1" 4 | //> using dep "org.apache.flink:flink-connector-files:1.18.1" 5 | //> using dep "org.apache.flink:flink-connector-kafka:3.0.2-1.18" 6 | //> using dep "org.apache.flink:flink-table-runtime:1.18.1" 7 | //> using dep "org.apache.flink:flink-table-planner-loader:1.18.1" 8 | 9 | import org.apache.flink.table.api._ 10 | import org.apache.flink.connector.datagen.table.DataGenConnectorOptions 11 | import org.apache.flinkx.api._ 12 | import org.apache.flinkx.api.serializers._ 13 | 14 | import java.lang.{Long => JLong} 15 | 16 | val env = StreamExecutionEnvironment.getExecutionEnvironment 17 | val settings = EnvironmentSettings.newInstance.inStreamingMode.build 18 | val table = TableEnvironment.create(settings) 19 | val schema = Schema.newBuilder 20 | .column("id", DataTypes.INT()) 21 | .column("bid_price", DataTypes.DOUBLE()) 22 | .column("order_time", DataTypes.TIMESTAMP(2)) 23 | .build 24 | 25 | table.createTemporaryTable( 26 | "SourceTable", 27 | TableDescriptor 28 | .forConnector("datagen") 29 | .schema(schema) 30 | .option(DataGenConnectorOptions.NUMBER_OF_ROWS, JLong(1000)) 31 | .option("fields.id.kind", "sequence") 32 | .option("fields.id.start", "10001") 33 | .option("fields.id.end", "20000") 34 | .build 35 | ) 36 | 37 | val brokers = "confluentkafka-cp-kafka:9092" 38 | 39 | table.createTemporaryTable( 40 | "SinkTable", 41 | TableDescriptor 42 | .forConnector("kafka") 43 | .schema(schema) 44 | .option("properties.bootstrap.servers", brokers) 45 | .option("topic", "bids") 46 | .option("format", "csv") 47 | .option("value.format", "csv") 48 | .build 49 | ) 50 | 51 | table.executeSql("insert into SinkTable select * from SourceTable").print 52 | -------------------------------------------------------------------------------- /modules/examples/scripts/hybrid-source.sc: -------------------------------------------------------------------------------- 1 | //> using dep "org.flinkextended::flink-scala-api:1.18.1_1.2.4" 2 | //> using dep "org.apache.flink:flink-clients:1.18.1" 3 | //> using dep "org.apache.flink:flink-csv:1.18.1" 4 | //> using dep "org.apache.flink:flink-connector-files:1.18.1" 5 | //> using dep "org.apache.flink:flink-connector-kafka:3.0.2-1.18" 6 | 7 | import org.apache.flinkx.api.* 8 | import org.apache.flinkx.api.serializers.* 9 | import org.apache.flink.connector.file.src.FileSource 10 | import org.apache.flink.connector.file.src.reader.TextLineInputFormat 11 | import org.apache.flink.connector.file.src.impl.StreamFormatAdapter 12 | import org.apache.flink.connector.kafka.source.KafkaSource 13 | import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer 14 | import org.apache.flink.connector.base.source.hybrid.HybridSource 15 | import org.apache.flink.api.common.serialization.SimpleStringSchema 16 | import org.apache.flink.api.common.eventtime.WatermarkStrategy 17 | import org.apache.flink.core.fs.Path 18 | 19 | val currentDirectory = java.io.File(".").getCanonicalPath 20 | 21 | val fileSource = FileSource 22 | .forBulkFileFormat( 23 | StreamFormatAdapter(TextLineInputFormat()), 24 | Path(s"$currentDirectory/sink-table") 25 | ) 26 | .build 27 | 28 | val switchTimestamp = -1L 29 | val brokers = "confluentkafka-cp-kafka:9092" 30 | 31 | val kafkaSource = KafkaSource 32 | .builder[String]() 33 | .setBootstrapServers(brokers) 34 | .setTopics("bids") 35 | .setStartingOffsets(OffsetsInitializer.timestamp(switchTimestamp + 1)) 36 | .setValueOnlyDeserializer(SimpleStringSchema()) 37 | .build 38 | 39 | val hybridSource = HybridSource 40 | .builder(fileSource) 41 | .addSource(kafkaSource) 42 | .build 43 | 44 | val env = StreamExecutionEnvironment.getExecutionEnvironment 45 | env 46 | .fromSource(hybridSource, WatermarkStrategy.noWatermarks(), "combined") 47 | .print() 48 | 49 | env.execute() 50 | -------------------------------------------------------------------------------- /modules/examples/scripts/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | true 5 | 6 | 7 | 8 | 9 | %date{yyyy-MM-dd HH:mm:ss.SSSZ, UTC} %-16level %-43thread %-24logger{24} %message%n%xException 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /modules/examples/src/main/protobuf/simple.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package com.example; 3 | option java_package = "com.example"; 4 | option java_multiple_files = true; 5 | 6 | message SimpleTest { 7 | optional int64 uid = 1; 8 | optional string name = 2; 9 | optional int32 category_type = 3; 10 | optional bytes content = 4; 11 | optional double price = 5; 12 | } -------------------------------------------------------------------------------- /modules/examples/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | true 5 | 6 | 7 | 8 | 9 | %date{yyyy-MM-dd HH:mm:ss.SSSZ, UTC} %-16level %-43thread %-24logger{24} %message%n%xException 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /modules/examples/src/main/scala/org/example/Job.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | import org.apache.flinkx.api.* 4 | import org.apache.flinkx.api.serializers.* 5 | 6 | class JobFailed(cause: Exception) extends Exception(cause) 7 | 8 | @main def job = 9 | try { 10 | val env = StreamExecutionEnvironment.getExecutionEnvironment 11 | env 12 | .fromElements(1, 2, 3, 4, 5, 6) 13 | .filter(_ % 2 == 1) 14 | .map(i => i * i) 15 | .print() 16 | throw new RuntimeException("boom") 17 | try env.execute() 18 | catch case e: Exception => throw JobFailed(e) 19 | } catch 20 | case e: JobFailed => 21 | throw e.getCause 22 | case e: Throwable => 23 | e.printStackTrace() 24 | // failure in main method, not in the Flink job 25 | val env = StreamExecutionEnvironment.getExecutionEnvironment 26 | env 27 | .fromElements("printing stacktrace") 28 | .print() 29 | env.execute() 30 | -------------------------------------------------------------------------------- /modules/examples/src/main/scala/org/example/SocketTextStreamWordCount.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | /* 4 | * Licensed to the Apache Software Foundation (ASF) under one 5 | * or more contributor license agreements. See the NOTICE file 6 | * distributed with this work for additional information 7 | * regarding copyright ownership. The ASF licenses this file 8 | * to you under the Apache License, Version 2.0 (the 9 | * "License"); you may not use this file except in compliance 10 | * with the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | import org.apache.flinkx.api.* 22 | import org.apache.flinkx.api.serializers.* 23 | import org.apache.flink.configuration.Configuration 24 | import org.apache.flink.configuration.ConfigConstants 25 | import org.apache.flink.configuration.RestOptions.BIND_PORT 26 | import scala.jdk.CollectionConverters.* 27 | 28 | /** This example shows an implementation of WordCount with data from a text socket. To run the example make sure that 29 | * the service providing the text data is already up and running. 30 | * 31 | * To start an example socket text stream on your local machine run netcat from a command line, where the parameter 32 | * specifies the port number: 33 | * 34 | * {{{ 35 | * nc -lk 9999 36 | * }}} 37 | * 38 | * Usage: 39 | * {{{ 40 | * SocketTextStreamWordCount 41 | * }}} 42 | * 43 | * This example shows how to: 44 | * 45 | * - use StreamExecutionEnvironment.socketTextStream 46 | * - write a simple Flink Streaming program in scala. 47 | * - write and use user-defined functions. 48 | */ 49 | @main def SocketTextStreamWordCount(hostName: String, port: Int) = 50 | val config = Configuration.fromMap( 51 | Map( 52 | // ConfigConstants.LOCAL_START_WEBSERVER -> "true", 53 | BIND_PORT.key -> "8080" 54 | ).asJava 55 | ) 56 | val flink = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config) 57 | 58 | flink 59 | .socketTextStream(hostName, port) 60 | .flatMap(_.toLowerCase.split("\\W+").filter(_.nonEmpty)) 61 | .map((_, 1)) 62 | .keyBy(_._1) 63 | .sum(1) 64 | .print() 65 | 66 | flink.execute("Scala SocketTextStreamWordCount Example") 67 | -------------------------------------------------------------------------------- /modules/examples/src/main/scala/org/example/connectedStreams.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | import org.apache.flink.streaming.api.functions.co.RichCoFlatMapFunction 4 | import org.apache.flink.util.Collector 5 | import org.apache.flink.api.common.typeinfo.TypeInformation 6 | import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} 7 | import org.apache.flinkx.api.* 8 | import org.apache.flinkx.api.serializers.* 9 | 10 | @main def ConnectedStreams = 11 | val env = StreamExecutionEnvironment.getExecutionEnvironment 12 | 13 | given tranTypeInfo: TypeInformation[Transaction] = deriveTypeInformation 14 | 15 | val control = env 16 | .addSource(TransactionsSource.iterator) 17 | .keyBy(_.accountId) 18 | 19 | val streamOfWords = env 20 | .addSource(TransactionsSource.iterator) 21 | .keyBy(_.accountId) 22 | 23 | control 24 | .connect(streamOfWords) 25 | .flatMap(ControlFunction()) 26 | .print() 27 | 28 | env.execute() 29 | 30 | class ControlFunction extends RichCoFlatMapFunction[Transaction, Transaction, Transaction]: 31 | 32 | @transient lazy val state: ValueState[Double] = getRuntimeContext.getState( 33 | new ValueStateDescriptor( 34 | "joined-transaction", 35 | classOf[Double] 36 | ) 37 | ) 38 | 39 | override def flatMap1( 40 | t: Transaction, 41 | out: Collector[Transaction] 42 | ): Unit = 43 | sumUp(t, out) 44 | 45 | override def flatMap2( 46 | t: Transaction, 47 | out: Collector[Transaction] 48 | ): Unit = 49 | sumUp(t, out) 50 | 51 | private def sumUp(t: Transaction, out: Collector[Transaction]): Unit = 52 | Option(state.value()) match 53 | case Some(v) => 54 | out.collect(t.copy(amount = t.amount + v)) 55 | state.clear() 56 | case None => 57 | state.update(t.amount) 58 | -------------------------------------------------------------------------------- /modules/examples/src/main/scala/org/example/fileFilter.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | import org.apache.flinkx.api.* 4 | import org.apache.flinkx.api.serializers.* 5 | 6 | import org.apache.flink.connector.file.src.FileSource 7 | import org.apache.flink.connector.file.src.reader.TextLineInputFormat 8 | import org.apache.flink.connector.file.src.enumerate.NonSplittingRecursiveEnumerator 9 | import org.apache.flink.core.fs.Path 10 | import org.apache.flink.api.common.eventtime.WatermarkStrategy 11 | import org.apache.flink.configuration.Configuration 12 | 13 | import java.io.File 14 | import java.time.Duration 15 | import java.util.function.Predicate 16 | 17 | class MyDefaultFileFilter extends Predicate[Path]: 18 | override def test(path: Path): Boolean = 19 | print(s"S3 FILE PATH: $path") 20 | 21 | val fileName = path.getName 22 | println(s", name: $fileName") 23 | 24 | fileName.headOption match 25 | case Some(first) => 26 | first != '.' && first != '_' && !fileName.startsWith("GRP") 27 | case None => false 28 | 29 | @main def filterFiles = 30 | val currentDirectory = File(".").getCanonicalPath 31 | val inputBasePath = Path(s"$currentDirectory/input-table") 32 | val fileSourceBuilder = 33 | FileSource.forRecordStreamFormat( 34 | TextLineInputFormat(), 35 | inputBasePath 36 | ) 37 | 38 | val fileSource = fileSourceBuilder 39 | .monitorContinuously(Duration.ofSeconds(2)) 40 | .setFileEnumerator(() => NonSplittingRecursiveEnumerator(MyDefaultFileFilter())) 41 | .build() 42 | val env = 43 | StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(Configuration()) 44 | 45 | env 46 | .fromSource(fileSource, WatermarkStrategy.noWatermarks(), "csvs") 47 | .map((_, 1)) 48 | .keyBy(_ => "count") 49 | .sum(1) 50 | .map { case (w, c) => 51 | println(s"count: $c") 52 | } 53 | 54 | env.execute("Filter Files") 55 | -------------------------------------------------------------------------------- /modules/examples/src/main/scala/org/example/fraud/RunningAverage.scala: -------------------------------------------------------------------------------- 1 | package org.example.fraud 2 | 3 | import org.apache.flinkx.api._ 4 | import org.apache.flinkx.api.serializers._ 5 | 6 | import org.apache.flink.api.common.functions.RichMapFunction 7 | import org.apache.flink.api.common.state.ValueStateDescriptor 8 | import org.apache.flink.api.common.typeinfo.TypeInformation 9 | import org.apache.flink.configuration.Configuration 10 | import org.example.Transaction 11 | 12 | class RunningAverage extends RichMapFunction[Transaction, (Transaction, Double)]: 13 | 14 | given tranTypeInfo: TypeInformation[Transaction] = 15 | TypeInformation.of(classOf[Transaction]) 16 | 17 | @transient lazy val runningAvg = getRuntimeContext.getState( 18 | ValueStateDescriptor( 19 | "running-average", 20 | classOf[Double], 21 | 0d 22 | ) 23 | ) 24 | 25 | @transient lazy val count = getRuntimeContext.getState( 26 | ValueStateDescriptor("count", classOf[Int], 0) 27 | ) 28 | 29 | private def threadName = Thread.currentThread.getName 30 | override def open(config: Configuration): Unit = 31 | println(s"open map: $threadName") 32 | 33 | override def map(t: Transaction): (Transaction, Double) = 34 | Option(count.value) match 35 | case Some(cnt) => count.update(cnt + 1) 36 | case _ => () 37 | 38 | Option(runningAvg.value) match 39 | case Some(avg) => runningAvg.update((avg + t.amount) / count.value) 40 | case _ => () 41 | 42 | (t, runningAvg.value) 43 | -------------------------------------------------------------------------------- /modules/examples/src/main/scala/org/example/wordCount.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | import org.apache.flinkx.api.* 4 | import org.apache.flinkx.api.serializers.* 5 | 6 | @main def wordCountExample = 7 | val env = StreamExecutionEnvironment.getExecutionEnvironment 8 | val text = env.fromElements( 9 | "To be, or not to be,--that is the question:--", 10 | "Whether 'tis nobler in the mind to suffer", 11 | "The slings and arrows of outrageous fortune", 12 | "Or to take arms against a sea of troubles," 13 | ) 14 | 15 | text 16 | .flatMap(_.toLowerCase.split("\\W+")) 17 | .map((_, 1)) 18 | .keyBy(_._1) 19 | .sum(1) 20 | .print() 21 | 22 | env.execute("wordCount") 23 | -------------------------------------------------------------------------------- /modules/examples/src/test/scala/org/example/ConnectedStreamsTest.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | import org.scalatest.flatspec.AnyFlatSpec 4 | import org.scalatest.matchers.should.Matchers 5 | 6 | import org.apache.flinkx.api.* 7 | import org.apache.flinkx.api.serializers.* 8 | import org.apache.flink.api.common.eventtime.WatermarkStrategy 9 | import org.apache.flink.streaming.api.functions.co.CoMapFunction 10 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction 11 | import org.apache.flink.util.Collector 12 | 13 | import java.util.concurrent.TimeUnit 14 | import java.time.Duration 15 | 16 | case class TestCommand(timestamp: Long, bag: List[String] = Nil) 17 | 18 | class ConnectedStreamsTest extends AnyFlatSpec with Matchers: 19 | 20 | it should "process in random order" in { 21 | val env = StreamExecutionEnvironment.getExecutionEnvironment 22 | env.setParallelism(1) 23 | 24 | val commands = Seq(TestCommand(1), TestCommand(2), TestCommand(3)) 25 | val events = Seq( 26 | TestEvent(2L, 1, 0), 27 | TestEvent(2L, 2, 0), 28 | TestEvent(2L, 3, 0) 29 | ) 30 | def strategy[T] = WatermarkStrategy 31 | .forBoundedOutOfOrderness[T](Duration.ofSeconds(1000)) 32 | 33 | env 34 | .fromCollection(commands) 35 | .assignTimestampsAndWatermarks( 36 | strategy 37 | .withTimestampAssigner((cmd: TestCommand, streamRecordTimestamp: Long) => cmd.timestamp) 38 | ) 39 | .connect( 40 | env 41 | .fromCollection(events) 42 | .assignTimestampsAndWatermarks( 43 | strategy 44 | .withTimestampAssigner((event: TestEvent, streamRecordTimestamp: Long) => event.timestamp) 45 | ) 46 | ) 47 | .process { 48 | new CoProcessFunction[TestCommand, TestEvent, String]: 49 | 50 | override def processElement1( 51 | value: TestCommand, 52 | ctx: CoProcessFunction[ 53 | org.example.TestCommand, 54 | org.example.TestEvent, 55 | String 56 | ]#Context, 57 | out: Collector[String] 58 | ): Unit = 59 | out.collect(s"cmd: ${value.timestamp}") 60 | 61 | override def processElement2( 62 | value: TestEvent, 63 | ctx: CoProcessFunction[ 64 | org.example.TestCommand, 65 | org.example.TestEvent, 66 | String 67 | ]#Context, 68 | out: Collector[String] 69 | ): Unit = 70 | out.collect(s"event: ${value.timestamp}") 71 | } 72 | .print() 73 | 74 | env.execute() 75 | } 76 | -------------------------------------------------------------------------------- /modules/examples/src/test/scala/org/example/MyKeyedProcessFunctionTest.scala: -------------------------------------------------------------------------------- 1 | package org.example 2 | 3 | import org.scalatest.flatspec.AnyFlatSpec 4 | import org.scalatest.matchers.should.Matchers 5 | 6 | import org.apache.flink.streaming.api.functions.KeyedProcessFunction 7 | import org.apache.flink.streaming.api.operators.KeyedProcessOperator 8 | import org.apache.flink.streaming.api.functions.KeyedProcessFunction 9 | import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness 10 | import org.apache.flink.api.common.typeinfo.TypeInformation 11 | import org.apache.flink.util.Collector 12 | 13 | @SerialVersionUID(1L) 14 | class MyKeyedProcessFunction extends KeyedProcessFunction[Long, TestEvent, Long]: 15 | 16 | @throws[Exception] 17 | override def processElement( 18 | e: TestEvent, 19 | context: KeyedProcessFunction[Long, TestEvent, Long]#Context, 20 | out: Collector[Long] 21 | ): Unit = 22 | out.collect(e.timestamp) 23 | 24 | class MyKeyedProcessFunctionTest extends AnyFlatSpec with Matchers: 25 | 26 | it should "test state" in { 27 | val operator = 28 | KeyedProcessOperator(MyKeyedProcessFunction()) 29 | val testHarness = 30 | KeyedOneInputStreamOperatorTestHarness[Long, TestEvent, Long]( 31 | operator, 32 | e => e.key, 33 | TypeInformation.of(classOf[Long]) 34 | ) 35 | 36 | testHarness.getExecutionConfig().setAutoWatermarkInterval(50) 37 | testHarness.setup() 38 | testHarness.open() 39 | 40 | testHarness.getOutput().size() should be(0) 41 | 42 | testHarness.processElement(TestEvent(2L, 1, 0), 100L) 43 | testHarness.getOutput().size() shouldNot be(0) 44 | } 45 | -------------------------------------------------------------------------------- /modules/examples/src/test/scala/org/example/fraud/FraudDetectorTest.scala: -------------------------------------------------------------------------------- 1 | package org.example.fraud 2 | 3 | import org.example.Transaction 4 | import org.example.Alert 5 | 6 | import org.scalatest.matchers.should.Matchers 7 | import org.scalatest.flatspec.AnyFlatSpec 8 | import org.apache.flink.util.Collector 9 | import org.apache.flink.streaming.api.functions.KeyedProcessFunction 10 | import org.apache.flink.util.OutputTag 11 | import org.apache.flink.streaming.api.TimerService 12 | import org.apache.flink.streaming.api.operators.StreamingRuntimeContext 13 | import org.apache.flink.api.common.functions.util.AbstractRuntimeUDFContext 14 | import org.apache.flink.streaming.api.operators.StreamFlatMap 15 | import org.apache.flink.configuration.Configuration 16 | 17 | import scala.collection.mutable.ListBuffer 18 | import org.example.fraud.FraudDetector 19 | 20 | class FraudDetectorTest extends AnyFlatSpec with Matchers: 21 | 22 | class FakeCollector extends Collector[Alert]: 23 | val state = ListBuffer.empty[Alert] 24 | 25 | override def collect(record: Alert): Unit = 26 | state += record 27 | override def close(): Unit = 28 | state.clear 29 | 30 | def makeContext(detector: FraudDetector) = 31 | new detector.Context: 32 | override def getCurrentKey(): Long = ??? 33 | override def output[X](outputTag: OutputTag[X], value: X): Unit = ??? 34 | override def timestamp(): java.lang.Long = 0L 35 | override def timerService(): TimerService = new TimerService: 36 | override def currentProcessingTime(): Long = 0L 37 | override def registerProcessingTimeTimer(time: Long): Unit = () 38 | override def currentWatermark(): Long = ??? 39 | override def deleteProcessingTimeTimer(time: Long): Unit = () 40 | override def registerEventTimeTimer(time: Long): Unit = ??? 41 | override def deleteEventTimeTimer(time: Long): Unit = ??? 42 | 43 | it should "detect fraud" in { 44 | // given 45 | val detector = FraudDetector() 46 | detector.setRuntimeContext(FakeRuntimeContext()) 47 | detector.open(new Configuration()) 48 | 49 | val ctx = makeContext(detector) 50 | val collector = FakeCollector() 51 | // when 52 | detector.processElement(Transaction(1, 1, 0.1), ctx, collector) 53 | // then 54 | collector.state should be(empty) 55 | // when 56 | detector.processElement(Transaction(1, 1, 500.1), ctx, collector) 57 | // then 58 | collector.state shouldNot be(empty) 59 | } 60 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/CloseableIterator.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.util.{CloseableIterator => JCloseableIterator} 4 | 5 | /** This interface represents an [[Iterator]] that is also [[AutoCloseable]]. A typical use-case for this interface are 6 | * iterators that are based on native-resources such as files, network, or database connections. Clients must call 7 | * close after using the iterator. 8 | */ 9 | trait CloseableIterator[T] extends Iterator[T] with AutoCloseable {} 10 | 11 | object CloseableIterator { 12 | 13 | def fromJava[T](it: JCloseableIterator[T]): CloseableIterator[T] = 14 | new CloseableIterator[T] { 15 | override def hasNext: Boolean = it.hasNext 16 | 17 | override def next(): T = it.next 18 | 19 | override def close(): Unit = it.close() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/DataStreamUtils.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.annotation.Experimental 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.java.functions.KeySelector 6 | import org.apache.flink.streaming.api.datastream.{DataStreamUtils => JavaStreamUtils} 7 | 8 | import scala.jdk.CollectionConverters._ 9 | import scala.reflect.ClassTag 10 | import ScalaStreamOps._ 11 | 12 | /** This class provides simple utility methods for collecting a [[DataStream]], effectively enriching it with the 13 | * functionality encapsulated by [[DataStreamUtils]]. 14 | * 15 | * This experimental class is relocated from flink-streaming-contrib. 16 | * 17 | * @param self 18 | * DataStream 19 | */ 20 | @Experimental 21 | class DataStreamUtils[T: TypeInformation: ClassTag](val self: DataStream[T]) { 22 | 23 | /** Returns a scala iterator to iterate over the elements of the DataStream. 24 | * @return 25 | * The iterator 26 | */ 27 | def collect(): Iterator[T] = 28 | self.javaStream.executeAndCollect().asScala 29 | 30 | /** Reinterprets the given [[DataStream]] as a [[KeyedStream]], which extracts keys with the given [[KeySelector]]. 31 | * 32 | * IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be partitioned 33 | * exactly in the same way as if it was created through a [[DataStream#keyBy(KeySelector)]]. 34 | * 35 | * @param keySelector 36 | * Function that defines how keys are extracted from the data stream. 37 | * @return 38 | * The reinterpretation of the [[DataStream]] as a [[KeyedStream]]. 39 | */ 40 | def reinterpretAsKeyedStream[K: TypeInformation](keySelector: T => K): KeyedStream[T, K] = { 41 | 42 | val keyTypeInfo = implicitly[TypeInformation[K]] 43 | val cleanSelector = clean(keySelector) 44 | val javaKeySelector = new JavaKeySelector[T, K](cleanSelector) 45 | 46 | asScalaStream(JavaStreamUtils.reinterpretAsKeyedStream(self.javaStream, javaKeySelector, keyTypeInfo)) 47 | } 48 | 49 | private[flinkx] def clean[F <: AnyRef](f: F): F = { 50 | new StreamExecutionEnvironment(self.javaStream.getExecutionEnvironment).scalaClean(f) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/OutputTag.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.annotation.PublicEvolving 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.util.{OutputTag => JOutputTag} 6 | 7 | /** An [[OutputTag]] is a typed and named tag to use for tagging side outputs of an operator. 8 | * 9 | * Example: 10 | * {{{ 11 | * val outputTag = OutputTag[String]("late-data") 12 | * }}} 13 | * 14 | * @tparam T 15 | * the type of elements in the side-output stream. 16 | */ 17 | @PublicEvolving 18 | class OutputTag[T: TypeInformation](id: String) extends JOutputTag[T](id, implicitly[TypeInformation[T]]) 19 | 20 | object OutputTag { 21 | def apply[T: TypeInformation](id: String): OutputTag[T] = new OutputTag(id) 22 | } 23 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/ScalaStreamOps.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flinkx.api.typeinfo.CaseClassTypeInfo 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream} 6 | import org.apache.flink.streaming.api.datastream.{ConnectedStreams => ConnectedJavaStreams} 7 | import org.apache.flink.streaming.api.datastream.{BroadcastConnectedStream => BroadcastConnectedJavaStreams} 8 | import org.apache.flink.streaming.api.datastream.{KeyedStream => KeyedJavaStream} 9 | 10 | import language.implicitConversions 11 | import language.experimental.macros 12 | 13 | object ScalaStreamOps { 14 | 15 | /** Converts an [[org.apache.flink.streaming.api.datastream.DataStream]] to a [[org.apache.flinkx.api.DataStream]]. 16 | */ 17 | def asScalaStream[R](stream: JavaStream[R]) = new DataStream[R](stream) 18 | 19 | /** Converts an [[org.apache.flink.streaming.api.datastream.KeyedStream]] to a [[org.apache.flinkx.api.KeyedStream]]. 20 | */ 21 | def asScalaStream[R, K](stream: KeyedJavaStream[R, K]) = new KeyedStream[R, K](stream) 22 | 23 | /** Converts an [[org.apache.flink.streaming.api.datastream.ConnectedStreams]] to a 24 | * [[org.apache.flinkx.api.ConnectedStreams]]. 25 | */ 26 | def asScalaStream[IN1, IN2](stream: ConnectedJavaStreams[IN1, IN2]) = new ConnectedStreams[IN1, IN2](stream) 27 | 28 | /** Converts an [[org.apache.flink.streaming.api.datastream.BroadcastConnectedStream]] to a 29 | * [[org.apache.flinkx.api.BroadcastConnectedStream]]. 30 | */ 31 | def asScalaStream[IN1, IN2](stream: BroadcastConnectedJavaStreams[IN1, IN2]) = 32 | new BroadcastConnectedStream[IN1, IN2](stream) 33 | 34 | private[flinkx] def fieldNames2Indices(typeInfo: TypeInformation[_], fields: Array[String]): Array[Int] = { 35 | typeInfo match { 36 | case ti: CaseClassTypeInfo[_] => 37 | val result = ti.getFieldIndices(fields) 38 | 39 | if (result.contains(-1)) { 40 | throw new IllegalArgumentException( 41 | "Fields '" + fields.mkString(", ") + 42 | "' are not valid for '" + ti.toString + "'." 43 | ) 44 | } 45 | 46 | result 47 | 48 | case _ => 49 | throw new UnsupportedOperationException( 50 | "Specifying fields by name is only" + 51 | "supported on Case Classes (for now)." 52 | ) 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/async/AsyncFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.annotation.PublicEvolving 22 | import org.apache.flink.api.common.functions.Function 23 | 24 | import java.util.concurrent.TimeoutException 25 | 26 | /** A function to trigger async I/O operations. 27 | * 28 | * For each asyncInvoke an async io operation can be triggered, and once it has been done, the result can be collected 29 | * by calling ResultFuture.complete. For each async operation, its context is stored in the operator immediately after 30 | * invoking asyncInvoke, avoiding blocking for each stream input as long as the internal buffer is not full. 31 | * 32 | * [[ResultFuture]] can be passed into callbacks or futures to collect the result data. An error can also be propagate 33 | * to the async IO operator by [[ResultFuture.completeExceptionally(Throwable)]]. 34 | * 35 | * @tparam IN 36 | * The type of the input element 37 | * @tparam OUT 38 | * The type of the output elements 39 | */ 40 | @PublicEvolving 41 | trait AsyncFunction[IN, OUT] extends Function { 42 | 43 | /** Trigger the async operation for each stream input 44 | * 45 | * @param input 46 | * element coming from an upstream task 47 | * @param resultFuture 48 | * to be completed with the result data 49 | */ 50 | def asyncInvoke(input: IN, resultFuture: ResultFuture[OUT]): Unit 51 | 52 | /** [[AsyncFunction.asyncInvoke]] timeout occurred. By default, the result future is exceptionally completed with a 53 | * timeout exception. 54 | * 55 | * @param input 56 | * element coming from an upstream task 57 | * @param resultFuture 58 | * to be completed with the result data 59 | */ 60 | def timeout(input: IN, resultFuture: ResultFuture[OUT]): Unit = { 61 | resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")) 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/async/JavaResultFutureWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.annotation.Internal 22 | import org.apache.flink.streaming.api.functions.async 23 | 24 | import scala.jdk.CollectionConverters._ 25 | 26 | /** Internal wrapper class to map a Flink's Java API [[org.apache.flink.streaming.api.functions.async.ResultFuture]] to 27 | * a Scala [[org.apache.flink.api.async.ResultFuture]]. 28 | * 29 | * @param javaResultFuture 30 | * to forward the calls to 31 | * @tparam OUT 32 | * type of the output elements 33 | */ 34 | @Internal 35 | class JavaResultFutureWrapper[OUT](val javaResultFuture: async.ResultFuture[OUT]) extends ResultFuture[OUT] { 36 | override def complete(result: Iterable[OUT]): Unit = { 37 | javaResultFuture.complete(result.asJavaCollection) 38 | } 39 | 40 | override def completeExceptionally(throwable: Throwable): Unit = { 41 | javaResultFuture.completeExceptionally(throwable) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/async/ResultFuture.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.annotation.PublicEvolving 22 | 23 | /** The result future collects data/errors from the user code while processing asynchronous I/O operations. 24 | * 25 | * @tparam OUT 26 | * type of the output element 27 | */ 28 | @PublicEvolving 29 | trait ResultFuture[OUT] { 30 | 31 | /** Complete the ResultFuture with a set of result elements. 32 | * 33 | * Note that it should be called for exactly one time in the user code. Calling this function for multiple times will 34 | * cause data lose. 35 | * 36 | * Put all results in a [[Iterable]] and then issue ResultFuture.complete(Iterable). 37 | * 38 | * @param result 39 | * to complete the async collector with 40 | */ 41 | def complete(result: Iterable[OUT]): Unit 42 | 43 | /** Complete this ResultFuture with an error. 44 | * 45 | * @param throwable 46 | * to complete the async collector with 47 | */ 48 | def completeExceptionally(throwable: Throwable): Unit 49 | } 50 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/async/RichAsyncFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.api.common.functions.AbstractRichFunction 22 | 23 | /** Rich variant of [[AsyncFunction]]. As a [[org.apache.flink.api.common.functions.RichFunction]], it gives access to 24 | * the [[org.apache.flink.api.common.functions.RuntimeContext]] and provides setup and teardown methods. 25 | * 26 | * State related apis in [[org.apache.flink.api.common.functions.RuntimeContext]] are not supported yet because the key 27 | * may get changed while accessing states in the working thread. 28 | * 29 | * [[org.apache.flink.api.common.functions.IterationRuntimeContext#getIterationAggregator(String)]] is not supported 30 | * since the aggregator may be modified by multiple threads. 31 | * 32 | * @tparam IN 33 | * The type of the input value. 34 | * @tparam OUT 35 | * The type of the output value. 36 | */ 37 | abstract class RichAsyncFunction[IN, OUT] extends AbstractRichFunction with AsyncFunction[IN, OUT] {} 38 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/async/ScalaRichAsyncFunctionWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.api.common.functions.RuntimeContext 22 | import org.apache.flink.configuration.Configuration 23 | import org.apache.flink.streaming.api.functions.async.{ 24 | ResultFuture => JResultFuture, 25 | RichAsyncFunction => JRichAsyncFunction 26 | } 27 | 28 | /** A wrapper function that exposes a Scala RichAsyncFunction as a Java Rich Async Function. 29 | * 30 | * The Scala and Java RichAsyncFunctions differ in their type of "ResultFuture" 31 | * - Scala RichAsyncFunction: [[org.apache.flink.api.async.ResultFuture]] 32 | * - Java RichAsyncFunction: [[org.apache.flink.streaming.api.functions.async.ResultFuture]] 33 | */ 34 | final class ScalaRichAsyncFunctionWrapper[IN, OUT](func: RichAsyncFunction[IN, OUT]) 35 | extends JRichAsyncFunction[IN, OUT] { 36 | 37 | override def asyncInvoke(input: IN, resultFuture: JResultFuture[OUT]): Unit = { 38 | func.asyncInvoke(input, new JavaResultFutureWrapper[OUT](resultFuture)) 39 | } 40 | 41 | override def timeout(input: IN, resultFuture: JResultFuture[OUT]): Unit = { 42 | func.timeout(input, new JavaResultFutureWrapper[OUT](resultFuture)) 43 | } 44 | 45 | override def open(parameters: Configuration): Unit = { 46 | func.open(parameters) 47 | } 48 | 49 | override def close(): Unit = { 50 | func.close() 51 | } 52 | 53 | override def setRuntimeContext(runtimeContext: RuntimeContext): Unit = { 54 | super.setRuntimeContext(runtimeContext) 55 | func.setRuntimeContext(super.getRuntimeContext) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/conv.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv} 4 | import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream} 5 | 6 | import scala.language.implicitConversions 7 | 8 | object conv { 9 | implicit def toJavaEnv(e: StreamExecutionEnvironment): JavaEnv = e.getJavaEnv 10 | 11 | implicit def toJavaDataStream[T](d: DataStream[T]): JavaStream[T] = d.javaStream 12 | } 13 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/extensions/impl/acceptPartialFunctions/OnJoinedStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.extensions.impl.acceptPartialFunctions 19 | 20 | import org.apache.flink.annotation.PublicEvolving 21 | import org.apache.flink.api.common.typeinfo.TypeInformation 22 | import org.apache.flinkx.api.{DataStream, JoinedStreams} 23 | import org.apache.flink.streaming.api.windowing.windows.Window 24 | 25 | /** Wraps a joined data stream, allowing to use anonymous partial functions to perform extraction of items in a tuple, 26 | * case class instance or collection 27 | * 28 | * @param stream 29 | * The wrapped data stream 30 | * @tparam L 31 | * The type of the data stream items from the left side of the join 32 | * @tparam R 33 | * The type of the data stream items from the right input of the join 34 | * @tparam K 35 | * The type of key 36 | * @tparam W 37 | * The type of the window 38 | */ 39 | class OnJoinedStream[L, R, K, W <: Window](stream: JoinedStreams[L, R]#Where[K]#EqualTo#WithWindow[W]) { 40 | 41 | /** Completes the join operation with the user function that is executed for windowed groups. 42 | * 43 | * @param fun 44 | * The function that defines the projection of the join 45 | * @tparam O 46 | * The return type of the projection, for which type information must be known 47 | * @return 48 | * A fully joined data set of Os 49 | */ 50 | @PublicEvolving 51 | def projecting[O: TypeInformation](fun: (L, R) => O): DataStream[O] = 52 | stream.apply(fun) 53 | 54 | } 55 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/extensions/impl/acceptPartialFunctions/OnKeyedStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.extensions.impl.acceptPartialFunctions 19 | 20 | import org.apache.flink.annotation.PublicEvolving 21 | import org.apache.flink.api.common.typeinfo.TypeInformation 22 | import org.apache.flinkx.api.{DataStream, KeyedStream} 23 | 24 | /** Wraps a keyed data stream, allowing to use anonymous partial functions to perform extraction of items in a tuple, 25 | * case class instance or collection 26 | * 27 | * @param stream 28 | * The wrapped data stream 29 | * @tparam T 30 | * The type of the data stream items 31 | * @tparam K 32 | * The type of key 33 | */ 34 | class OnKeyedStream[T, K](stream: KeyedStream[T, K]) { 35 | 36 | /** Applies a reducer `fun` to the stream 37 | * 38 | * @param fun 39 | * The reducing function to be applied on the keyed stream 40 | * @return 41 | * A data set of Ts 42 | */ 43 | @PublicEvolving 44 | def reduceWith(fun: (T, T) => T): DataStream[T] = 45 | stream.reduce(fun) 46 | } 47 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/extensions/impl/acceptPartialFunctions/OnWindowedStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.extensions.impl.acceptPartialFunctions 19 | 20 | import org.apache.flink.annotation.PublicEvolving 21 | import org.apache.flink.api.common.typeinfo.TypeInformation 22 | import org.apache.flinkx.api.{DataStream, WindowedStream} 23 | import org.apache.flink.streaming.api.windowing.windows.Window 24 | import org.apache.flink.util.Collector 25 | 26 | /** Wraps a joined data stream, allowing to use anonymous partial functions to perform extraction of items in a tuple, 27 | * case class instance or collection 28 | * 29 | * @param stream 30 | * The wrapped data stream 31 | * @tparam T 32 | * The type of the data stream items from the right input of the join 33 | * @tparam K 34 | * The type of key 35 | * @tparam W 36 | * The type of the window 37 | */ 38 | class OnWindowedStream[T, K, W <: Window](stream: WindowedStream[T, K, W]) { 39 | 40 | /** Applies a reduce function to the window. The window function is called for each evaluation of the window for each 41 | * key individually. The output of the reduce function is interpreted as a regular non-windowed stream. 42 | * 43 | * This window will try and pre-aggregate data as much as the window policies permit. For example,tumbling time 44 | * windows can perfectly pre-aggregate the data, meaning that only one element per key is stored. Sliding time 45 | * windows will pre-aggregate on the granularity of the slide interval, so a few elements are stored per key (one per 46 | * slide interval). Custom windows may not be able to pre-aggregate, or may need to store extra values in an 47 | * aggregation tree. 48 | * 49 | * @param function 50 | * The reduce function. 51 | * @return 52 | * The data stream that is the result of applying the reduce function to the window. 53 | */ 54 | @PublicEvolving 55 | def reduceWith(function: (T, T) => T): DataStream[T] = 56 | stream.reduce(function) 57 | 58 | } 59 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/AllWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.function 19 | 20 | import org.apache.flink.annotation.Public 21 | import org.apache.flink.api.common.functions.Function 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import java.io.Serializable 26 | 27 | /** Base interface for functions that are evaluated over non-grouped windows, i.e., windows over all stream partitions. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | */ 34 | @Public 35 | trait AllWindowFunction[IN, OUT, W <: Window] extends Function with Serializable { 36 | 37 | /** Evaluates the window and outputs none or several elements. 38 | * 39 | * @param window 40 | * The window that is being evaluated. 41 | * @param input 42 | * The elements in the window being evaluated. 43 | * @param out 44 | * A collector for emitting elements. 45 | * @throws Exception 46 | * The function may throw exceptions to fail the program and trigger recovery. 47 | */ 48 | def apply(window: W, input: Iterable[IN], out: Collector[OUT]): Unit 49 | } 50 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/RichAllWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function 20 | 21 | import org.apache.flink.api.common.functions.AbstractRichFunction 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | 24 | /** Rich variant of the [[org.apache.flinkx.api.function.AllWindowFunction]]. 25 | * 26 | * As a [[org.apache.flink.api.common.functions.RichFunction]], it gives access to the 27 | * [[org.apache.flink.api.common.functions.RuntimeContext]] and provides setup and tear-down methods. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | * @tparam W 34 | * The type of Window that this window function can be applied on. 35 | */ 36 | abstract class RichAllWindowFunction[IN, OUT, W <: Window] 37 | extends AbstractRichFunction 38 | with AllWindowFunction[IN, OUT, W] {} 39 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/RichWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function 20 | 21 | import org.apache.flink.api.common.functions.AbstractRichFunction 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | 24 | /** Rich variant of the [[org.apache.flinkx.api.function.WindowFunction]]. 25 | * 26 | * As a [[org.apache.flink.api.common.functions.RichFunction]], it gives access to the 27 | * [[org.apache.flink.api.common.functions.RuntimeContext]] and provides setup and tear-down methods. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | * @tparam KEY 34 | * The type of the key. 35 | * @tparam W 36 | * The type of Window that this window function can be applied on. 37 | */ 38 | abstract class RichWindowFunction[IN, OUT, KEY, W <: Window] 39 | extends AbstractRichFunction 40 | with WindowFunction[IN, OUT, KEY, W] {} 41 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/StatefulFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function 20 | 21 | import org.apache.flink.annotation.Public 22 | import org.apache.flink.api.common.functions.RichFunction 23 | import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} 24 | import org.apache.flink.api.common.typeutils.TypeSerializer 25 | import org.apache.flink.configuration.Configuration 26 | 27 | /** Trait implementing the functionality necessary to apply stateful functions in RichFunctions without exposing the 28 | * OperatorStates to the user. The user should call the applyWithState method in his own RichFunction implementation. 29 | */ 30 | @Public 31 | trait StatefulFunction[I, O, S] extends RichFunction { 32 | 33 | protected val stateSerializer: TypeSerializer[S] 34 | 35 | private[this] var state: ValueState[S] = _ 36 | 37 | def applyWithState(in: I, fun: (I, Option[S]) => (O, Option[S])): O = { 38 | val (o, s: Option[S]) = fun(in, Option(state.value())) 39 | s match { 40 | case Some(v) => state.update(v) 41 | case None => state.update(null.asInstanceOf[S]) 42 | } 43 | o 44 | } 45 | 46 | override def open(c: Configuration): Unit = { 47 | val info = new ValueStateDescriptor[S]("state", stateSerializer) 48 | state = getRuntimeContext.getState(info) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/WindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.function 19 | 20 | import org.apache.flink.annotation.Public 21 | import org.apache.flink.api.common.functions.Function 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import java.io.Serializable 26 | 27 | /** Base interface for functions that are evaluated over keyed (grouped) windows. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | * @tparam KEY 34 | * The type of the key. 35 | */ 36 | @Public 37 | trait WindowFunction[IN, OUT, KEY, W <: Window] extends Function with Serializable { 38 | 39 | /** Evaluates the window and outputs none or several elements. 40 | * 41 | * @param key 42 | * The key for which this window is evaluated. 43 | * @param window 44 | * The window that is being evaluated. 45 | * @param input 46 | * The elements in the window being evaluated. 47 | * @param out 48 | * A collector for emitting elements. 49 | * @throws Exception 50 | * The function may throw exceptions to fail the program and trigger recovery. 51 | */ 52 | def apply(key: KEY, window: W, input: Iterable[IN], out: Collector[OUT]): Unit 53 | } 54 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaAllWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction => JAllWindowFunction} 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import scala.jdk.CollectionConverters._ 26 | 27 | /** A wrapper function that exposes a Scala Function3 as a Java AllWindowFunction. 28 | */ 29 | final class ScalaAllWindowFunction[IN, OUT, W <: Window]( 30 | private[this] val function: (W, Iterable[IN], Collector[OUT]) => Unit 31 | ) extends JAllWindowFunction[IN, OUT, W] { 32 | 33 | @throws(classOf[Exception]) 34 | override def apply(window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 35 | function.apply(window, input.asScala, out) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaAllWindowFunctionWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext} 22 | import org.apache.flink.api.java.operators.translation.WrappingFunction 23 | import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction => JAllWindowFunction} 24 | import org.apache.flinkx.api.function.AllWindowFunction 25 | import org.apache.flink.streaming.api.windowing.windows.Window 26 | import org.apache.flink.util.Collector 27 | 28 | import scala.jdk.CollectionConverters._ 29 | 30 | /** A wrapper function that exposes a Scala WindowFunction as a JavaWindow function. 31 | * 32 | * The Scala and Java Window functions differ in their type of "Iterable": 33 | * - Scala WindowFunction: scala.Iterable 34 | * - Java WindowFunction: java.lang.Iterable 35 | */ 36 | final class ScalaAllWindowFunctionWrapper[IN, OUT, W <: Window](func: AllWindowFunction[IN, OUT, W]) 37 | extends WrappingFunction[AllWindowFunction[IN, OUT, W]](func) 38 | with JAllWindowFunction[IN, OUT, W] 39 | with RichFunction { 40 | 41 | @throws(classOf[Exception]) 42 | override def apply(window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 43 | wrappedFunction.apply(window, input.asScala, out) 44 | } 45 | 46 | override def getRuntimeContext: RuntimeContext = { 47 | throw new RuntimeException("This should never be called") 48 | } 49 | 50 | override def getIterationRuntimeContext: IterationRuntimeContext = { 51 | throw new RuntimeException("This should never be called") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaReduceFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.api.common.functions.ReduceFunction 22 | 23 | /** A wrapper function that exposes a Scala Function2 as a [[ReduceFunction]]. 24 | */ 25 | final class ScalaReduceFunction[T](private[this] val function: (T, T) => T) extends ReduceFunction[T] { 26 | 27 | @throws(classOf[Exception]) 28 | override def reduce(a: T, b: T): T = { 29 | function(a, b) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.streaming.api.functions.windowing.{WindowFunction => JWindowFunction} 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import scala.jdk.CollectionConverters._ 26 | 27 | /** A wrapper function that exposes a Scala Function4 as a Java WindowFunction. 28 | */ 29 | final class ScalaWindowFunction[IN, OUT, KEY, W <: Window]( 30 | private[this] val function: (KEY, W, Iterable[IN], Collector[OUT]) => Unit 31 | ) extends JWindowFunction[IN, OUT, KEY, W] { 32 | 33 | @throws(classOf[Exception]) 34 | override def apply(key: KEY, window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 35 | function.apply(key, window, input.asScala, out) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaWindowFunctionWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flinkx.api.function.WindowFunction 22 | import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext} 23 | import org.apache.flink.api.java.operators.translation.WrappingFunction 24 | import org.apache.flink.streaming.api.functions.windowing.{WindowFunction => JWindowFunction} 25 | import org.apache.flink.streaming.api.windowing.windows.Window 26 | import org.apache.flink.util.Collector 27 | 28 | import scala.jdk.CollectionConverters._ 29 | 30 | /** A wrapper function that exposes a Scala WindowFunction as a JavaWindow function. 31 | * 32 | * The Scala and Java Window functions differ in their type of "Iterable": 33 | * - Scala WindowFunction: scala.Iterable 34 | * - Java WindowFunction: java.lang.Iterable 35 | */ 36 | final class ScalaWindowFunctionWrapper[IN, OUT, KEY, W <: Window](func: WindowFunction[IN, OUT, KEY, W]) 37 | extends WrappingFunction[WindowFunction[IN, OUT, KEY, W]](func) 38 | with JWindowFunction[IN, OUT, KEY, W] 39 | with RichFunction { 40 | 41 | @throws(classOf[Exception]) 42 | override def apply(key: KEY, window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 43 | wrappedFunction.apply(key, window, input.asScala, out) 44 | } 45 | 46 | override def getRuntimeContext: RuntimeContext = { 47 | throw new RuntimeException("This should never be called") 48 | } 49 | 50 | override def getIterationRuntimeContext: IterationRuntimeContext = { 51 | throw new RuntimeException("This should never be called") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/mapper/BigDecMapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.mapper 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 4 | 5 | class BigDecMapper extends TypeMapper[scala.BigDecimal, java.math.BigDecimal] { 6 | override def map(a: BigDecimal): java.math.BigDecimal = a.bigDecimal 7 | override def contramap(b: java.math.BigDecimal): BigDecimal = BigDecimal(b) 8 | } 9 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/mapper/BigIntMapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.mapper 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 4 | 5 | import java.math.BigInteger 6 | 7 | class BigIntMapper() extends TypeMapper[scala.BigInt, java.math.BigInteger] { 8 | override def contramap(b: BigInteger): BigInt = BigInt(b) 9 | override def map(a: BigInt): BigInteger = a.bigInteger 10 | } 11 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/mapper/UuidMapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.mapper 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 4 | 5 | import java.nio.ByteBuffer 6 | import java.util.UUID 7 | 8 | class UuidMapper() extends TypeMapper[UUID, Array[Byte]] { 9 | override def map(a: UUID): Array[Byte] = { 10 | val buffer = ByteBuffer.allocate(16) 11 | buffer.putLong(a.getMostSignificantBits) 12 | buffer.putLong(a.getLeastSignificantBits) 13 | buffer.array() 14 | } 15 | 16 | override def contramap(b: Array[Byte]): UUID = { 17 | val buffer = ByteBuffer.wrap(b) 18 | val mostSignificantBits = buffer.getLong() 19 | val leastSignificantBits = buffer.getLong() 20 | new UUID(mostSignificantBits, leastSignificantBits) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/ArraySerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | import scala.reflect.ClassTag 7 | 8 | class ArraySerializer[T](val child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Array[T]] { 9 | 10 | implicit val classTag: ClassTag[T] = ClassTag(clazz) 11 | 12 | override def createInstance(): Array[T] = Array.empty[T] 13 | 14 | override def copy(from: Array[T]): Array[T] = { 15 | if (from == null) { 16 | from 17 | } else { 18 | val length = from.length 19 | val copy = Array.copyOf(from, length) 20 | if (!child.isImmutableType) { 21 | var i = 0 22 | while (i < length) { 23 | val element = copy(i) 24 | if (element != null) copy(i) = child.copy(element) 25 | i += 1 26 | } 27 | } 28 | copy 29 | } 30 | } 31 | 32 | override def duplicate(): ArraySerializer[T] = { 33 | val duplicatedChild = child.duplicate() 34 | if (duplicatedChild.eq(child)) { 35 | this 36 | } else { 37 | new ArraySerializer[T](duplicatedChild, clazz) 38 | } 39 | } 40 | 41 | override def getLength: Int = -1 42 | 43 | override def deserialize(source: DataInputView): Array[T] = { 44 | val length = source.readInt() 45 | val array = new Array[T](length) 46 | var i = 0 47 | while (i < length) { 48 | array(i) = child.deserialize(source) 49 | i += 1 50 | } 51 | array 52 | } 53 | 54 | override def serialize(record: Array[T], target: DataOutputView): Unit = { 55 | val length = record.length 56 | target.writeInt(length) 57 | var i = 0 58 | while (i < length) { // while loop is significantly faster than foreach when working on arrays 59 | val element = record(i) 60 | child.serialize(element, target) 61 | i += 1 62 | } 63 | } 64 | 65 | override def snapshotConfiguration(): TypeSerializerSnapshot[Array[T]] = 66 | new CollectionSerializerSnapshot[Array, T, ArraySerializer[T]](child, classOf[ArraySerializer[T]], clazz) 67 | 68 | } 69 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/ListCCSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class ListCCSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[::[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: ::[T]): ::[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | val result = from.map(child.copy) 15 | ::(result.head, result.tail) 16 | } 17 | } 18 | 19 | override def duplicate(): ListCCSerializer[T] = { 20 | val duplicatedChild = child.duplicate() 21 | if (duplicatedChild.eq(child)) { 22 | this 23 | } else { 24 | new ListCCSerializer[T](duplicatedChild, clazz) 25 | } 26 | } 27 | 28 | override def createInstance(): ::[T] = throw new IllegalArgumentException("cannot create instance of non-empty list") 29 | override def getLength: Int = -1 30 | override def deserialize(source: DataInputView): ::[T] = { 31 | val count = source.readInt() 32 | val result = (0 until count) 33 | .map(_ => child.deserialize(source)) 34 | 35 | ::(result.head, result.tail.toList) 36 | } 37 | override def serialize(record: ::[T], target: DataOutputView): Unit = { 38 | target.writeInt(record.size) 39 | record.foreach(element => child.serialize(element, target)) 40 | } 41 | override def snapshotConfiguration(): TypeSerializerSnapshot[::[T]] = 42 | new CollectionSerializerSnapshot[::, T, ListCCSerializer[T]](child, classOf[ListCCSerializer[T]], clazz) 43 | 44 | } 45 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/ListSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class ListSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[List[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: List[T]): List[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): ListSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new ListSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): List[T] = List.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): List[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result.toList 37 | } 38 | override def serialize(record: List[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[List[T]] = 43 | new CollectionSerializerSnapshot[List, T, ListSerializer[T]](child, classOf[ListSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/ScalaCaseObjectSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flinkx.api.serializer.ScalaCaseObjectSerializer.ScalaCaseObjectSerializerSnapshot 4 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSchemaCompatibility, TypeSerializerSnapshot} 5 | import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton 6 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 7 | import org.apache.flink.util.InstantiationUtil 8 | 9 | class ScalaCaseObjectSerializer[T](clazz: Class[T]) extends ImmutableSerializer[T] { 10 | override def copy(source: DataInputView, target: DataOutputView): Unit = {} 11 | override def createInstance(): T = clazz.getField("MODULE$").get(null).asInstanceOf[T] 12 | override def getLength: Int = 0 13 | override def serialize(record: T, target: DataOutputView): Unit = {} 14 | 15 | override def deserialize(source: DataInputView): T = { 16 | clazz.getField("MODULE$").get(null).asInstanceOf[T] 17 | } 18 | 19 | override def snapshotConfiguration(): TypeSerializerSnapshot[T] = 20 | new ScalaCaseObjectSerializerSnapshot(clazz) 21 | } 22 | 23 | object ScalaCaseObjectSerializer { 24 | class ScalaCaseObjectSerializerSnapshot[T](var clazz: Class[T]) extends TypeSerializerSnapshot[T] { 25 | def this() = this(null) 26 | 27 | override def readSnapshot(readVersion: Int, in: DataInputView, userCodeClassLoader: ClassLoader): Unit = { 28 | clazz = InstantiationUtil.resolveClassByName(in, userCodeClassLoader) 29 | } 30 | 31 | override def writeSnapshot(out: DataOutputView): Unit = { 32 | out.writeUTF(clazz.getName) 33 | } 34 | 35 | override def getCurrentVersion: Int = 1 36 | override def resolveSchemaCompatibility(newSerializer: TypeSerializer[T]): TypeSerializerSchemaCompatibility[T] = 37 | TypeSerializerSchemaCompatibility.compatibleAsIs() 38 | 39 | override def restoreSerializer(): TypeSerializer[T] = 40 | new ScalaCaseObjectSerializer[T](clazz) 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/SeqSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class SeqSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Seq[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: Seq[T]): Seq[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): SeqSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new SeqSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): Seq[T] = Seq.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): Seq[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result 37 | } 38 | override def serialize(record: Seq[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[Seq[T]] = 43 | new CollectionSerializerSnapshot[Seq, T, SeqSerializer[T]](child, classOf[SeqSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/SetSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class SetSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Set[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: Set[T]): Set[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): SetSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new SetSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): Set[T] = Set.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): Set[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result.toSet 37 | } 38 | override def serialize(record: Set[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[Set[T]] = 43 | new CollectionSerializerSnapshot[Set, T, SetSerializer[T]](child, classOf[SetSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/serializer/VectorSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class VectorSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Vector[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: Vector[T]): Vector[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): VectorSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new VectorSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): Vector[T] = Vector.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): Vector[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result.toVector 37 | } 38 | override def serialize(record: Vector[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[Vector[T]] = 43 | new CollectionSerializerSnapshot[Vector, T, VectorSerializer[T]](child, classOf[VectorSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/typeinfo/CollectionTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | import scala.reflect.{ClassTag, classTag} 8 | 9 | case class CollectionTypeInformation[T: ClassTag](serializer: TypeSerializer[T]) extends TypeInformation[T] { 10 | val clazz = classTag[T].runtimeClass.asInstanceOf[Class[T]] 11 | override def createSerializer(config: ExecutionConfig): TypeSerializer[T] = 12 | serializer.duplicate() 13 | override def isBasicType: Boolean = false 14 | override def isTupleType: Boolean = false 15 | override def isKeyType: Boolean = false 16 | override def getTotalFields: Int = 1 17 | override def getTypeClass: Class[T] = clazz 18 | override def getArity: Int = 1 19 | } 20 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/typeinfo/CoproductTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | case class CoproductTypeInformation[T](c: Class[T], ser: TypeSerializer[T]) extends TypeInformation[T] { 8 | override def createSerializer(config: ExecutionConfig): TypeSerializer[T] = 9 | ser.duplicate() 10 | override def isBasicType: Boolean = false 11 | override def isTupleType: Boolean = false 12 | override def isKeyType: Boolean = false 13 | override def getTotalFields: Int = 1 14 | override def getTypeClass: Class[T] = c 15 | override def getArity: Int = 1 16 | } 17 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/typeinfo/MappedTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer 4 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 5 | import org.apache.flink.api.common.ExecutionConfig 6 | import org.apache.flink.api.common.typeinfo.TypeInformation 7 | import org.apache.flink.api.common.typeutils.TypeSerializer 8 | 9 | import scala.reflect.{ClassTag, classTag} 10 | 11 | case class MappedTypeInformation[A: ClassTag, B](mapper: TypeMapper[A, B], nested: TypeInformation[B]) 12 | extends TypeInformation[A] { 13 | override def createSerializer(config: ExecutionConfig): TypeSerializer[A] = 14 | new MappedSerializer(mapper, nested.createSerializer(config)) 15 | override def isKeyType: Boolean = nested.isKeyType 16 | override def getTotalFields: Int = nested.getTotalFields 17 | override def isTupleType: Boolean = nested.isTupleType 18 | 19 | override def canEqual(obj: Any): Boolean = obj match { 20 | case m: MappedTypeInformation[_, _] => true 21 | case _ => false 22 | } 23 | override def getTypeClass: Class[A] = classTag[A].runtimeClass.asInstanceOf[Class[A]] 24 | override def getArity: Int = nested.getArity 25 | override def isBasicType: Boolean = nested.isBasicType 26 | 27 | override def toString: String = nested.toString 28 | 29 | override def equals(obj: Any): Boolean = obj match { 30 | case m: MappedTypeInformation[_, _] => (m.nested == nested) && m.mapper.equals(mapper) 31 | case _ => false 32 | } 33 | 34 | override def hashCode(): Int = nested.hashCode() 35 | 36 | } 37 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/typeinfo/ProductTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | class ProductTypeInformation[T <: Product]( 8 | c: Class[T], 9 | fieldTypes: Seq[TypeInformation[_]], 10 | fieldNames: Seq[String], 11 | ser: TypeSerializer[T] 12 | ) extends CaseClassTypeInfo[T]( 13 | clazz = c, 14 | typeParamTypeInfos = Array(), 15 | fieldTypes, 16 | fieldNames 17 | ) { 18 | override def createSerializer(config: ExecutionConfig): TypeSerializer[T] = ser.duplicate() 19 | 20 | } 21 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/typeinfo/SimpleTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.ExecutionConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | import scala.reflect.{classTag, ClassTag} 8 | 9 | abstract class SimpleTypeInformation[T: ClassTag: TypeSerializer] extends TypeInformation[T] { 10 | override def createSerializer(config: ExecutionConfig): TypeSerializer[T] = { 11 | val ser = implicitly[TypeSerializer[T]] 12 | ser.duplicate() 13 | } 14 | override def isBasicType: Boolean = false 15 | override def isTupleType: Boolean = false 16 | override def isKeyType: Boolean = false 17 | override def getTotalFields: Int = 1 18 | override def getTypeClass: Class[T] = classTag[T].runtimeClass.asInstanceOf[Class[T]] 19 | override def getArity: Int = 1 20 | } 21 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/main/scala/org/apache/flinkx/api/typeinfo/UnitTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flinkx.api.serializer.UnitSerializer 4 | import org.apache.flink.api.common.ExecutionConfig 5 | import org.apache.flink.api.common.typeinfo.TypeInformation 6 | import org.apache.flink.api.common.typeutils.TypeSerializer 7 | 8 | class UnitTypeInformation extends TypeInformation[Unit] { 9 | override def createSerializer(config: ExecutionConfig): TypeSerializer[Unit] = new UnitSerializer() 10 | override def isKeyType: Boolean = true 11 | override def getTotalFields: Int = 0 12 | override def isTupleType: Boolean = false 13 | override def canEqual(obj: Any): Boolean = obj.isInstanceOf[UnitTypeInformation] 14 | override def getTypeClass: Class[Unit] = classOf[Unit] 15 | override def getArity: Int = 0 16 | override def isBasicType: Boolean = false 17 | 18 | override def toString: String = "{}" 19 | 20 | override def equals(obj: Any): Boolean = obj match { 21 | case _: UnitTypeInformation => true 22 | case _ => false 23 | } 24 | 25 | override def hashCode(): Int = ().hashCode() 26 | } 27 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/resources/click.dat: -------------------------------------------------------------------------------- 1 | p1a 2021-01-01b 2021-01-01c 2021-01-01test1 -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | true 5 | 6 | 7 | 8 | 9 | %date{yyyy-MM-dd HH:mm:ss.SSSZ, UTC} %-16level %-43thread %-24logger{24} %message%n%xException 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/resources/old-serializer-snapshot.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flink-extended/flink-scala-api/90c69ea3df4bbb8dd9d1b75aea3b7e63ace7178b/modules/flink-1-api/src/test/resources/old-serializer-snapshot.dat -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/resources/without-arity-test.dat: -------------------------------------------------------------------------------- 1 | test -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala-3/org/apache/flinkx/api/Scala3EnumTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.scalatest.flatspec.AnyFlatSpec 4 | import org.scalatest.matchers.should.Matchers 5 | import org.apache.flink.api.common.typeinfo.TypeInformation 6 | 7 | import org.apache.flinkx.api.serializers.* 8 | 9 | class Scala3EnumTest extends AnyFlatSpec with Matchers { 10 | import Scala3EnumTest.Example 11 | 12 | it should "derive type information for a Scala 3 enum" in { 13 | drop(implicitly[TypeInformation[Example]]) 14 | } 15 | } 16 | 17 | object Scala3EnumTest { 18 | enum Example { 19 | case Foo(a: String, b: Int) 20 | case Bar 21 | } 22 | 23 | object Example { 24 | implicit val exampleTi: TypeInformation[Example] = deriveTypeInformation 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/AnyTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import cats.data.NonEmptyList 4 | import org.apache.flinkx.api.serializers._ 5 | import org.apache.flinkx.api.AnyTest._ 6 | import org.apache.flinkx.api.AnyTest.FAny.FValueAny.FTerm 7 | import org.apache.flinkx.api.AnyTest.FAny.FValueAny.FTerm.StringTerm 8 | import org.apache.flink.api.common.typeinfo.TypeInformation 9 | import org.apache.flink.api.common.ExecutionConfig 10 | import org.scalatest.flatspec.AnyFlatSpec 11 | import org.scalatest.matchers.should.Matchers 12 | 13 | class AnyTest extends AnyFlatSpec with Matchers with TestUtils { 14 | val ec = new ExecutionConfig() 15 | 16 | def createSerializer[T: TypeInformation] = 17 | implicitly[TypeInformation[T]].createSerializer(ec) 18 | 19 | it should "serialize concrete class" in { 20 | val ser = createSerializer[StringTerm] 21 | roundtrip(ser, StringTerm("fo")) 22 | } 23 | 24 | it should "serialize ADT" in { 25 | val ser = createSerializer[FAny] 26 | roundtrip(ser, StringTerm("fo")) 27 | } 28 | 29 | it should "serialize NEL" in { 30 | val ser = createSerializer[NonEmptyList[FTerm]] 31 | roundtrip(ser, NonEmptyList.one(StringTerm("fo"))) 32 | } 33 | 34 | it should "serialize nested nel" in { 35 | val ser = createSerializer[TermFilter] 36 | roundtrip(ser, TermFilter("a", NonEmptyList.one(StringTerm("fo")))) 37 | } 38 | } 39 | 40 | object AnyTest { 41 | sealed trait FAny 42 | 43 | object FAny { 44 | sealed trait FValueAny extends FAny { 45 | def value: Any 46 | } 47 | 48 | object FValueAny { 49 | sealed trait FTerm extends FValueAny 50 | 51 | object FTerm { 52 | case class StringTerm(value: String) extends FTerm { 53 | type T = String 54 | } 55 | 56 | object StringTerm { 57 | implicit val stringTermTi: TypeInformation[StringTerm] = deriveTypeInformation 58 | } 59 | 60 | case class NumericTerm(value: Double) extends FTerm { 61 | type T = Double 62 | } 63 | 64 | object NumericTerm { 65 | implicit val numericTermTi: TypeInformation[NumericTerm] = deriveTypeInformation 66 | } 67 | 68 | implicit val fTermTi: TypeInformation[FTerm] = deriveTypeInformation 69 | } 70 | 71 | implicit val fValueAnyTi: TypeInformation[FValueAny] = deriveTypeInformation 72 | } 73 | 74 | implicit val fAnyTi: TypeInformation[FAny] = deriveTypeInformation 75 | } 76 | 77 | case class TermFilter(field: String, values: NonEmptyList[FTerm]) 78 | 79 | object TermFilter { 80 | implicit val termFilterTi: TypeInformation[TermFilter] = deriveTypeInformation 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/CatsTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import cats.data.NonEmptyList 4 | import org.scalatest.flatspec.AnyFlatSpec 5 | import org.scalatest.matchers.should.Matchers 6 | 7 | import org.apache.flink.api.common.ExecutionConfig 8 | import org.apache.flink.api.common.typeinfo.TypeInformation 9 | import org.apache.flinkx.api.serializers._ 10 | 11 | class CatsTest extends AnyFlatSpec with Matchers with TestUtils { 12 | implicit val stringListTi: TypeInformation[NonEmptyList[String]] = deriveTypeInformation 13 | implicit val intListTi: TypeInformation[NonEmptyList[Int]] = deriveTypeInformation 14 | 15 | def createSerializer[T: TypeInformation] = 16 | implicitly[TypeInformation[T]].createSerializer(new ExecutionConfig()) 17 | 18 | it should "derive for NEL[String]" in { 19 | val ser = createSerializer[NonEmptyList[String]] 20 | roundtrip(ser, NonEmptyList.one("doo")) 21 | } 22 | 23 | it should "derive for NEL[Int]" in { 24 | val ser = createSerializer[NonEmptyList[Int]] 25 | roundtrip(ser, NonEmptyList.one(1)) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/CoGroupedStreamsTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows 4 | import org.apache.flink.streaming.api.windowing.time.Time 5 | import org.apache.flinkx.api.serializers._ 6 | import org.scalatest.flatspec.AnyFlatSpec 7 | import org.scalatest.matchers.should.Matchers 8 | 9 | class CoGroupedStreamsTest extends AnyFlatSpec with Matchers with IntegrationTest { 10 | private val dataStream1 = env.fromElements("a1", "a2", "a3") 11 | private val dataStream2 = env.fromElements("a1", "a2") 12 | private val keySelector = (s: String) => s 13 | private val tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1)) 14 | 15 | it should "set allowed lateness" in { 16 | val lateness = Time.milliseconds(42) 17 | 18 | val withLateness = dataStream1 19 | .coGroup(dataStream2) 20 | .where(keySelector) 21 | .equalTo(keySelector) 22 | .window(tsAssigner) 23 | .allowedLateness(lateness) 24 | 25 | withLateness.allowedLateness.toMilliseconds shouldBe lateness.toMilliseconds 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/ExampleTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation 4 | import org.apache.flinkx.api.serializers._ 5 | import org.scalatest.flatspec.AnyFlatSpec 6 | import org.scalatest.matchers.should.Matchers 7 | 8 | import java.util.UUID 9 | 10 | class ExampleTest extends AnyFlatSpec with Matchers with IntegrationTest { 11 | import ExampleTest._ 12 | 13 | it should "serialize elements with type mappers across operators" in { 14 | val result = env 15 | .fromElements( 16 | Event.View(UUID.randomUUID()), 17 | Event.Click("1"), 18 | Event.Purchase(1.0) 19 | ) 20 | .keyBy(_.hashCode) 21 | .collect() 22 | 23 | result should have size 3 24 | } 25 | } 26 | 27 | object ExampleTest { 28 | sealed trait Event extends Product with Serializable 29 | 30 | object Event { 31 | final case class View(id: UUID) extends Event 32 | final case class Click(id: String) extends Event 33 | final case class Purchase(price: BigDecimal) extends Event 34 | 35 | implicit val eventTypeInfo: TypeInformation[Event] = deriveTypeInformation[Event] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/IntegrationTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.api.common.RuntimeExecutionMode 4 | import org.apache.flink.api.common.restartstrategy.RestartStrategies 5 | import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration 6 | import org.apache.flink.test.util.MiniClusterWithClientResource 7 | import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite} 8 | 9 | trait IntegrationTest extends BeforeAndAfterEach with BeforeAndAfterAll { 10 | this: Suite => 11 | 12 | // It is recommended to always test your pipelines locally with a parallelism > 1 to identify bugs which only 13 | // surface for the pipelines executed in parallel. Setting number of slots per task manager and number of task 14 | // managers to 2 and parallelism to a multiple of them is a good starting point. 15 | val numberSlotsPerTaskManager = 2 16 | val numberTaskManagers = 2 17 | val parallelism: Int = numberSlotsPerTaskManager * numberTaskManagers 18 | 19 | val cluster = new MiniClusterWithClientResource( 20 | new MiniClusterResourceConfiguration.Builder() 21 | .setNumberSlotsPerTaskManager(numberSlotsPerTaskManager) 22 | .setNumberTaskManagers(numberTaskManagers) 23 | .build() 24 | ) 25 | 26 | val env: StreamExecutionEnvironment = { 27 | val env = StreamExecutionEnvironment.getExecutionEnvironment 28 | env.setParallelism(parallelism) 29 | env.setRuntimeMode(RuntimeExecutionMode.STREAMING) 30 | env.enableCheckpointing(1000) 31 | env.setRestartStrategy(RestartStrategies.noRestart()) 32 | env.getConfig.disableGenericTypes() 33 | env 34 | } 35 | 36 | override protected def beforeAll(): Unit = { 37 | super.beforeAll() 38 | cluster.before() 39 | } 40 | 41 | override protected def beforeEach(): Unit = { 42 | super.beforeEach() 43 | IntegrationTestSink.values.clear() 44 | } 45 | 46 | override def afterAll(): Unit = { 47 | cluster.after() 48 | super.afterAll() 49 | } 50 | 51 | implicit final class DataStreamOps[T](private val dataStream: DataStream[T]) { 52 | import scala.collection.JavaConverters._ 53 | 54 | def collect(): List[T] = { 55 | dataStream.addSink(new IntegrationTestSink[T]) 56 | env.execute() 57 | IntegrationTestSink.values.asScala.toList.map(_.asInstanceOf[T]) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/IntegrationTestSink.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.streaming.api.functions.sink.SinkFunction 4 | 5 | class IntegrationTestSink[T] extends SinkFunction[T] { 6 | override def invoke(value: T, context: SinkFunction.Context): Unit = IntegrationTestSink.values.add(value) 7 | } 8 | 9 | object IntegrationTestSink { 10 | val values: java.util.List[Any] = java.util.Collections.synchronizedList(new java.util.ArrayList()) 11 | } 12 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/JoinedStreamsTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows 4 | import org.apache.flink.streaming.api.windowing.time.Time 5 | import org.apache.flinkx.api.serializers._ 6 | import org.scalatest.flatspec.AnyFlatSpec 7 | import org.scalatest.matchers.should.Matchers 8 | 9 | class JoinedStreamsTest extends AnyFlatSpec with Matchers with IntegrationTest { 10 | private val dataStream1 = env.fromElements("a1", "a2", "a3") 11 | private val dataStream2 = env.fromElements("a1", "a2") 12 | private val keySelector = (s: String) => s 13 | private val tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1)) 14 | 15 | it should "set allowed lateness" in { 16 | val lateness = Time.milliseconds(42) 17 | 18 | val withLateness = dataStream1 19 | .join(dataStream2) 20 | .where(keySelector) 21 | .equalTo(keySelector) 22 | .window(tsAssigner) 23 | .allowedLateness(lateness) 24 | 25 | withLateness.allowedLateness.toMilliseconds shouldBe lateness.toMilliseconds 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/MappedTypeInfoTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation 4 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 5 | import org.apache.flinkx.api.serializers._ 6 | import org.scalatest.flatspec.AnyFlatSpec 7 | import org.scalatest.matchers.should.Matchers 8 | 9 | class MappedTypeInfoTest extends AnyFlatSpec with Matchers with TestUtils { 10 | import MappedTypeInfoTest._ 11 | it should "derive TI for non-serializable classes" in { 12 | drop(implicitly[TypeInformation[WrappedString]]) 13 | } 14 | } 15 | 16 | object MappedTypeInfoTest { 17 | class WrappedMapper extends TypeMapper[WrappedString, String] { 18 | override def map(a: WrappedString): String = a.get 19 | 20 | override def contramap(b: String): WrappedString = { 21 | val str = new WrappedString 22 | str.put(b) 23 | str 24 | } 25 | } 26 | implicit val mapper: TypeMapper[WrappedString, String] = new WrappedMapper() 27 | 28 | class WrappedString { 29 | private var internal: String = "" 30 | 31 | override def equals(obj: Any): Boolean = obj match { 32 | case s: WrappedString => s.get == internal 33 | case _ => false 34 | } 35 | def get: String = internal 36 | def put(value: String) = { 37 | internal = value 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/ProcessFunctionTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flinkx.api.serializers._ 4 | import org.apache.flinkx.api.ProcessFunctionTest._ 5 | import org.apache.flink.api.common.ExecutionConfig 6 | import org.apache.flink.api.common.typeinfo.Types 7 | import org.apache.flink.api.common.typeutils.TypeSerializer 8 | import org.apache.flink.streaming.api.functions.KeyedProcessFunction 9 | import org.apache.flink.streaming.api.operators.KeyedProcessOperator 10 | import org.apache.flink.streaming.util._ 11 | import org.apache.flink.util.Collector 12 | import org.scalatest.flatspec.AnyFlatSpec 13 | import org.scalatest.matchers.should.Matchers 14 | 15 | import scala.jdk.CollectionConverters._ 16 | 17 | class ProcessFunctionTest extends AnyFlatSpec with Matchers { 18 | 19 | "ProcessFunction" should "return same values unchanged" in { 20 | val serializer = deriveTypeInformation[DataPackage].createSerializer(new ExecutionConfig()) 21 | val harness = prepareTestHarness(new SimpleFunction(), serializer) 22 | sources.foreach(data => harness.processElement(data, harness.getProcessingTime)) 23 | val result = harness.extractOutputValues().asScala.toList 24 | result should contain theSameElementsAs sources 25 | } 26 | } 27 | 28 | object ProcessFunctionTest { 29 | 30 | case class DataPackage(id: String, ints: List[Int]) extends Serializable 31 | 32 | val sources: List[DataPackage] = List( 33 | DataPackage("3", List(3, 2, 1)), 34 | DataPackage("2", List(2)), 35 | DataPackage("1", Nil) 36 | ) 37 | 38 | private def prepareTestHarness[D <: DataPackage, R <: DataPackage]( 39 | processFunction: KeyedProcessFunction[String, D, R], 40 | serializer: TypeSerializer[R] 41 | ): KeyedOneInputStreamOperatorTestHarness[String, D, R] = { 42 | val harness = new KeyedOneInputStreamOperatorTestHarness[String, D, R]( 43 | new KeyedProcessOperator[String, D, R](processFunction), 44 | (data: D) => data.id, 45 | Types.STRING, 46 | 1, 47 | 1, 48 | 0 49 | ) 50 | harness.setup(serializer) 51 | harness.open() 52 | harness 53 | } 54 | 55 | @SerialVersionUID(1) 56 | final class SimpleFunction extends KeyedProcessFunction[String, DataPackage, DataPackage] { 57 | 58 | type ContextT = KeyedProcessFunction[String, DataPackage, DataPackage]#Context 59 | 60 | def processElement(value: DataPackage, ctx: ContextT, out: Collector[DataPackage]): Unit = { 61 | out.collect(value) 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/StreamExecutionEnvironmentTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.connector.source.Boundedness 6 | import org.apache.flink.api.connector.source.mocks.MockSource 7 | import org.apache.flink.api.java.typeutils.GenericTypeInfo 8 | import org.apache.flink.util.FlinkRuntimeException 9 | import org.scalatest.flatspec.AnyFlatSpec 10 | import org.scalatest.matchers.should.Matchers 11 | 12 | import scala.util.{Failure, Success, Try} 13 | 14 | class StreamExecutionEnvironmentTest extends AnyFlatSpec with Matchers with IntegrationTest { 15 | 16 | it should "create a stream from a source" in { 17 | implicit val typeInfo: TypeInformation[Integer] = new MockTypeInfo() 18 | 19 | val stream = env.fromSource( 20 | new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), 21 | WatermarkStrategy.noWatermarks(), 22 | "test source" 23 | ) 24 | 25 | stream.dataType shouldBe typeInfo 26 | } 27 | 28 | it should "create a stream from a sequence" in { 29 | import org.apache.flinkx.api.serializers._ 30 | val typeInfo = implicitly[TypeInformation[Long]] 31 | 32 | val stream = env.fromSequence(1, 100) 33 | 34 | stream.dataType shouldBe typeInfo 35 | } 36 | 37 | "From Flink 1.19, TypeInformation.of(Class)" should "fail-fast trying to resolve Scala type" in { 38 | Try(Class.forName("org.apache.flink.configuration.PipelineOptions").getField("SERIALIZATION_CONFIG")) match { 39 | case Failure(_) => // Before Flink 1.19: no fail-fast, exception happens at execution 40 | implicit val typeInfo: TypeInformation[Option[Int]] = TypeInformation.of(classOf[Option[Int]]) 41 | val stream = env.fromElements(Some(1), None, Some(100)) 42 | val exception = intercept[UnsupportedOperationException] { 43 | stream.executeAndCollect(3) 44 | } 45 | exception.getMessage should startWith( 46 | "Generic types have been disabled in the ExecutionConfig and type scala.Option is treated as a generic type." 47 | ) 48 | 49 | case Success(_) => // From Flink 1.19: fail-fast at Scala type resolution 50 | val exception = intercept[FlinkRuntimeException] { 51 | TypeInformation.of(classOf[Option[Int]]) 52 | } 53 | exception.getMessage should startWith("You are using a 'Class' to resolve 'scala.Option' Scala type.") 54 | } 55 | } 56 | 57 | // -------------------------------------------------------------------------- 58 | // mocks 59 | // -------------------------------------------------------------------------- 60 | 61 | private class MockTypeInfo extends GenericTypeInfo[Integer](classOf[Integer]) {} 62 | } 63 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/TestUtils.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flinkx.api.serializer.CaseClassSerializer 4 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 5 | import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer 6 | import org.apache.flink.core.memory.{DataInputViewStreamWrapper, DataOutputViewStreamWrapper} 7 | import org.scalatest.{Assertion, Inspectors} 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectOutputStream} 11 | 12 | trait TestUtils extends Matchers with Inspectors { 13 | def roundtrip[T](ser: TypeSerializer[T], in: T): Assertion = { 14 | val out = new ByteArrayOutputStream() 15 | ser.serialize(in, new DataOutputViewStreamWrapper(out)) 16 | val snapBytes = new ByteArrayOutputStream() 17 | TypeSerializerSnapshot.writeVersionedSnapshot( 18 | new DataOutputViewStreamWrapper(snapBytes), 19 | ser.snapshotConfiguration() 20 | ) 21 | val restoredSnapshot = TypeSerializerSnapshot.readVersionedSnapshot[T]( 22 | new DataInputViewStreamWrapper(new ByteArrayInputStream(snapBytes.toByteArray)), 23 | ser.getClass.getClassLoader 24 | ) 25 | val restoredSerializer = restoredSnapshot.restoreSerializer() 26 | val copy = restoredSerializer.deserialize(new DataInputViewStreamWrapper(new ByteArrayInputStream(out.toByteArray))) 27 | in shouldBe copy 28 | } 29 | 30 | def noKryo[T](ser: TypeSerializer[T]): Unit = 31 | ser match { 32 | case p: CaseClassSerializer[_] => 33 | forAll(p.getFieldSerializers) { param => 34 | noKryo(param) 35 | } 36 | case _: KryoSerializer[_] => 37 | throw new IllegalArgumentException("kryo detected") 38 | case _ => // ok 39 | } 40 | 41 | def serializable[T](ser: TypeSerializer[T]): Unit = { 42 | val stream = new ObjectOutputStream(new ByteArrayOutputStream()) 43 | stream.writeObject(ser) 44 | } 45 | 46 | def all[T](ser: TypeSerializer[T], in: T): Unit = { 47 | roundtrip(ser, in) 48 | noKryo(ser) 49 | serializable(ser) 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/TypeInfoTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flinkx.api.TypeInfoTest.{ADT, ListedArray, ListedList, ListedMap, Parameterized, Simple} 4 | import org.scalatest.flatspec.AnyFlatSpec 5 | import org.scalatest.matchers.should.Matchers 6 | import org.apache.flinkx.api.serializers._ 7 | 8 | class TypeInfoTest extends AnyFlatSpec with Matchers { 9 | 10 | it should "derive simple classes" in { 11 | drop(deriveTypeInformation[Simple]) 12 | } 13 | 14 | it should "derive parameterized classes" in { 15 | drop(deriveTypeInformation[Parameterized[String]]) 16 | } 17 | 18 | it should "derive lists" in { 19 | drop(deriveTypeInformation[ListedList]) 20 | } 21 | 22 | it should "derive arrays" in { 23 | drop(deriveTypeInformation[ListedArray]) 24 | } 25 | 26 | it should "derive maps" in { 27 | drop(deriveTypeInformation[ListedMap]) 28 | } 29 | 30 | it should "derive ADT" in { 31 | drop(deriveTypeInformation[ADT]) 32 | } 33 | } 34 | 35 | object TypeInfoTest { 36 | case class Parameterized[T](a: T) 37 | case class Simple(a: String, b: Int) 38 | case class ListedList(x: List[String]) 39 | case class ListedArray(x: Array[String]) 40 | case class ListedMap(x: Map[String, String]) 41 | 42 | sealed trait ADT 43 | case class Foo(a: String) extends ADT 44 | case class Bar(a: Int) extends ADT 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-1-api/src/test/scala/org/apache/flinkx/api/serializer/ArraySerializerTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.java.typeutils.runtime.RowSerializer 4 | import org.apache.flink.types.Row 5 | import org.scalatest.flatspec.AnyFlatSpec 6 | import org.scalatest.matchers.should.Matchers 7 | 8 | class ArraySerializerTest extends AnyFlatSpec with Matchers { 9 | 10 | it should "copy to new array with same immutable elements" in { 11 | val intArraySerializer = new ArraySerializer[Int](org.apache.flinkx.api.serializers.intSerializer, classOf[Int]) 12 | val expectedData = Array(1, 2, 3) 13 | 14 | val resultData = intArraySerializer.copy(expectedData) 15 | resultData shouldNot be theSameInstanceAs expectedData 16 | resultData shouldEqual expectedData 17 | } 18 | 19 | it should "copy to new array with a copy of mutable elements" in { 20 | val intArraySerializer = new ArraySerializer[Int](org.apache.flinkx.api.serializers.intSerializer, classOf[Int]) 21 | val arrayOfIntArraySerializer = new ArraySerializer[Array[Int]](intArraySerializer, classOf[Array[Int]]) 22 | val expectedData = Array(Array(1, 2), Array(3, 4)) 23 | 24 | val resultData = arrayOfIntArraySerializer.copy(expectedData) 25 | resultData shouldNot be theSameInstanceAs expectedData 26 | resultData(0) shouldNot be theSameInstanceAs expectedData(0) 27 | resultData(1) shouldNot be theSameInstanceAs expectedData(1) 28 | resultData shouldEqual expectedData 29 | } 30 | 31 | it should "return itself when duplicate an immutable serializer" in { 32 | val intArraySerializer = new ArraySerializer[Int](org.apache.flinkx.api.serializers.intSerializer, classOf[Int]) 33 | 34 | val duplicatedIntArraySerializer = intArraySerializer.duplicate() 35 | duplicatedIntArraySerializer should be theSameInstanceAs intArraySerializer 36 | } 37 | 38 | it should "duplicate itself when the serializer is mutable" in { 39 | val rowArraySerializer = new ArraySerializer[Row](new RowSerializer(Array.empty), classOf[Row]) 40 | 41 | val duplicatedIntArraySerializer = rowArraySerializer.duplicate() 42 | 43 | duplicatedIntArraySerializer shouldNot be theSameInstanceAs rowArraySerializer 44 | duplicatedIntArraySerializer should be(rowArraySerializer) 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/CloseableIterator.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.util.{CloseableIterator => JCloseableIterator} 4 | 5 | /** This interface represents an [[Iterator]] that is also [[AutoCloseable]]. A typical use-case for this interface are 6 | * iterators that are based on native-resources such as files, network, or database connections. Clients must call 7 | * close after using the iterator. 8 | */ 9 | trait CloseableIterator[T] extends Iterator[T] with AutoCloseable {} 10 | 11 | object CloseableIterator { 12 | 13 | def fromJava[T](it: JCloseableIterator[T]): CloseableIterator[T] = 14 | new CloseableIterator[T] { 15 | override def hasNext: Boolean = it.hasNext 16 | 17 | override def next(): T = it.next 18 | 19 | override def close(): Unit = it.close() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/DataStreamUtils.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.annotation.Experimental 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.java.functions.KeySelector 6 | import org.apache.flink.streaming.api.datastream.{DataStreamUtils => JavaStreamUtils} 7 | 8 | import scala.jdk.CollectionConverters._ 9 | import scala.reflect.ClassTag 10 | import ScalaStreamOps._ 11 | 12 | /** This class provides simple utility methods for collecting a [[DataStream]], effectively enriching it with the 13 | * functionality encapsulated by [[DataStreamUtils]]. 14 | * 15 | * This experimental class is relocated from flink-streaming-contrib. 16 | * 17 | * @param self 18 | * DataStream 19 | */ 20 | @Experimental 21 | class DataStreamUtils[T: TypeInformation: ClassTag](val self: DataStream[T]) { 22 | 23 | /** Returns a scala iterator to iterate over the elements of the DataStream. 24 | * @return 25 | * The iterator 26 | */ 27 | def collect(): Iterator[T] = 28 | self.javaStream.executeAndCollect().asScala 29 | 30 | /** Reinterprets the given [[DataStream]] as a [[KeyedStream]], which extracts keys with the given [[KeySelector]]. 31 | * 32 | * IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be partitioned 33 | * exactly in the same way as if it was created through a [[DataStream#keyBy(KeySelector)]]. 34 | * 35 | * @param keySelector 36 | * Function that defines how keys are extracted from the data stream. 37 | * @return 38 | * The reinterpretation of the [[DataStream]] as a [[KeyedStream]]. 39 | */ 40 | def reinterpretAsKeyedStream[K: TypeInformation](keySelector: T => K): KeyedStream[T, K] = { 41 | 42 | val keyTypeInfo = implicitly[TypeInformation[K]] 43 | val cleanSelector = clean(keySelector) 44 | val javaKeySelector = new JavaKeySelector[T, K](cleanSelector) 45 | 46 | asScalaStream(JavaStreamUtils.reinterpretAsKeyedStream(self.javaStream, javaKeySelector, keyTypeInfo)) 47 | } 48 | 49 | private[flinkx] def clean[F <: AnyRef](f: F): F = { 50 | new StreamExecutionEnvironment(self.javaStream.getExecutionEnvironment).scalaClean(f) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/OutputTag.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.annotation.PublicEvolving 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.util.{OutputTag => JOutputTag} 6 | 7 | /** An [[OutputTag]] is a typed and named tag to use for tagging side outputs of an operator. 8 | * 9 | * Example: 10 | * {{{ 11 | * val outputTag = OutputTag[String]("late-data") 12 | * }}} 13 | * 14 | * @tparam T 15 | * the type of elements in the side-output stream. 16 | */ 17 | @PublicEvolving 18 | class OutputTag[T: TypeInformation](id: String) extends JOutputTag[T](id, implicitly[TypeInformation[T]]) 19 | 20 | object OutputTag { 21 | def apply[T: TypeInformation](id: String): OutputTag[T] = new OutputTag(id) 22 | } 23 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/ScalaStreamOps.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flinkx.api.typeinfo.CaseClassTypeInfo 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream} 6 | import org.apache.flink.streaming.api.datastream.{ConnectedStreams => ConnectedJavaStreams} 7 | import org.apache.flink.streaming.api.datastream.{BroadcastConnectedStream => BroadcastConnectedJavaStreams} 8 | import org.apache.flink.streaming.api.datastream.{KeyedStream => KeyedJavaStream} 9 | 10 | import language.implicitConversions 11 | import language.experimental.macros 12 | 13 | object ScalaStreamOps { 14 | 15 | /** Converts an [[org.apache.flink.streaming.api.datastream.DataStream]] to a [[org.apache.flinkx.api.DataStream]]. 16 | */ 17 | def asScalaStream[R](stream: JavaStream[R]) = new DataStream[R](stream) 18 | 19 | /** Converts an [[org.apache.flink.streaming.api.datastream.KeyedStream]] to a [[org.apache.flinkx.api.KeyedStream]]. 20 | */ 21 | def asScalaStream[R, K](stream: KeyedJavaStream[R, K]) = new KeyedStream[R, K](stream) 22 | 23 | /** Converts an [[org.apache.flink.streaming.api.datastream.ConnectedStreams]] to a 24 | * [[org.apache.flinkx.api.ConnectedStreams]]. 25 | */ 26 | def asScalaStream[IN1, IN2](stream: ConnectedJavaStreams[IN1, IN2]) = new ConnectedStreams[IN1, IN2](stream) 27 | 28 | /** Converts an [[org.apache.flink.streaming.api.datastream.BroadcastConnectedStream]] to a 29 | * [[org.apache.flinkx.api.BroadcastConnectedStream]]. 30 | */ 31 | def asScalaStream[IN1, IN2](stream: BroadcastConnectedJavaStreams[IN1, IN2]) = 32 | new BroadcastConnectedStream[IN1, IN2](stream) 33 | 34 | private[flinkx] def fieldNames2Indices(typeInfo: TypeInformation[_], fields: Array[String]): Array[Int] = { 35 | typeInfo match { 36 | case ti: CaseClassTypeInfo[_] => 37 | val result = ti.getFieldIndices(fields) 38 | 39 | if (result.contains(-1)) { 40 | throw new IllegalArgumentException( 41 | "Fields '" + fields.mkString(", ") + 42 | "' are not valid for '" + ti.toString + "'." 43 | ) 44 | } 45 | 46 | result 47 | 48 | case _ => 49 | throw new UnsupportedOperationException( 50 | "Specifying fields by name is only" + 51 | "supported on Case Classes (for now)." 52 | ) 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/async/AsyncFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.annotation.PublicEvolving 22 | import org.apache.flink.api.common.functions.Function 23 | 24 | import java.util.concurrent.TimeoutException 25 | 26 | /** A function to trigger async I/O operations. 27 | * 28 | * For each asyncInvoke an async io operation can be triggered, and once it has been done, the result can be collected 29 | * by calling ResultFuture.complete. For each async operation, its context is stored in the operator immediately after 30 | * invoking asyncInvoke, avoiding blocking for each stream input as long as the internal buffer is not full. 31 | * 32 | * [[ResultFuture]] can be passed into callbacks or futures to collect the result data. An error can also be propagate 33 | * to the async IO operator by [[ResultFuture.completeExceptionally(Throwable)]]. 34 | * 35 | * @tparam IN 36 | * The type of the input element 37 | * @tparam OUT 38 | * The type of the output elements 39 | */ 40 | @PublicEvolving 41 | trait AsyncFunction[IN, OUT] extends Function { 42 | 43 | /** Trigger the async operation for each stream input 44 | * 45 | * @param input 46 | * element coming from an upstream task 47 | * @param resultFuture 48 | * to be completed with the result data 49 | */ 50 | def asyncInvoke(input: IN, resultFuture: ResultFuture[OUT]): Unit 51 | 52 | /** [[AsyncFunction.asyncInvoke]] timeout occurred. By default, the result future is exceptionally completed with a 53 | * timeout exception. 54 | * 55 | * @param input 56 | * element coming from an upstream task 57 | * @param resultFuture 58 | * to be completed with the result data 59 | */ 60 | def timeout(input: IN, resultFuture: ResultFuture[OUT]): Unit = { 61 | resultFuture.completeExceptionally(new TimeoutException("Async function call has timed out.")) 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/async/JavaResultFutureWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.annotation.Internal 22 | import org.apache.flink.streaming.api.functions.async 23 | 24 | import scala.jdk.CollectionConverters._ 25 | 26 | /** Internal wrapper class to map a Flink's Java API [[org.apache.flink.streaming.api.functions.async.ResultFuture]] to 27 | * a Scala [[org.apache.flink.api.async.ResultFuture]]. 28 | * 29 | * @param javaResultFuture 30 | * to forward the calls to 31 | * @tparam OUT 32 | * type of the output elements 33 | */ 34 | @Internal 35 | class JavaResultFutureWrapper[OUT](val javaResultFuture: async.ResultFuture[OUT]) extends ResultFuture[OUT] { 36 | override def complete(result: Iterable[OUT]): Unit = { 37 | javaResultFuture.complete(result.asJavaCollection) 38 | } 39 | 40 | override def completeExceptionally(throwable: Throwable): Unit = { 41 | javaResultFuture.completeExceptionally(throwable) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/async/ResultFuture.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.annotation.PublicEvolving 22 | 23 | /** The result future collects data/errors from the user code while processing asynchronous I/O operations. 24 | * 25 | * @tparam OUT 26 | * type of the output element 27 | */ 28 | @PublicEvolving 29 | trait ResultFuture[OUT] { 30 | 31 | /** Complete the ResultFuture with a set of result elements. 32 | * 33 | * Note that it should be called for exactly one time in the user code. Calling this function for multiple times will 34 | * cause data lose. 35 | * 36 | * Put all results in a [[Iterable]] and then issue ResultFuture.complete(Iterable). 37 | * 38 | * @param result 39 | * to complete the async collector with 40 | */ 41 | def complete(result: Iterable[OUT]): Unit 42 | 43 | /** Complete this ResultFuture with an error. 44 | * 45 | * @param throwable 46 | * to complete the async collector with 47 | */ 48 | def completeExceptionally(throwable: Throwable): Unit 49 | } 50 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/async/RichAsyncFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.api.common.functions.AbstractRichFunction 22 | 23 | /** Rich variant of [[AsyncFunction]]. As a [[org.apache.flink.api.common.functions.RichFunction]], it gives access to 24 | * the [[org.apache.flink.api.common.functions.RuntimeContext]] and provides setup and teardown methods. 25 | * 26 | * State related apis in [[org.apache.flink.api.common.functions.RuntimeContext]] are not supported yet because the key 27 | * may get changed while accessing states in the working thread. 28 | * 29 | * [[org.apache.flink.api.common.functions.IterationRuntimeContext#getIterationAggregator(String)]] is not supported 30 | * since the aggregator may be modified by multiple threads. 31 | * 32 | * @tparam IN 33 | * The type of the input value. 34 | * @tparam OUT 35 | * The type of the output value. 36 | */ 37 | abstract class RichAsyncFunction[IN, OUT] extends AbstractRichFunction with AsyncFunction[IN, OUT] {} 38 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/async/ScalaRichAsyncFunctionWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.async 20 | 21 | import org.apache.flink.api.common.functions.RuntimeContext 22 | import org.apache.flink.api.common.functions.OpenContext 23 | import org.apache.flink.streaming.api.functions.async.{ 24 | ResultFuture => JResultFuture, 25 | RichAsyncFunction => JRichAsyncFunction 26 | } 27 | 28 | /** A wrapper function that exposes a Scala RichAsyncFunction as a Java Rich Async Function. 29 | * 30 | * The Scala and Java RichAsyncFunctions differ in their type of "ResultFuture" 31 | * - Scala RichAsyncFunction: [[org.apache.flink.api.async.ResultFuture]] 32 | * - Java RichAsyncFunction: [[org.apache.flink.streaming.api.functions.async.ResultFuture]] 33 | */ 34 | final class ScalaRichAsyncFunctionWrapper[IN, OUT](func: RichAsyncFunction[IN, OUT]) 35 | extends JRichAsyncFunction[IN, OUT] { 36 | 37 | override def asyncInvoke(input: IN, resultFuture: JResultFuture[OUT]): Unit = { 38 | func.asyncInvoke(input, new JavaResultFutureWrapper[OUT](resultFuture)) 39 | } 40 | 41 | override def timeout(input: IN, resultFuture: JResultFuture[OUT]): Unit = { 42 | func.timeout(input, new JavaResultFutureWrapper[OUT](resultFuture)) 43 | } 44 | 45 | override def open(context: OpenContext): Unit = { 46 | func.open(context) 47 | } 48 | 49 | override def close(): Unit = { 50 | func.close() 51 | } 52 | 53 | override def setRuntimeContext(runtimeContext: RuntimeContext): Unit = { 54 | super.setRuntimeContext(runtimeContext) 55 | func.setRuntimeContext(super.getRuntimeContext) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/conv.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv} 4 | import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream} 5 | 6 | import scala.language.implicitConversions 7 | 8 | object conv { 9 | implicit def toJavaEnv(e: StreamExecutionEnvironment): JavaEnv = e.getJavaEnv 10 | 11 | implicit def toJavaDataStream[T](d: DataStream[T]): JavaStream[T] = d.javaStream 12 | } 13 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/extensions/impl/acceptPartialFunctions/OnJoinedStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.extensions.impl.acceptPartialFunctions 19 | 20 | import org.apache.flink.annotation.PublicEvolving 21 | import org.apache.flink.api.common.typeinfo.TypeInformation 22 | import org.apache.flinkx.api.{DataStream, JoinedStreams} 23 | import org.apache.flink.streaming.api.windowing.windows.Window 24 | 25 | /** Wraps a joined data stream, allowing to use anonymous partial functions to perform extraction of items in a tuple, 26 | * case class instance or collection 27 | * 28 | * @param stream 29 | * The wrapped data stream 30 | * @tparam L 31 | * The type of the data stream items from the left side of the join 32 | * @tparam R 33 | * The type of the data stream items from the right input of the join 34 | * @tparam K 35 | * The type of key 36 | * @tparam W 37 | * The type of the window 38 | */ 39 | class OnJoinedStream[L, R, K, W <: Window](stream: JoinedStreams[L, R]#Where[K]#EqualTo#WithWindow[W]) { 40 | 41 | /** Completes the join operation with the user function that is executed for windowed groups. 42 | * 43 | * @param fun 44 | * The function that defines the projection of the join 45 | * @tparam O 46 | * The return type of the projection, for which type information must be known 47 | * @return 48 | * A fully joined data set of Os 49 | */ 50 | @PublicEvolving 51 | def projecting[O: TypeInformation](fun: (L, R) => O): DataStream[O] = 52 | stream.apply(fun) 53 | 54 | } 55 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/extensions/impl/acceptPartialFunctions/OnKeyedStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.extensions.impl.acceptPartialFunctions 19 | 20 | import org.apache.flink.annotation.PublicEvolving 21 | import org.apache.flink.api.common.typeinfo.TypeInformation 22 | import org.apache.flinkx.api.{DataStream, KeyedStream} 23 | 24 | /** Wraps a keyed data stream, allowing to use anonymous partial functions to perform extraction of items in a tuple, 25 | * case class instance or collection 26 | * 27 | * @param stream 28 | * The wrapped data stream 29 | * @tparam T 30 | * The type of the data stream items 31 | * @tparam K 32 | * The type of key 33 | */ 34 | class OnKeyedStream[T, K](stream: KeyedStream[T, K]) { 35 | 36 | /** Applies a reducer `fun` to the stream 37 | * 38 | * @param fun 39 | * The reducing function to be applied on the keyed stream 40 | * @return 41 | * A data set of Ts 42 | */ 43 | @PublicEvolving 44 | def reduceWith(fun: (T, T) => T): DataStream[T] = 45 | stream.reduce(fun) 46 | } 47 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/AllWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.function 19 | 20 | import org.apache.flink.annotation.Public 21 | import org.apache.flink.api.common.functions.Function 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import java.io.Serializable 26 | 27 | /** Base interface for functions that are evaluated over non-grouped windows, i.e., windows over all stream partitions. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | */ 34 | @Public 35 | trait AllWindowFunction[IN, OUT, W <: Window] extends Function with Serializable { 36 | 37 | /** Evaluates the window and outputs none or several elements. 38 | * 39 | * @param window 40 | * The window that is being evaluated. 41 | * @param input 42 | * The elements in the window being evaluated. 43 | * @param out 44 | * A collector for emitting elements. 45 | * @throws Exception 46 | * The function may throw exceptions to fail the program and trigger recovery. 47 | */ 48 | def apply(window: W, input: Iterable[IN], out: Collector[OUT]): Unit 49 | } 50 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/RichAllWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function 20 | 21 | import org.apache.flink.api.common.functions.AbstractRichFunction 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | 24 | /** Rich variant of the [[org.apache.flinkx.api.function.AllWindowFunction]]. 25 | * 26 | * As a [[org.apache.flink.api.common.functions.RichFunction]], it gives access to the 27 | * [[org.apache.flink.api.common.functions.RuntimeContext]] and provides setup and tear-down methods. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | * @tparam W 34 | * The type of Window that this window function can be applied on. 35 | */ 36 | abstract class RichAllWindowFunction[IN, OUT, W <: Window] 37 | extends AbstractRichFunction 38 | with AllWindowFunction[IN, OUT, W] {} 39 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/RichWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function 20 | 21 | import org.apache.flink.api.common.functions.AbstractRichFunction 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | 24 | /** Rich variant of the [[org.apache.flinkx.api.function.WindowFunction]]. 25 | * 26 | * As a [[org.apache.flink.api.common.functions.RichFunction]], it gives access to the 27 | * [[org.apache.flink.api.common.functions.RuntimeContext]] and provides setup and tear-down methods. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | * @tparam KEY 34 | * The type of the key. 35 | * @tparam W 36 | * The type of Window that this window function can be applied on. 37 | */ 38 | abstract class RichWindowFunction[IN, OUT, KEY, W <: Window] 39 | extends AbstractRichFunction 40 | with WindowFunction[IN, OUT, KEY, W] {} 41 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/StatefulFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function 20 | 21 | import org.apache.flink.annotation.Public 22 | import org.apache.flink.api.common.functions.RichFunction 23 | import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} 24 | import org.apache.flink.api.common.typeutils.TypeSerializer 25 | import org.apache.flink.api.common.functions.OpenContext 26 | 27 | /** Trait implementing the functionality necessary to apply stateful functions in RichFunctions without exposing the 28 | * OperatorStates to the user. The user should call the applyWithState method in his own RichFunction implementation. 29 | */ 30 | @Public 31 | trait StatefulFunction[I, O, S] extends RichFunction { 32 | 33 | protected val stateSerializer: TypeSerializer[S] 34 | 35 | private[this] var state: ValueState[S] = _ 36 | 37 | def applyWithState(in: I, fun: (I, Option[S]) => (O, Option[S])): O = { 38 | val (o, s: Option[S]) = fun(in, Option(state.value())) 39 | s match { 40 | case Some(v) => state.update(v) 41 | case None => state.update(null.asInstanceOf[S]) 42 | } 43 | o 44 | } 45 | 46 | override def open(context: OpenContext): Unit = { 47 | val info = new ValueStateDescriptor[S]("state", stateSerializer) 48 | state = getRuntimeContext.getState(info) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/WindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.flinkx.api.function 19 | 20 | import org.apache.flink.annotation.Public 21 | import org.apache.flink.api.common.functions.Function 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import java.io.Serializable 26 | 27 | /** Base interface for functions that are evaluated over keyed (grouped) windows. 28 | * 29 | * @tparam IN 30 | * The type of the input value. 31 | * @tparam OUT 32 | * The type of the output value. 33 | * @tparam KEY 34 | * The type of the key. 35 | */ 36 | @Public 37 | trait WindowFunction[IN, OUT, KEY, W <: Window] extends Function with Serializable { 38 | 39 | /** Evaluates the window and outputs none or several elements. 40 | * 41 | * @param key 42 | * The key for which this window is evaluated. 43 | * @param window 44 | * The window that is being evaluated. 45 | * @param input 46 | * The elements in the window being evaluated. 47 | * @param out 48 | * A collector for emitting elements. 49 | * @throws Exception 50 | * The function may throw exceptions to fail the program and trigger recovery. 51 | */ 52 | def apply(key: KEY, window: W, input: Iterable[IN], out: Collector[OUT]): Unit 53 | } 54 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaAllWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction => JAllWindowFunction} 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import scala.jdk.CollectionConverters._ 26 | 27 | /** A wrapper function that exposes a Scala Function3 as a Java AllWindowFunction. 28 | */ 29 | final class ScalaAllWindowFunction[IN, OUT, W <: Window]( 30 | private[this] val function: (W, Iterable[IN], Collector[OUT]) => Unit 31 | ) extends JAllWindowFunction[IN, OUT, W] { 32 | 33 | @throws(classOf[Exception]) 34 | override def apply(window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 35 | function.apply(window, input.asScala, out) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaAllWindowFunctionWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext} 22 | import org.apache.flink.api.common.functions.WrappingFunction 23 | import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction => JAllWindowFunction} 24 | import org.apache.flinkx.api.function.AllWindowFunction 25 | import org.apache.flink.streaming.api.windowing.windows.Window 26 | import org.apache.flink.util.Collector 27 | 28 | import scala.jdk.CollectionConverters._ 29 | 30 | /** A wrapper function that exposes a Scala WindowFunction as a JavaWindow function. 31 | * 32 | * The Scala and Java Window functions differ in their type of "Iterable": 33 | * - Scala WindowFunction: scala.Iterable 34 | * - Java WindowFunction: java.lang.Iterable 35 | */ 36 | final class ScalaAllWindowFunctionWrapper[IN, OUT, W <: Window](func: AllWindowFunction[IN, OUT, W]) 37 | extends WrappingFunction[AllWindowFunction[IN, OUT, W]](func) 38 | with JAllWindowFunction[IN, OUT, W] 39 | with RichFunction { 40 | 41 | @throws(classOf[Exception]) 42 | override def apply(window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 43 | wrappedFunction.apply(window, input.asScala, out) 44 | } 45 | 46 | override def getRuntimeContext: RuntimeContext = { 47 | throw new RuntimeException("This should never be called") 48 | } 49 | 50 | override def getIterationRuntimeContext: IterationRuntimeContext = { 51 | throw new RuntimeException("This should never be called") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaReduceFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.api.common.functions.ReduceFunction 22 | 23 | /** A wrapper function that exposes a Scala Function2 as a [[ReduceFunction]]. 24 | */ 25 | final class ScalaReduceFunction[T](private[this] val function: (T, T) => T) extends ReduceFunction[T] { 26 | 27 | @throws(classOf[Exception]) 28 | override def reduce(a: T, b: T): T = { 29 | function(a, b) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaWindowFunction.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flink.streaming.api.functions.windowing.{WindowFunction => JWindowFunction} 22 | import org.apache.flink.streaming.api.windowing.windows.Window 23 | import org.apache.flink.util.Collector 24 | 25 | import scala.jdk.CollectionConverters._ 26 | 27 | /** A wrapper function that exposes a Scala Function4 as a Java WindowFunction. 28 | */ 29 | final class ScalaWindowFunction[IN, OUT, KEY, W <: Window]( 30 | private[this] val function: (KEY, W, Iterable[IN], Collector[OUT]) => Unit 31 | ) extends JWindowFunction[IN, OUT, KEY, W] { 32 | 33 | @throws(classOf[Exception]) 34 | override def apply(key: KEY, window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 35 | function.apply(key, window, input.asScala, out) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/function/util/ScalaWindowFunctionWrapper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.function.util 20 | 21 | import org.apache.flinkx.api.function.WindowFunction 22 | import org.apache.flink.api.common.functions.{IterationRuntimeContext, RichFunction, RuntimeContext} 23 | import org.apache.flink.api.common.functions.WrappingFunction 24 | import org.apache.flink.streaming.api.functions.windowing.{WindowFunction => JWindowFunction} 25 | import org.apache.flink.streaming.api.windowing.windows.Window 26 | import org.apache.flink.util.Collector 27 | 28 | import scala.jdk.CollectionConverters._ 29 | 30 | /** A wrapper function that exposes a Scala WindowFunction as a JavaWindow function. 31 | * 32 | * The Scala and Java Window functions differ in their type of "Iterable": 33 | * - Scala WindowFunction: scala.Iterable 34 | * - Java WindowFunction: java.lang.Iterable 35 | */ 36 | final class ScalaWindowFunctionWrapper[IN, OUT, KEY, W <: Window](func: WindowFunction[IN, OUT, KEY, W]) 37 | extends WrappingFunction[WindowFunction[IN, OUT, KEY, W]](func) 38 | with JWindowFunction[IN, OUT, KEY, W] 39 | with RichFunction { 40 | 41 | @throws(classOf[Exception]) 42 | override def apply(key: KEY, window: W, input: java.lang.Iterable[IN], out: Collector[OUT]) = { 43 | wrappedFunction.apply(key, window, input.asScala, out) 44 | } 45 | 46 | override def getRuntimeContext: RuntimeContext = { 47 | throw new RuntimeException("This should never be called") 48 | } 49 | 50 | override def getIterationRuntimeContext: IterationRuntimeContext = { 51 | throw new RuntimeException("This should never be called") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/mapper/BigDecMapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.mapper 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 4 | 5 | class BigDecMapper extends TypeMapper[scala.BigDecimal, java.math.BigDecimal] { 6 | override def map(a: BigDecimal): java.math.BigDecimal = a.bigDecimal 7 | override def contramap(b: java.math.BigDecimal): BigDecimal = BigDecimal(b) 8 | } 9 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/mapper/BigIntMapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.mapper 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 4 | 5 | import java.math.BigInteger 6 | 7 | class BigIntMapper() extends TypeMapper[scala.BigInt, java.math.BigInteger] { 8 | override def contramap(b: BigInteger): BigInt = BigInt(b) 9 | override def map(a: BigInt): BigInteger = a.bigInteger 10 | } 11 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/mapper/UuidMapper.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.mapper 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 4 | 5 | import java.nio.ByteBuffer 6 | import java.util.UUID 7 | 8 | class UuidMapper() extends TypeMapper[UUID, Array[Byte]] { 9 | override def map(a: UUID): Array[Byte] = { 10 | val buffer = ByteBuffer.allocate(16) 11 | buffer.putLong(a.getMostSignificantBits) 12 | buffer.putLong(a.getLeastSignificantBits) 13 | buffer.array() 14 | } 15 | 16 | override def contramap(b: Array[Byte]): UUID = { 17 | val buffer = ByteBuffer.wrap(b) 18 | val mostSignificantBits = buffer.getLong() 19 | val leastSignificantBits = buffer.getLong() 20 | new UUID(mostSignificantBits, leastSignificantBits) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/ArraySerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | import scala.reflect.ClassTag 7 | 8 | class ArraySerializer[T](val child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Array[T]] { 9 | 10 | implicit val classTag: ClassTag[T] = ClassTag(clazz) 11 | 12 | override def createInstance(): Array[T] = Array.empty[T] 13 | 14 | override def copy(from: Array[T]): Array[T] = { 15 | if (from == null) { 16 | from 17 | } else { 18 | val length = from.length 19 | val copy = Array.copyOf(from, length) 20 | if (!child.isImmutableType) { 21 | var i = 0 22 | while (i < length) { 23 | val element = copy(i) 24 | if (element != null) copy(i) = child.copy(element) 25 | i += 1 26 | } 27 | } 28 | copy 29 | } 30 | } 31 | 32 | override def duplicate(): ArraySerializer[T] = { 33 | val duplicatedChild = child.duplicate() 34 | if (duplicatedChild.eq(child)) { 35 | this 36 | } else { 37 | new ArraySerializer[T](duplicatedChild, clazz) 38 | } 39 | } 40 | 41 | override def getLength: Int = -1 42 | 43 | override def deserialize(source: DataInputView): Array[T] = { 44 | val length = source.readInt() 45 | val array = new Array[T](length) 46 | var i = 0 47 | while (i < length) { 48 | array(i) = child.deserialize(source) 49 | i += 1 50 | } 51 | array 52 | } 53 | 54 | override def serialize(record: Array[T], target: DataOutputView): Unit = { 55 | val length = record.length 56 | target.writeInt(length) 57 | var i = 0 58 | while (i < length) { // while loop is significantly faster than foreach when working on arrays 59 | val element = record(i) 60 | child.serialize(element, target) 61 | i += 1 62 | } 63 | } 64 | 65 | override def snapshotConfiguration(): TypeSerializerSnapshot[Array[T]] = 66 | new CollectionSerializerSnapshot[Array, T, ArraySerializer[T]](child, classOf[ArraySerializer[T]], clazz) 67 | 68 | } 69 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/CollectionSerializerSnapshot.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSchemaCompatibility, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | import org.apache.flink.util.InstantiationUtil 6 | 7 | /** Generic serializer snapshot for collection. 8 | * 9 | * @param nestedSerializer 10 | * the serializer of `T` 11 | * @param clazz 12 | * the class of `S` 13 | * @param vclazz 14 | * the class of `T` 15 | * @tparam F 16 | * the type of the serialized collection 17 | * @tparam T 18 | * the type of the collection's elements 19 | * @tparam S 20 | * the type of the collection serializer 21 | */ 22 | class CollectionSerializerSnapshot[F[_], T, S <: TypeSerializer[F[T]]]( 23 | var nestedSerializer: TypeSerializer[T], 24 | var clazz: Class[S], 25 | var vclazz: Class[T] 26 | ) extends TypeSerializerSnapshot[F[T]] { 27 | 28 | // Empty constructor is required to instantiate this class during deserialization. 29 | def this() = this(null, null, null) 30 | 31 | override def getCurrentVersion: Int = 2 32 | 33 | override def readSnapshot(readVersion: Int, in: DataInputView, userCodeClassLoader: ClassLoader): Unit = { 34 | clazz = InstantiationUtil.resolveClassByName[S](in, userCodeClassLoader) 35 | vclazz = InstantiationUtil.resolveClassByName[T](in, userCodeClassLoader) 36 | nestedSerializer = TypeSerializerSnapshot.readVersionedSnapshot[T](in, userCodeClassLoader).restoreSerializer() 37 | } 38 | 39 | override def writeSnapshot(out: DataOutputView): Unit = { 40 | out.writeUTF(clazz.getName) 41 | vclazz.getName match { 42 | case "double" => out.writeUTF("java.lang.Double") 43 | case "float" => out.writeUTF("java.lang.Float") 44 | case "int" => out.writeUTF("java.lang.Integer") 45 | case "long" => out.writeUTF("java.lang.Long") 46 | case "byte" => out.writeUTF("java.lang.Byte") 47 | case "short" => out.writeUTF("java.lang.Short") 48 | case "char" => out.writeUTF("java.lang.Char") 49 | case "boolean" => out.writeUTF("java.lang.Boolean") 50 | case other => out.writeUTF(other) 51 | } 52 | TypeSerializerSnapshot.writeVersionedSnapshot(out, nestedSerializer.snapshotConfiguration()) 53 | } 54 | 55 | override def resolveSchemaCompatibility( 56 | oldSerializerSnapshot: TypeSerializerSnapshot[F[T]] 57 | ): TypeSerializerSchemaCompatibility[F[T]] = TypeSerializerSchemaCompatibility.compatibleAsIs() 58 | 59 | override def restoreSerializer(): TypeSerializer[F[T]] = { 60 | val constructor = clazz.getConstructors()(0) 61 | constructor.newInstance(nestedSerializer, vclazz).asInstanceOf[TypeSerializer[F[T]]] 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/ListCCSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class ListCCSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[::[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: ::[T]): ::[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | val result = from.map(child.copy) 15 | ::(result.head, result.tail) 16 | } 17 | } 18 | 19 | override def duplicate(): ListCCSerializer[T] = { 20 | val duplicatedChild = child.duplicate() 21 | if (duplicatedChild.eq(child)) { 22 | this 23 | } else { 24 | new ListCCSerializer[T](duplicatedChild, clazz) 25 | } 26 | } 27 | 28 | override def createInstance(): ::[T] = throw new IllegalArgumentException("cannot create instance of non-empty list") 29 | override def getLength: Int = -1 30 | override def deserialize(source: DataInputView): ::[T] = { 31 | val count = source.readInt() 32 | val result = (0 until count) 33 | .map(_ => child.deserialize(source)) 34 | 35 | ::(result.head, result.tail.toList) 36 | } 37 | override def serialize(record: ::[T], target: DataOutputView): Unit = { 38 | target.writeInt(record.size) 39 | record.foreach(element => child.serialize(element, target)) 40 | } 41 | override def snapshotConfiguration(): TypeSerializerSnapshot[::[T]] = 42 | new CollectionSerializerSnapshot[::, T, ListCCSerializer[T]](child, classOf[ListCCSerializer[T]], clazz) 43 | 44 | } 45 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/ListSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class ListSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[List[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: List[T]): List[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): ListSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new ListSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): List[T] = List.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): List[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result.toList 37 | } 38 | override def serialize(record: List[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[List[T]] = 43 | new CollectionSerializerSnapshot[List, T, ListSerializer[T]](child, classOf[ListSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/ScalaCaseObjectSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton 4 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSchemaCompatibility, TypeSerializerSnapshot} 5 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 6 | import org.apache.flink.util.InstantiationUtil 7 | import org.apache.flinkx.api.serializer.ScalaCaseObjectSerializer.ScalaCaseObjectSerializerSnapshot 8 | 9 | class ScalaCaseObjectSerializer[T](clazz: Class[T]) extends ImmutableSerializer[T] { 10 | override def copy(source: DataInputView, target: DataOutputView): Unit = {} 11 | override def createInstance(): T = clazz.getField("MODULE$").get(null).asInstanceOf[T] 12 | override def getLength: Int = 0 13 | override def serialize(record: T, target: DataOutputView): Unit = {} 14 | 15 | override def deserialize(source: DataInputView): T = { 16 | clazz.getField("MODULE$").get(null).asInstanceOf[T] 17 | } 18 | 19 | override def snapshotConfiguration(): TypeSerializerSnapshot[T] = 20 | new ScalaCaseObjectSerializerSnapshot(clazz) 21 | } 22 | 23 | object ScalaCaseObjectSerializer { 24 | class ScalaCaseObjectSerializerSnapshot[T](var clazz: Class[T]) extends TypeSerializerSnapshot[T] { 25 | def this() = this(null) 26 | 27 | override def readSnapshot(readVersion: Int, in: DataInputView, userCodeClassLoader: ClassLoader): Unit = { 28 | clazz = InstantiationUtil.resolveClassByName(in, userCodeClassLoader) 29 | } 30 | 31 | override def writeSnapshot(out: DataOutputView): Unit = { 32 | out.writeUTF(clazz.getName) 33 | } 34 | 35 | override def getCurrentVersion: Int = 1 36 | override def resolveSchemaCompatibility( 37 | oldSerializerSnapshot: TypeSerializerSnapshot[T] 38 | ): TypeSerializerSchemaCompatibility[T] = 39 | TypeSerializerSchemaCompatibility.compatibleAsIs() 40 | 41 | override def restoreSerializer(): TypeSerializer[T] = 42 | new ScalaCaseObjectSerializer[T](clazz) 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/SeqSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class SeqSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Seq[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: Seq[T]): Seq[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): SeqSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new SeqSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): Seq[T] = Seq.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): Seq[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result 37 | } 38 | override def serialize(record: Seq[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[Seq[T]] = 43 | new CollectionSerializerSnapshot[Seq, T, SeqSerializer[T]](child, classOf[SeqSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/SetSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class SetSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Set[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: Set[T]): Set[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): SetSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new SetSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): Set[T] = Set.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): Set[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result.toSet 37 | } 38 | override def serialize(record: Set[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[Set[T]] = 43 | new CollectionSerializerSnapshot[Set, T, SetSerializer[T]](child, classOf[SetSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/serializer/VectorSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | class VectorSerializer[T](child: TypeSerializer[T], clazz: Class[T]) extends MutableSerializer[Vector[T]] { 7 | 8 | override val isImmutableType: Boolean = child.isImmutableType 9 | 10 | override def copy(from: Vector[T]): Vector[T] = { 11 | if (from == null || isImmutableType) { 12 | from 13 | } else { 14 | from.map(child.copy) 15 | } 16 | } 17 | 18 | override def duplicate(): VectorSerializer[T] = { 19 | val duplicatedChild = child.duplicate() 20 | if (duplicatedChild.eq(child)) { 21 | this 22 | } else { 23 | new VectorSerializer[T](duplicatedChild, clazz) 24 | } 25 | } 26 | 27 | override def createInstance(): Vector[T] = Vector.empty[T] 28 | override def getLength: Int = -1 29 | override def deserialize(source: DataInputView): Vector[T] = { 30 | val count = source.readInt() 31 | val result = for { 32 | _ <- 0 until count 33 | } yield { 34 | child.deserialize(source) 35 | } 36 | result.toVector 37 | } 38 | override def serialize(record: Vector[T], target: DataOutputView): Unit = { 39 | target.writeInt(record.size) 40 | record.foreach(element => child.serialize(element, target)) 41 | } 42 | override def snapshotConfiguration(): TypeSerializerSnapshot[Vector[T]] = 43 | new CollectionSerializerSnapshot[Vector, T, VectorSerializer[T]](child, classOf[VectorSerializer[T]], clazz) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/typeinfo/CollectionTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.serialization.SerializerConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | import scala.reflect.{ClassTag, classTag} 8 | 9 | case class CollectionTypeInformation[T: ClassTag](serializer: TypeSerializer[T]) extends TypeInformation[T] { 10 | val clazz = classTag[T].runtimeClass.asInstanceOf[Class[T]] 11 | override def createSerializer(config: SerializerConfig): TypeSerializer[T] = 12 | serializer.duplicate() 13 | override def isBasicType: Boolean = false 14 | override def isTupleType: Boolean = false 15 | override def isKeyType: Boolean = false 16 | override def getTotalFields: Int = 1 17 | override def getTypeClass: Class[T] = clazz 18 | override def getArity: Int = 1 19 | } 20 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/typeinfo/CoproductTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.serialization.SerializerConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | case class CoproductTypeInformation[T](c: Class[T], ser: TypeSerializer[T]) extends TypeInformation[T] { 8 | override def createSerializer(config: SerializerConfig): TypeSerializer[T] = 9 | ser.duplicate() 10 | override def isBasicType: Boolean = false 11 | override def isTupleType: Boolean = false 12 | override def isKeyType: Boolean = false 13 | override def getTotalFields: Int = 1 14 | override def getTypeClass: Class[T] = c 15 | override def getArity: Int = 1 16 | } 17 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/typeinfo/MappedTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flinkx.api.serializer.MappedSerializer 4 | import org.apache.flinkx.api.serializer.MappedSerializer.TypeMapper 5 | import org.apache.flink.api.common.serialization.SerializerConfig 6 | import org.apache.flink.api.common.typeinfo.TypeInformation 7 | import org.apache.flink.api.common.typeutils.TypeSerializer 8 | 9 | import scala.reflect.{ClassTag, classTag} 10 | 11 | case class MappedTypeInformation[A: ClassTag, B](mapper: TypeMapper[A, B], nested: TypeInformation[B]) 12 | extends TypeInformation[A] { 13 | override def createSerializer(config: SerializerConfig): TypeSerializer[A] = 14 | new MappedSerializer(mapper, nested.createSerializer(config)) 15 | override def isKeyType: Boolean = nested.isKeyType 16 | override def getTotalFields: Int = nested.getTotalFields 17 | override def isTupleType: Boolean = nested.isTupleType 18 | 19 | override def canEqual(obj: Any): Boolean = obj match { 20 | case m: MappedTypeInformation[_, _] => true 21 | case _ => false 22 | } 23 | override def getTypeClass: Class[A] = classTag[A].runtimeClass.asInstanceOf[Class[A]] 24 | override def getArity: Int = nested.getArity 25 | override def isBasicType: Boolean = nested.isBasicType 26 | 27 | override def toString: String = nested.toString 28 | 29 | override def equals(obj: Any): Boolean = obj match { 30 | case m: MappedTypeInformation[_, _] => (m.nested == nested) && m.mapper.equals(mapper) 31 | case _ => false 32 | } 33 | 34 | override def hashCode(): Int = nested.hashCode() 35 | 36 | } 37 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/typeinfo/ProductTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.serialization.SerializerConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | class ProductTypeInformation[T <: Product]( 8 | c: Class[T], 9 | fieldTypes: Seq[TypeInformation[_]], 10 | fieldNames: Seq[String], 11 | ser: TypeSerializer[T] 12 | ) extends CaseClassTypeInfo[T]( 13 | clazz = c, 14 | typeParamTypeInfos = Array(), 15 | fieldTypes, 16 | fieldNames 17 | ) { 18 | override def createSerializer(config: SerializerConfig): TypeSerializer[T] = ser.duplicate() 19 | 20 | } 21 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/typeinfo/SimpleTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.serialization.SerializerConfig 4 | import org.apache.flink.api.common.typeinfo.TypeInformation 5 | import org.apache.flink.api.common.typeutils.TypeSerializer 6 | 7 | import scala.reflect.{classTag, ClassTag} 8 | 9 | abstract class SimpleTypeInformation[T: ClassTag: TypeSerializer] extends TypeInformation[T] { 10 | override def createSerializer(config: SerializerConfig): TypeSerializer[T] = { 11 | val ser = implicitly[TypeSerializer[T]] 12 | ser.duplicate() 13 | } 14 | override def isBasicType: Boolean = false 15 | override def isTupleType: Boolean = false 16 | override def isKeyType: Boolean = false 17 | override def getTotalFields: Int = 1 18 | override def getTypeClass: Class[T] = classTag[T].runtimeClass.asInstanceOf[Class[T]] 19 | override def getArity: Int = 1 20 | } 21 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/main/scala/org/apache/flinkx/api/typeinfo/UnitTypeInformation.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flinkx.api.serializer.UnitSerializer 4 | import org.apache.flink.api.common.serialization.SerializerConfig 5 | import org.apache.flink.api.common.typeinfo.TypeInformation 6 | import org.apache.flink.api.common.typeutils.TypeSerializer 7 | 8 | class UnitTypeInformation extends TypeInformation[Unit] { 9 | override def createSerializer(config: SerializerConfig): TypeSerializer[Unit] = new UnitSerializer() 10 | override def isKeyType: Boolean = true 11 | override def getTotalFields: Int = 0 12 | override def isTupleType: Boolean = false 13 | override def canEqual(obj: Any): Boolean = obj.isInstanceOf[UnitTypeInformation] 14 | override def getTypeClass: Class[Unit] = classOf[Unit] 15 | override def getArity: Int = 0 16 | override def isBasicType: Boolean = false 17 | 18 | override def toString: String = "{}" 19 | 20 | override def equals(obj: Any): Boolean = obj match { 21 | case _: UnitTypeInformation => true 22 | case _ => false 23 | } 24 | 25 | override def hashCode(): Int = ().hashCode() 26 | } 27 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/test/scala/org/apache/flinkx/api/serializer/CollectionSerializerSnapshotTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputDeserializer, DataOutputSerializer} 5 | import org.apache.flinkx.api.serializers.* 6 | import org.scalatest.flatspec.AnyFlatSpec 7 | import org.scalatest.matchers.should.Matchers 8 | 9 | class CollectionSerializerSnapshotTest extends AnyFlatSpec with Matchers { 10 | 11 | it should "serialize then deserialize" in { 12 | // Create SerializerSnapshot 13 | val tSerializer = implicitly[TypeSerializer[String]] 14 | val serializerSnapshot: CollectionSerializerSnapshot[Set, String, SetSerializer[String]] = 15 | new CollectionSerializerSnapshot(tSerializer, classOf[SetSerializer[String]], classOf[String]) 16 | 17 | val expectedSerializer = serializerSnapshot.restoreSerializer() 18 | 19 | // Serialize SerializerSnapshot 20 | val snapshotOutput = new DataOutputSerializer(1024 * 1024) 21 | TypeSerializerSnapshot.writeVersionedSnapshot(snapshotOutput, serializerSnapshot) 22 | val snapshotInput = new DataInputDeserializer(snapshotOutput.getSharedBuffer) 23 | 24 | // Deserialize SerializerSnapshot 25 | val deserializedSnapshot = TypeSerializerSnapshot 26 | .readVersionedSnapshot[SetSerializer[String]](snapshotInput, getClass.getClassLoader) 27 | 28 | val deserializedSerializer = deserializedSnapshot.restoreSerializer() 29 | deserializedSerializer shouldNot be theSameInstanceAs expectedSerializer 30 | deserializedSerializer should be(expectedSerializer) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/test/scala/org/apache/flinkx/api/serializer/CoproductSerializerSnapshotTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputDeserializer, DataOutputSerializer} 5 | import org.apache.flinkx.api.serializer.CoproductSerializerSnapshotTest.{ADT, Bar, Foo} 6 | import org.apache.flinkx.api.serializers.* 7 | import org.scalatest.flatspec.AnyFlatSpec 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | class CoproductSerializerSnapshotTest extends AnyFlatSpec with Matchers { 11 | 12 | it should "serialize then deserialize" in { 13 | // Create SerializerSnapshot 14 | val subtypeClasses: Array[Class[?]] = Array(classOf[Foo], classOf[Bar]) 15 | val subtypeSerializers: Array[TypeSerializer[?]] = Array( 16 | implicitly[TypeSerializer[Foo]], 17 | implicitly[TypeSerializer[Bar]] 18 | ) 19 | val serializerSnapshot: CoproductSerializer.CoproductSerializerSnapshot[ADT] = 20 | new CoproductSerializer.CoproductSerializerSnapshot(subtypeClasses, subtypeSerializers) 21 | 22 | val expectedSerializer = serializerSnapshot.restoreSerializer() 23 | 24 | // Serialize SerializerSnapshot 25 | val snapshotOutput = new DataOutputSerializer(1024 * 1024) 26 | TypeSerializerSnapshot.writeVersionedSnapshot(snapshotOutput, serializerSnapshot) 27 | val snapshotInput = new DataInputDeserializer(snapshotOutput.getSharedBuffer) 28 | 29 | // Deserialize SerializerSnapshot 30 | val deserializedSnapshot = TypeSerializerSnapshot 31 | .readVersionedSnapshot[SetSerializer[String]](snapshotInput, getClass.getClassLoader) 32 | 33 | val deserializedSerializer = deserializedSnapshot.restoreSerializer() 34 | deserializedSerializer shouldNot be theSameInstanceAs expectedSerializer 35 | deserializedSerializer should be(expectedSerializer) 36 | } 37 | 38 | } 39 | 40 | object CoproductSerializerSnapshotTest { 41 | sealed trait ADT 42 | case class Foo(a: String) extends ADT 43 | case class Bar(b: Int) extends ADT 44 | } 45 | -------------------------------------------------------------------------------- /modules/flink-2-api/src/test/scala/org/apache/flinkx/api/serializer/MappedSerializerSnapshotTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeSerializerSnapshot} 4 | import org.apache.flink.core.memory.{DataInputDeserializer, DataOutputSerializer} 5 | import org.apache.flinkx.api.mapper.BigDecMapper 6 | import org.apache.flinkx.api.serializers.* 7 | import org.scalatest.flatspec.AnyFlatSpec 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | import java.math.BigDecimal as JBigDecimal 11 | 12 | class MappedSerializerSnapshotTest extends AnyFlatSpec with Matchers { 13 | 14 | it should "serialize then deserialize" in { 15 | // Create SerializerSnapshot 16 | val mapper = new BigDecMapper() 17 | val tSerializer = implicitly[TypeSerializer[JBigDecimal]] 18 | val serializerSnapshot: MappedSerializer.MappedSerializerSnapshot[scala.BigDecimal, JBigDecimal] = 19 | new MappedSerializer.MappedSerializerSnapshot(mapper, tSerializer) 20 | 21 | val expectedSerializer = serializerSnapshot.restoreSerializer() 22 | 23 | // Serialize SerializerSnapshot 24 | val snapshotOutput = new DataOutputSerializer(1024 * 1024) 25 | TypeSerializerSnapshot.writeVersionedSnapshot(snapshotOutput, serializerSnapshot) 26 | val snapshotInput = new DataInputDeserializer(snapshotOutput.getSharedBuffer) 27 | 28 | // Deserialize SerializerSnapshot 29 | val deserializedSnapshot = TypeSerializerSnapshot 30 | .readVersionedSnapshot[SetSerializer[String]](snapshotInput, getClass.getClassLoader) 31 | .asInstanceOf[MappedSerializer.MappedSerializerSnapshot[scala.BigDecimal, JBigDecimal]] 32 | 33 | deserializedSnapshot.restoreSerializer() shouldNot be theSameInstanceAs expectedSerializer 34 | deserializedSnapshot.ser should be(serializerSnapshot.ser) 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/java/org/apache/flinkx/api/serializer/ScalaEitherSerializerSnapshot.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.serializer; 20 | 21 | import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; 22 | import org.apache.flink.api.common.typeutils.TypeSerializer; 23 | 24 | import scala.util.Either; 25 | 26 | /** 27 | * Configuration snapshot for serializers of Scala's {@link Either} type, containing configuration 28 | * snapshots of the Left and Right serializers. 29 | */ 30 | public class ScalaEitherSerializerSnapshot 31 | extends CompositeTypeSerializerSnapshot, EitherSerializer> { 32 | 33 | private static final int CURRENT_VERSION = 1; 34 | 35 | /** Constructor for read instantiation. */ 36 | public ScalaEitherSerializerSnapshot() { 37 | super(EitherSerializer.class); 38 | } 39 | 40 | /** Constructor to create the snapshot for writing. */ 41 | public ScalaEitherSerializerSnapshot(EitherSerializer eitherSerializer) { 42 | super(eitherSerializer); 43 | } 44 | 45 | @Override 46 | public int getCurrentOuterSnapshotVersion() { 47 | return CURRENT_VERSION; 48 | } 49 | 50 | @Override 51 | protected EitherSerializer createOuterSerializerWithNestedSerializers( 52 | TypeSerializer[] nestedSerializers) { 53 | @SuppressWarnings("unchecked") 54 | TypeSerializer leftSerializer = (TypeSerializer) nestedSerializers[0]; 55 | 56 | @SuppressWarnings("unchecked") 57 | TypeSerializer rightSerializer = (TypeSerializer) nestedSerializers[1]; 58 | 59 | return new EitherSerializer<>(leftSerializer, rightSerializer); 60 | } 61 | 62 | @Override 63 | protected TypeSerializer[] getNestedSerializers(EitherSerializer outerSerializer) { 64 | return new TypeSerializer[] { 65 | outerSerializer.getLeftSerializer(), outerSerializer.getRightSerializer() 66 | }; 67 | } 68 | } 69 | 70 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/java/org/apache/flinkx/api/serializer/ScalaOptionSerializerSnapshot.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.flinkx.api.serializer; 20 | 21 | import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; 22 | import org.apache.flink.api.common.typeutils.TypeSerializer; 23 | import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; 24 | 25 | import scala.Option; 26 | 27 | /** A {@link TypeSerializerSnapshot} for the Scala {@link OptionSerializer}. */ 28 | public final class ScalaOptionSerializerSnapshot 29 | extends CompositeTypeSerializerSnapshot, OptionSerializer> { 30 | 31 | private static final int VERSION = 2; 32 | 33 | @SuppressWarnings("WeakerAccess") 34 | public ScalaOptionSerializerSnapshot() { 35 | super(OptionSerializer.class); 36 | } 37 | 38 | public ScalaOptionSerializerSnapshot(OptionSerializer serializerInstance) { 39 | super(serializerInstance); 40 | } 41 | 42 | @Override 43 | protected int getCurrentOuterSnapshotVersion() { 44 | return VERSION; 45 | } 46 | 47 | @Override 48 | protected TypeSerializer[] getNestedSerializers(OptionSerializer outerSerializer) { 49 | return new TypeSerializer[] {outerSerializer.elemSerializer()}; 50 | } 51 | 52 | @Override 53 | protected OptionSerializer createOuterSerializerWithNestedSerializers( 54 | TypeSerializer[] nestedSerializers) { 55 | @SuppressWarnings("unchecked") 56 | TypeSerializer nestedSerializer = (TypeSerializer) nestedSerializers[0]; 57 | return new OptionSerializer<>(nestedSerializer); 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala-3/org/apache/flinkx/api/TypeTag.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import scala.quoted.* 4 | 5 | // A basic replacement for `TypeTag`, which is absent in Scala 3. 6 | trait TypeTag[A]: 7 | // Is the type a module, i.e. is it a case object? 8 | def isModule: Boolean 9 | def isCachable: Boolean 10 | def toString: String 11 | 12 | object TypeTag: 13 | def apply[A: TypeTag]: TypeTag[A] = summon 14 | 15 | // Fine to use `inline given` here, since usage is exclusive to Scala 3. 16 | inline given derived[A]: TypeTag[A] = ${ TypeTagMacro.gen[A] } 17 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala-3/org/apache/flinkx/api/TypeTagMacro.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api 2 | 3 | import scala.quoted.* 4 | 5 | // Simple macro to derive a TypeTag for any type. 6 | object TypeTagMacro: 7 | def gen[A: Type](using q: Quotes): Expr[TypeTag[A]] = 8 | import q.reflect.* 9 | 10 | def check(r: TypeRepr): Boolean = 11 | r match { 12 | case a: AppliedType => 13 | !a.args.exists { t => t.typeSymbol.isAbstractType } && a.args.forall { t => check(t) } 14 | case _ => true 15 | } 16 | 17 | val A = TypeRepr.of[A] 18 | val symA = A.typeSymbol 19 | val flagsA = symA.flags 20 | val isModuleExpr = Expr(flagsA.is(Flags.Module)) 21 | val isCachableExpr = Expr(check(A)) 22 | 23 | val toStringExpr = Expr(A.show) 24 | 25 | '{ 26 | new TypeTag[A]: 27 | override lazy val isModule: Boolean = ${ isModuleExpr } 28 | override lazy val isCachable: Boolean = ${ isCachableExpr } 29 | override lazy val toString: String = ${ toStringExpr } 30 | } 31 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala-3/org/apache/flinkx/api/serializer/ConstructorCompat.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import java.lang.reflect.Modifier 4 | 5 | import scala.util.control.NonFatal 6 | 7 | private[serializer] trait ConstructorCompat: 8 | // As of Scala version 3.1.2, there is no direct support for runtime reflection. 9 | // This is in contrast to Scala 2, which has its own APIs for reflecting on classes. 10 | // Thus, fallback to Java reflection and look up the constructor matching the required signature. 11 | final def lookupConstructor[T](cls: Class[T], numFields: Int): Array[AnyRef] => T = 12 | // Types of parameters can fail to match when (un)boxing is used. 13 | // Say you have a class `final case class Foo(a: String, b: Int)`. 14 | // The first parameter is an alias for `java.lang.String`, which the constructor uses. 15 | // The second parameter is an alias for `java.lang.Integer`, but the constructor actually takes an unboxed `int`. 16 | val constructor = 17 | try cls.getConstructors.collectFirst { case c if c.getParameterCount == numFields => c }.get 18 | catch 19 | case NonFatal(e) => 20 | throw new IllegalArgumentException( 21 | s""" 22 | |The class ${cls.getSimpleName} does not have a matching constructor. 23 | |It could be an instance class, meaning it is not a member of a 24 | |toplevel object, or of an object contained in a toplevel object, 25 | |therefore it requires an outer instance to be instantiated, but we don't have a 26 | |reference to the outer instance. Please consider changing the outer class to an object. 27 | |""".stripMargin, 28 | e 29 | ) 30 | 31 | { (args: Array[AnyRef]) => 32 | { 33 | lazy val defaultArgs = cls 34 | .getMethods() 35 | .filter( 36 | _.getName() 37 | .startsWith("$lessinit$greater$default") 38 | ) 39 | .sortBy(_.getName()) 40 | .map(_.invoke(null)) 41 | .takeRight(numFields - args.length) // read default values for missing arguments 42 | 43 | val allArgs = args.toList ++ (if (args.length == numFields) Nil else defaultArgs) 44 | constructor.newInstance(allArgs*).asInstanceOf[T] 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala/org/apache/flinkx/api/serializer/ImmutableSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | abstract class ImmutableSerializer[T] extends TypeSerializerSingleton[T] { 7 | override def isImmutableType: Boolean = true 8 | override def copy(from: T): T = from 9 | override def copy(from: T, reuse: T): T = from 10 | override def deserialize(reuse: T, source: DataInputView): T = deserialize(source) 11 | override def copy(source: DataInputView, target: DataOutputView): Unit = serialize(deserialize(source), target) 12 | } 13 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala/org/apache/flinkx/api/serializer/MutableSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton 4 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 5 | 6 | abstract class MutableSerializer[T] extends TypeSerializerSingleton[T] { 7 | override def isImmutableType: Boolean = false 8 | override def copy(from: T, reuse: T): T = copy(from) 9 | override def deserialize(reuse: T, source: DataInputView): T = deserialize(source) 10 | override def copy(source: DataInputView, target: DataOutputView): Unit = serialize(deserialize(source), target) 11 | } 12 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala/org/apache/flinkx/api/serializer/UnitSerializer.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.serializer 2 | 3 | import org.apache.flinkx.api.serializer.UnitSerializer.UnitSerializerSnapshot 4 | import org.apache.flink.api.common.typeutils.{SimpleTypeSerializerSnapshot, TypeSerializer, TypeSerializerSnapshot} 5 | import org.apache.flink.core.memory.{DataInputView, DataOutputView} 6 | 7 | import java.util.function.Supplier 8 | 9 | class UnitSerializer extends ImmutableSerializer[Unit] { 10 | override def getLength: Int = 0 11 | 12 | override def serialize(record: Unit, target: DataOutputView): Unit = {} 13 | 14 | override def deserialize(reuse: Unit, source: DataInputView): Unit = {} 15 | 16 | override def deserialize(source: DataInputView): Unit = {} 17 | 18 | override def snapshotConfiguration(): TypeSerializerSnapshot[Unit] = new UnitSerializerSnapshot() 19 | 20 | override def createInstance(): Unit = {} 21 | } 22 | 23 | object UnitSerializer { 24 | class UnitSerializerSnapshot 25 | extends SimpleTypeSerializerSnapshot[Unit]( 26 | new Supplier[TypeSerializer[Unit]] { 27 | override def get(): TypeSerializer[Unit] = new UnitSerializer 28 | } 29 | ) 30 | } 31 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala/org/apache/flinkx/api/typeinfo/FailFastTypeInfoFactory.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.typeinfo 2 | 3 | import org.apache.flink.api.common.typeinfo.{TypeInfoFactory, TypeInformation} 4 | import org.apache.flink.util.FlinkRuntimeException 5 | import org.apache.flinkx.api.typeinfo.FailFastTypeInfoFactory.formatType 6 | 7 | import java.lang.reflect.Type 8 | import java.util 9 | import scala.jdk.CollectionConverters._ 10 | 11 | class FailFastTypeInfoFactory extends TypeInfoFactory[Nothing] { 12 | 13 | override def createTypeInfo(t: Type, params: util.Map[String, TypeInformation[_]]): TypeInformation[Nothing] = 14 | throw new FlinkRuntimeException( 15 | s"""You are using a 'Class' to resolve '${formatType( 16 | t, 17 | params 18 | )}' Scala type. flink-scala-api has no control over this kind of type resolution which may lead to silently fallback to generic Kryo serializers. 19 | |Use type information instead: import 'org.apache.flinkx.api.serializers._' to make implicitly available in the scope required 'TypeInformation' to resolve Scala types. 20 | |To disable this check, set 'DISABLE_FAIL_FAST_ON_SCALA_TYPE_RESOLUTION_WITH_CLASS' environment variable to 'true'.""".stripMargin 21 | ) 22 | 23 | } 24 | 25 | object FailFastTypeInfoFactory { 26 | 27 | private def formatType(t: Type, params: util.Map[String, TypeInformation[_]]): String = if (params.isEmpty) { 28 | t.getTypeName 29 | } else { 30 | params.keySet().asScala.mkString(s"${t.getTypeName}[", ", ", "]") 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/main/scala/org/apache/flinkx/api/util/ClassUtil.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.util 2 | 3 | import java.lang.reflect.{Field, Modifier} 4 | 5 | object ClassUtil { 6 | 7 | def isFieldFinal(fields: Array[Field], className: String, fieldName: String): Boolean = 8 | Modifier.isFinal( 9 | fields 10 | .find(f => f.getName == fieldName) 11 | .orElse(fields.find(f => f.getName == s"${className.replace('.', '$')}$$$$$fieldName")) 12 | .getOrElse(throw new NoSuchFieldException(fieldName)) // Same as Class.getDeclaredField 13 | .getModifiers 14 | ) 15 | 16 | } 17 | -------------------------------------------------------------------------------- /modules/flink-common-api/src/test/scala/org/apache/flinkx/api/util/ClassUtilTest.scala: -------------------------------------------------------------------------------- 1 | package org.apache.flinkx.api.util 2 | 3 | import org.apache.flinkx.api.util.ClassUtilTest._ 4 | import org.scalatest.flatspec.AnyFlatSpec 5 | import org.scalatest.matchers.should.Matchers 6 | 7 | class ClassUtilTest extends AnyFlatSpec with Matchers { 8 | 9 | it should "return true when the field is a val" in { 10 | val aFinal = classOf[Final] 11 | ClassUtil.isFieldFinal(aFinal.getDeclaredFields, aFinal.getName, "a") shouldBe true 12 | } 13 | 14 | it should "return false when the field is a var" in { 15 | val aNonFinal = classOf[NonFinal] 16 | ClassUtil.isFieldFinal(aNonFinal.getDeclaredFields, aNonFinal.getName, "a") shouldBe false 17 | } 18 | 19 | it should "return true when the field is a private val" in { 20 | val aPrivateFinal = classOf[PrivateFinal] 21 | ClassUtil.isFieldFinal(aPrivateFinal.getDeclaredFields, aPrivateFinal.getName, "a") shouldBe true 22 | } 23 | 24 | it should "return false when the field is a private var" in { 25 | val aPrivateNonFinal = classOf[PrivateNonFinal] 26 | ClassUtil.isFieldFinal(aPrivateNonFinal.getDeclaredFields, aPrivateNonFinal.getName, "a") shouldBe false 27 | } 28 | 29 | it should "return true when the field is a disrupted private val" in { 30 | val aDisruptedPrivateFinal = classOf[DisruptedPrivateFinal] 31 | ClassUtil.isFieldFinal(aDisruptedPrivateFinal.getDeclaredFields, aDisruptedPrivateFinal.getName, "a") shouldBe true 32 | } 33 | 34 | it should "return false when the field is a disrupted private var" in { 35 | val aDisruptedPrivateNonFinal = classOf[DisruptedPrivateNonFinal] 36 | ClassUtil.isFieldFinal( 37 | aDisruptedPrivateNonFinal.getDeclaredFields, 38 | aDisruptedPrivateNonFinal.getName, 39 | "a" 40 | ) shouldBe false 41 | } 42 | 43 | it should "throw NoSuchFieldException when the field doesn't exist" in { 44 | val aFinal = classOf[Final] 45 | assertThrows[NoSuchFieldException] { 46 | ClassUtil.isFieldFinal(aFinal.getDeclaredFields, aFinal.getName, "wrongField") shouldBe true 47 | } 48 | } 49 | 50 | } 51 | 52 | object ClassUtilTest { 53 | 54 | case class Final(a: String) 55 | case class NonFinal(var a: String) 56 | case class PrivateFinal(private val a: String) 57 | case class PrivateNonFinal(private var a: String) 58 | object DisruptiveObject { 59 | def apply(value: Int): DisruptedPrivateFinal = DisruptedPrivateFinal(String.valueOf(value)) 60 | def apply(value: Long): DisruptedPrivateNonFinal = DisruptedPrivateNonFinal(String.valueOf(value)) 61 | } 62 | case class DisruptedPrivateFinal(private val a: String) 63 | case class DisruptedPrivateNonFinal(private var a: String) 64 | } 65 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.11.1 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") 2 | addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") 3 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4") 4 | addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.7.1") 5 | addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") 6 | addSbtPlugin("com.github.sbt" % "sbt-git" % "2.1.0") 7 | addSbtPlugin("com.github.sbt" % "sbt-protobuf" % "0.8.2") 8 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | RELEASE_VERSION_BUMP=true sbt -Dgpg.passphrase=$GPG_PASSPHRASE test "; release with-defaults" 4 | wait -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / version := "1.2.8-SNAPSHOT" 2 | --------------------------------------------------------------------------------