├── .gitignore
├── .scalafmt.conf
├── .travis.yml
├── CHANGELOG.md
├── LICENSE
├── README.md
├── build.sbt
├── flink-jpmml-assets
└── src
│ └── main
│ └── resources
│ ├── kmeans.xml
│ ├── kmeans32.xml
│ ├── kmeans40.xml
│ ├── kmeans41.xml
│ ├── kmeans42.xml
│ ├── kmeans_empty.xml
│ ├── kmeans_nooutput.xml
│ ├── kmeans_nooutput_notarget.xml
│ └── kmeans_stringfields.xml
├── flink-jpmml-examples
├── README.md
└── src
│ └── main
│ └── scala
│ └── io
│ └── radicalbit
│ └── examples
│ ├── CheckpointEvaluate.scala
│ ├── DynamicEvaluateKmeans.scala
│ ├── EvaluateKmeans.scala
│ ├── QuickEvaluateKmeans.scala
│ ├── model
│ ├── Iris.scala
│ └── Utils.scala
│ ├── sources
│ ├── ControlSource.scala
│ ├── FiniteSource.scala
│ ├── InfiniteSource.scala
│ └── IrisSource.scala
│ └── util
│ ├── DynamicParams.scala
│ └── EnsureParameters.scala
├── flink-jpmml-scala
└── src
│ ├── main
│ ├── resources
│ │ └── log4j.properties
│ └── scala
│ │ └── io
│ │ └── radicalbit
│ │ └── flink
│ │ └── pmml
│ │ └── scala
│ │ ├── api
│ │ ├── Evaluator.scala
│ │ ├── PmmlModel.scala
│ │ ├── converter
│ │ │ └── VectorConverter.scala
│ │ ├── exceptions
│ │ │ └── package.scala
│ │ ├── functions
│ │ │ ├── EvaluationCoFunction.scala
│ │ │ └── EvaluationFunction.scala
│ │ ├── managers
│ │ │ ├── MetadataManager.scala
│ │ │ └── ModelsManager.scala
│ │ ├── package.scala
│ │ ├── pipeline
│ │ │ └── Pipeline.scala
│ │ └── reader
│ │ │ ├── FsReader.scala
│ │ │ └── ModelReader.scala
│ │ ├── logging
│ │ └── LazyLogging.scala
│ │ ├── models
│ │ ├── control
│ │ │ └── ServingMessage.scala
│ │ ├── core
│ │ │ ├── ModelId.scala
│ │ │ └── ModelInfo.scala
│ │ ├── input
│ │ │ └── BaseEvent.scala
│ │ ├── prediction
│ │ │ ├── Prediction.scala
│ │ │ └── Target.scala
│ │ └── state
│ │ │ └── CheckpointType.scala
│ │ └── package.scala
│ └── test
│ ├── resources
│ └── log4j.properties
│ └── scala
│ └── io
│ └── radicalbit
│ └── flink
│ └── pmml
│ └── scala
│ ├── QuickDataStreamSpec.scala
│ ├── RichConnectedStreamSpec.scala
│ ├── RichDataStreamSpec.scala
│ ├── api
│ ├── EvaluatorSpec.scala
│ ├── PmmlModelSpec.scala
│ ├── converter
│ │ └── VectorConverterSpec.scala
│ ├── functions
│ │ ├── EvaluationCoFunctionSpec.scala
│ │ └── EvaluationFunctionSpec.scala
│ ├── managers
│ │ ├── MetadataManagerSpec.scala
│ │ └── ModelsManagerSpec.scala
│ └── reader
│ │ └── ModelReaderSpec.scala
│ ├── models
│ ├── core
│ │ └── ModelIdSpec.scala
│ └── prediction
│ │ ├── PredictionSpec.scala
│ │ └── TargetSpec.scala
│ ├── sources
│ └── TemporizedSourceFunction.scala
│ └── utils
│ ├── FlinkTestKits.scala
│ ├── PmmlEvaluatorKit.scala
│ ├── PmmlLoaderKit.scala
│ └── models
│ └── Input.scala
├── idea.sbt
├── images
└── architecture.png
├── project
├── Commons.scala
├── Dependencies.scala
├── LicenseSetting.scala
├── PublishSettings.scala
├── assembly.sbt
├── build.properties
├── plugins.sbt
└── unidoc.sbt
└── version.sbt
/.gitignore:
--------------------------------------------------------------------------------
1 | /*.iml
2 | /flink-jpmml-handson.iml
3 | ### Java template
4 |
5 | # OSx files
6 | *.DS_Store
7 |
8 | # Mobile Tools for Java (J2ME)
9 | .mtj.tmp/
10 |
11 | # Package Files #
12 | *.jar
13 | *.war
14 | *.ear
15 |
16 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
17 | hs_err_pid*
18 | ### Scala template
19 | *.log
20 |
21 | # sbt specific
22 | .cache
23 | .history
24 | .lib/
25 | dist/*
26 | **/target
27 |
28 | lib_managed/
29 | src_managed/
30 | project/boot/
31 | project/plugins/project/
32 | project/project/
33 |
34 |
35 | # logger backend
36 | flink-jpmml-scala/src/test/resources/
37 | flink-jpmml-scala/src/main/resources/
38 |
39 | mainRunner
40 | # Scala-IDE specific
41 | .scala_dependencies
42 | .worksheet
43 |
44 | # IntelliJ specific
45 | .idea
--------------------------------------------------------------------------------
/.scalafmt.conf:
--------------------------------------------------------------------------------
1 | maxColumn = 120
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: scala
2 | scala:
3 | - 2.10.6
4 | - 2.11.11
5 | jdk:
6 | - oraclejdk8
7 | script:
8 | - sbt clean coverage test coverageReport coverageAggregate
9 | after_success:
10 | - bash <(curl -s https://codecov.io/bash)
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Change Log
2 |
3 | ## [Unreleased](https://github.com/FlinkML/flink-jpmml/tree/HEAD)
4 |
5 | [Full Changelog](https://github.com/FlinkML/flink-jpmml/compare/v0.6.1...HEAD)
6 |
7 | **Implemented enhancements:**
8 |
9 | - Publish the 0.6.1 version on a public repo [\#3](https://github.com/FlinkML/flink-jpmml/issues/3)
10 |
11 | ## [v0.6.1](https://github.com/FlinkML/flink-jpmml/tree/v0.6.1) (2017-09-22)
12 | [Full Changelog](https://github.com/FlinkML/flink-jpmml/compare/v0.6.0...v0.6.1)
13 |
14 | **Closed issues:**
15 |
16 | - Adding Changelog file [\#42](https://github.com/FlinkML/flink-jpmml/issues/42)
17 | - Releasing flink-jpmml 0.6.0 [\#30](https://github.com/FlinkML/flink-jpmml/issues/30)
18 |
19 | **Merged pull requests:**
20 |
21 | - Update publish setting for sonatype [\#44](https://github.com/FlinkML/flink-jpmml/pull/44) ([francescofrontera](https://github.com/francescofrontera))
22 | - Adding Changelog file [\#43](https://github.com/FlinkML/flink-jpmml/pull/43) ([spi-x-i](https://github.com/spi-x-i))
23 |
24 | ## [v0.6.0](https://github.com/FlinkML/flink-jpmml/tree/v0.6.0) (2017-09-21)
25 | [Full Changelog](https://github.com/FlinkML/flink-jpmml/compare/v0.5.1...v0.6.0)
26 |
27 | **Implemented enhancements:**
28 |
29 | - Move all the exceptions in a separate package [\#35](https://github.com/FlinkML/flink-jpmml/issues/35)
30 | - Introducing updated flink-jpmml examples [\#28](https://github.com/FlinkML/flink-jpmml/issues/28)
31 | - Updating flink-jpmml project dependencies [\#19](https://github.com/FlinkML/flink-jpmml/issues/19)
32 |
33 | **Fixed bugs:**
34 |
35 | - Updating README to version 0.6 [\#29](https://github.com/FlinkML/flink-jpmml/issues/29)
36 |
37 | **Closed issues:**
38 |
39 | - Introducing Dynamic Model Evaluation [\#27](https://github.com/FlinkML/flink-jpmml/issues/27)
40 | - Extending evaluator ADT with Empty predictions [\#26](https://github.com/FlinkML/flink-jpmml/issues/26)
41 | - Releasing 0.5.1 version [\#24](https://github.com/FlinkML/flink-jpmml/issues/24)
42 |
43 | **Merged pull requests:**
44 |
45 | - \[\#29\] Update readme to 0.6.0 [\#41](https://github.com/FlinkML/flink-jpmml/pull/41) ([spi-x-i](https://github.com/spi-x-i))
46 | - \[\#28\] Updated flink jpmml examples [\#39](https://github.com/FlinkML/flink-jpmml/pull/39) ([francescofrontera](https://github.com/francescofrontera))
47 | - Feature/27 introducing dynamic model evaluation [\#38](https://github.com/FlinkML/flink-jpmml/pull/38) ([riccardo14](https://github.com/riccardo14))
48 | - Enhancement/35 move all the exceptions in a separate package [\#36](https://github.com/FlinkML/flink-jpmml/pull/36) ([riccardo14](https://github.com/riccardo14))
49 | - Feature/26 extending evaluator adt with empty predictions [\#34](https://github.com/FlinkML/flink-jpmml/pull/34) ([riccardo14](https://github.com/riccardo14))
50 | - Update the version of all the dependencies [\#33](https://github.com/FlinkML/flink-jpmml/pull/33) ([riccardo14](https://github.com/riccardo14))
51 |
52 | ## [v0.5.1](https://github.com/FlinkML/flink-jpmml/tree/v0.5.1) (2017-09-19)
53 | [Full Changelog](https://github.com/FlinkML/flink-jpmml/compare/v0.5.0...v0.5.1)
54 |
55 | **Implemented enhancements:**
56 |
57 | - Updating Flink to 1.3.1 [\#17](https://github.com/FlinkML/flink-jpmml/issues/17)
58 | - Integrate Travis CI [\#2](https://github.com/FlinkML/flink-jpmml/issues/2)
59 | - Bump master to 0.6.0-SNAPSHOT [\#1](https://github.com/FlinkML/flink-jpmml/issues/1)
60 | - Added license apply informations [\#21](https://github.com/FlinkML/flink-jpmml/pull/21) ([maocorte](https://github.com/maocorte))
61 |
62 | **Closed issues:**
63 |
64 | - Avoiding boilerplate to Vector handler Type Class [\#22](https://github.com/FlinkML/flink-jpmml/issues/22)
65 | - Fixing flink-jpmml-examples README file [\#13](https://github.com/FlinkML/flink-jpmml/issues/13)
66 |
67 | **Merged pull requests:**
68 |
69 | - Updating to new minor version [\#25](https://github.com/FlinkML/flink-jpmml/pull/25) ([spi-x-i](https://github.com/spi-x-i))
70 | - \#22 - Avoiding boilerplate to Vector handler Type Class [\#23](https://github.com/FlinkML/flink-jpmml/pull/23) ([francescofrontera](https://github.com/francescofrontera))
71 | - Updating flink version to 1.3.1 [\#20](https://github.com/FlinkML/flink-jpmml/pull/20) ([spi-x-i](https://github.com/spi-x-i))
72 | - Updated Travis CI build badge link [\#18](https://github.com/FlinkML/flink-jpmml/pull/18) ([maocorte](https://github.com/maocorte))
73 | - Cleaned travis-ci configuration file [\#16](https://github.com/FlinkML/flink-jpmml/pull/16) ([maocorte](https://github.com/maocorte))
74 | - Fixing explanation about model loading phase [\#15](https://github.com/FlinkML/flink-jpmml/pull/15) ([spi-x-i](https://github.com/spi-x-i))
75 | - \[\#13\]fix Readme examples [\#14](https://github.com/FlinkML/flink-jpmml/pull/14) ([francescofrontera](https://github.com/francescofrontera))
76 | - Updating Flink version to 1.3.0 [\#12](https://github.com/FlinkML/flink-jpmml/pull/12) ([spi-x-i](https://github.com/spi-x-i))
77 | - \[hotfix\] Add author info for @stefanobaghino [\#11](https://github.com/FlinkML/flink-jpmml/pull/11) ([stefanobaghino](https://github.com/stefanobaghino))
78 | - Fixing sbt-header [\#10](https://github.com/FlinkML/flink-jpmml/pull/10) ([francescofrontera](https://github.com/francescofrontera))
79 | - Added travis ci build status to README [\#8](https://github.com/FlinkML/flink-jpmml/pull/8) ([maocorte](https://github.com/maocorte))
80 | - Added github usernames [\#7](https://github.com/FlinkML/flink-jpmml/pull/7) ([francescofrontera](https://github.com/francescofrontera))
81 | - \[\#1\] - Update Version [\#6](https://github.com/FlinkML/flink-jpmml/pull/6) ([francescofrontera](https://github.com/francescofrontera))
82 | - \[ISSUE-2\] added travis configuration file [\#5](https://github.com/FlinkML/flink-jpmml/pull/5) ([maocorte](https://github.com/maocorte))
83 | - added simone robutti contribution [\#4](https://github.com/FlinkML/flink-jpmml/pull/4) ([chobeat](https://github.com/chobeat))
84 |
85 | ## [v0.5.0](https://github.com/FlinkML/flink-jpmml/tree/v0.5.0) (2017-05-25)
86 |
87 |
88 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*
--------------------------------------------------------------------------------
/build.sbt:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | resolvers in ThisBuild ++= Seq(
23 | "Apache Development Snapshot Repository" at "https://repository.apache.org/content/repositories/snapshots/",
24 | Resolver.mavenLocal
25 | )
26 |
27 | lazy val noPublishSetting = Seq(
28 | publish := {},
29 | publishLocal := {},
30 | publishArtifact := false
31 | )
32 |
33 | lazy val root = project
34 | .in(file("."))
35 | .settings(noPublishSetting)
36 | .enablePlugins(ScalaUnidocPlugin)
37 | .settings(
38 | name := "flink-jpmml",
39 | crossScalaVersions := Seq("2.10.6", "2.11.11"),
40 | unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(`flink-jpmml-examples`,
41 | `flink-jpmml-assets`)
42 | )
43 | .aggregate(`flink-jpmml-examples`, `flink-jpmml-scala`, `flink-jpmml-assets`)
44 |
45 | lazy val `flink-jpmml-assets` = project
46 | .enablePlugins(AutomateHeaderPlugin)
47 | .settings(LicenseSetting.settings: _*)
48 | .settings(Commons.settings: _*)
49 | .settings(PublishSettings.settings: _*)
50 |
51 | lazy val `flink-jpmml-examples` = project
52 | .enablePlugins(AutomateHeaderPlugin)
53 | .settings(LicenseSetting.settings: _*)
54 | .settings(Commons.settings: _*)
55 | .settings(PublishSettings.settings: _*)
56 | .settings(libraryDependencies ++= Dependencies.Examples.libraries)
57 | .dependsOn(`flink-jpmml-scala`)
58 |
59 | lazy val `flink-jpmml-scala` = project
60 | .enablePlugins(AutomateHeaderPlugin)
61 | .settings(LicenseSetting.settings: _*)
62 | .settings(Commons.settings: _*)
63 | .settings(PublishSettings.settings: _*)
64 | .settings(libraryDependencies ++= Dependencies.Scala.libraries)
65 | .dependsOn(`flink-jpmml-assets`)
66 |
67 | onLoad in Global := (Command.process("scalafmt", _: State)) compose (onLoad in Global).value
68 |
69 | // make run command include the provided dependencies
70 | run in Compile := Defaults.runTask(fullClasspath in Compile, mainClass in (Compile, run), runner in (Compile, run))
71 |
72 | // exclude Scala library from assembly
73 | assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
74 |
75 | // assign default options to JUnit test execution
76 | testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v")
77 |
78 | fork in test := false
79 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 | 6.9125000000000005 3.099999999999999 5.846874999999999 2.1312499999999996
60 |
61 |
62 | 6.23658536585366 2.8585365853658535 4.807317073170731 1.6219512195121943
63 |
64 |
65 | 5.005999999999999 3.4180000000000006 1.464 0.2439999999999999
66 |
67 |
68 | 5.529629629629629 2.6222222222222222 3.940740740740741 1.2185185185185188
69 |
70 |
73 |
74 |
75 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans32.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
22 |
23 |
24 | 2012-09-27 13:19:09
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 | 6.8538461538461535 3.076923076923076 5.715384615384614 2.0538461538461537
54 |
55 |
56 | 5.883606557377049 2.740983606557377 4.388524590163936 1.4344262295081966
57 |
58 |
59 | 5.005999999999999 3.4180000000000006 1.4640000000000002 0.2439999999999999
60 |
61 |
64 |
65 |
66 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans40.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 | 6.8538461538461535 3.076923076923076 5.715384615384614 2.0538461538461537
60 |
61 |
62 | 5.883606557377049 2.740983606557377 4.388524590163936 1.4344262295081966
63 |
64 |
65 | 5.005999999999999 3.4180000000000006 1.4640000000000002 0.2439999999999999
66 |
67 |
70 |
71 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans41.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 | 6.9125000000000005 3.099999999999999 5.846874999999999 2.1312499999999996
60 |
61 |
62 | 6.23658536585366 2.8585365853658535 4.807317073170731 1.6219512195121943
63 |
64 |
65 | 5.005999999999999 3.4180000000000006 1.464 0.2439999999999999
66 |
67 |
68 | 5.529629629629629 2.6222222222222222 3.940740740740741 1.2185185185185188
69 |
70 |
73 |
74 |
75 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans42.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
22 |
23 | 2015-04-28T07:25:30
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 | 6.8538461538461535 3.076923076923076 5.715384615384614 2.0538461538461537
53 |
54 |
55 | 5.883606557377049 2.740983606557377 4.388524590163936 1.4344262295081966
56 |
57 |
58 | 5.005999999999999 3.4180000000000006 1.4640000000000002 0.2439999999999999
59 |
60 |
63 |
64 |
65 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans_empty.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans_nooutput.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 | 6.9125000000000005 3.099999999999999 5.846874999999999 2.1312499999999996
60 |
61 |
62 | 6.23658536585366 2.8585365853658535 4.807317073170731 1.6219512195121943
63 |
64 |
65 | 5.005999999999999 3.4180000000000006 1.464 0.2439999999999999
66 |
67 |
68 | 5.529629629629629 2.6222222222222222 3.940740740740741 1.2185185185185188
69 |
70 |
71 |
72 |
73 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans_nooutput_notarget.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 | 6.9125000000000005 3.099999999999999 5.846874999999999 2.1312499999999996
59 |
60 |
61 | 6.23658536585366 2.8585365853658535 4.807317073170731 1.6219512195121943
62 |
63 |
64 | 5.005999999999999 3.4180000000000006 1.464 0.2439999999999999
65 |
66 |
67 | 5.529629629629629 2.6222222222222222 3.940740740740741 1.2185185185185188
68 |
69 |
70 |
71 |
72 |
--------------------------------------------------------------------------------
/flink-jpmml-assets/src/main/resources/kmeans_stringfields.xml:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 | 6.9125000000000005 3.099999999999999 5.846874999999999 2.1312499999999996
60 |
61 |
62 | 6.23658536585366 2.8585365853658535 4.807317073170731 1.6219512195121943
63 |
64 |
65 | 5.005999999999999 3.4180000000000006 1.464 0.2439999999999999
66 |
67 |
68 | 5.529629629629629 2.6222222222222222 3.940740740740741 1.2185185185185188
69 |
70 |
73 |
74 |
75 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/README.md:
--------------------------------------------------------------------------------
1 | ## Running simple examples
2 | This module contains the examples running simple predictions from an iris Source.
3 | The source emits the following data:
4 | ```
5 | Iris(sepalLength: Double, sepalWidth: Double, petalLength: Double, petalWidth: Double)
6 | ```
7 | In order to keep the example run
8 |
9 | 1) Run `sbt` command in project dir and select project:
10 | ```
11 | project flink-jpmml-handson
12 | ```
13 |
14 | 2) Create a `.jar`:
15 | ```
16 | assembly
17 | ```
18 |
19 | 3) Run the examples. If you want full predictions:
20 | ```
21 | ./path/to/bin/flink run -c io.radicalbit.examples.EvaluateKmeans /flink-jpmml/flink-jpmml-examples/target/scala-2.x/flink-jpmml-examples-assembly-0.7.0-SNAPSHOT.jar --model path/to/pmml/model.pmml --output /path/to/output
22 | ```
23 | Either you can employ the _quick_ predictor:
24 | ```
25 | ./path/to/bin/flink run -c io.radicalbit.examples.QuickEvaluateKmeans /flink-jpmml/flink-jpmml-examples/target/scala-2.x/flink-jpmml-examples-assembly-0.7.0-SNAPSHOT.jar --model path/to/pmml/model.pmml --output /path/to/output
26 | ```
27 |
28 | ## Fault-Tolerance
29 |
30 | _if you like testing the fault-tolerance behaviour of the operator you can run a `CheckpointEvaluate` example._
31 |
32 | In order to do that:
33 |
34 | 1) Create a socket in your local machine:
35 | ```
36 | nc -l -k 9999
37 | ```
38 |
39 | 2) Run the flink-cluster( [Flink 1.3.2](http://flink.apache.org/downloads.html#binaries) is required ):
40 | ```
41 | ./path/to/flink-1.3.2/start-cluster.sh
42 | ```
43 |
44 | 3) run the flink job:
45 | ```
46 | ./path/to/bin/flink run -c io.radicalbit.examples.CheckpointEvaluate /flink-jpmml/flink-jpmml-examples/target/scala-2.x/flink-jpmml-examples-assembly-0.7.0-SNAPSHOT.jar --output /path/to/output
47 | ```
48 |
49 | 4) Send the model via socket, in this case you can use the models in `flink-jpmml-assets`:
50 | ```
51 | /flink-jpmml/flink-jpmml-assets/resources/kmeans.xml
52 | /flink-jpmml/flink-jpmml-assets/resources/kmeans_nooutput.xml
53 |
54 | ```
55 |
56 | 6) Stop the task manager:
57 | ```
58 | ./path/to/flink-1.3.2/bin/taskmanager.sh stop
59 | ```
60 | _you can see the job's status in Flink UI on http://localhost:8081_
61 |
62 |
63 | 7) Restart the task manager:
64 | ```
65 | ./path/to/flink-1.3.2/bin/taskmanager.sh start
66 | ```
67 |
68 | _At this point, when the job is restarted, there's no need to re-send the models info by control stream because you should see the models from the last checkpoint_
69 |
70 |
71 |
72 | Note: Both above jobs log out predictions to output path.
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/CheckpointEvaluate.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples
21 |
22 | import io.radicalbit.examples.model.Utils
23 | import io.radicalbit.examples.models.Iris
24 | import io.radicalbit.examples.util.DynamicParams
25 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
26 | import io.radicalbit.flink.pmml.scala.models.control.AddMessage
27 | import io.radicalbit.flink.pmml.scala.models.core.ModelId
28 | import org.apache.flink.api.java.utils.ParameterTool
29 | import org.apache.flink.core.fs.FileSystem.WriteMode
30 | import org.apache.flink.streaming.api.CheckpointingMode
31 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
32 | import org.apache.flink.streaming.api.scala._
33 |
34 | import scala.util.Random
35 |
36 | object CheckpointEvaluate {
37 |
38 | private final lazy val idSet = Set(
39 | "4897c9f4-5226-43c7-8f2d-f9fd388cf2bc",
40 | "5f919c52-2ef8-4ff2-94b2-2e64bb85005e"
41 | )
42 |
43 | def main(args: Array[String]): Unit = {
44 | val parameterTool = ParameterTool.fromArgs(args)
45 |
46 | val outputPath = parameterTool.getRequired("output")
47 |
48 | val env = StreamExecutionEnvironment.getExecutionEnvironment
49 |
50 | val parameters = DynamicParams.fromParameterTool(parameterTool)
51 |
52 | //Enable checkpoint to keep control stream
53 | env.enableCheckpointing(parameters.ckpInterval, CheckpointingMode.EXACTLY_ONCE)
54 |
55 | //Create source for Iris data
56 | val eventStream = env.addSource((sc: SourceContext[Iris]) => {
57 | val NumberOfParameters = 4
58 | lazy val RandomGenerator = scala.util.Random
59 | val RandomMin = 0.2
60 | val RandomMax = 6.0
61 |
62 | @inline def truncateDouble(n: Double) = (math floor n * 10) / 10
63 |
64 | while (true) {
65 | def randomVal = RandomMin + (RandomMax - RandomMin) * RandomGenerator.nextDouble()
66 | val dataForIris = Seq.fill(NumberOfParameters)(truncateDouble(randomVal))
67 | val iris =
68 | Iris(idSet.toVector(Random.nextInt(idSet.size)) + ModelId.separatorSymbol + "1",
69 | dataForIris.head,
70 | dataForIris(1),
71 | dataForIris(2),
72 | dataForIris.last,
73 | Utils.now())
74 | sc.collect(iris)
75 | Thread.sleep(1000)
76 | }
77 | })
78 |
79 | //Create a stream for socket
80 | val controlStream = env
81 | .socketTextStream("localhost", 9999)
82 | .map(path => AddMessage(idSet.toVector(Random.nextInt(idSet.size)), 1L, path, Utils.now()))
83 |
84 | /*
85 | * Make a prediction withSupportStream that represents the stream from the socket
86 | * evaluate the model with model upload in ControlStream
87 | *
88 | * */
89 | val predictions = eventStream
90 | .withSupportStream(controlStream)
91 | .evaluate { (event: Iris, model: PmmlModel) =>
92 | val vectorized = event.toVector
93 | val prediction = model.predict(vectorized, Some(0.0))
94 | (event, prediction.value)
95 | }
96 |
97 | predictions
98 | .writeAsText(outputPath, WriteMode.OVERWRITE)
99 |
100 | env.execute("Checkpoint Evaluate Example")
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/DynamicEvaluateKmeans.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples
21 |
22 | import io.radicalbit.examples.models.Iris
23 | import io.radicalbit.examples.sources.{ControlSource, IrisSource}
24 | import io.radicalbit.examples.util.DynamicParams
25 | import io.radicalbit.flink.pmml.scala._
26 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
27 | import org.apache.flink.api.java.utils.ParameterTool
28 | import org.apache.flink.streaming.api.CheckpointingMode
29 | import org.apache.flink.streaming.api.scala._
30 |
31 | /**
32 | * Toy Job about Stateful Dynamic Model Serving:
33 | * The job owns two input streams:
34 | * - event stream: The main input events stream
35 | * - control stream: Control messages about model repository server current state
36 | *
37 | */
38 | object DynamicEvaluateKmeans {
39 |
40 | def main(args: Array[String]): Unit = {
41 |
42 | val parameterTool = ParameterTool.fromArgs(args)
43 |
44 | val parameters = DynamicParams.fromParameterTool(parameterTool)
45 |
46 | val env = StreamExecutionEnvironment.getExecutionEnvironment
47 |
48 | env.enableCheckpointing(parameters.ckpInterval, CheckpointingMode.EXACTLY_ONCE)
49 |
50 | val eventStream = IrisSource.irisSource(env, Option(parameters.availableIds))
51 | val controlStream =
52 | ControlSource.generateStream(env, parameters.genPolicy, parameters.pathAndIds, parameters.ctrlGenInterval)
53 |
54 | val predictions = eventStream
55 | .withSupportStream(controlStream)
56 | .evaluate { (event: Iris, model: PmmlModel) =>
57 | val vectorized = event.toVector
58 | val prediction = model.predict(vectorized, Some(0.0))
59 | (event, prediction.value)
60 | }
61 |
62 | predictions.writeAsText(parameters.outputPath)
63 |
64 | env.execute("Dynamic Clustering Example")
65 | }
66 |
67 | }
68 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/EvaluateKmeans.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples
21 |
22 | import io.radicalbit.examples.sources.IrisSource._
23 | import io.radicalbit.examples.util.EnsureParameters
24 | import io.radicalbit.flink.pmml.scala._
25 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
26 | import org.apache.flink.api.java.utils.ParameterTool
27 | import org.apache.flink.streaming.api.scala._
28 |
29 | object EvaluateKmeans extends EnsureParameters {
30 |
31 | def main(args: Array[String]): Unit = {
32 | val params: ParameterTool = ParameterTool.fromArgs(args)
33 | implicit val env = StreamExecutionEnvironment.getExecutionEnvironment
34 |
35 | env.getConfig.setGlobalJobParameters(params)
36 | val (inputModel, output) = ensureParams(params)
37 |
38 | //Read data from custom iris source
39 | val irisDataStream = irisSource(env, None)
40 |
41 | //Load model
42 | val modelReader = ModelReader(inputModel)
43 |
44 | //Using evaluate operator
45 | val prediction = irisDataStream.evaluate(modelReader) {
46 | //Iris data and modelReader instance
47 | case (event, model) =>
48 | val vectorized = event.toVector
49 | val prediction = model.predict(vectorized, Some(0.0))
50 | (event, prediction.value.getOrElse(-1.0))
51 | }
52 |
53 | prediction.writeAsText(output)
54 |
55 | env.execute("Clustering example")
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/QuickEvaluateKmeans.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples
21 |
22 | import io.radicalbit.examples.sources.IrisSource._
23 | import io.radicalbit.examples.util.EnsureParameters
24 | import io.radicalbit.flink.pmml.scala._
25 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
26 | import org.apache.flink.api.java.utils.ParameterTool
27 | import org.apache.flink.streaming.api.scala._
28 |
29 | object QuickEvaluateKmeans extends EnsureParameters {
30 |
31 | def main(args: Array[String]): Unit = {
32 | val params: ParameterTool = ParameterTool.fromArgs(args)
33 | val env = StreamExecutionEnvironment.getExecutionEnvironment
34 |
35 | env.getConfig.setGlobalJobParameters(params)
36 | val (inputModel, output) = ensureParams(params)
37 |
38 | //Read data from custom iris source
39 | val irisDataStream = irisSource(env, None)
40 |
41 | //Convert iris to DenseVector
42 | val irisToVector = irisDataStream.map(iris => iris.toVector)
43 |
44 | //Load PMML model
45 | val model = ModelReader(inputModel)
46 |
47 | irisToVector
48 | .quickEvaluate(model)
49 | .writeAsText(output)
50 |
51 | env.execute("Quick evaluator Clustering")
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/model/Iris.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.models
21 |
22 | import io.radicalbit.flink.pmml.scala.models.input.BaseEvent
23 | import org.apache.flink.ml.math.DenseVector
24 |
25 | case class Iris(modelId: String,
26 | sepalLength: Double,
27 | sepalWidth: Double,
28 | petalLength: Double,
29 | petalWidth: Double,
30 | occurredOn: Long)
31 | extends BaseEvent {
32 | def toVector = DenseVector(sepalLength, sepalWidth, petalLength, petalWidth)
33 | }
34 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/model/Utils.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.model
21 |
22 | import java.util.UUID
23 |
24 | import io.radicalbit.flink.pmml.scala.models.core.ModelId
25 |
26 | object Utils {
27 |
28 | final val modelVersion = 1.toString
29 |
30 | def retrieveMappingIdPath(modelPaths: Seq[String]): Map[String, String] =
31 | modelPaths.map(path => (UUID.randomUUID().toString, path)).toMap
32 |
33 | def retrieveAvailableId(mappingIdPath: Map[String, String]): Seq[String] =
34 | mappingIdPath.keys.map(name => name + ModelId.separatorSymbol + modelVersion).toSeq
35 |
36 | def now(): Long = System.currentTimeMillis()
37 | }
38 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/sources/ControlSource.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.sources
21 |
22 | import io.radicalbit.flink.pmml.scala.models.control.ServingMessage
23 | import org.apache.flink.streaming.api.scala._
24 |
25 | object ControlSource {
26 |
27 | sealed trait Mode
28 |
29 | case object Loop extends Mode
30 | case object Finite extends Mode
31 | case object Random extends Mode
32 |
33 | val procedures = Seq(Loop, Finite, Random)
34 |
35 | /**
36 | * Generation control stream method: it has three main generation logic (called _policies_):
37 | * - INFINITE:
38 | * - Loop Generation : The events are generating by infinitely looping over the paths' sequence
39 | * - Random Generation : The events are randomly generated by picking from the paths' sequence
40 | * - FINITE:
41 | * - Finite Generation : The events are generating 1-to-1 with the paths' sequence
42 | * @param env The Stream execution environment
43 | * @param mode Generation policy
44 | * @param mappingIdPath The list of the id. and path of the models that users want to employ into generation
45 | * @param maxInterval Max Interval between controls' generation
46 | * @return
47 | */
48 | def generateStream(env: StreamExecutionEnvironment,
49 | mode: Mode,
50 | mappingIdPath: Map[String, String],
51 | maxInterval: Long): DataStream[ServingMessage] =
52 | mode match {
53 | case Loop => env.addSource(new InfiniteSource(mappingIdPath, Loop, maxInterval))
54 | case Random => env.addSource(new InfiniteSource(mappingIdPath, Random, maxInterval))
55 | case Finite => env.addSource(new FiniteSource(mappingIdPath, maxInterval))
56 | case _ => env.addSource(new InfiniteSource(mappingIdPath, Random, maxInterval))
57 | }
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/sources/FiniteSource.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.sources
21 |
22 | import io.radicalbit.examples.model.Utils
23 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, ServingMessage}
24 | import org.apache.flink.streaming.api.functions.source.SourceFunction
25 |
26 | import scala.util.Random
27 |
28 | /**
29 | * Finite Control Messages Sources
30 | * @param mappingIdPath The Id, models path
31 | * @param maxInterval The Max interval of generation between events
32 | */
33 | class FiniteSource(mappingIdPath: Map[String, String], maxInterval: Long) extends SourceFunction[ServingMessage] {
34 |
35 | private val rand: Random = scala.util.Random
36 |
37 | override def cancel(): Unit = {}
38 |
39 | override def run(ctx: SourceFunction.SourceContext[ServingMessage]): Unit =
40 | mappingIdPath.foreach { idPath =>
41 | val (id, path) = idPath
42 | ctx.getCheckpointLock.synchronized {
43 | ctx.collect(AddMessage(id, 1, path, Utils.now))
44 | }
45 |
46 | Thread.sleep(rand.nextDouble() * maxInterval toLong)
47 | }
48 |
49 | }
50 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/sources/InfiniteSource.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.sources
21 |
22 | import java.util.concurrent.atomic.AtomicBoolean
23 |
24 | import io.radicalbit.examples.model.Utils
25 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, ServingMessage}
26 | import org.apache.flink.streaming.api.functions.source.SourceFunction
27 |
28 | import scala.util.Random
29 |
30 | /**
31 | * Infinite Control Source
32 | * @param mappingIdPath The list of id, models paths mapping
33 | * @param policy The generation policy
34 | */
35 | class InfiniteSource(mappingIdPath: Map[String, String], policy: ControlSource.Mode, maxInterval: Long)
36 | extends SourceFunction[ServingMessage] {
37 |
38 | private val isRunning: AtomicBoolean = new AtomicBoolean(true)
39 |
40 | private val rand: Random = scala.util.Random
41 |
42 | override def cancel(): Unit = isRunning.set(false)
43 |
44 | /**
45 | * Since it's unbounded source, he generates events as long as the job lives, abiding by user defined policy
46 | * @param ctx The Flink SourceContext
47 | */
48 | override def run(ctx: SourceFunction.SourceContext[ServingMessage]): Unit =
49 | if (policy == ControlSource.Loop) loopedGeneration(ctx) else randomGeneration(ctx)
50 |
51 | private def loopedGeneration(context: SourceFunction.SourceContext[ServingMessage]) =
52 | while (isRunning.get()) {
53 | mappingIdPath.foreach { tuple =>
54 | val (id, path) = tuple
55 | context.getCheckpointLock.synchronized {
56 | context.collect(AddMessage(id, 1, path, Utils.now))
57 | }
58 |
59 | Thread.sleep(rand.nextDouble() * maxInterval toLong)
60 | }
61 | }
62 |
63 | private def randomGeneration(context: SourceFunction.SourceContext[ServingMessage]) =
64 | while (isRunning.get()) {
65 | val (currentId, currentPath) = rand.shuffle(mappingIdPath).head
66 | context.getCheckpointLock.synchronized {
67 | context.collect(AddMessage(currentId, 1, currentPath, Utils.now))
68 | }
69 |
70 | Thread.sleep(rand.nextDouble() * maxInterval toLong)
71 | }
72 |
73 | }
74 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/sources/IrisSource.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.sources
21 |
22 | import io.radicalbit.examples.model.Utils
23 | import io.radicalbit.examples.models.Iris
24 | import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
25 | import org.apache.flink.streaming.api.scala._
26 |
27 | import scala.util.Random
28 |
29 | object IrisSource {
30 | private final val NumberOfParameters = 4
31 | private final lazy val RandomGenerator = scala.util.Random
32 | private final val RandomMin = 0.2
33 | private final val RandomMax = 6.0
34 |
35 | private final def truncateDouble(n: Double) = (math floor n * 10) / 10
36 |
37 | @throws(classOf[Exception])
38 | def irisSource(env: StreamExecutionEnvironment, availableModelIdOp: Option[Seq[String]]): DataStream[Iris] = {
39 | val availableModelId = availableModelIdOp.getOrElse(Seq.empty[String])
40 | env.addSource((sc: SourceContext[Iris]) => {
41 | while (true) {
42 | def randomVal = RandomMin + (RandomMax - RandomMin) * RandomGenerator.nextDouble()
43 | val dataForIris = Seq.fill(NumberOfParameters)(truncateDouble(randomVal))
44 | val iris =
45 | Iris(availableModelId(Random.nextInt(availableModelId.size)),
46 | dataForIris(0),
47 | dataForIris(1),
48 | dataForIris(2),
49 | dataForIris(3),
50 | Utils.now())
51 | sc.collect(iris)
52 | Thread.sleep(1000)
53 | }
54 | })
55 |
56 | }
57 |
58 | }
59 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/util/DynamicParams.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.util
21 |
22 | import io.radicalbit.examples.model.Utils
23 | import io.radicalbit.examples.sources.ControlSource
24 | import org.apache.flink.api.java.utils.ParameterTool
25 |
26 | object DynamicParams {
27 |
28 | def fromParameterTool(params: ParameterTool): DynamicParams = {
29 |
30 | val outputPath = params.getRequired("output")
31 |
32 | val pathsAndIds = retrievePathsAndIds(params.getRequired("models"))
33 |
34 | val policy = computeGenPolicy(params.get("gen-policy", "random"))
35 |
36 | val availableIdModels = computeAvailableIds(pathsAndIds)
37 |
38 | val intervalCheckpoint = params.get("intervalCheckpoint", 1000.toString).toLong
39 |
40 | val maxIntervalControlStream = params.get("maxIntervalControlStream", 5000L.toString).toLong
41 |
42 | DynamicParams(outputPath, policy, pathsAndIds, availableIdModels, intervalCheckpoint, maxIntervalControlStream)
43 | }
44 |
45 | private def retrievePathsAndIds(paths: String) = {
46 | val rawModelsPaths = paths.split(",")
47 | Utils.retrieveMappingIdPath(rawModelsPaths)
48 | }
49 |
50 | private def computeGenPolicy(rawPolicy: String) =
51 | rawPolicy match {
52 | case "random" => ControlSource.Random
53 | case "loop" => ControlSource.Loop
54 | case "finite" => ControlSource.Finite
55 | case _ => throw new IllegalArgumentException(s"$rawPolicy is not recognized generation policy.")
56 | }
57 |
58 | private def computeAvailableIds(pathsAndIds: Map[String, String]) =
59 | Utils.retrieveAvailableId(pathsAndIds)
60 |
61 | }
62 |
63 | case class DynamicParams(outputPath: String,
64 | genPolicy: ControlSource.Mode,
65 | pathAndIds: Map[String, String],
66 | availableIds: Seq[String],
67 | ckpInterval: Long,
68 | ctrlGenInterval: Long)
69 |
--------------------------------------------------------------------------------
/flink-jpmml-examples/src/main/scala/io/radicalbit/examples/util/EnsureParameters.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.examples.util
21 |
22 | import org.apache.flink.api.java.utils.ParameterTool
23 |
24 | trait EnsureParameters {
25 |
26 | def ensureParams(params: ParameterTool) =
27 | (params.getRequired("model"), params.getRequired("output"))
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Copyright (C) 2017 Radicalbit
2 | #
3 | # This file is part of flink-JPMML
4 | #
5 | # flink-JPMML is free software: you can redistribute it and/or modify
6 | # it under the terms of the GNU Affero General Public License as
7 | # published by the Free Software Foundation, either version 3 of the
8 | # License, or (at your option) any later version.
9 | #
10 | # flink-JPMML is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 | # GNU Affero General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Affero General Public License
16 | # along with flink-JPMML. If not, see .
17 |
18 | log4j.rootLogger=INFO, console
19 | log4j.logger.deng=INFO
20 | log4j.appender.console=org.apache.log4j.ConsoleAppender
21 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
22 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/Evaluator.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api
21 |
22 | import io.radicalbit.flink.pmml.scala.api.exceptions.EmptyEvaluatorException
23 | import org.dmg.pmml.Model
24 | import org.jpmml.evaluator.ModelEvaluator
25 |
26 | /**
27 | * Represents the Evaluator of a PmmlModel
28 | */
29 | object Evaluator {
30 |
31 | /** If the evaluator exists, it returns a [[PmmlEvaluator]], [[EmptyEvaluator]] otherwise.
32 | *
33 | * @param evaluator An instance of [[org.jpmml.evaluator.ModelEvaluator]]
34 | * @return An instance of [[Evaluator]]
35 | */
36 | def apply(evaluator: ModelEvaluator[_ <: Model]): Evaluator = PmmlEvaluator(evaluator)
37 |
38 | /** Returns an empty instance of [[Evaluator]]
39 | *
40 | * @return An [[EmptyEvaluator]]
41 | */
42 | def empty: Evaluator = EmptyEvaluator
43 |
44 | }
45 |
46 | /**
47 | * ADT sealed trait providing getters for ModelEvaluator
48 | */
49 | sealed trait Evaluator {
50 |
51 | def model: ModelEvaluator[_ <: Model]
52 |
53 | /** Returns [[org.jpmml.evaluator.ModelEvaluator]]] if evaluator has value, default value otherwise
54 | *
55 | * @param default the defined default value
56 | * @return the current evaluator if it has value, default otherwise
57 | */
58 | def getOrElse(default: => ModelEvaluator[_ <: Model]) =
59 | this match {
60 | case PmmlEvaluator(evaluator) => evaluator
61 | case EmptyEvaluator => default
62 | }
63 |
64 | }
65 |
66 | /**
67 | * Represents the Evaluator if it is not present
68 | */
69 | case object EmptyEvaluator extends Evaluator {
70 |
71 | /** Implements model method when the [[PmmlEvaluator]] is empty
72 | *
73 | * @return scala.NoSuchElementException for `EmptyEvaluator` instance.
74 | */
75 | override def model: ModelEvaluator[_ <: Model] = throw new EmptyEvaluatorException("EmptyEvaluator.None")
76 |
77 | }
78 |
79 | /**
80 | * Represents the Evaluator if it is present
81 | * @param modelEval the evaluator for the Pmml Model
82 | */
83 | final case class PmmlEvaluator(modelEval: ModelEvaluator[_ <: Model]) extends Evaluator {
84 |
85 | /**
86 | * Retrieving the evaluator of the JpmmlEvaluator
87 | * @return the [[org.jpmml.evaluator.ModelEvaluator]]
88 | */
89 | override def model: ModelEvaluator[_ <: Model] = modelEval
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/PmmlModel.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api
21 |
22 | import java.io.StringReader
23 | import java.util
24 |
25 | import io.radicalbit.flink.pmml.scala.api.exceptions.{InputValidationException, JPMMLExtractionException}
26 | import io.radicalbit.flink.pmml.scala.api.pipeline.Pipeline
27 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
28 | import io.radicalbit.flink.pmml.scala.models.prediction.Prediction
29 | import org.apache.flink.ml.math.Vector
30 | import org.dmg.pmml.FieldName
31 | import org.jpmml.evaluator._
32 | import org.jpmml.model.{ImportFilter, JAXBUtil}
33 | import org.xml.sax.InputSource
34 |
35 | import scala.collection.JavaConversions._
36 | import scala.util.Try
37 |
38 | /** Contains [[PmmlModel]] `fromReader` factory method.
39 | *
40 | * As singleton, it guarantees one and only copy of the model when the latter is requested.
41 | *
42 | */
43 | object PmmlModel {
44 |
45 | private val evaluatorInstance: ModelEvaluatorFactory = ModelEvaluatorFactory.newInstance()
46 |
47 | /** Loads the distributed path by [[io.radicalbit.flink.pmml.scala.api.reader.FsReader.buildDistributedPath]]
48 | * and construct a [[PmmlModel]] instance starting from `evalauatorInstance`
49 | *
50 | * @param reader The instance providing method in order to read the model from distributed backends lazily
51 | * @return [[PmmlModel]] instance from the Reader
52 | */
53 | private[api] def fromReader(reader: ModelReader): PmmlModel = {
54 | val readerFromFs = reader.buildDistributedPath
55 | val result = fromFilteredSource(readerFromFs)
56 |
57 | new PmmlModel(Evaluator(evaluatorInstance.newModelEvaluator(JAXBUtil.unmarshalPMML(result))))
58 | }
59 |
60 | private def fromFilteredSource(PMMLPath: String) =
61 | JAXBUtil.createFilteredSource(new InputSource(new StringReader(PMMLPath)), new ImportFilter())
62 |
63 | /** It provides a new instance of the [[PmmlModel]] with [[EmptyEvaluator]]
64 | *
65 | * @return [[PmmlModel]] with [[EmptyEvaluator]]
66 | */
67 | private[api] def empty = new PmmlModel(Evaluator.empty)
68 |
69 | }
70 |
71 | /** Provides to the user the model instance and its methods.
72 | *
73 | * Users get the instance along the [[io.radicalbit.flink.pmml.scala.RichDataStream.evaluate]] UDF input.
74 | *
75 | * {{{
76 | * toEvaluateStream.evaluate(reader) { (input, model) =>
77 | * val pmmlModel: PmmlModel = model
78 | * val prediction = model.predict(vectorized(input))
79 | * }
80 | * }}}
81 | *
82 | * This class contains [[PmmlModel#predict]] method and implements the core logic of the project.
83 | *
84 | * @param evaluator The PMML model instance
85 | */
86 | class PmmlModel(private[api] val evaluator: Evaluator) extends Pipeline {
87 |
88 | import io.radicalbit.flink.pmml.scala.api.converter.VectorConverter._
89 |
90 | final def modelName: String = evaluator.model.getModel.getModelName
91 |
92 | /** Implements the entire prediction pipeline, which can be described as 4 main steps:
93 | *
94 | * - `validateInput` validates the input to be conform to PMML model size
95 | *
96 | * - `prepareInput` prepares the input in full compliance to [[org.jpmml.evaluator.EvaluatorUtil.prepare]] JPMML method
97 | *
98 | * - `evaluateInput` evaluates the input against inner PMML model instance and returns a Java Map output
99 | *
100 | * - `extractTarget` extracts the target from evaluation result.
101 | *
102 | * As final action the pipelined statement is executed by [[Prediction]]
103 | *
104 | * @param inputVector the input event as a [[org.apache.flink.ml.math.Vector]] instance
105 | * @param replaceNan A [[scala.Option]] describing a replace value for not defined vector values
106 | * @tparam V subclass of [[org.apache.flink.ml.math.Vector]]
107 | * @return [[Prediction]] instance
108 | */
109 | final def predict[V <: Vector](inputVector: V, replaceNan: Option[Double] = None): Prediction = {
110 | val result = Try {
111 | val validatedInput = validateInput(inputVector)
112 | val preparedInput = prepareInput(validatedInput, replaceNan)
113 | val evaluationResult = evaluateInput(preparedInput)
114 | val extractResult = extractTarget(evaluationResult)
115 | extractResult
116 | }
117 |
118 | Prediction.extractPrediction(result)
119 | }
120 |
121 | /** Validates the input vector in size terms and converts it as a `Map[String, Any]` (see [[PmmlInput]])
122 | *
123 | * @param v The raw input vector
124 | * @param vec2Pmml The conversion function
125 | * @return The converted instance
126 | */
127 | private[api] def validateInput(v: Vector)(implicit vec2Pmml: (Vector, Evaluator) => PmmlInput): PmmlInput = {
128 | val modelSize = evaluator.model.getActiveFields.size
129 |
130 | if (v.size != modelSize)
131 | throw new InputValidationException(s"input vector $v size ${v.size} is not conform to model size $modelSize")
132 | else
133 | vec2Pmml(v, evaluator)
134 | }
135 |
136 | /** Binds each field with input value and prepare the record to be evaluated
137 | * by [[EvaluatorUtil.prepare]] method.
138 | *
139 | * @param input Validated input as a [[Map]] keyed by field name
140 | * @param replaceNaN Optional replace value in case of missing values
141 | * @return Prepared input to be evaluated
142 | */
143 | private[api] def prepareInput(input: PmmlInput, replaceNaN: Option[Double]): Map[FieldName, FieldValue] = {
144 |
145 | val activeFields = evaluator.model.getActiveFields
146 |
147 | activeFields.map { field =>
148 | val rawValue = input.get(field.getName.getValue).orElse(replaceNaN).orNull
149 | prepareAndEmit(Try { EvaluatorUtil.prepare(field, rawValue) }, field.getName)
150 | }.toMap
151 |
152 | }
153 |
154 | /** Evaluates the prepared input against the PMML model
155 | *
156 | * @param preparedInput JPMML prepared input as `Map[FieldName, FieldValue]`
157 | * @return JPMML output result as a Java map
158 | */
159 | private[api] def evaluateInput(preparedInput: Map[FieldName, FieldValue]): util.Map[FieldName, _] =
160 | evaluator.model.evaluate(preparedInput)
161 |
162 | /** Extracts the target from evaluation result
163 | * @throws JPMMLExtractionException if the target couldn't be extracted
164 | * @param evaluationResult outcome from JPMML evaluation
165 | * @return The prediction value as a [Double]
166 | */
167 | private[api] def extractTarget(evaluationResult: java.util.Map[FieldName, _]): Double = {
168 | val targets = extractTargetFields(evaluationResult)
169 |
170 | targets.headOption.flatMap {
171 | case (_, target) => extractTargetValue(target)
172 | } getOrElse (throw new JPMMLExtractionException("Target value is null."))
173 |
174 | }
175 |
176 | }
177 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/converter/VectorConverter.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.converter
21 |
22 | import io.radicalbit.flink.pmml.scala.api.{Evaluator, PmmlInput}
23 | import org.apache.flink.ml.math.{DenseVector, SparseVector, Vector}
24 |
25 | import scala.collection.JavaConversions._
26 |
27 | /** Type Class Pattern implementing converters from Flink [[org.apache.flink.ml.math.Vector]] instances to
28 | * internal types; the existing fields (i.e. value defined fields) are modeled as [[scala.collection.mutable.Map]]s; the
29 | * not existing fields (i.e. NaN values) will not be mapped within the Internal type
30 | */
31 | sealed trait VectorConverter[-T] extends Serializable {
32 | def serializeVector(v: T, eval: Evaluator): Map[String, Any]
33 | }
34 |
35 | private[api] object VectorConverter {
36 |
37 | private def apply[A: VectorConverter] = implicitly[VectorConverter[A]]
38 |
39 | /**
40 | * Create an instance for specific [[VectorConverter]]
41 | *
42 | * @param serialize The function producing a [[PmmlInput]]
43 | * @return The vector converter instance
44 | *
45 | * */
46 | private def createConverter[IN](serialize: (IN, Evaluator) => PmmlInput): VectorConverter[IN] =
47 | new VectorConverter[IN] {
48 | override def serializeVector(v: IN, eval: Evaluator): PmmlInput =
49 | serialize(v, eval)
50 | }
51 |
52 | /** Type class pattern entry-point: it delivers right converter depending
53 | * on the input type (i.e. Dense or Sparse)
54 | *
55 | * @return The specific converter instance for type [[Vector]]
56 | *
57 | */
58 | private[api] implicit val vectorConversion: VectorConverter[Vector] =
59 | createConverter {
60 | case (vec: DenseVector, evaluator) => denseVector2Map.serializeVector(vec, evaluator)
61 | case (vec: SparseVector, evaluator) => sparseVector2Map.serializeVector(vec, evaluator)
62 | }
63 |
64 | /** Converts a [[DenseVector]] to the internal type by mapping PMML model fields to vector values.
65 | *
66 | * @return The converted instance
67 | */
68 | private[api] implicit val denseVector2Map: VectorConverter[DenseVector] =
69 | createConverter { (vec, evaluator) =>
70 | getKeyFromModel(evaluator)
71 | .zip(vec.data)
72 | .toMap
73 | }
74 |
75 | /** Converts a [[SparseVector]] to the internal type by mapping PMML model fields to vector values.
76 | * Note that only existing values will be mapped
77 | *
78 | * @return The converted instance
79 | */
80 | private[api] implicit val sparseVector2Map: VectorConverter[SparseVector] =
81 | createConverter { (vec, evaluator) =>
82 | getKeyFromModel(evaluator)
83 | .zip(toDenseData(vec))
84 | .collect { case (key, Some(value)) => (key, value) }
85 | .toMap
86 | }
87 |
88 | private[api] implicit def applyConversion[T: VectorConverter, E <: Evaluator](dataVector: T, evaluator: E) =
89 | implicitly[VectorConverter[T]].serializeVector(dataVector, evaluator)
90 |
91 | /** Extracts the key values of the model fields from [[Evaluator]] instance.
92 | *
93 | * @param evaluator PMML evaluator instance
94 | * @return the keys as a Scala [[Seq]]
95 | *
96 | */
97 | private def getKeyFromModel(evaluator: Evaluator) =
98 | evaluator.model.getActiveFields.map(_.getName.getValue)
99 |
100 | private def toDenseData(sparseVector: SparseVector): Seq[Option[Any]] =
101 | (0 until sparseVector.size).map { index =>
102 | if (sparseVector.indices.contains(index)) Some(sparseVector(index)) else None
103 | }
104 |
105 | }
106 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/exceptions/package.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api
21 |
22 | package object exceptions {
23 |
24 | /** Models conformity failure between PMML model and input [[org.apache.flink.streaming.api.scala.DataStream]]
25 | *
26 | */
27 | private[scala] class InputValidationException(msg: String) extends RuntimeException(msg)
28 |
29 | /** Models [[org.jpmml.evaluator.EvaluatorUtil.prepare()]] method failure
30 | *
31 | */
32 | private[scala] class InputPreparationException(msg: String) extends RuntimeException(msg)
33 |
34 | /** Models empty result from [[org.jpmml.evaluator.ModelEvaluator]] evaluation
35 | *
36 | */
37 | private[scala] class JPMMLExtractionException(msg: String) extends RuntimeException(msg)
38 |
39 | /** Models failure on loading PMML model from distributed system
40 | *
41 | */
42 | private[scala] class ModelLoadingException(msg: String, throwable: Throwable)
43 | extends RuntimeException(msg, throwable)
44 |
45 | /** Prediction failure due to [[io.radicalbit.flink.pmml.scala.api.EmptyEvaluator]]
46 | *
47 | */
48 | private[scala] class EmptyEvaluatorException(msg: String) extends NoSuchElementException(msg)
49 |
50 | /** Parsing of ModelId has failed
51 | *
52 | */
53 | private[scala] class WrongModelIdFormat(msg: String) extends ArrayIndexOutOfBoundsException(msg)
54 |
55 | }
56 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/functions/EvaluationCoFunction.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.functions
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.api.exceptions.ModelLoadingException
24 | import io.radicalbit.flink.pmml.scala.api.managers.{MetadataManager, ModelsManager}
25 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
26 | import io.radicalbit.flink.pmml.scala.logging.LazyLogging
27 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, DelMessage, ServingMessage}
28 | import io.radicalbit.flink.pmml.scala.models.core.{ModelId, ModelInfo}
29 | import io.radicalbit.flink.pmml.scala.models.state.CheckpointType.MetadataCheckpoint
30 | import org.apache.flink.api.common.state.{ListState, ListStateDescriptor}
31 | import org.apache.flink.api.common.typeinfo.{TypeHint, TypeInformation}
32 | import org.apache.flink.configuration.Configuration
33 | import org.apache.flink.runtime.state.{FunctionInitializationContext, FunctionSnapshotContext}
34 | import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction
35 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction
36 | import org.apache.flink.util.Collector
37 |
38 | import scala.collection.JavaConverters._
39 | import scala.collection.{immutable, mutable}
40 | import scala.util.{Failure, Success, Try}
41 |
42 | /** Abstract class extending a [[CoProcessFunction]]; it provides:
43 | * two maps for caching both [[PmmlModel]] and Metadata of each Model
44 | * the open method in order to initialize the Metadata Map
45 | * the processElement2 in order to handle models and metadata against a control stream
46 | *
47 | * Abstract class extends [[CheckpointedFunction]] and provides therefore:
48 | * the snapshotState method in order to checkpoint the current state of the operator
49 | * the initializeState in order to provide an initial state at the operator and/or restore the latest
50 | *
51 | * @tparam EVENT The input Type of the event to predict
52 | * @tparam CTRL The control stream Type. Note: It must extend [[io.radicalbit.flink.pmml.scala.models.control.ServingMessage]]
53 | * @tparam OUT The output Type
54 | */
55 | private[scala] abstract class EvaluationCoFunction[EVENT, CTRL <: ServingMessage, OUT]
56 | extends CoProcessFunction[EVENT, CTRL, OUT]
57 | with CheckpointedFunction
58 | with LazyLogging {
59 |
60 | @transient
61 | private var snapshotMetadata: ListState[MetadataCheckpoint] = _
62 |
63 | @transient
64 | final protected var servingMetadata: immutable.Map[ModelId, ModelInfo] = _
65 |
66 | final protected lazy val servingModels: mutable.WeakHashMap[Int, PmmlModel] =
67 | mutable.WeakHashMap.empty[Int, PmmlModel]
68 |
69 | override def processElement2(control: CTRL,
70 | ctx: CoProcessFunction[EVENT, CTRL, OUT]#Context,
71 | out: Collector[OUT]): Unit = {
72 | manageModels(control)
73 | manageMetadata(control)
74 | }
75 |
76 | override def snapshotState(context: FunctionSnapshotContext): Unit = {
77 | snapshotMetadata.clear()
78 | snapshotMetadata.add(new MetadataCheckpoint(servingMetadata.asJava))
79 | }
80 |
81 | override def initializeState(context: FunctionInitializationContext): Unit = {
82 | servingMetadata = immutable.Map.empty[ModelId, ModelInfo]
83 |
84 | val description = new ListStateDescriptor[MetadataCheckpoint](
85 | "metadata-snapshot",
86 | TypeInformation.of(new TypeHint[MetadataCheckpoint]() {}))
87 |
88 | snapshotMetadata = context.getOperatorStateStore.getUnionListState(description)
89 |
90 | if (context.isRestored) {
91 | Try(snapshotMetadata.get()) match {
92 | case Success(state) => servingMetadata ++= state.asScala.toSet[MetadataCheckpoint].flatMap(_.asScala).toMap
93 | case Failure(_) => logger.info("Not available state in ListState!")
94 | }
95 | }
96 | }
97 |
98 | final def loadModel(path: String): PmmlModel =
99 | Try(PmmlModel.fromReader(ModelReader(path))) match {
100 | case Success(model) =>
101 | logger.info("Model has been successfully loaded, model name: {}", model.modelName)
102 | model
103 | case Failure(e) => throw new ModelLoadingException(e.getMessage, e)
104 | }
105 |
106 | final def fromMetadata(modelId: String): PmmlModel = {
107 | val currentModelId: ModelId = ModelId.fromIdentifier(modelId)
108 | if (!servingMetadata.contains(currentModelId)) PmmlModel.empty
109 | else addAndRetrieveModel(modelId, currentModelId)
110 | }
111 |
112 | final private def addAndRetrieveModel(modelId: String, currentModelId: ModelId): PmmlModel = {
113 | val currentModelPath = servingMetadata(currentModelId).path
114 | val loadedModel = loadModel(currentModelPath)
115 | servingModels += (modelId.hashCode -> loadedModel)
116 | loadedModel
117 | }
118 |
119 | final private def manageMetadata(control: CTRL): Unit =
120 | control match {
121 | case add: AddMessage =>
122 | servingMetadata = MetadataManager(add, servingMetadata)
123 | case del: DelMessage =>
124 | servingMetadata = MetadataManager(del, servingMetadata)
125 | }
126 |
127 | final private def manageModels(control: CTRL): Unit =
128 | control match {
129 | case del: DelMessage => servingModels --= ModelsManager(del, servingModels)
130 | case _ =>
131 | }
132 |
133 | }
134 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/functions/EvaluationFunction.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.functions
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.api.exceptions.ModelLoadingException
24 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
25 | import io.radicalbit.flink.pmml.scala.logging.LazyLogging
26 | import org.apache.flink.api.common.functions.RichFlatMapFunction
27 | import org.apache.flink.configuration.Configuration
28 |
29 | import scala.util.{Failure, Success, Try}
30 |
31 | /** Abstract class extending a [[RichFlatMapFunction]]; it provides:
32 | * the `evaluator` lazy evaluated object as instance of [[PmmlModel]]
33 | * the open method as a builder of the evaluator instance at operator initialization time
34 | *
35 | * @param reader The model reader instance coupled with the model source path
36 | * @tparam IN The input Type
37 | * @tparam OUT The output Type
38 | */
39 | private[scala] abstract class EvaluationFunction[IN, OUT](reader: ModelReader)
40 | extends RichFlatMapFunction[IN, OUT]
41 | with LazyLogging {
42 |
43 | protected lazy val evaluator: PmmlModel = PmmlModel.fromReader(reader)
44 |
45 | override def open(parameters: Configuration): Unit = Try(evaluator.evaluator.model.getModel) match {
46 | case Success(model) => logger.info(s"Model has been read successfully, model name: {}", model.getModelName)
47 | case Failure(e) => throw new ModelLoadingException(e.getMessage, e)
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/managers/MetadataManager.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.managers
21 |
22 | import io.radicalbit.flink.pmml.scala.logging.LazyLogging
23 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, DelMessage}
24 | import io.radicalbit.flink.pmml.scala.models.core.{ModelId, ModelInfo}
25 |
26 | import scala.collection.immutable
27 |
28 | /**
29 | * Type class in order to enrich Message Protocol ADT with proper methods for acting on metadata.
30 | *
31 | * @tparam T Specific Message ADT subType; it's contro-variant in order to accept super classes, then generics.
32 | */
33 | sealed trait MetadataManager[T] {
34 |
35 | def manageMetadata(command: T, metadata: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo]
36 |
37 | }
38 |
39 | object MetadataManager extends LazyLogging {
40 |
41 | implicit def apply[T: MetadataManager](
42 | command: T,
43 | metadata: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo] =
44 | implicitly[MetadataManager[T]].manageMetadata(command, metadata)
45 |
46 | /**
47 | * Implicit value aimed to Adding model information to metadata.
48 | *
49 | * If a new model is coming (where new means a model bind to a previously unknown identifier)
50 | * so the model information is added to metadata.
51 | *
52 | * If a not new model is coming (where not new means the model has an already present identifier)
53 | * so a WARN is logged to the system; indeed, if the user wants to update a model, he should provide
54 | * a newer version for it.
55 | *
56 | * Add messages don't remove elements from metadata for any reason.
57 | *
58 | */
59 | implicit val addMetadataServing = new MetadataManager[AddMessage] {
60 |
61 | def manageMetadata(addMessage: AddMessage,
62 | metadata: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo] = {
63 | if (metadata.contains(addMessage.modelId)) {
64 | logger.warn("ADD action on existing models is not possible (newer version needed). {} given.", addMessage)
65 | metadata
66 | } else metadata + (addMessage.modelId -> addMessage.modelInfo)
67 | }
68 |
69 | }
70 |
71 | /**
72 | * Implicit object aimed to removing model information from metadata.
73 | *
74 | * Del messages don't add elements to metadata for any reason.
75 | *
76 | * If a metadata element needs to be removed a key Set of the
77 | * to-be-removed elements is returned.
78 | *
79 | */
80 | implicit val deleteMetadataServing = new MetadataManager[DelMessage] {
81 |
82 | def manageMetadata(delMessage: DelMessage,
83 | metadata: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo] =
84 | metadata - delMessage.modelId
85 |
86 | }
87 |
88 | }
89 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/managers/ModelsManager.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.managers
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.logging.LazyLogging
24 | import io.radicalbit.flink.pmml.scala.models.control.DelMessage
25 |
26 | import scala.collection.{immutable, mutable}
27 |
28 | /**
29 | * Type class in order to enrich Message Protocol ADT with proper methods for acting on models.
30 | *
31 | * @tparam T Specific Message ADT subType; it's contro-variant in order to accept super classes, then generics.
32 | */
33 | sealed trait ModelsManager[T] {
34 |
35 | def manageModels(command: T, models: mutable.Map[Int, PmmlModel]): immutable.Set[Int]
36 |
37 | }
38 |
39 | object ModelsManager extends LazyLogging {
40 |
41 | def apply[T: ModelsManager](command: T, models: mutable.Map[Int, PmmlModel]): Set[Int] =
42 | implicitly[ModelsManager[T]].manageModels(command, models)
43 |
44 | /**
45 | * Implicit value aimed to removing models from internal operator state on DelMessage.
46 | *
47 | * [[deleteModelsServing.manageModels]] returns the elements key set which need to be
48 | * removed from models.
49 | *
50 | */
51 | implicit val deleteModelsServing = new ModelsManager[DelMessage] {
52 |
53 | def manageModels(delMessage: DelMessage, models: mutable.Map[Int, PmmlModel]): immutable.Set[Int] = {
54 | models.keySet.intersect(Set(delMessage.modelId.hashCode)).toSet
55 | }
56 |
57 | }
58 |
59 | }
60 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/package.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala
21 |
22 | import org.dmg.pmml.Model
23 | import org.jpmml.evaluator.ModelEvaluator
24 |
25 | /** Provides features implementation.
26 | *
27 | * The `api` package object contains inner types definition.
28 | *
29 | * [[io.radicalbit.flink.pmml.scala.api.PmmlInput]] represents internal input type
30 | */
31 | package object api {
32 |
33 | type PmmlInput = Map[String, Any]
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/pipeline/Pipeline.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.pipeline
21 |
22 | import io.radicalbit.flink.pmml.scala.api._
23 | import io.radicalbit.flink.pmml.scala.api.exceptions.InputPreparationException
24 | import io.radicalbit.flink.pmml.scala.models.prediction.Prediction
25 | import org.apache.flink.ml.math.Vector
26 | import org.dmg.pmml.FieldName
27 | import org.jpmml.evaluator.{EvaluatorUtil, FieldValue, ModelField}
28 |
29 | import scala.collection.JavaConversions._
30 | import scala.collection.mutable
31 | import scala.util.{Failure, Success, Try}
32 |
33 | /** Self type trait extending [[PmmlModel]] instance methods; they offer
34 | * additional features for the preparation and extraction pipeline steps.
35 | *
36 | */
37 | private[api] trait Pipeline { self: PmmlModel =>
38 |
39 | def predict[V <: Vector](inputVector: V, replaceNan: Option[Double] = None): Prediction
40 |
41 | /** Emits prepared input if JPMML preparation went fine.
42 | *
43 | * @throws InputPreparationException if the JPMML preparation fails.
44 | *
45 | * @param outcome `Try` evaluation of [[EvaluatorUtil.prepare]] method output value
46 | * @param field The field name related to the value
47 | * @return Prepared Input
48 | */
49 | private[api] def prepareAndEmit(outcome: Try[FieldValue], field: FieldName): (FieldName, FieldValue) =
50 | outcome match {
51 | case Success(value) => (field, value)
52 | case Failure(_) =>
53 | throw new InputPreparationException(s"The ${field.getValue} field JPMML finalization failed.")
54 | }
55 |
56 | /** Extracts all the target fields specified by the PMML document.
57 | *
58 | * @param evaluationResult evaluation step outcome
59 | * @return extracted output
60 | */
61 | private[api] def extractTargetFields(evaluationResult: java.util.Map[FieldName, _]): Seq[(String, Any)] =
62 | extractFields(evaluator.model.getTargetFields, evaluationResult, evaluator)
63 |
64 | /** Extracts all the output fields specified by the PMML document.
65 | *
66 | * @param evaluationResult evaluation step outcome
67 | * @return extracted output
68 | */
69 | private[api] def extractOutputFields(evaluationResult: java.util.Map[FieldName, _]): Seq[(String, Any)] =
70 | extractFields(evaluator.model.getOutputFields, evaluationResult, evaluator)
71 |
72 | /** Calls [[EvaluatorUtil.decode]] for each field demanded for extraction.
73 | *
74 | * @param fields demanded for extraction
75 | * @param evaluationResult evaluation outcome container
76 | * @param evaluator The PMML instance as a [[org.jpmml.evaluator.ModelEvaluator]]
77 | * @return
78 | */
79 | private[api] def extractFields(fields: java.util.List[_ <: ModelField],
80 | evaluationResult: java.util.Map[FieldName, _],
81 | evaluator: Evaluator): mutable.Buffer[(String, AnyRef)] =
82 | for {
83 | field <- fields
84 | fieldName <- Option(field.getName)
85 | } yield { fieldName.getValue -> EvaluatorUtil.decode(evaluationResult.get(fieldName)) }
86 |
87 | /** Casts a String to Double if the outcome is a String, returns the Double otherwise.
88 | *
89 | * @param target The extracted target
90 | * @throws scala.ClassCastException if the outcome could not be casted to Double
91 | * @return The outcome as a Double
92 | */
93 | @throws(classOf[ClassCastException])
94 | protected def extractTargetValue(target: Any): Option[Double] = target match {
95 | case s: String => Some(s.toDouble)
96 | case d: Double => Some(d)
97 | case _ => None
98 | }
99 |
100 | }
101 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/reader/FsReader.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.reader
21 |
22 | import java.io.Closeable
23 |
24 | import org.apache.flink.core.fs.Path
25 |
26 | import scala.util.control.Exception.allCatch
27 |
28 | /** Self type trait extending [[ModelReader]] features by providing automatic
29 | * path building, allowing to load models from any Flink supported distributed backend
30 | *
31 | */
32 | private[api] trait FsReader { self: ModelReader =>
33 |
34 | private def closable[T <: Closeable, R](t: T)(f: T => R): R =
35 | allCatch.andFinally(t.close()).apply(f(t))
36 |
37 | /** Loan pattern ensuring the resource is loaded once and then closed.
38 | *
39 | * @return
40 | */
41 | private[api] def buildDistributedPath: String = {
42 | val pathFs = new Path(self.sourcePath)
43 | val fs = pathFs.getFileSystem
44 |
45 | closable(fs.open(pathFs)) { is =>
46 | scala.io.Source.fromInputStream(is).mkString
47 | }
48 |
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/api/reader/ModelReader.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.reader
21 |
22 | /** Provides the abstraction for PMML model input reader. It extends [[FsReader]] in order
23 | * to provide distributed backend reading methods.
24 | *
25 | * @param sourcePath the PMML source path addressing the model
26 | */
27 | case class ModelReader(sourcePath: String) extends FsReader
28 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/logging/LazyLogging.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.logging
21 |
22 | import org.slf4j.LoggerFactory
23 |
24 | /** Guarantees lazy logging; it is necessary in order to provide the feature along
25 | * the code also if users employ `scala-2.10` scala version because `scala-logging` is
26 | * no more published.
27 | *
28 | */
29 | trait LazyLogging {
30 |
31 | protected lazy val logger = LoggerFactory.getLogger(getClass.getName)
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/control/ServingMessage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.control
21 |
22 | import io.radicalbit.flink.pmml.scala.models.core.{ModelId, ModelInfo}
23 |
24 | /** Defines the mandatory fields that events control stream must implement.
25 | *
26 | */
27 | sealed trait ServingMessage {
28 |
29 | def name: String
30 |
31 | def occurredOn: Long
32 |
33 | }
34 |
35 | /** Defines a event control message in order to add a new model
36 | *
37 | * @param name of the model
38 | * @param version of the model
39 | * @param path of the model
40 | * @param occurredOn represents when the event occurred
41 | */
42 | final case class AddMessage(name: String, version: Long, path: String, occurredOn: Long) extends ServingMessage {
43 |
44 | def modelId: ModelId = ModelId(name, version)
45 |
46 | def modelInfo: ModelInfo = ModelInfo(path)
47 |
48 | }
49 |
50 | /** Defines a event control message in order to delete a model
51 | *
52 | * @param name of the model
53 | * @param version of the model
54 | * @param occurredOn represents when the event occurred
55 | */
56 | final case class DelMessage(name: String, version: Long, occurredOn: Long) extends ServingMessage {
57 |
58 | def modelId: ModelId = ModelId(name, version)
59 |
60 | }
61 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/core/ModelId.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.core
21 |
22 | import io.radicalbit.flink.pmml.scala.api.exceptions.WrongModelIdFormat
23 |
24 | import scala.util.parsing.combinator.RegexParsers
25 |
26 | /** Provides the regular expressions for name, version and id of a model;
27 | * Name is a UUID
28 | * Version is a Long
29 | * the id of a model is modelled by the following pattern: name + separatorSymbol(`_`) + version
30 | *
31 | */
32 | object ModelId extends RegexParsers {
33 |
34 | final val separatorSymbol = "_"
35 |
36 | lazy val name = """[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}""".r ^^ { _.toString }
37 | lazy val version = """^\d+$""".r ^^ { _.toLong }
38 | lazy val nameAndVersion = name ~ separatorSymbol ~ version ^^ {
39 | case name ~ _ ~ version => ModelId(name, version)
40 | }
41 |
42 | /** Parses and validates the model id
43 | *
44 | * @param id name and version of the model as id
45 | * @return [[ModelId]] according to the provided id
46 | */
47 | def fromIdentifier(id: String): ModelId =
48 | parse(nameAndVersion, id) match {
49 | case Success(checked, _) => checked
50 | case NoSuccess(msg, _) => throw new WrongModelIdFormat(msg)
51 | }
52 |
53 | }
54 |
55 | /** Represents an instance of the [[ModelId]]
56 | *
57 | * @param name Name identifying the model
58 | * @param version The version of the model
59 | */
60 | final case class ModelId(name: String, version: Long) {
61 |
62 | override def hashCode: Int = (name + ModelId.separatorSymbol + version).hashCode
63 |
64 | }
65 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/core/ModelInfo.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.core
21 |
22 | /** Contains the metadata of a model
23 | *
24 | * @param path of the model
25 | */
26 | final case class ModelInfo(path: String)
27 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/input/BaseEvent.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.input
21 |
22 | /** Defines fields that an event of the eventToPredict stream must implement.
23 | *
24 | */
25 | trait BaseEvent {
26 |
27 | def modelId: String
28 |
29 | def occurredOn: Long
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/prediction/Prediction.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.prediction
21 |
22 | import io.radicalbit.flink.pmml.scala.api.exceptions._
23 | import io.radicalbit.flink.pmml.scala.logging.LazyLogging
24 | import org.jpmml.evaluator.EvaluationException
25 |
26 | import scala.util.control.NonFatal
27 | import scala.util.{Failure, Success, Try}
28 |
29 | /** Factory for [[Prediction]] case class instances */
30 | object Prediction extends LazyLogging {
31 |
32 | /** Evaluates [[Try]] statement executed at [[io.radicalbit.flink.pmml.scala.api.PmmlModel.predict]]
33 | *
34 | * @param out containing evaluation pipeline execution
35 | * @return
36 | */
37 | private[scala] def extractPrediction(out: Try[Double]): Prediction = out match {
38 | case Success(result) => Prediction(Score(result))
39 | case Failure(throwable) => onFailedPrediction(throwable)
40 | }
41 |
42 | /** Pattern matches failures arisen by [[io.radicalbit.flink.pmml.scala.api.PmmlModel.predict]]
43 | *
44 | * @param throwable
45 | * @return
46 | */
47 | private[scala] def onFailedPrediction(throwable: Throwable): Prediction = {
48 |
49 | throwable match {
50 | case e: JPMMLExtractionException =>
51 | logger.warn("Error while extracting results", e.getMessage)
52 | case e: InputPreparationException =>
53 | logger.warn("Error while preparing input", e.getMessage)
54 | case e: InputValidationException => logger.warn("Error while validate input", e.getMessage)
55 | case e: EvaluationException => logger.warn("Error while evaluate model", e.getMessage)
56 | case e: ClassCastException => logger.error("Error while extract target", e)
57 | case NonFatal(e) => logger.error("Error", e)
58 | }
59 |
60 | emptyTarget
61 |
62 | }
63 |
64 | private val emptyTarget = Prediction(EmptyScore)
65 |
66 | }
67 |
68 | /** Models the result output container
69 | *
70 | * @param value contains the extracted prediction of [[Target]] type
71 | */
72 | case class Prediction(value: Target)
73 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/prediction/Target.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.prediction
21 |
22 | /** Represents the result output value; if the target is present has [[Score]] value,
23 | * [[EmptyScore]] otherwise.
24 | */
25 | object Target {
26 |
27 | /** Factory method
28 | *
29 | * @param v
30 | * @return
31 | */
32 | def apply(v: Double): Target = Score(v)
33 |
34 | /** Returns an empty instance of [[Target]]
35 | *
36 | * @return
37 | */
38 | def empty = EmptyScore
39 | }
40 |
41 | /** ADT sealed trait providing getters for values.
42 | *
43 | */
44 | sealed trait Target extends Serializable {
45 |
46 | /** Returns [[Score]]] if target has value, default value otherwise
47 | *
48 | * @param default the user defined default value
49 | * @return
50 | */
51 | final def getOrElse(default: => Double): Double =
52 | this match {
53 | case Score(v) => v
54 | case EmptyScore => default
55 | }
56 |
57 | def get: Double
58 | }
59 |
60 | /** Represents the Target value if it is present
61 | *
62 | * @param value the prediction value as [[scala.Double]]
63 | */
64 | final case class Score(value: Double) extends Target {
65 |
66 | /** Returns the [[Score]] value
67 | *
68 | * @return
69 | */
70 | def get: Double = value
71 | }
72 |
73 | /** Represents the Target value if it is not present
74 | *
75 | */
76 | case object EmptyScore extends Target {
77 |
78 | /** Implements get method when the [[Score]] is empty
79 | *
80 | * @return
81 | * @throws scala.NoSuchElementException for `EmptyScore` instance.
82 | */
83 | def get: Double = throw new NoSuchElementException("EmptyScore.nan")
84 | }
85 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/models/state/CheckpointType.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.state
21 |
22 | import java.util
23 |
24 | import io.radicalbit.flink.pmml.scala.models.core.{ModelId, ModelInfo}
25 |
26 | /** Provides Types for Checkpointed State of the [[io.radicalbit.flink.pmml.scala.api.functions.EvaluationCoFunction]]
27 | *
28 | */
29 | object CheckpointType {
30 | type MetadataCheckpoint = util.HashMap[ModelId, ModelInfo]
31 | type MetadataCheckpointedList = util.List[MetadataCheckpoint]
32 | }
33 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/main/scala/io/radicalbit/flink/pmml/scala/package.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.api.functions.{EvaluationCoFunction, EvaluationFunction}
24 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
25 | import io.radicalbit.flink.pmml.scala.models.control.ServingMessage
26 | import io.radicalbit.flink.pmml.scala.models.input.BaseEvent
27 | import io.radicalbit.flink.pmml.scala.models.prediction.Prediction
28 | import org.apache.flink.api.common.typeinfo.TypeInformation
29 | import org.apache.flink.ml.math.Vector
30 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction
31 | import org.apache.flink.streaming.api.scala._
32 | import org.apache.flink.util.Collector
33 |
34 | import _root_.scala.reflect.ClassTag
35 |
36 | /** Main library package, it contains the core of the library.
37 | *
38 | * The `scala` package object provides implicit classes enriching Flink
39 | * [[org.apache.flink.streaming.api.scala.DataStream]] in order to compute evaluations against streams.
40 | *
41 | */
42 | package object scala {
43 |
44 | /** Enriches Flink [[org.apache.flink.streaming.api.scala.DataStream]] with [[evaluate]] method, as
45 | *
46 | * {{{
47 | * case class Input(values: Seq[Double])
48 | * val inputStream = env.fromCollection(Seq(Input(Seq(1.0)), Input(Seq(3.0)))
49 | * inputStream.evaluate(reader) { (event, model) =>
50 | * val prediction = model.predict(event.toVector)
51 | * prediction.value
52 | * }
53 | * }}}
54 | *
55 | * @param stream The input stream
56 | * @tparam T The input stream inner Type
57 | */
58 | implicit class RichDataStream[T: TypeInformation: ClassTag](stream: DataStream[T]) {
59 |
60 | /**
61 | * It connects the main `DataStream` with the `ControlStream`
62 | */
63 | def withSupportStream[CTRL <: ServingMessage: TypeInformation](
64 | supportStream: DataStream[CTRL]): ConnectedStreams[T, CTRL] =
65 | stream.connect(supportStream.broadcast)
66 |
67 | /** It evaluates the `DataStream` against the model pointed out by
68 | * [[io.radicalbit.flink.pmml.scala.api.reader.ModelReader]]; it takes as input an UDF `(T, PmmlModel) => R)` .
69 | * It's modeled on top of `EvaluationFunction`.
70 | *
71 | * @param modelReader the [[io.radicalbit.flink.pmml.scala.api.reader.ModelReader]] instance
72 | * @param f UDF function
73 | * @tparam R The output type
74 | * @return `R`
75 | */
76 | def evaluate[R: TypeInformation](modelReader: ModelReader)(f: (T, PmmlModel) => R): DataStream[R] = {
77 | val abstractOperator = new EvaluationFunction[T, R](modelReader) {
78 | override def flatMap(value: T, out: Collector[R]): Unit = out.collect(f(value, evaluator))
79 | }
80 |
81 | stream.flatMap(abstractOperator)
82 | }
83 |
84 | }
85 |
86 | /**
87 | * It wraps the connected `` stream and provides the evaluate function.
88 | *
89 | * @param connectedStream the connected stream: it chains the event Stream and the models control Stream
90 | * @tparam T Type information relative to the main event stream
91 | */
92 | implicit class RichConnectedStream[T <: BaseEvent: TypeInformation: ClassTag, CTRL <: ServingMessage](
93 | connectedStream: ConnectedStreams[T, CTRL]) {
94 |
95 | /**
96 | * It provides the evaluation function by applying
97 | * [[io.radicalbit.flink.pmml.scala.api.functions.EvaluationCoFunction]] to the connected streams.
98 | *
99 | * The first flatMap handles the event stream and applies the UDF (i.e. executing the punctual prediction)
100 | * The second flatMap handles models control stream and records the information relative to current model
101 | * and update the model instance
102 | *
103 | * @param f UDF for prediction manipulation and pre/post-processing logic
104 | * @tparam R UDF return type
105 | * @return The prediction output as defined by the UDF
106 | */
107 | def evaluate[R: TypeInformation](f: (T, PmmlModel) => R): DataStream[R] = {
108 |
109 | val abstractOperator = new EvaluationCoFunction[T, CTRL, R] {
110 |
111 | override def processElement1(event: T, ctx: CoProcessFunction[T, CTRL, R]#Context, out: Collector[R]): Unit = {
112 | val model = servingModels.getOrElse(event.modelId.hashCode, fromMetadata(event.modelId))
113 | out.collect(f(event, model))
114 | }
115 |
116 | }
117 |
118 | connectedStream.process(abstractOperator)
119 | }
120 |
121 | }
122 |
123 | /** Enriches Flink DataStream with [[evaluate]] on
124 | * [[https://ci.apache.org/projects/flink/flink-docs-release-1.2/dev/libs/ml/index.html FlinkML]]
125 | * [[org.apache.flink.ml.math.Vector]] input stream
126 | *
127 | * @param stream The input stream
128 | * @tparam V The input stream inner type; it is subclass of [[org.apache.flink.ml.math.Vector]]
129 | */
130 | implicit class QuickDataStream[V <: Vector: TypeInformation: ClassTag](stream: DataStream[V]) {
131 |
132 | /** Evaluates the `DataStream` against PmmlModel by invoking [[RichDataStream]] `evaluate` method.
133 | * It returns directly the prediction along with the input vector.
134 | *
135 | * @param modelReader The reader instance coupled to model source path.
136 | * @return (Prediction, V)
137 | */
138 | def quickEvaluate(modelReader: ModelReader): DataStream[(Prediction, V)] =
139 | new RichDataStream[V](stream).evaluate(modelReader) { (vec, model) =>
140 | val result: Prediction = model.predict(vec, None)
141 | (result, vec)
142 | }
143 | }
144 |
145 | }
146 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Copyright (C) 2017 Radicalbit
2 | #
3 | # This file is part of flink-JPMML
4 | #
5 | # flink-JPMML is free software: you can redistribute it and/or modify
6 | # it under the terms of the GNU Affero General Public License as
7 | # published by the Free Software Foundation, either version 3 of the
8 | # License, or (at your option) any later version.
9 | #
10 | # flink-JPMML is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 | # GNU Affero General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Affero General Public License
16 | # along with flink-JPMML. If not, see .
17 |
18 | log4j.rootLogger=ERROR, console
19 | log4j.logger.deng=ERROR
20 | log4j.appender.console=org.apache.log4j.ConsoleAppender
21 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
22 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/QuickDataStreamSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala
21 |
22 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
23 | import io.radicalbit.flink.pmml.scala.models.prediction.{Prediction, Score, Target}
24 | import io.radicalbit.flink.pmml.scala.utils.PmmlLoaderKit
25 | import io.radicalbit.flink.streaming.spec.core.{FlinkPipelineTestKit, FlinkTestKitCompanion}
26 | import org.apache.flink.api.scala.ClosureCleaner
27 | import org.apache.flink.ml.math.{DenseVector, SparseVector, Vector}
28 | import org.apache.flink.runtime.client.JobExecutionException
29 | import org.apache.flink.streaming.api.scala._
30 | import org.scalatest.{Matchers, WordSpecLike}
31 |
32 | object QuickDataStreamSpec extends FlinkTestKitCompanion[(Prediction, Vector)]
33 |
34 | class QuickDataStreamSpec
35 | extends FlinkPipelineTestKit[Vector, (Prediction, Vector)]
36 | with WordSpecLike
37 | with Matchers
38 | with PmmlLoaderKit {
39 |
40 | private implicit val companion = QuickDataStreamSpec
41 |
42 | private val defaultInput: Vector = DenseVector(1.0, 1.0, 1.0, 1.0)
43 | private val defaultSparseInput: Vector = SparseVector(4, Array(0, 1, 2, 3), Array(1.0, 1.0, 1.0, 1.0))
44 |
45 | private val defaultPrediction = (Prediction(Score(3.0)), defaultInput)
46 | private val sparsePrediction = (Prediction(Score(3.0)), defaultSparseInput)
47 | private val emptyPrediction = (Prediction(Target.empty), defaultInput)
48 |
49 | private def pipelineBuilder(source: Option[String]) = {
50 | val reader = ModelReader(source getOrElse getPMMLSource(Source.KmeansPmml))
51 |
52 | (in: DataStream[Vector]) =>
53 | in.quickEvaluate(reader)
54 | }
55 |
56 | "QuickDataStream" should {
57 |
58 | "quick DataStream should be serializable" in {
59 | noException should be thrownBy ClosureCleaner.clean(pipelineBuilder(None), checkSerializable = true)
60 | }
61 |
62 | "return correct output sequence on heterogeneous input" in {
63 | val in: Seq[Vector] = Seq(defaultInput, defaultSparseInput)
64 | val out = Seq(defaultPrediction, sparsePrediction)
65 | executePipeline(in)(pipelineBuilder(None)) shouldBe out
66 | }
67 |
68 | "compute quick prediction with any dense input vector" in {
69 | val in: Seq[Vector] = Seq(defaultInput)
70 | val out = Seq(defaultPrediction)
71 |
72 | executePipeline(in)(pipelineBuilder(None)) shouldBe out
73 | }
74 |
75 | "compute quick predictions with any sparse input vector" in {
76 | val in: Seq[Vector] = Seq(defaultSparseInput)
77 | val out = Seq(sparsePrediction)
78 |
79 | executePipeline(in)(pipelineBuilder(None)) shouldBe out
80 | }
81 |
82 | "throw JobExecutionException if the model path cannot be loaded" in {
83 | val invalidSource = Source.NotExistingPath
84 |
85 | an[JobExecutionException] should be thrownBy {
86 | executePipeline(Seq(defaultInput))(pipelineBuilder(Some(invalidSource))) shouldBe Seq(defaultPrediction)
87 | }
88 | }
89 |
90 | "Emit empty prediction if the input is not valid" in {
91 | val shortInput: Vector = SparseVector(2, Array(0, 3), Array(1.0, 1.0))
92 |
93 | executePipeline(Seq(shortInput))(pipelineBuilder(None)) shouldBe Seq((Prediction(Target.empty), shortInput))
94 | }
95 |
96 | "Emit empty prediction if the model is not valid" in {
97 | val invalidModelSource = getPMMLSource(Source.KmeansPmmlEmpty)
98 |
99 | an[JobExecutionException] should be thrownBy {
100 | executePipeline(Seq(defaultInput))(pipelineBuilder(Some(invalidModelSource))) shouldBe Seq(emptyPrediction)
101 | }
102 | }
103 |
104 | }
105 |
106 | }
107 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/RichConnectedStreamSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala
21 |
22 | import java.util.UUID
23 |
24 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
25 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, DelMessage, ServingMessage}
26 | import io.radicalbit.flink.pmml.scala.models.prediction.{Prediction, Score, Target}
27 | import io.radicalbit.flink.pmml.scala.utils.{FlinkSourcedPipelineTestKit, PmmlLoaderKit}
28 | import io.radicalbit.flink.pmml.scala.utils.models.{BaseInput, DynamicInput}
29 | import io.radicalbit.flink.streaming.spec.core.FlinkTestKitCompanion
30 | import org.apache.flink.runtime.client.JobExecutionException
31 | import org.apache.flink.streaming.api.scala._
32 | import org.scalatest.{Matchers, WordSpecLike}
33 |
34 | object RichConnectedStreamSpec extends FlinkTestKitCompanion[Prediction] {
35 |
36 | private val defaultDenseEvalFunction: (DynamicInput, PmmlModel) => Prediction = {
37 | (in: DynamicInput, model: PmmlModel) =>
38 | model.predict(BaseInput.toDenseVector(in), None)
39 | }
40 |
41 | }
42 |
43 | class RichConnectedStreamSpec
44 | extends FlinkSourcedPipelineTestKit[DynamicInput, ServingMessage, Prediction]
45 | with WordSpecLike
46 | with Matchers
47 | with PmmlLoaderKit {
48 |
49 | import RichConnectedStreamSpec._
50 |
51 | private implicit val companion = RichConnectedStreamSpec
52 |
53 | private val defaultPrediction = Prediction(Score(3.0))
54 | private val emptyPrediction = Prediction(Target.empty)
55 |
56 | private def pipeline(inputStream: DataStream[DynamicInput], controlStream: DataStream[ServingMessage]) =
57 | inputStream
58 | .withSupportStream(controlStream)
59 | .evaluate(defaultDenseEvalFunction)
60 |
61 | val nameModel1 = UUID.randomUUID().toString
62 | val nameModel2 = UUID.randomUUID().toString
63 |
64 | val version = "1"
65 |
66 | val idModel1 = BaseInput.toIdentifier(nameModel1, version)
67 | val idModel2 = BaseInput.toIdentifier(nameModel2, version)
68 |
69 | "RichConnectedStreamSpec" should {
70 |
71 | "compute the right prediction (-> model -> event = prediction)" in {
72 |
73 | val in1 = Seq((1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), 0)))
74 |
75 | val in2: Seq[(Long, ServingMessage)] =
76 | Seq((0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())))
77 |
78 | val out = Seq(defaultPrediction)
79 |
80 | executePipeline(in1, in2)(pipeline) shouldBe out
81 | }
82 |
83 | "compute the right prediction (-> event -> model -> event = empty prediction, default prediction)" in {
84 |
85 | val in1 = Seq(
86 | (0L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), 0)),
87 | (2L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), 2))
88 | )
89 | val in2: Seq[(Long, ServingMessage)] =
90 | Seq((1L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())))
91 |
92 | val out = Seq(emptyPrediction, defaultPrediction)
93 |
94 | executePipeline(in1, in2)(pipeline) shouldBe out
95 | }
96 |
97 | "compute the right prediction (-> model1 -> model2 -> event1 -> event2 = two predictions)" in {
98 |
99 | val in1 = Seq(
100 | (2L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
101 | (3L, DynamicInput(idModel2, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
102 | )
103 |
104 | val in2: Seq[(Long, ServingMessage)] = Seq(
105 | (0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
106 | (1L,
107 | AddMessage(nameModel2,
108 | version.toLong,
109 | getPMMLSource(Source.KmeansPmmlNoOutNoTrg),
110 | System.currentTimeMillis()))
111 | )
112 |
113 | val out = Seq(defaultPrediction, emptyPrediction)
114 |
115 | executePipeline(in1, in2)(pipeline) shouldBe out
116 | }
117 |
118 | "return the correct predictions (-> event1 -> model1 -> event2 -> model2 -> event1 -> event2 = two empty predictions and two predictions)" in {
119 |
120 | val in1 = Seq(
121 | (0L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
122 | (2L, DynamicInput(idModel2, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
123 | (4L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
124 | (5L, DynamicInput(idModel2, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
125 | )
126 | val in2: Seq[(Long, ServingMessage)] = Seq(
127 | (1L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
128 | (3L, AddMessage(nameModel2, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis()))
129 | )
130 |
131 | val output = Seq(
132 | emptyPrediction,
133 | emptyPrediction,
134 | defaultPrediction,
135 | defaultPrediction
136 | )
137 |
138 | executePipeline(in1, in2)(pipeline) shouldBe output
139 | }
140 |
141 | "return the correct predictions with only events coming" in {
142 |
143 | val in1 = Seq(
144 | (0L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
145 | (1L, DynamicInput(idModel2, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
146 | )
147 |
148 | val in2: Seq[(Long, ServingMessage)] = Seq.empty
149 |
150 | val out = Seq(emptyPrediction, emptyPrediction)
151 |
152 | executePipeline(in1, in2)(pipeline) shouldBe out
153 | }
154 |
155 | "return the correct predictions with only models coming" in {
156 |
157 | val in1 = Seq.empty[(Long, DynamicInput)]
158 |
159 | val in2: Seq[(Long, ServingMessage)] = Seq(
160 | (0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
161 | (1L,
162 | AddMessage(nameModel2,
163 | version.toLong,
164 | getPMMLSource(Source.KmeansPmmlNoOutNoTrg),
165 | System.currentTimeMillis()))
166 | )
167 |
168 | val out = Seq.empty[Prediction]
169 |
170 | executePipeline(in1, in2)(pipeline) shouldBe out
171 |
172 | }
173 |
174 | "return EmptyScore if a DelMessage delete with no models" in {
175 |
176 | val in1: Seq[(Long, DynamicInput)] =
177 | Seq((1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())))
178 |
179 | val in2: Seq[(Long, ServingMessage)] =
180 | Seq((0L, DelMessage(nameModel1, version.toLong, System.currentTimeMillis())))
181 |
182 | val out = Seq(emptyPrediction)
183 |
184 | executePipeline(in1, in2)(pipeline) shouldBe out
185 | }
186 |
187 | "return EmptyScore if a DelMessage delete the current model" in {
188 |
189 | val in1: Seq[(Long, DynamicInput)] =
190 | Seq(
191 | (1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
192 | (3L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
193 | )
194 |
195 | val in2: Seq[(Long, ServingMessage)] =
196 | Seq(
197 | (0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
198 | (2L, DelMessage(nameModel1, version.toLong, System.currentTimeMillis()))
199 | )
200 |
201 | val out = Seq(defaultPrediction, emptyPrediction)
202 |
203 | executePipeline(in1, in2)(pipeline) shouldBe out
204 | }
205 |
206 | "return Prediction if a DelMessage delete a not existing model" in {
207 |
208 | val in1: Seq[(Long, DynamicInput)] =
209 | Seq(
210 | (1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
211 | (3L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
212 | )
213 |
214 | val in2: Seq[(Long, ServingMessage)] =
215 | Seq(
216 | (0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
217 | (2L, DelMessage(nameModel2, version.toLong, System.currentTimeMillis()))
218 | )
219 |
220 | val out = Seq(defaultPrediction, defaultPrediction)
221 |
222 | executePipeline(in1, in2)(pipeline) shouldBe out
223 | }
224 |
225 | "return Prediction on Add --> Delete --> Add pattern" in {
226 |
227 | val in1: Seq[(Long, DynamicInput)] =
228 | Seq(
229 | (1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
230 | (3L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
231 | (5L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
232 | )
233 |
234 | val in2: Seq[(Long, ServingMessage)] =
235 | Seq(
236 | (0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
237 | (2L, DelMessage(nameModel1, version.toLong, System.currentTimeMillis())),
238 | (4L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis()))
239 | )
240 |
241 | val out = Seq(defaultPrediction, emptyPrediction, defaultPrediction)
242 |
243 | executePipeline(in1, in2)(pipeline) shouldBe out
244 | }
245 |
246 | "return Prediction on Delete --> Add --> Delete pattern" in {
247 |
248 | val in1: Seq[(Long, DynamicInput)] =
249 | Seq(
250 | (1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
251 | (3L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
252 | (5L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
253 | )
254 |
255 | val in2: Seq[(Long, ServingMessage)] =
256 | Seq(
257 | (0L, DelMessage(nameModel1, version.toLong, System.currentTimeMillis())),
258 | (2L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
259 | (4L, DelMessage(nameModel1, version.toLong, System.currentTimeMillis()))
260 | )
261 |
262 | val out = Seq(emptyPrediction, defaultPrediction, emptyPrediction)
263 |
264 | executePipeline(in1, in2)(pipeline) shouldBe out
265 | }
266 |
267 | "discard ADD message on same Identifier (Add message with newer version needed here)." in {
268 |
269 | val in1 = Seq(
270 | (1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())),
271 | (3L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis()))
272 | )
273 |
274 | val in2: Seq[(Long, ServingMessage)] = Seq(
275 | (0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())),
276 | (2L,
277 | AddMessage(nameModel1,
278 | version.toLong,
279 | getPMMLSource(Source.KmeansPmmlNoOutNoTrg),
280 | System.currentTimeMillis()))
281 | )
282 |
283 | val out = Seq(defaultPrediction, defaultPrediction)
284 |
285 | executePipeline(in1, in2)(pipeline) shouldBe out
286 | }
287 |
288 | "throw JobExecutionException if the model path cannot be loaded" in {
289 |
290 | val in1 = Seq((1L, DynamicInput(idModel1, List(1.0, 1.0, 1.0, 1.0), System.currentTimeMillis())))
291 |
292 | val in2: Seq[(Long, ServingMessage)] =
293 | Seq((0L, AddMessage(nameModel1, version.toLong, Source.NotExistingPath, System.currentTimeMillis())))
294 |
295 | val out = Seq(defaultPrediction)
296 |
297 | an[JobExecutionException] should be thrownBy {
298 | executePipeline(in1, in2)(pipeline) shouldBe out
299 | }
300 | }
301 |
302 | "Emit empty prediction if the input is not valid" in {
303 | val evalFunction = { (in: DynamicInput, model: PmmlModel) =>
304 | model.predict(BaseInput.toSparseVector(in, 2), None)
305 | }
306 |
307 | def customPipeline(inputStream: DataStream[DynamicInput], controlStream: DataStream[ServingMessage]) =
308 | inputStream
309 | .withSupportStream(controlStream)
310 | .evaluate(evalFunction)
311 |
312 | val in1 = Seq((1L, DynamicInput(idModel1, List(1.0, 1.0), System.currentTimeMillis())))
313 |
314 | val in2: Seq[(Long, ServingMessage)] =
315 | Seq((0L, AddMessage(nameModel1, version.toLong, getPMMLSource(Source.KmeansPmml), System.currentTimeMillis())))
316 |
317 | val out = Seq(emptyPrediction)
318 |
319 | executePipeline(in1, in2)(pipeline) shouldBe out
320 | }
321 |
322 | "Emit empty prediction if the model is not valid" in {
323 | val invalidModelSource = getPMMLSource(Source.KmeansPmmlEmpty)
324 |
325 | val in1 = Seq((1L, DynamicInput(idModel1, List(1.0, 1.0), System.currentTimeMillis())))
326 |
327 | val in2: Seq[(Long, ServingMessage)] =
328 | Seq((0L, AddMessage(nameModel1, version.toLong, invalidModelSource, System.currentTimeMillis())))
329 |
330 | val out = Seq(emptyPrediction)
331 |
332 | an[JobExecutionException] should be thrownBy {
333 | executePipeline(in1, in2)(pipeline) shouldBe out
334 | }
335 | }
336 |
337 | }
338 |
339 | }
340 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/RichDataStreamSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
24 | import io.radicalbit.flink.pmml.scala.models.prediction.{Prediction, Score, Target}
25 | import io.radicalbit.flink.pmml.scala.utils.models.{BaseInput, Input}
26 | import io.radicalbit.flink.pmml.scala.utils.PmmlLoaderKit
27 | import io.radicalbit.flink.streaming.spec.core.{FlinkPipelineTestKit, FlinkTestKitCompanion}
28 | import org.apache.flink.api.scala.ClosureCleaner
29 | import org.apache.flink.runtime.client.JobExecutionException
30 | import org.apache.flink.streaming.api.scala._
31 | import org.scalatest.{Matchers, WordSpecLike}
32 |
33 | object RichDataStreamSpec extends FlinkTestKitCompanion[Prediction] {
34 |
35 | private val defaultDenseEvalFunction = { (in: Input, model: PmmlModel) =>
36 | model.predict(BaseInput.toDenseVector(in), None)
37 | }
38 |
39 | }
40 |
41 | class RichDataStreamSpec
42 | extends FlinkPipelineTestKit[Input, Prediction]
43 | with WordSpecLike
44 | with Matchers
45 | with PmmlLoaderKit {
46 |
47 | import RichDataStreamSpec._
48 |
49 | private implicit val companion = RichDataStreamSpec
50 |
51 | private val defaultInput = Input(1.0, 1.0, 1.0, 1.0)
52 |
53 | private val defaultPrediction = Prediction(Score(3.0))
54 | private val emptyPrediction = Prediction(Target.empty)
55 |
56 | private def pipelineBuilder(source: Option[String])(
57 | f: (Input, PmmlModel) => Prediction): DataStream[Input] => DataStream[Prediction] = {
58 |
59 | val evaluator = ModelReader(source getOrElse getPMMLSource(Source.KmeansPmml))
60 |
61 | (dataInput: DataStream[Input]) =>
62 | dataInput.evaluate(evaluator)(f)
63 | }
64 |
65 | "flink-jpmml" should {
66 |
67 | "richDataStream should be serializable" in {
68 | val evalFunction = defaultDenseEvalFunction
69 |
70 | noException should be thrownBy ClosureCleaner.clean(pipelineBuilder(None)(evalFunction))
71 | }
72 |
73 | "compute predictions with any input and an evaluation function" in {
74 | val in = Seq(defaultInput)
75 | val out = Seq(defaultPrediction)
76 |
77 | val evalFunction = defaultDenseEvalFunction
78 |
79 | executePipeline(in)(pipelineBuilder(None)(evalFunction)) shouldBe out
80 | }
81 |
82 | "throw JobExecutionException if the model path cannot be loaded" in {
83 | val randomSource = Source.NotExistingPath
84 |
85 | an[JobExecutionException] should be thrownBy {
86 | executePipeline(Seq(defaultInput))(pipelineBuilder(Some(randomSource))(defaultDenseEvalFunction)) shouldBe Seq(
87 | defaultPrediction)
88 | }
89 | }
90 |
91 | "Emit empty prediction if the input is not valid" in {
92 | val evalFunction = { (in: Input, model: PmmlModel) =>
93 | model.predict(BaseInput.toSparseVector(in, 2), None)
94 | }
95 | executePipeline(Seq(Input(1.0, 3.0)))(pipelineBuilder(None)(evalFunction)) shouldBe Seq(emptyPrediction)
96 | }
97 |
98 | "Emit empty prediction if the model is not valid" in {
99 | val invalidModelSource = getPMMLSource(Source.KmeansPmmlEmpty)
100 |
101 | an[JobExecutionException] should be thrownBy {
102 | executePipeline(Seq(defaultInput))(pipelineBuilder(Some(invalidModelSource))(defaultDenseEvalFunction)) shouldBe Seq(
103 | emptyPrediction)
104 | }
105 | }
106 |
107 | }
108 |
109 | }
110 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/EvaluatorSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api
21 |
22 | import io.radicalbit.flink.pmml.scala.api.exceptions.EmptyEvaluatorException
23 | import io.radicalbit.flink.pmml.scala.utils.{PmmlEvaluatorKit, PmmlLoaderKit}
24 | import org.jpmml.evaluator.ModelEvaluatorFactory
25 | import org.scalatest.{Matchers, WordSpec}
26 |
27 | class EvaluatorSpec extends WordSpec with Matchers with PmmlEvaluatorKit with PmmlLoaderKit {
28 |
29 | private val pmmlModel = ModelEvaluatorFactory.newInstance.newModelEvaluator(getPMMLResource(Source.KmeansPmml))
30 | private val pmmlEvaluator = Evaluator(pmmlModel)
31 |
32 | private val emptyEvaluator =
33 | Evaluator.empty
34 |
35 | "Evaluator" should {
36 |
37 | "have right box called PmmlEvaluator and with right model" in {
38 | pmmlEvaluator shouldBe PmmlEvaluator(pmmlModel)
39 | }
40 |
41 | "have right empty box called EmptyEvaluator and with right value" in {
42 | emptyEvaluator shouldBe EmptyEvaluator
43 | }
44 |
45 | "have a model method that return the pmml model" in {
46 | pmmlEvaluator.model shouldBe pmmlModel
47 | }
48 |
49 | "have a getOrElse method that return pmml model" in {
50 | emptyEvaluator.getOrElse(pmmlModel) shouldBe pmmlModel
51 | pmmlEvaluator.getOrElse(pmmlModel) shouldBe pmmlModel
52 | }
53 |
54 | "throw an EmptyEvaluatorException if call model on emptyEvaluator" in {
55 | an[EmptyEvaluatorException] should be thrownBy emptyEvaluator.model
56 | }
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/converter/VectorConverterSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.converter
21 |
22 | import io.radicalbit.flink.pmml.scala.api.Evaluator
23 | import io.radicalbit.flink.pmml.scala.utils.{PmmlEvaluatorKit, PmmlLoaderKit}
24 | import org.apache.flink.ml.math.{DenseVector, SparseVector, Vector}
25 | import org.scalatest.{Matchers, WordSpec}
26 |
27 | import scala.collection.JavaConversions._
28 |
29 | class VectorConverterSpec extends WordSpec with Matchers with PmmlEvaluatorKit with PmmlLoaderKit {
30 |
31 | import VectorConverter._
32 |
33 | private val evaluator = buildEvaluator(getPMMLResource(Source.KmeansPmml))
34 |
35 | private val modelKeys: Seq[String] = evaluator.model.getActiveFields.map(_.getName.getValue)
36 |
37 | private def implicitTestConverter(input: Vector, evaluator: Evaluator)(
38 | implicit f: (Vector, Evaluator) => Map[String, Any]) = f(input, evaluator)
39 |
40 | "VectorConverter" should {
41 |
42 | "convert a DenseVector to Map[String, Double]" in {
43 | val inputVector = DenseVector(1.0, 2.0, -1.0, 0.2)
44 | val outputValues = inputVector.data
45 |
46 | implicitTestConverter(inputVector, evaluator) shouldBe modelKeys.zip(outputValues).toMap
47 | }
48 |
49 | "convert a trivial SparseVector to Map[String, Double]" in {
50 | val inputVector = SparseVector(4, Array(0, 1, 2, 3), Array(1.0, 2.0, 3.0, 4.0))
51 | val outputValues = inputVector.toDenseVector.data
52 |
53 | implicitTestConverter(inputVector, evaluator) shouldBe modelKeys.zip(outputValues).toMap
54 | }
55 |
56 | "convert a non-trivial SparseVector to Map[String, Double]" in {
57 | val inputVector = SparseVector(4, Array(0, 2), Array(1.0, 2.0))
58 | val outputValues = Array.fill(4)(None: Option[Double])
59 |
60 | inputVector.indices.foreach(index => outputValues(index) = Some(inputVector(index)))
61 |
62 | implicitTestConverter(inputVector, evaluator) shouldBe modelKeys
63 | .zip(outputValues)
64 | .collect { case (fieldKey, Some(fieldValue)) => (fieldKey, fieldValue) }
65 | .toMap
66 |
67 | }
68 |
69 | "convert a short DenseVector to incomplete Map[String, Double]" in {
70 | val inputVector = DenseVector(4.0, -1.0)
71 | val outputVector = inputVector.data
72 |
73 | implicitTestConverter(inputVector, evaluator) shouldBe modelKeys.zip(outputVector).toMap
74 | }
75 |
76 | "return a key for binding map" in {
77 | modelKeys shouldBe Seq("sepal_length", "sepal_width", "petal_length", "petal_width")
78 | }
79 |
80 | }
81 |
82 | }
83 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/functions/EvaluationCoFunctionSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.functions
21 |
22 | import java.util.UUID
23 |
24 | import io.radicalbit.flink.pmml.scala.api.exceptions.{ModelLoadingException, WrongModelIdFormat}
25 | import io.radicalbit.flink.pmml.scala.api.{Evaluator, PmmlModel}
26 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, ServingMessage}
27 | import io.radicalbit.flink.pmml.scala.models.core.{ModelId, ModelInfo}
28 | import io.radicalbit.flink.pmml.scala.models.prediction.{Prediction, Score}
29 | import io.radicalbit.flink.pmml.scala.utils.models.{BaseInput, DynamicInput}
30 | import io.radicalbit.flink.pmml.scala.utils._
31 | import io.radicalbit.flink.streaming.spec.core.FlinkTestKitCompanion
32 | import org.apache.flink.api.scala.ClosureCleaner
33 | import org.apache.flink.streaming.api.functions.co.CoProcessFunction
34 | import org.apache.flink.streaming.api.scala._
35 | import org.apache.flink.util.Collector
36 | import org.scalatest.{Matchers, WordSpecLike}
37 |
38 | import scala.collection.immutable
39 |
40 | object EvaluationCoFunctionSpec extends FlinkTestKitCompanion[Prediction] {
41 |
42 | private def evaluationCoOperator(f: (DynamicInput, PmmlModel) => Prediction) =
43 | new EvaluationCoFunction[DynamicInput, ServingMessage, Prediction] {
44 |
45 | servingMetadata = immutable.Map.empty[ModelId, ModelInfo]
46 |
47 | override def processElement1(event: DynamicInput,
48 | ctx: CoProcessFunction[DynamicInput, ServingMessage, Prediction]#Context,
49 | out: Collector[Prediction]): Unit =
50 | out.collect(f(event, servingModels.getOrElse(event.modelId.hashCode, fromMetadata(event.modelId))))
51 | }
52 |
53 | private val defaultPrediction = Prediction(Score(1.0))
54 |
55 | private val operator = evaluationCoOperator((_: DynamicInput, _: PmmlModel) => defaultPrediction)
56 |
57 | def operatorFunction: (DynamicInput, PmmlModel) => Prediction = (_, _) => defaultPrediction
58 |
59 | }
60 |
61 | class EvaluationCoFunctionSpec
62 | extends FlinkSourcedPipelineTestKit[DynamicInput, ServingMessage, Prediction]
63 | with WordSpecLike
64 | with Matchers
65 | with PmmlLoaderKit
66 | with PmmlEvaluatorKit {
67 |
68 | import EvaluationCoFunctionSpec._
69 |
70 | private implicit val companion = EvaluationCoFunctionSpec
71 |
72 | private val modelPath = getPMMLSource(Source.KmeansPmml)
73 |
74 | private def pipeline(events: DataStream[DynamicInput], control: DataStream[ServingMessage]) =
75 | events.withSupportStream(control).process(operator)
76 |
77 | val nameModel: String = UUID.randomUUID().toString
78 | val version = "1"
79 | val modelId: String = BaseInput.toIdentifier(nameModel, version)
80 |
81 | "EvaluationCoFunction" should {
82 |
83 | "be Serializable" in {
84 | noException should be thrownBy ClosureCleaner.clean(operator, checkSerializable = true)
85 | }
86 |
87 | "return expected behavior on given function" in {
88 | val eventsStream = Seq(
89 | (0L, DynamicInput(modelId, List(1.0, 4.0, 2.0), System.currentTimeMillis())),
90 | (2L, DynamicInput(modelId, List(2.0, -1.0, 0.0), System.currentTimeMillis()))
91 | )
92 | val controlStream: Seq[(Long, ServingMessage)] = Seq(
93 | (1L, AddMessage(nameModel, version.toLong, modelPath, System.currentTimeMillis())),
94 | (3L, AddMessage(nameModel, version.toLong, modelPath, System.currentTimeMillis()))
95 | )
96 | executePipeline(eventsStream, controlStream)(pipeline) shouldBe Seq(defaultPrediction, defaultPrediction)
97 | }
98 |
99 | "load successfully a model" in {
100 |
101 | val modelInstance = buildEvaluator(getPMMLResource(Source.KmeansPmml))
102 |
103 | operator.loadModel(getPMMLSource(Source.KmeansPmml)).evaluator === modelInstance
104 | }
105 |
106 | "restore successfully a model" in {
107 |
108 | val modelInstance = buildEvaluator(getPMMLResource(Source.KmeansPmml))
109 |
110 | operator.loadModel(getPMMLSource(Source.KmeansPmml)).evaluator.model === modelInstance
111 | }
112 |
113 | "raise exception on wrong model" in {
114 |
115 | an[ModelLoadingException] shouldBe thrownBy { operator.loadModel(Source.NotExistingPath) }
116 | }
117 |
118 | }
119 |
120 | "fromMetadata function" should {
121 |
122 | "return an EmptyEvaluator if the servingMetadata is empty" in {
123 |
124 | val randModelId = BaseInput.toIdentifier(UUID.randomUUID().toString, version)
125 |
126 | val emptyEvaluator = new PmmlModel(Evaluator.empty).evaluator
127 |
128 | evaluationCoOperator(operatorFunction).fromMetadata(randModelId).evaluator shouldBe emptyEvaluator
129 |
130 | }
131 |
132 | "throw a WrongIdModelFormat if the id model format is wrong" in {
133 |
134 | val wrongIdModel = UUID.randomUUID().toString
135 |
136 | a[WrongModelIdFormat] shouldBe thrownBy { evaluationCoOperator(operatorFunction).fromMetadata(wrongIdModel) }
137 | }
138 |
139 | "throw a WrongIdModelFormat if the id model format is wrong 2" in {
140 |
141 | val wrongIdModel = BaseInput.toIdentifier(UUID.randomUUID().toString, UUID.randomUUID().toString)
142 |
143 | a[WrongModelIdFormat] shouldBe thrownBy { evaluationCoOperator(operatorFunction).fromMetadata(wrongIdModel) }
144 | }
145 |
146 | }
147 |
148 | }
149 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/functions/EvaluationFunctionSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.functions
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
24 | import io.radicalbit.flink.pmml.scala.models.prediction.{Prediction, Score}
25 | import io.radicalbit.flink.pmml.scala.utils.models.Input
26 | import io.radicalbit.flink.pmml.scala.utils.PmmlLoaderKit
27 | import io.radicalbit.flink.streaming.spec.core.{FlinkPipelineTestKit, FlinkTestKitCompanion}
28 | import org.apache.flink.api.scala.ClosureCleaner
29 | import org.apache.flink.streaming.api.scala._
30 | import org.apache.flink.util.Collector
31 | import org.scalatest.{Matchers, WordSpecLike}
32 |
33 | object EvaluationFunctionSpec extends FlinkTestKitCompanion[Prediction]
34 |
35 | class EvaluationFunctionSpec
36 | extends FlinkPipelineTestKit[Input, Prediction]
37 | with WordSpecLike
38 | with Matchers
39 | with PmmlLoaderKit {
40 |
41 | private implicit val companion = EvaluationFunctionSpec
42 |
43 | private val reader = ModelReader(getPMMLSource(Source.KmeansPmml))
44 |
45 | private def evaluationOperator[T](source: ModelReader)(f: (T, PmmlModel) => Prediction) =
46 | new EvaluationFunction[T, Prediction](source) {
47 | override def flatMap(value: T, out: Collector[Prediction]): Unit = out.collect(f(value, evaluator))
48 | }
49 |
50 | private val operator = evaluationOperator(reader) { (in: Input, model: PmmlModel) =>
51 | Prediction(Score(1.0))
52 | }
53 |
54 | private def pipeline(source: DataStream[Input]): DataStream[Prediction] = source.flatMap(operator)
55 |
56 | "EvaluationFunction" should {
57 |
58 | "be Serializable" in {
59 | noException should be thrownBy ClosureCleaner.clean(operator, checkSerializable = true)
60 | }
61 |
62 | "return expected behavior on given function" in {
63 | executePipeline(Seq(Input(1.0, 2.0)))(pipeline) shouldBe Seq(Prediction(Score(1.0)))
64 | }
65 |
66 | }
67 |
68 | }
69 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/managers/MetadataManagerSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.managers
21 |
22 | import io.radicalbit.flink.pmml.scala.models.control.{AddMessage, DelMessage}
23 | import io.radicalbit.flink.pmml.scala.models.core.{ModelId, ModelInfo}
24 | import io.radicalbit.flink.pmml.scala.utils.PmmlLoaderKit
25 | import org.scalatest.{Matchers, WordSpec}
26 |
27 | import scala.collection.immutable
28 |
29 | abstract class MetadataManagerSpec[M: MetadataManager] extends WordSpec with Matchers with PmmlLoaderKit {
30 |
31 | val modelName = "model"
32 | val modelVersion = 1
33 | val modelPath: String = getPMMLSource(Source.KmeansPmml)
34 |
35 | val modelId: ModelId = ModelId(modelName, modelVersion)
36 | val modelInfo = ModelInfo(modelPath)
37 |
38 | val in = immutable.Map(modelId -> modelInfo)
39 | val unknownIn = immutable.Map(ModelId("unknown-id", scala.util.Random.nextLong()) -> modelInfo)
40 |
41 | def outOnKnown: immutable.Map[ModelId, ModelInfo] = toOut(in)
42 | def outOnUnknown: immutable.Map[ModelId, ModelInfo] = toOut(unknownIn)
43 |
44 | def toOut(in: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo]
45 | def controlMessage: M
46 |
47 | "MetadataManager" should {
48 |
49 | "manage metadata correctly if targeted model is not already in metadata (Add add metadata, Del returns input)" in {
50 | MetadataManager(controlMessage, unknownIn) shouldBe outOnUnknown
51 | }
52 |
53 | "manage metadata correctly if targeted model already exists (Add returns input, Del removes metadata)" in {
54 | MetadataManager(controlMessage, in) shouldBe outOnKnown
55 | }
56 |
57 | }
58 |
59 | }
60 |
61 | class AddMetadataManagerSpec extends MetadataManagerSpec[AddMessage] {
62 | override lazy val controlMessage: AddMessage =
63 | AddMessage(modelName, modelVersion, modelPath, System.currentTimeMillis())
64 |
65 | override def toOut(in: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo] =
66 | in.get(modelId) match {
67 | case Some(_) => in
68 | case None => in + (modelId -> modelInfo)
69 | }
70 | }
71 |
72 | class RemoveMetadataManagerSpec extends MetadataManagerSpec[DelMessage] {
73 | override lazy val controlMessage: DelMessage = DelMessage(modelName, modelVersion, System.currentTimeMillis())
74 |
75 | override def toOut(in: immutable.Map[ModelId, ModelInfo]): immutable.Map[ModelId, ModelInfo] =
76 | in - ModelId(modelName, modelVersion)
77 | }
78 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/managers/ModelsManagerSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.managers
21 |
22 | import io.radicalbit.flink.pmml.scala.api.PmmlModel
23 | import io.radicalbit.flink.pmml.scala.api.reader.ModelReader
24 | import io.radicalbit.flink.pmml.scala.models.control.DelMessage
25 | import io.radicalbit.flink.pmml.scala.models.core.ModelId
26 | import io.radicalbit.flink.pmml.scala.utils.PmmlLoaderKit
27 | import org.scalatest.{Matchers, WordSpec}
28 |
29 | import scala.collection.{immutable, mutable}
30 |
31 | abstract class ModelsManagerSpec[M: ModelsManager] extends WordSpec with Matchers with PmmlLoaderKit {
32 |
33 | val modelName = "model"
34 | val modelVersion = 1
35 | val modelPath: String = getPMMLSource(Source.KmeansPmml)
36 |
37 | val modelId: Int = ModelId(modelName, modelVersion).hashCode
38 |
39 | private val pmmlModel = PmmlModel.fromReader(ModelReader(getPMMLSource(Source.KmeansPmml42)))
40 | private val unknownIn = mutable.Map(scala.util.Random.nextInt() -> pmmlModel)
41 | private val in = mutable.Map(modelId -> pmmlModel)
42 |
43 | def controlMessage: M
44 |
45 | def onOut(in: mutable.Map[Int, PmmlModel]): immutable.Set[Int]
46 |
47 | def knownOut: immutable.Set[Int] = onOut(in)
48 | def unknownOut: immutable.Set[Int] = onOut(unknownIn)
49 |
50 | "ModelsManager" should {
51 |
52 | "remove models method correctly on known models identifier" in {
53 | ModelsManager(controlMessage, in) shouldBe knownOut
54 | }
55 |
56 | "remove nothing on unknown models identifier" in {
57 | ModelsManager(controlMessage, unknownIn) shouldBe unknownOut
58 | }
59 |
60 | }
61 |
62 | }
63 |
64 | class ModelsManagerDelMessageSpec extends ModelsManagerSpec[DelMessage] {
65 |
66 | override lazy val controlMessage: DelMessage = DelMessage(modelName, modelVersion, System.currentTimeMillis())
67 |
68 | override def onOut(in: mutable.Map[Int, PmmlModel]): Set[Int] =
69 | in.keySet.intersect(Set(controlMessage.modelId.hashCode)).toSet
70 |
71 | }
72 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/api/reader/ModelReaderSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.api.reader
21 |
22 | import java.io.{File, IOException}
23 |
24 | import org.apache.flink.core.fs.Path
25 | import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem
26 | import org.apache.hadoop.conf.Configuration
27 | import org.apache.hadoop.fs.FileUtil
28 | import org.apache.hadoop.hdfs.MiniDFSCluster
29 | import org.scalatest.{BeforeAndAfter, Matchers, WordSpec}
30 |
31 | import scala.util.Try
32 |
33 | object ModelReaderSpec {
34 | private case class FakeReader()
35 |
36 | private val modelString =
37 | """|
38 | |
39 | |
42 | |
43 | |
44 | |
45 | |
46 | |
47 | |
48 | |
49 | |
50 | |
51 | |
52 | |
53 | |
54 | |
55 | |
56 | |
57 | |
58 | |
59 | |
60 | |
61 | |
62 | |
63 | |
64 | |
65 | |
66 | |
67 | |
68 | |
69 | |
70 | |
71 | |
72 | |
73 | |
74 | |
75 | |
76 | |
77 | | 6.9125000000000005 3.099999999999999 5.846874999999999 2.1312499999999996
78 | |
79 | |
80 | | 6.23658536585366 2.8585365853658535 4.807317073170731 1.6219512195121943
81 | |
82 | |
83 | | 5.005999999999999 3.4180000000000006 1.464 0.2439999999999999
84 | |
85 | |
86 | | 5.529629629629629 2.6222222222222222 3.940740740740741 1.2185185185185188
87 | |
88 | |
91 | |
92 | |
93 | |
94 | """.stripMargin
95 |
96 | }
97 |
98 | class ModelReaderSpec extends WordSpec with BeforeAndAfter with Matchers {
99 |
100 | import ModelReaderSpec._
101 |
102 | protected var hdfsURI: String = _
103 | private var hdfsCluster: MiniDFSCluster = _
104 | private var hdfsPath: org.apache.hadoop.fs.Path = _
105 | protected var hdfs: org.apache.hadoop.fs.FileSystem = _
106 |
107 | before {
108 | try {
109 | val conf = new Configuration()
110 | val baseDir = new File("./target/hdfs/hdfsTestModel").getAbsoluteFile
111 | FileUtil.fullyDelete(baseDir)
112 | conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath)
113 |
114 | val builder = new MiniDFSCluster.Builder(conf)
115 | hdfsCluster = builder.build()
116 |
117 | hdfsURI = "hdfs://" + hdfsCluster.getURI.getHost + ":" + hdfsCluster.getNameNodePort + "/"
118 |
119 | hdfsPath = new org.apache.hadoop.fs.Path("/model.xml")
120 | hdfs = hdfsPath.getFileSystem(conf)
121 | val stream = hdfs.create(hdfsPath)
122 | stream.write(modelString.getBytes("UTF-8"))
123 | stream.close()
124 | } catch {
125 | case (e: Throwable) =>
126 | e.printStackTrace()
127 | fail("Test fail" + e.getMessage)
128 | }
129 | }
130 |
131 | after {
132 | try {
133 | hdfs.delete(hdfsPath, false)
134 | hdfsCluster.shutdown()
135 | } catch {
136 | case (e: IOException) =>
137 | throw new RuntimeException(e)
138 | }
139 | }
140 |
141 | "Model reader on HDFS" should {
142 |
143 | "control GetFileSystem type" in {
144 | val pathFile = new Path(hdfsURI + hdfsPath)
145 | val fs = pathFile.getFileSystem
146 | fs shouldBe a[HadoopFileSystem]
147 | }
148 |
149 | "support correctly FsReader" in {
150 | val pathFile = hdfsURI + hdfsPath
151 | val reader = new ModelReader(pathFile) with FsReader
152 | val reuslt = Try(reader.buildDistributedPath)
153 | reuslt.getOrElse("") should equal(modelString)
154 | }
155 |
156 | "support FsReader on not found path model" in {
157 | val wrongPath = hdfsURI + "/wrong/path"
158 | val reader = new ModelReader(wrongPath) with FsReader
159 | val result = Try(reader.buildDistributedPath)
160 | result.getOrElse("") should equal("")
161 | }
162 | }
163 | }
164 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/core/ModelIdSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.core
21 |
22 | import io.radicalbit.flink.pmml.scala.api.exceptions.WrongModelIdFormat
23 | import org.scalatest.{Matchers, WordSpec}
24 |
25 | class ModelIdSpec extends WordSpec with Matchers {
26 | private val uuid = "2bc91c20-bf64-4860-ba11-de97a95d05cf"
27 | private val version = "23"
28 |
29 | "Parser" should {
30 |
31 | "matched uuid" in {
32 | val modelId = uuid + ModelId.separatorSymbol + version
33 | val modelIdExpected = ModelId(uuid, 23)
34 |
35 | ModelId.fromIdentifier(modelId) shouldBe modelIdExpected
36 | }
37 |
38 | "wrong uuid" in {
39 | val wrongUuid = "23441-" + ModelId.separatorSymbol + version
40 | an[WrongModelIdFormat] should be thrownBy ModelId.fromIdentifier(wrongUuid)
41 | }
42 |
43 | }
44 |
45 | "ModelId" should {
46 |
47 | "return the right hash code" in {
48 | val fullId = uuid + ModelId.separatorSymbol + version
49 | val modelId = ModelId(uuid, 23)
50 |
51 | modelId.hashCode shouldBe fullId.hashCode
52 | }
53 | }
54 |
55 | }
56 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/prediction/PredictionSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.prediction
21 |
22 | import io.radicalbit.flink.pmml.scala.api.exceptions._
23 | import org.jpmml.evaluator.EvaluationException
24 | import org.scalatest.{Matchers, WordSpec}
25 |
26 | import scala.util.Try
27 |
28 | class PredictionSpec extends WordSpec with Matchers {
29 |
30 | private def throwableFunc[E <: Exception](exception: E)(f: PartialFunction[Throwable, Prediction]) =
31 | Try(throw exception).recover(f).get
32 |
33 | "Prediction" should {
34 |
35 | "extract prediction if the extraction is Success" in {
36 | Prediction.extractPrediction(Try(2.0)) shouldBe Prediction(Score(2.0))
37 | }
38 |
39 | "testing prediction getOrElse EmptyScore" in {
40 | val emptyPrediction: Prediction = Prediction(Target.empty)
41 | emptyPrediction.value.getOrElse(-1.0) shouldBe -1.0
42 | }
43 |
44 | "tesing prediction getOrElse with Score" in {
45 | val prediction: Prediction = Prediction(Score(3.0))
46 | prediction.value.getOrElse(-1.0) shouldBe 3.0
47 | }
48 |
49 | "extract empty prediction if the extraction is Failure" in {
50 | Prediction.extractPrediction(Try(2 / 0)) shouldBe Prediction(EmptyScore)
51 | }
52 |
53 | "return None if onFailedPrediction is active and JPMMLExtractionException" in {
54 | throwableFunc(new JPMMLExtractionException("")) {
55 | case e: Throwable => Prediction.onFailedPrediction(e)
56 | } shouldBe Prediction(EmptyScore)
57 | }
58 |
59 | "return None if onFailedPrediction is active and InputPreparationException" in {
60 | throwableFunc(new InputPreparationException("")) {
61 | case e: Throwable => Prediction.onFailedPrediction(e)
62 | } shouldBe Prediction(EmptyScore)
63 | }
64 |
65 | "return None if onFailedPrediction is active and InputValidationException" in {
66 | throwableFunc(new InputValidationException("")) {
67 | case e: Throwable => Prediction.onFailedPrediction(e)
68 | } shouldBe Prediction(EmptyScore)
69 | }
70 |
71 | "return None if onFailedPrediction is active and EvaluationException" in {
72 | throwableFunc(new EvaluationException) {
73 | case e: Throwable => Prediction.onFailedPrediction(e)
74 | } shouldBe Prediction(EmptyScore)
75 | }
76 |
77 | "return None if onFailedPrediction is active and ClassCastException" in {
78 | throwableFunc(new ClassCastException) {
79 | case e: Throwable => Prediction.onFailedPrediction(e)
80 | } shouldBe Prediction(EmptyScore)
81 | }
82 |
83 | "return None if onFailedPrediction is active and whatever Exception" in {
84 | throwableFunc(new Exception) {
85 | case e: Throwable => Prediction.onFailedPrediction(e)
86 | } shouldBe Prediction(EmptyScore)
87 | }
88 |
89 | }
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/models/prediction/TargetSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.models.prediction
21 |
22 | import org.scalatest.{Matchers, WordSpec}
23 |
24 | class TargetSpec extends WordSpec with Matchers {
25 | private val target = Target(3.0)
26 | private val emptyTarget = Target.empty
27 |
28 | "Target" should {
29 |
30 | "have right box called Score and with right value" in {
31 | target shouldBe Score(3.0)
32 | }
33 |
34 | "have right empty box called EmptyScore and with right value " in {
35 | emptyTarget shouldBe EmptyScore
36 | }
37 |
38 | "have a get method" in {
39 | target.get shouldBe 3.0
40 | }
41 |
42 | "throw a NoSuchElementException if get on emptyTarget" in {
43 | an[NoSuchElementException] should be thrownBy emptyTarget.get
44 | }
45 |
46 | "have a getOrElse method and return vale if is score" in {
47 | target.getOrElse(-1.0) shouldBe 3.0
48 | }
49 |
50 | "have a getOrElse method and return empty value if is empty score" in {
51 | emptyTarget.getOrElse(-1.0) shouldBe -1.0
52 | }
53 |
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/sources/TemporizedSourceFunction.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.sources
21 |
22 | import org.apache.flink.streaming.api.functions.source.SourceFunction
23 | import org.apache.flink.util.FlinkRuntimeException
24 |
25 | /**
26 | * This class can be used when you need to test connect operator in Flink
27 | * It streams a sequence of item according to the order of the inputList
28 | * interval defines the time gap between streaming two items
29 | *
30 | * @param events Zipped sequence where the first element describes the order of sourcing and
31 | * the second is the event to be sourced
32 | * @tparam L The type of the first event which needs to be sourced
33 | * @tparam R The type of the second event which needs to be sourced
34 | */
35 | class TemporizedSourceFunction[L, R](events: Seq[(Option[L], Option[R])]) extends SourceFunction[Either[L, R]] {
36 |
37 | override def run(ctx: SourceFunction.SourceContext[Either[L, R]]): Unit = {
38 |
39 | events.foreach { tuple: (Option[L], Option[R]) =>
40 | val outputEvent: Either[L, R] = tuple match {
41 | case (Some(left), None) => Left(left)
42 | case (None, Some(right)) => Right(right)
43 | case _ => throw new FlinkRuntimeException(s"Temporized source function was not able to decode input.")
44 | }
45 | Thread.sleep(500l)
46 | ctx.getCheckpointLock.synchronized {
47 | ctx.collect(outputEvent)
48 | }
49 |
50 | }
51 |
52 | }
53 |
54 | override def cancel(): Unit = {}
55 |
56 | }
57 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/FlinkTestKits.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.utils
21 |
22 | import io.radicalbit.flink.pmml.scala.sources.TemporizedSourceFunction
23 | import io.radicalbit.flink.streaming.spec.core.FlinkTestKitCompanion
24 | import org.apache.flink.api.common.typeinfo.TypeInformation
25 | import org.apache.flink.streaming.api.functions.sink.SinkFunction
26 | import org.apache.flink.streaming.api.scala._
27 | import org.apache.flink.test.util.AbstractTestBase
28 |
29 | import scala.collection.mutable
30 | import scala.reflect.ClassTag
31 |
32 | trait FlinkSourcedPipelineTestKit[IN1, IN2, OUT] extends AbstractTestBase {
33 |
34 | def executePipeline[IN1: TypeInformation: ClassTag, IN2: TypeInformation: ClassTag](
35 | in1: Seq[(Long, IN1)],
36 | in2: Seq[(Long, IN2)])(pipeline: (DataStream[IN1], DataStream[IN2]) => DataStream[OUT])(
37 | implicit companion: FlinkTestKitCompanion[OUT]) = {
38 |
39 | companion.testResults = mutable.MutableList[OUT]()
40 |
41 | val env = StreamExecutionEnvironment.getExecutionEnvironment
42 | env.setParallelism(1)
43 |
44 | val events = in1
45 | .union(in2)
46 | .sortBy(_._1)
47 | .collect {
48 | case (_, left: IN1) => (Some(left), None)
49 | case (_, right: IN2) => (None, Some(right))
50 | }
51 |
52 | val stream = env.addSource(new TemporizedSourceFunction[IN1, IN2](events))
53 |
54 | val stream1: DataStream[IN1] = stream.filter(either => either.isLeft).map(either => either.left.get)
55 | val stream2: DataStream[IN2] = stream.filter(either => either.isRight).map(either => either.right.get)
56 |
57 | pipeline(stream1, stream2)
58 | .addSink(new SinkFunction[OUT] {
59 | override def invoke(in: OUT) = {
60 | companion.testResults += in
61 | }
62 | })
63 |
64 | env.execute(this.getClass.getSimpleName)
65 |
66 | companion.testResults
67 | }
68 |
69 | }
70 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlEvaluatorKit.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.utils
21 |
22 | import io.radicalbit.flink.pmml.scala.api.Evaluator
23 | import org.apache.flink.ml.math.{DenseVector, SparseVector, Vector}
24 | import org.dmg.pmml.{FieldName, PMML}
25 | import org.jpmml.evaluator.{FieldValueUtil, ModelEvaluatorFactory}
26 |
27 | trait PmmlEvaluatorKit {
28 |
29 | final protected def buildEvaluator(pmml: PMML): Evaluator =
30 | Evaluator(ModelEvaluatorFactory.newInstance.newModelEvaluator(pmml))
31 |
32 | final protected def buildExpectedInputMap(in: Vector, keys: Seq[String]) = {
33 | val data: Seq[Option[Double]] = in match {
34 | case dv: DenseVector => dv.data.map(Option(_))
35 | case sv: SparseVector => (0 to keys.size).map(index => if (sv.indices.contains(index)) Some(sv(index)) else None)
36 | }
37 |
38 | keys.zip(data).collect { case (k, Some(v)) => k -> v } toMap
39 | }
40 |
41 | final protected def buildExpectedPreparedMap(in: Map[String, Any], keys: Seq[String], replaceValue: Option[Double]) =
42 | keys.map {
43 | case k if in.contains(k) => new FieldName(k) -> FieldValueUtil.create(null, null, in(k))
44 | case emptyKey => new FieldName(emptyKey) -> FieldValueUtil.create(null, null, replaceValue.orNull)
45 | } toMap
46 |
47 | }
48 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/PmmlLoaderKit.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.utils
21 |
22 | import org.dmg.pmml.PMML
23 | import org.jpmml.model.{ImportFilter, JAXBUtil}
24 | import org.xml.sax.InputSource
25 |
26 | trait PmmlLoaderKit {
27 |
28 | protected case object Source {
29 | val KmeansPmml = "/kmeans.xml"
30 | val KmeansPmml41 = "/kmeans41.xml"
31 | val KmeansPmml40 = "/kmeans40.xml"
32 | val KmeansPmml42 = "/kmeans42.xml"
33 | val KmeansPmml32 = "/kmeans41.xml"
34 |
35 | val KmeansPmmlEmpty = "/kmeans_empty.xml"
36 | val KmeansPmmlNoOut = "/kmeans_nooutput.xml"
37 | val KmeansPmmlStringFields = "/kmeans_stringfields.xml"
38 | val KmeansPmmlNoOutNoTrg = "/kmeans_nooutput_notarget.xml"
39 | val NotExistingPath: String = "/not/existing/" + scala.util.Random.nextString(4)
40 | }
41 |
42 | final protected def getPMMLSource(path: String): String =
43 | getClass.getResource(path).getPath
44 |
45 | final protected def getPMMLResource(path: String): PMML = {
46 | val source = scala.io.Source.fromURL(getClass.getResource(path)).reader()
47 | JAXBUtil.unmarshalPMML(ImportFilter.apply(new InputSource(source)))
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/flink-jpmml-scala/src/test/scala/io/radicalbit/flink/pmml/scala/utils/models/Input.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2017 Radicalbit
3 | *
4 | * This file is part of flink-JPMML
5 | *
6 | * flink-JPMML is free software: you can redistribute it and/or modify
7 | * it under the terms of the GNU Affero General Public License as
8 | * published by the Free Software Foundation, either version 3 of the
9 | * License, or (at your option) any later version.
10 | *
11 | * flink-JPMML is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU Affero General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU Affero General Public License
17 | * along with flink-JPMML. If not, see .
18 | */
19 |
20 | package io.radicalbit.flink.pmml.scala.utils.models
21 |
22 | import io.radicalbit.flink.pmml.scala.models.core.ModelId
23 | import io.radicalbit.flink.pmml.scala.models.input.BaseEvent
24 | import org.apache.flink.ml.math.{DenseVector, SparseVector}
25 |
26 | object BaseInput {
27 |
28 | def toDenseVector(input: BaseInput): DenseVector =
29 | DenseVector(input.values.toArray)
30 |
31 | def toSparseVector(input: BaseInput, size: Int): SparseVector =
32 | SparseVector(size, Array.range(0, input.size - 1), input.values.toArray)
33 |
34 | def toIdentifier(name: String, version: String): String = name + ModelId.separatorSymbol + version
35 |
36 | }
37 | sealed trait BaseInput {
38 |
39 | def values: Seq[Double]
40 |
41 | def size: Int
42 | }
43 |
44 | object Input {
45 | def apply(rawValues: Double*): Input = Input(values = rawValues.toList)
46 | }
47 | case class Input(values: List[Double]) extends BaseInput {
48 | def size: Int = values.size
49 | }
50 |
51 | final case class DynamicInput(modelId: String, values: List[Double], occurredOn: Long)
52 | extends BaseEvent
53 | with BaseInput {
54 | def size: Int = values.size
55 | }
56 |
--------------------------------------------------------------------------------
/idea.sbt:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | lazy val mainRunner = project
23 | .in(file("mainRunner"))
24 | .dependsOn(
25 | RootProject(file(".")),
26 | ProjectRef(file("."), "flink-jpmml-examples")
27 | )
28 | .settings(
29 | // we set all provided dependencies to none, so that they are included in the classpath of mainRunner
30 | libraryDependencies := (libraryDependencies in RootProject(file("."))).value.map { module =>
31 | if (module.configurations.equals(Some("provided"))) {
32 | module.copy(configurations = None)
33 | } else {
34 | module
35 | }
36 | },
37 | libraryDependencies := (libraryDependencies in ProjectRef(file("."), "flink-jpmml-examples")).value.map { module =>
38 | if (module.configurations.equals(Some("provided"))) {
39 | module.copy(configurations = None)
40 | } else {
41 | module
42 | }
43 | }
44 | )
45 |
--------------------------------------------------------------------------------
/images/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/FlinkML/flink-jpmml/53f31dd557f7b4c473f87e6ac0ef4d3e723ca43e/images/architecture.png
--------------------------------------------------------------------------------
/project/Commons.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | import sbt.Keys._
23 | import sbt._
24 |
25 | object Commons {
26 |
27 | val settings: Seq[Def.Setting[_]] = Seq(
28 | organization := "io.radicalbit",
29 | scalaVersion in ThisBuild := "2.11.12",
30 | resolvers in ThisBuild ++= Seq(
31 | "Radicalbit Releases" at "https://tools.radicalbit.io/artifactory/public-release/"
32 | )
33 | )
34 | }
35 |
--------------------------------------------------------------------------------
/project/Dependencies.scala:
--------------------------------------------------------------------------------
1 | import sbt._
2 | import sbt.Keys._
3 | /*
4 | *
5 | * Copyright (c) 2017 Radicalbit
6 | *
7 | * This file is part of flink-JPMML
8 | *
9 | * flink-JPMML is free software: you can redistribute it and/or modify
10 | * it under the terms of the GNU Affero General Public License as published by
11 | * the Free Software Foundation, either version 3 of the License, or
12 | * (at your option) any later version.
13 | *
14 | * flink-JPMML is distributed in the hope that it will be useful,
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 | * GNU Affero General Public License for more details.
18 | *
19 | * You should have received a copy of the GNU Affero General Public License
20 | * along with flink-JPMML. If not, see .
21 | *
22 | */
23 |
24 | object Dependencies {
25 |
26 | object Scala {
27 |
28 | lazy val libraries = Seq(
29 | flink.scalaCore % Provided,
30 | flink.streaming % Provided,
31 | flink.clients % Provided,
32 | flink.ml,
33 | jpmml.evaluator,
34 | logging.slf4j,
35 | // Test utils
36 | asm.assembly % Test,
37 | flink.utils % Test,
38 | hadoop.common % "test" classifier "tests",
39 | hadoop.hdfs % "test" classifier "tests",
40 | hadoop.mincluster % "test",
41 | scalatest.scalatest % Test,
42 | `flink-streaming-spec`.core % Test
43 | )
44 | }
45 |
46 | object Examples {
47 |
48 | lazy val libraries = Seq(
49 | flink.scalaCore % Provided,
50 | flink.streaming % Provided
51 | )
52 |
53 | }
54 |
55 | private object flink {
56 | lazy val namespace = "org.apache.flink"
57 | lazy val version = "1.6.2"
58 | lazy val core = namespace % "flink-core" % version
59 | lazy val scalaCore = namespace %% "flink-scala" % version
60 | lazy val streaming = namespace %% "flink-streaming-scala" % version
61 | lazy val ml = namespace %% "flink-ml" % version
62 | lazy val clients = namespace %% "flink-clients" % version
63 | lazy val utils = namespace %% "flink-test-utils" % version
64 | }
65 |
66 | private object jpmml {
67 | lazy val namespace = "org.jpmml"
68 | lazy val version = "1.3.9"
69 | lazy val evaluator = namespace % "pmml-evaluator" % version
70 | }
71 |
72 | object logging {
73 | lazy val namespace = "org.slf4j"
74 | lazy val version = "1.7.7"
75 | lazy val slf4j = namespace % "slf4j-api" % version
76 | }
77 |
78 | /*** Test utils ***/
79 | private object asm {
80 | lazy val namespace = "asm"
81 | lazy val version = "3.3.1"
82 | lazy val assembly = namespace % "asm" % version
83 | }
84 |
85 | private object hadoop {
86 | lazy val namespace = "org.apache.hadoop"
87 | lazy val version = "2.3.0"
88 | lazy val hdfs = namespace % "hadoop-hdfs" % version
89 | lazy val common = namespace % "hadoop-common" % version
90 | lazy val mincluster = namespace % "hadoop-minicluster" % version
91 | }
92 |
93 | private object junitinterface {
94 | lazy val namespace = "com.novocode"
95 | lazy val version = "0.11"
96 | lazy val interface = namespace % "junit-interface" % version
97 | }
98 |
99 | private object scalatest {
100 | lazy val namespace = "org.scalatest"
101 | lazy val version = "3.0.1"
102 | lazy val scalatest = namespace %% "scalatest" % version
103 | }
104 |
105 | private object `flink-streaming-spec` {
106 | lazy val namespace = "io.radicalbit"
107 | lazy val version = "0.3.0"
108 | lazy val core = namespace %% "flink-streaming-spec" % version
109 | }
110 |
111 | }
112 |
--------------------------------------------------------------------------------
/project/LicenseSetting.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | import de.heikoseeberger.sbtheader.FileType
23 | import de.heikoseeberger.sbtheader.HeaderPlugin.autoImport._
24 | import sbt.Def
25 |
26 | object LicenseSetting {
27 |
28 | lazy val settings: Seq[Def.Setting[_]] = Seq(
29 | headerLicense := Some(
30 | HeaderLicense.Custom(
31 | """|Copyright (C) 2017 Radicalbit
32 | |
33 | |This file is part of flink-JPMML
34 | |
35 | |flink-JPMML is free software: you can redistribute it and/or modify
36 | |it under the terms of the GNU Affero General Public License as
37 | |published by the Free Software Foundation, either version 3 of the
38 | |License, or (at your option) any later version.
39 | |
40 | |flink-JPMML is distributed in the hope that it will be useful,
41 | |but WITHOUT ANY WARRANTY; without even the implied warranty of
42 | |MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
43 | |GNU Affero General Public License for more details.
44 | |
45 | |You should have received a copy of the GNU Affero General Public License
46 | |along with flink-JPMML. If not, see .
47 | |""".stripMargin
48 | )),
49 | headerMappings := headerMappings.value + (
50 | HeaderFileType.xml -> HeaderCommentStyle.XmlStyleBlockComment,
51 | FileType("properties") -> HeaderCommentStyle.HashLineComment
52 | )
53 | )
54 |
55 | }
56 |
--------------------------------------------------------------------------------
/project/PublishSettings.scala:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | import com.typesafe.sbt.SbtPgp.autoImportImpl.PgpKeys
23 | import sbt.Keys.{publishMavenStyle, _}
24 | import sbt.{Def, url, _}
25 | import sbtrelease.ReleasePlugin.autoImport.{releaseCrossBuild, releasePublishArtifactsAction}
26 | import xerial.sbt.Sonatype._
27 |
28 | object PublishSettings {
29 |
30 | lazy val settings: Seq[Def.Setting[_]] = sonatypeSettings ++ Seq(
31 | publishTo := Some(
32 | if (isSnapshot.value)
33 | Opts.resolver.sonatypeSnapshots
34 | else
35 | Opts.resolver.sonatypeStaging
36 | ),
37 | publishMavenStyle := true,
38 | licenses := Seq("AGPL-3.0" -> url("https://opensource.org/licenses/AGPL-3.0")),
39 | homepage := Some(url("https://github.com/FlinkML/flink-jpmml")),
40 | scmInfo := Some(
41 | ScmInfo(
42 | url("https://github.com/FlinkML/flink-jpmml"),
43 | "scm:git:git@github.com:FlinkML/flink-jpmml.git"
44 | )
45 | ),
46 | developers := List(
47 | Developer(id = "spi-x-i",
48 | name = "Andrea Spina",
49 | email = "andrea.spina@radicalbit.io",
50 | url = url("https://github.com/spi-x-i")),
51 | Developer(id = "francescofrontera",
52 | name = "Francesco Frontera",
53 | email = "francesco.frontera@radicalbit.io",
54 | url = url("https://github.com/francescofrontera")),
55 | Developer(id = "riccardo14",
56 | name = "Riccardo Diomedi",
57 | email = "riccardo.diomedi@radicalbit.io",
58 | url = url("https://github.com/riccardo14")),
59 | Developer(id = "maocorte",
60 | name = "Mauro Cortellazzi",
61 | email = "mauro.cortellazzi@radicalbit.io",
62 | url = url("https://github.com/maocorte"))
63 | ),
64 | autoAPIMappings := true,
65 | releaseCrossBuild := true,
66 | releasePublishArtifactsAction := PgpKeys.publishSigned.value
67 | )
68 | }
69 |
--------------------------------------------------------------------------------
/project/assembly.sbt:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4")
--------------------------------------------------------------------------------
/project/build.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright (c) 2017 Radicalbit
3 | #
4 | # This file is part of flink-JPMML
5 | #
6 | # flink-JPMML is free software: you can redistribute it and/or modify
7 | # it under the terms of the GNU Affero General Public License as published by
8 | # the Free Software Foundation, either version 3 of the License, or
9 | # (at your option) any later version.
10 | #
11 | # flink-JPMML is distributed in the hope that it will be useful,
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | # GNU Affero General Public License for more details.
15 | #
16 | # You should have received a copy of the GNU Affero General Public License
17 | # along with flink-JPMML. If not, see .
18 | #
19 |
20 | sbt.version=0.13.15
21 |
--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | addSbtPlugin("com.geirsson" % "sbt-scalafmt" % "0.6.3")
23 |
24 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0")
25 |
26 | addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "latest.release")
27 |
28 | addSbtPlugin("de.heikoseeberger" % "sbt-header" % "2.0.0")
29 |
30 | addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6")
31 |
32 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
33 |
34 | addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
35 |
--------------------------------------------------------------------------------
/project/unidoc.sbt:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0")
--------------------------------------------------------------------------------
/version.sbt:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * Copyright (c) 2017 Radicalbit
4 | *
5 | * This file is part of flink-JPMML
6 | *
7 | * flink-JPMML is free software: you can redistribute it and/or modify
8 | * it under the terms of the GNU Affero General Public License as published by
9 | * the Free Software Foundation, either version 3 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * flink-JPMML is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU Affero General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU Affero General Public License
18 | * along with flink-JPMML. If not, see .
19 | *
20 | */
21 |
22 | version in ThisBuild := "0.7.0-SNAPSHOT"
--------------------------------------------------------------------------------