├── VERSION ├── evaluator-onnx ├── src │ ├── main │ │ ├── resources │ │ │ └── reference.conf │ │ └── java │ │ │ └── com │ │ │ └── ovh │ │ │ └── mls │ │ │ └── serving │ │ │ └── runtime │ │ │ └── onnx │ │ │ └── OnnxEvaluatorManifest.java │ └── test │ │ └── resources │ │ └── onnx │ │ ├── iris │ │ ├── single_gen.json │ │ ├── batch_gen.json │ │ ├── iris.onnx │ │ ├── single.json │ │ ├── batch.json │ │ ├── manifest.json │ │ └── manifest-same-output.json │ │ ├── titanic │ │ ├── single.json │ │ ├── single_gen.json │ │ ├── pipeline_titanic.onnx │ │ ├── batch.json │ │ ├── batch_gen.json │ │ └── manifest.json │ │ └── adult │ │ ├── transformation.onnx │ │ └── single_gen.json ├── pom.xml └── python │ └── iris.py ├── CHANGELOG.md ├── evaluator-torch ├── converter │ └── requirements.txt ├── src │ ├── test │ │ ├── resources │ │ │ ├── model.ts │ │ │ └── manifest.json │ │ └── java │ │ │ └── com │ │ │ └── ovh │ │ │ └── mls │ │ │ └── serving │ │ │ └── runtime │ │ │ └── torch │ │ │ └── TorchScriptEvaluatorManifestTest.java │ └── main │ │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ └── torch │ │ └── TorchScriptEvaluatorManifest.java ├── README.md ├── Makefile └── pom.xml ├── examples ├── Dockerfile ├── 2d_savedmodel.zip ├── pipeline_titanic.onnx └── manifest.json ├── api └── src │ ├── test │ ├── resources │ │ ├── tensorflow │ │ │ ├── mnist │ │ │ │ ├── model │ │ │ │ │ ├── variables │ │ │ │ │ │ ├── variables.data-00000-of-00002 │ │ │ │ │ │ ├── variables.index │ │ │ │ │ │ └── variables.data-00001-of-00002 │ │ │ │ │ └── saved_model.pb │ │ │ │ ├── inputs │ │ │ │ │ ├── 0.png │ │ │ │ │ ├── 1.png │ │ │ │ │ ├── 2.png │ │ │ │ │ ├── 3.png │ │ │ │ │ ├── 4.png │ │ │ │ │ ├── 5.png │ │ │ │ │ ├── 6.png │ │ │ │ │ ├── 7.png │ │ │ │ │ ├── 8.png │ │ │ │ │ ├── 9.png │ │ │ │ │ └── 0.json │ │ │ │ ├── outputs │ │ │ │ │ ├── 0.png │ │ │ │ │ ├── 0.json │ │ │ │ │ ├── 0.html │ │ │ │ │ ├── 0.input0.png.html │ │ │ │ │ └── 0.multipart │ │ │ │ └── api.conf │ │ │ └── 2d_savedmodel │ │ │ │ ├── 2d_savedmodel.zip │ │ │ │ ├── api.conf │ │ │ │ ├── 2d_input.json │ │ │ │ └── 2d-tensorflow-model-manifest.json │ │ ├── onnx │ │ │ ├── pipeline_titanic.onnx │ │ │ ├── api.conf │ │ │ └── batch_gen.json │ │ ├── huggingface │ │ │ ├── api.conf │ │ │ └── manifest.json │ │ └── torch │ │ │ ├── simple_model │ │ │ ├── model.ts │ │ │ ├── api.conf │ │ │ └── manifest.json │ │ │ └── multiple_input_output_model │ │ │ ├── model.ts │ │ │ ├── api.conf │ │ │ └── manifest.json │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ ├── IsCloseTo.java │ │ ├── torch │ │ ├── TorchSimpleIT.java │ │ └── TorchMultipleInputOutputIT.java │ │ ├── tensorflow │ │ └── Tensorflow2DIT.java │ │ └── onnx │ │ └── OnnxIT.java │ └── main │ ├── resources │ ├── reference.conf │ ├── log4j2.xml │ └── swagger │ │ └── swagger.html │ └── java │ └── com │ └── ovh │ └── mls │ └── serving │ └── runtime │ ├── Main.java │ ├── exceptions │ ├── ErrorMessage.java │ ├── JsonParseExceptionMapper.java │ ├── JsonMappingExceptionMapper.java │ ├── WebApplicationExceptionMapper.java │ ├── EvaluationExceptionMapper.java │ └── RestExceptionMapper.java │ ├── core │ ├── builder │ │ ├── from │ │ │ ├── TensorIOIntoJsonBinary.java │ │ │ ├── TensorIOIntoImageBinary.java │ │ │ ├── TensorIOIntoMultipartBinary.java │ │ │ └── TensorIOIntoHTMLBinary.java │ │ └── into │ │ │ └── InputStreamIntoTensorIO.java │ └── LogFilter.java │ ├── swagger │ └── SwaggerHomeResource.java │ ├── utils │ └── MultipartUtils.java │ └── EvaluationService.java ├── evaluator-tensorflow ├── src │ ├── test │ │ ├── resources │ │ │ └── tensorflow │ │ │ │ ├── test_h5 │ │ │ │ ├── single_gen.json │ │ │ │ └── test.h5 │ │ │ │ ├── 2d_savedmodel │ │ │ │ ├── 2d_savedmodel.zip │ │ │ │ ├── batch.json │ │ │ │ ├── batch_tensor.json │ │ │ │ ├── 2d-tensorflow-model-manifest.json │ │ │ │ └── 2d-tensorflow-model-manifest-same-output.json │ │ │ │ └── 3d_savedmodel │ │ │ │ ├── 3d_savedmodel.zip │ │ │ │ ├── batch.json │ │ │ │ ├── batch_tensor.json │ │ │ │ └── 3d-tensorflow-model-manifest.json │ │ └── java │ │ │ └── com │ │ │ └── ovh │ │ │ └── mls │ │ │ └── serving │ │ │ └── runtime │ │ │ └── tensorflow │ │ │ └── H5Test.java │ └── main │ │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ └── tensorflow │ │ ├── TensorflowPbGenerator.java │ │ ├── TensorflowEvaluatorManifest.java │ │ └── TensorflowH5Generator.java ├── h5_converter │ ├── requirements.txt │ ├── hooks │ │ ├── __pycache__ │ │ │ ├── hook-tensorflow_core.cpython-36.pyc │ │ │ └── hook-tensorflow_core.cpython-37.pyc │ │ └── hook-tensorflow_core.py │ └── Makefile └── pom.xml ├── commons ├── src │ ├── test │ │ ├── resources │ │ │ ├── core │ │ │ │ ├── tensor-input │ │ │ │ │ ├── vector-multiple.json │ │ │ │ │ ├── tensor-multiple.json │ │ │ │ │ └── tensor-single.json │ │ │ │ └── test-manifest.json │ │ │ └── utils │ │ │ │ └── img │ │ │ │ ├── amber.jpg │ │ │ │ └── amber_100px_100px.png │ │ └── java │ │ │ └── com │ │ │ └── ovh │ │ │ └── mls │ │ │ └── serving │ │ │ └── runtime │ │ │ ├── core │ │ │ ├── io │ │ │ │ └── TensorIOTest.java │ │ │ ├── EvaluatorUtilTest.java │ │ │ ├── tensor │ │ │ │ └── TensorShapeTest.java │ │ │ └── builder │ │ │ │ └── TensorIntoImagesTest.java │ │ │ ├── validation │ │ │ └── ValidatorTest.java │ │ │ └── utils │ │ │ └── img │ │ │ └── ImagesTensorConversionTest.java │ └── main │ │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ ├── utils │ │ └── img │ │ │ ├── ImgProperties.java │ │ │ ├── ImgChanelProperties.java │ │ │ └── BinaryContent.java │ │ ├── validation │ │ ├── NumberOnly.java │ │ └── Validator.java │ │ ├── core │ │ ├── EvaluatorGenerator.java │ │ ├── builder │ │ │ ├── Builder.java │ │ │ └── InputStreamJsonIntoTensorIO.java │ │ ├── IncludeAsEvaluatorGenerator.java │ │ ├── IncludeAsEvaluatorManifest.java │ │ ├── EvaluatorManifest.java │ │ ├── Interval.java │ │ ├── AbstractEvaluatorManifest.java │ │ ├── Evaluator.java │ │ ├── FlowEvaluatorManifest.java │ │ ├── io │ │ │ └── Part.java │ │ ├── tensor │ │ │ ├── TensorIndex.java │ │ │ └── TensorField.java │ │ ├── Field.java │ │ ├── EvaluationContext.java │ │ └── transformer │ │ │ └── ImageTransformerInfo.java │ │ └── exceptions │ │ ├── EvaluationException.java │ │ ├── deserialization │ │ ├── TableDeserializationException.java │ │ ├── MissingFieldException.java │ │ ├── DifferentColumnLengthException.java │ │ └── UnexpectedValueForColumnException.java │ │ ├── EvaluatorException.java │ │ └── SwiftConfigurationException.java └── pom.xml ├── .dockerignore ├── evaluator-huggingface ├── huggingface-tokenizer-jni │ ├── Makefile │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── src │ └── main │ │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ └── huggingface │ │ └── tokenizer │ │ ├── Encoding.java │ │ ├── Offset.java │ │ ├── HuggingFaceTokenizerEvaluatorManifest.java │ │ └── Tokenizer.java └── pom.xml ├── evaluator-processors ├── src │ ├── test │ │ ├── resources │ │ │ └── processors │ │ │ │ ├── standard-scaler-manifest.json │ │ │ │ └── standard-scaler-same-output-manifest.json │ │ └── java │ │ │ └── com │ │ │ └── ovh │ │ │ └── mls │ │ │ └── serving │ │ │ └── runtime │ │ │ └── processors │ │ │ ├── StandardScalerTest.java │ │ │ └── StandardScalerWithSameOutputTest.java │ └── main │ │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ └── processors │ │ ├── MeanStd.java │ │ └── StandardScalerManifest.java └── pom.xml ├── .gitignore ├── evaluator-timeseries ├── src │ ├── test │ │ └── resources │ │ │ └── timeseries │ │ │ ├── datetime-timestamp-manifest.json │ │ │ ├── prediction-interval-manifest.json │ │ │ ├── prediction-interval-manifest-partial.json │ │ │ └── datetime-string-manifest.json │ └── main │ │ └── java │ │ └── com │ │ └── ovh │ │ └── mls │ │ └── serving │ │ └── runtime │ │ └── timeseries │ │ ├── DatetimeEvaluatorManifest.java │ │ ├── PredictionIntervalEvaluatorManifest.java │ │ └── DatetimeEvaluator.java └── pom.xml ├── AUTHORS ├── MAINTAINERS ├── CONTRIBUTORS ├── .github └── workflows │ ├── test.yml │ └── deploy-packages.yml ├── LICENSE ├── Makefile ├── dockerfiles ├── onnx.Dockerfile ├── full.Dockerfile └── tensorflow.Dockerfile └── CONTRIBUTING.md /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.2 -------------------------------------------------------------------------------- /evaluator-onnx/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | evaluator: {} 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | ## Next Release 3 | - Init evaluator interface -------------------------------------------------------------------------------- /evaluator-torch/converter/requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=1.5,<=1.6 2 | tqdm>=4,<=5 -------------------------------------------------------------------------------- /examples/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM serving-runtime-base:latest 2 | 3 | COPY . /deployments -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/model/variables/variables.data-00000-of-00002: -------------------------------------------------------------------------------- 1 | ' -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/single_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "float_input": [6.5, 3, 5.5, 1.8] 3 | } -------------------------------------------------------------------------------- /examples/2d_savedmodel.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/examples/2d_savedmodel.zip -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/test_h5/single_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputs": [6.4, 3.2, 4.5, 1.5] 3 | } -------------------------------------------------------------------------------- /examples/pipeline_titanic.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/examples/pipeline_titanic.onnx -------------------------------------------------------------------------------- /commons/src/test/resources/core/tensor-input/vector-multiple.json: -------------------------------------------------------------------------------- 1 | { 2 | "key1": [1, 2, 3], 3 | "key2": [4, 5, 6] 4 | } -------------------------------------------------------------------------------- /evaluator-torch/src/test/resources/model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-torch/src/test/resources/model.ts -------------------------------------------------------------------------------- /commons/src/test/resources/utils/img/amber.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/commons/src/test/resources/utils/img/amber.jpg -------------------------------------------------------------------------------- /api/src/test/resources/onnx/pipeline_titanic.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/onnx/pipeline_titanic.onnx -------------------------------------------------------------------------------- /api/src/test/resources/huggingface/api.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8088 3 | port: 8089 4 | } 5 | 6 | files.path: "src/test/resources/huggingface/" -------------------------------------------------------------------------------- /api/src/test/resources/torch/simple_model/model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/torch/simple_model/model.ts -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/batch_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "float_input": [ 3 | [4.9, 2.5, 4.5, 1.7], 4 | [7.4, 2.8, 6.1, 1.9] 5 | ] 6 | } -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/0.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/1.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/2.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/3.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/4.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/5.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/6.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/7.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/8.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/inputs/9.png -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/outputs/0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/outputs/0.png -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/iris.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-onnx/src/test/resources/onnx/iris/iris.onnx -------------------------------------------------------------------------------- /evaluator-tensorflow/h5_converter/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow==1.15.4 2 | pyinstaller>=3.5,<=3.6 3 | absl-py==0.8.1 4 | setuptools>=41.0.1,<45.0.0 5 | h5py==2.10.0 -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/single.json: -------------------------------------------------------------------------------- 1 | { 2 | "sepal_length": 6.5, 3 | "sepal_width": 3, 4 | "petal_length": 5.5, 5 | "petal_width": 1.8 6 | } -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/titanic/single.json: -------------------------------------------------------------------------------- 1 | { 2 | "pclass": "3", 3 | "sex": "male", 4 | "age": 30, 5 | "fare": 7.8958, 6 | "embarked": "S" 7 | } -------------------------------------------------------------------------------- /api/src/test/resources/huggingface/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "huggingface_tokenizer", 3 | "saved_model_uri": "src/test/resources/huggingface/tokenizer.json" 4 | } 5 | -------------------------------------------------------------------------------- /api/src/test/resources/torch/simple_model/api.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8092 3 | port: 8093 4 | } 5 | 6 | files.path: "src/test/resources/torch/simple_model" 7 | -------------------------------------------------------------------------------- /commons/src/test/resources/utils/img/amber_100px_100px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/commons/src/test/resources/utils/img/amber_100px_100px.png -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/titanic/single_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "pclass": "3", 3 | "sex": "male", 4 | "age": 30, 5 | "fare": 7.8958, 6 | "embarked": "S" 7 | } -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/model/saved_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/model/saved_model.pb -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/adult/transformation.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-onnx/src/test/resources/onnx/adult/transformation.onnx -------------------------------------------------------------------------------- /api/src/test/resources/onnx/api.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8082 3 | port: 8083 4 | } 5 | 6 | files.path: "src/test/resources/onnx/" 7 | evaluator.tensorflow.h5_converter.path: "" -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/2d_savedmodel/2d_savedmodel.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/2d_savedmodel/2d_savedmodel.zip -------------------------------------------------------------------------------- /api/src/test/resources/torch/multiple_input_output_model/model.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/torch/multiple_input_output_model/model.ts -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/test_h5/test.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-tensorflow/src/test/resources/tensorflow/test_h5/test.h5 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | */target/** 2 | **/*.iml 3 | **/tmp 4 | .git 5 | examples/ 6 | dockerfiles/ 7 | **.md 8 | .idea 9 | MAINTAINERS 10 | AUTHORS 11 | CONTRIBUTORS 12 | LICENSE 13 | VERSION -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/titanic/pipeline_titanic.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-onnx/src/test/resources/onnx/titanic/pipeline_titanic.onnx -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/model/variables/variables.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/model/variables/variables.index -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/batch.json: -------------------------------------------------------------------------------- 1 | { 2 | "sepal_length": [4.9, 7.4], 3 | "sepal_width": [2.5, 2.8], 4 | "petal_length": [4.5, 6.1], 5 | "petal_width": [1.7, 1.9] 6 | } -------------------------------------------------------------------------------- /api/src/test/resources/torch/multiple_input_output_model/api.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8092 3 | port: 8093 4 | } 5 | 6 | files.path: "src/test/resources/torch/multiple_input_output_model" 7 | -------------------------------------------------------------------------------- /evaluator-huggingface/huggingface-tokenizer-jni/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := build 2 | 3 | WORKDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) 4 | 5 | .PHONY: build 6 | build: 7 | cargo build --release 8 | -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/titanic/batch.json: -------------------------------------------------------------------------------- 1 | { 2 | "pclass": ["3", "3"], 3 | "sex": ["male", "female"], 4 | "age": [30, 48], 5 | "fare": [7.8958, 34.375], 6 | "embarked": ["S", "S"] 7 | } -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/api.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8086 3 | port: 8087 4 | } 5 | 6 | files.path: "src/test/resources/tensorflow/mnist/model/" 7 | evaluator.tensorflow.h5_converter.path: "" -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d_savedmodel.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d_savedmodel.zip -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/3d_savedmodel.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/3d_savedmodel.zip -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/2d_savedmodel/api.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8084 3 | port: 8085 4 | } 5 | 6 | files.path: "src/test/resources/tensorflow/2d_savedmodel/" 7 | evaluator.tensorflow.h5_converter.path: "" -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/model/variables/variables.data-00001-of-00002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/api/src/test/resources/tensorflow/mnist/model/variables/variables.data-00001-of-00002 -------------------------------------------------------------------------------- /evaluator-tensorflow/h5_converter/hooks/__pycache__/hook-tensorflow_core.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-tensorflow/h5_converter/hooks/__pycache__/hook-tensorflow_core.cpython-36.pyc -------------------------------------------------------------------------------- /evaluator-tensorflow/h5_converter/hooks/__pycache__/hook-tensorflow_core.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovh/serving-runtime/HEAD/evaluator-tensorflow/h5_converter/hooks/__pycache__/hook-tensorflow_core.cpython-37.pyc -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/utils/img/ImgProperties.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.utils.img; 2 | 3 | public enum ImgProperties { 4 | BATCH_SIZE, 5 | WIDTH, 6 | HEIGHT, 7 | CHANEL 8 | } 9 | -------------------------------------------------------------------------------- /commons/src/test/resources/core/tensor-input/tensor-multiple.json: -------------------------------------------------------------------------------- 1 | { 2 | "tensor_input_1": [ 3 | [1, 2, 3], 4 | [4, 5, 6] 5 | ], 6 | "tensor_input_2": [ 7 | [7, 8], 8 | [9, 10], 9 | [11, 12] 10 | ] 11 | } -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/utils/img/ImgChanelProperties.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.utils.img; 2 | 3 | public enum ImgChanelProperties { 4 | RED, 5 | BLUE, 6 | GREEN, 7 | GRAY_SCALE 8 | } 9 | -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/outputs/0.json: -------------------------------------------------------------------------------- 1 | {"class_ids":0,"logits":[3435.8083,-64.110596,107.91369,-479.41333,-688.7862,-592.7844,-576.5078,-386.3663,-845.04346,-607.1198],"classes":"0","probabilities":[1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]} -------------------------------------------------------------------------------- /commons/src/test/resources/core/test-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "test", 3 | "test_value": "someTestString", 4 | "inputs": [ 5 | { 6 | "name": "int", 7 | "type": "integer" 8 | } 9 | ], 10 | "outputs": [ 11 | ] 12 | } -------------------------------------------------------------------------------- /commons/src/test/resources/core/tensor-input/tensor-single.json: -------------------------------------------------------------------------------- 1 | { 2 | "tensor_input_1": [ 3 | [ 4 | [1, 2], 5 | [3, 4] 6 | ], 7 | [ 8 | [5, 6], 9 | [7, 8] 10 | ], 11 | [ 12 | [9, 10], 13 | [11, 12] 14 | ] 15 | ] 16 | } -------------------------------------------------------------------------------- /evaluator-tensorflow/h5_converter/hooks/hook-tensorflow_core.py: -------------------------------------------------------------------------------- 1 | from PyInstaller.utils.hooks import collect_submodules, collect_data_files 2 | 3 | hiddenimports = collect_submodules('tensorflow_core') 4 | datas = collect_data_files('tensorflow_core', subdir=None, include_py_files=True) 5 | -------------------------------------------------------------------------------- /api/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | server { 2 | metrics.port: 8081 3 | bind: "0.0.0.0" 4 | port: 8080 5 | } 6 | 7 | swagger { 8 | title: "Model Name", 9 | description: "Inference Model" 10 | version: "1" 11 | path: "/" 12 | } 13 | 14 | files.path: "examples/" 15 | 16 | evaluator: {} -------------------------------------------------------------------------------- /api/src/test/resources/onnx/batch_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "pclass": [ 3 | "3", 4 | "3" 5 | ], 6 | "sex": [ 7 | "male", 8 | "female" 9 | ], 10 | "age": [ 11 | 30, 12 | 48 13 | ], 14 | "fare": [ 15 | 7.8958, 16 | 34.375 17 | ], 18 | "embarked": [ 19 | "S", 20 | "S" 21 | ] 22 | } -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/titanic/batch_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "pclass": [ 3 | "3", 4 | "3" 5 | ], 6 | "sex": [ 7 | "male", 8 | "female" 9 | ], 10 | "age": [ 11 | 30, 12 | 48 13 | ], 14 | "fare": [ 15 | 7.8958, 16 | 34.375 17 | ], 18 | "embarked": [ 19 | "S", 20 | "S" 21 | ] 22 | } -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/2d_savedmodel/2d_input.json: -------------------------------------------------------------------------------- 1 | { 2 | "scaled_imputed_t": [2.0196744706314087, 2.045239970259654], 3 | "scaled_imputed_label": [-0.17866892904935183, 1.8181895427895707], 4 | "scaled_imputed_feature1": [-0.12179146375316918, 0.083989161980639], 5 | "scaled_imputed_feature2": [-0.40634564254730754, -0.20722287490242464] 6 | } -------------------------------------------------------------------------------- /evaluator-tensorflow/h5_converter/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := build 2 | 3 | WORKDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) 4 | 5 | .PHONY: build 6 | build: 7 | pip install -r requirements.txt 8 | bash -c "pyinstaller --clean --additional-hooks-dir=./hooks -F h5_converter.py --log-level=INFO --add-data $$(python -c 'import os;import astor;print(os.path.dirname(astor.__file__))')/VERSION:astor/" 9 | -------------------------------------------------------------------------------- /evaluator-processors/src/test/resources/processors/standard-scaler-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "mean_std_map": { 3 | "value": { 4 | "mean": 7.0, 5 | "std": 2.0 6 | } 7 | }, 8 | "inputs": [ 9 | { 10 | "name": "value", 11 | "type": "double" 12 | } 13 | ], 14 | "outputs": [ 15 | { 16 | "name": "scaled_value", 17 | "type": "double" 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | *.iml 3 | target/** 4 | **/target/** 5 | 6 | **/.factorypath 7 | **/.classpath 8 | **/.project 9 | **/.settings 10 | **/.DS_Store 11 | 12 | evaluator-tensorflow/h5_converter/__pycache__/* 13 | evaluator-tensorflow/h5_converter/dist/* 14 | evaluator-tensorflow/h5_converter/build/* 15 | evaluator-tensorflow/tmp/* 16 | evaluator-tensorflow/h5_converter/h5_converter.spec 17 | 18 | evaluator-torch/libtorch-* -------------------------------------------------------------------------------- /evaluator-torch/src/test/resources/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "torch_script", 3 | "saved_model_uri": "model.ts", 4 | "inputs": [ 5 | { 6 | "name": "input_0", 7 | "shape": [ 8 | 2 9 | ], 10 | "type": "float" 11 | } 12 | ], 13 | "outputs": [ 14 | { 15 | "name": "output_0", 16 | "shape": [ 17 | 2 18 | ], 19 | "type": "float" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /api/src/test/resources/torch/simple_model/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "torch_script", 3 | "saved_model_uri": "model.ts", 4 | "inputs": [ 5 | { 6 | "name": "input_0", 7 | "shape": [ 8 | 2 9 | ], 10 | "type": "float" 11 | } 12 | ], 13 | "outputs": [ 14 | { 15 | "name": "output_0", 16 | "shape": [ 17 | 2 18 | ], 19 | "type": "float" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/batch.json: -------------------------------------------------------------------------------- 1 | { 2 | "scaled_imputed_t": [2.0196744706314087, 2.045239970259654, 2.045239970259654], 3 | "scaled_imputed_label": [-0.17866892904935183, 1.8181895427895707, 2.045239970259654], 4 | "scaled_imputed_feature1": [-0.12179146375316918, 0.083989161980639, 2.045239970259654], 5 | "scaled_imputed_feature2": [-0.40634564254730754, -0.20722287490242464, 2.045239970259654] 6 | } 7 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/batch_tensor.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": [ 3 | [2.0196744706314087, -0.17866892904935183, -0.12179146375316918, -0.40634564254730754, 2.045239970259654, 1.8181895427895707, 0.083989161980639, -0.20722287490242464], 4 | [2.045239970259654, 1.8181895427895707, 0.083989161980639, -0.20722287490242464, 2.045239970259654, 2.045239970259654, 2.045239970259654, 2.045239970259654] 5 | ] 6 | } -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/batch.json: -------------------------------------------------------------------------------- 1 | { 2 | "scaled_imputed_t": [2.0196744706314087, 2.045239970259654, 2.045239970259654], 3 | "scaled_imputed_label": [-0.17866892904935183, 1.8181895427895707, 2.045239970259654], 4 | "scaled_imputed_feature1": [-0.12179146375316918, 0.083989161980639, 2.045239970259654], 5 | "scaled_imputed_feature2": [-0.40634564254730754, -0.20722287490242464, 2.045239970259654] 6 | } 7 | -------------------------------------------------------------------------------- /evaluator-timeseries/src/test/resources/timeseries/datetime-timestamp-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputs": [ 3 | { 4 | "name": "timestamp-seconds", 5 | "type": "long", 6 | "shape": [-1, -1], 7 | "format": "TS_S" 8 | } 9 | ], 10 | "outputs": [ 11 | { 12 | "name": "date", 13 | "type": "string", 14 | "shape": [-1, -1], 15 | "format": "yyyy-MM-dd HH:mm:ss" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/outputs/0.html: -------------------------------------------------------------------------------- 1 |

class_ids

0

logits

classes

0

probabilities

-------------------------------------------------------------------------------- /evaluator-timeseries/src/test/resources/timeseries/prediction-interval-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "residuals_std": { 3 | "label": 0.12 4 | }, 5 | "inputs": [ 6 | { 7 | "name": "label", 8 | "type": "double" 9 | } 10 | ], 11 | "outputs": [ 12 | { 13 | "name": "label_quantile_inf", 14 | "type": "double" 15 | }, 16 | { 17 | "name": "label_quantile_sup", 18 | "type": "double" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/Main.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime; 2 | 3 | import com.ovh.mls.serving.runtime.core.ApiServer; 4 | import com.typesafe.config.Config; 5 | import com.typesafe.config.ConfigFactory; 6 | 7 | public class Main { 8 | public static void main(String[] args) { 9 | Config config = ConfigFactory.load(); 10 | 11 | new ApiServer(config) 12 | .start() 13 | .join(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/validation/NumberOnly.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.validation; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | /** 7 | * Annotation used for validation purposes. Any Evaluator annotated with this annotation can only handle Number inputs. 8 | * 9 | * @see Validator 10 | */ 11 | @Retention(RetentionPolicy.RUNTIME) 12 | public @interface NumberOnly { 13 | } 14 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/EvaluatorGenerator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 4 | import com.typesafe.config.Config; 5 | 6 | import java.io.File; 7 | import java.io.FileNotFoundException; 8 | 9 | public interface EvaluatorGenerator { 10 | 11 | Evaluator generate(File filename, Config evaluatorConfig) throws EvaluatorException, FileNotFoundException; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/builder/Builder.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder; 2 | 3 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 4 | 5 | /** 6 | * Builder is a generic interface for converting an Object into another 7 | * @param The input object class 8 | * @param The output object class 9 | */ 10 | public interface Builder { 11 | 12 | O build(I input) throws EvaluationException; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/EvaluationException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | public class EvaluationException extends RuntimeException { 4 | public EvaluationException(String message) { 5 | super(message); 6 | } 7 | 8 | public EvaluationException(Exception e) { 9 | super(e); 10 | } 11 | 12 | public EvaluationException(String message, Exception e) { 13 | super(message, e); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/deserialization/TableDeserializationException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions.deserialization; 2 | 3 | public class TableDeserializationException extends RuntimeException { 4 | 5 | public TableDeserializationException(String message) { 6 | super(message); 7 | } 8 | 9 | public TableDeserializationException(String message, Throwable cause) { 10 | super(message, cause); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of OVH Serving Runtime authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files 3 | # and it lists the copyright holders only. 4 | 5 | # Names should be added to this file as one of 6 | # Organization's name 7 | # Individual's name 8 | # Individual's name 9 | # See CONTRIBUTORS for the meaning of multiple email addresses. 10 | 11 | # Please keep the list sorted. 12 | 13 | OVH SAS -------------------------------------------------------------------------------- /evaluator-huggingface/huggingface-tokenizer-jni/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "huggingface_tokenizer_jni" 3 | version = "0.1.0" 4 | authors = ["Corentin Regal "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [lib] 10 | name = "huggingface_tokenizer_jni" 11 | crate-type = ["cdylib"] 12 | 13 | [dependencies] 14 | jni = "0.16" 15 | tokenizers = { git = "https://github.com/huggingface/tokenizers" } 16 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/batch_tensor.json: -------------------------------------------------------------------------------- 1 | { 2 | "input": [ 3 | [ 4 | [2.0196744706314087, -0.17866892904935183, -0.12179146375316918, -0.40634564254730754], 5 | [2.045239970259654, 1.8181895427895707, 0.083989161980639, -0.20722287490242464] 6 | ], 7 | [ 8 | [2.045239970259654, 1.8181895427895707, 0.083989161980639, -0.20722287490242464], 9 | [2.045239970259654, 2.045239970259654, 2.045239970259654, 2.045239970259654] 10 | ] 11 | ] 12 | } -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/IncludeAsEvaluatorGenerator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | /** 7 | * Class annotated with EvaluatorGenerator need to add this annotation and declare a unique extension name 8 | * 9 | * @see EvaluatorGenerator 10 | */ 11 | @Retention(RetentionPolicy.RUNTIME) 12 | public @interface IncludeAsEvaluatorGenerator { 13 | String extension(); 14 | } 15 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/EvaluatorException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | public class EvaluatorException extends RuntimeException { 4 | public EvaluatorException(String message) { 5 | super(message); 6 | } 7 | 8 | public EvaluatorException(Throwable throwable) { 9 | super(throwable); 10 | } 11 | 12 | public EvaluatorException(String message, Throwable throwable) { 13 | super(message, throwable); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/deserialization/MissingFieldException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions.deserialization; 2 | 3 | import java.util.Set; 4 | 5 | public class MissingFieldException extends TableDeserializationException { 6 | 7 | public MissingFieldException(Set missingFields) { 8 | super(String.format( 9 | "The following fields were expected but not found : %s", 10 | missingFields.toString() 11 | )); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/adult/single_gen.json: -------------------------------------------------------------------------------- 1 | { 2 | "age": [[39]], 3 | "workclass": [["State-gov"]], 4 | "fnlwgt": [[77516]], 5 | "education": [["qsdfgqsfdqs"]], 6 | "education-num": [[13]], 7 | "marital-status": [["Married-civ-spouse"]], 8 | "occupation": [["Adm-clerical"]], 9 | "relationship": [["Not-in-family"]], 10 | "race": [["White"]], 11 | "sex" :[["Male"]], 12 | "capital-gain": [[2174]], 13 | "capital-loss": [[0]], 14 | "hours-per-week": [[40]], 15 | "native-country": [["United-States"]] 16 | } -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/outputs/0.input0.png.html: -------------------------------------------------------------------------------- 1 |

image

-------------------------------------------------------------------------------- /api/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | # This is the official list of the project maintainers. 2 | # This is mostly useful for contributors that want to push 3 | # significant pull requests or for project management issues. 4 | # 5 | # 6 | # Names should be added to this file like so: 7 | # Individual's name 8 | # Individual's name 9 | # 10 | # Please keep the list sorted. 11 | # 12 | 13 | Adrien Carreira 14 | Christophe Rannou 15 | Corentin Regal 16 | Maël Le Gal -------------------------------------------------------------------------------- /evaluator-processors/src/main/java/com/ovh/mls/serving/runtime/processors/MeanStd.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.processors; 2 | 3 | public class MeanStd { 4 | 5 | private Double mean; 6 | 7 | private Double std; 8 | 9 | public Double getMean() { 10 | return mean; 11 | } 12 | 13 | public MeanStd setMean(Double mean) { 14 | this.mean = mean; 15 | return this; 16 | } 17 | 18 | public Double getStd() { 19 | return std; 20 | } 21 | 22 | public MeanStd setStd(Double std) { 23 | this.std = std; 24 | return this; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/deserialization/DifferentColumnLengthException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions.deserialization; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | public class DifferentColumnLengthException extends TableDeserializationException { 7 | 8 | public DifferentColumnLengthException(Map> errorMap) { 9 | super( 10 | String.format( 11 | "Not all the column have the same size : %s", 12 | errorMap.toString() 13 | ) 14 | ); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /evaluator-processors/src/test/resources/processors/standard-scaler-same-output-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "mean_std_map": { 3 | "value": { 4 | "mean": 5.0, 5 | "std": 2.0 6 | }, 7 | "label": { 8 | "mean": 10.0, 9 | "std": 4.0 10 | } 11 | }, 12 | "inputs": [ 13 | { 14 | "name": "value", 15 | "type": "double" 16 | }, 17 | { 18 | "name": "label", 19 | "type": "double" 20 | } 21 | ], 22 | "outputs": [ 23 | { 24 | "name": "value", 25 | "type": "double" 26 | }, 27 | { 28 | "name": "label", 29 | "type": "double" 30 | } 31 | ] 32 | } -------------------------------------------------------------------------------- /evaluator-timeseries/src/test/resources/timeseries/prediction-interval-manifest-partial.json: -------------------------------------------------------------------------------- 1 | { 2 | "residuals_std": { 3 | "label": 0.12, 4 | "label_t+1": 0.3 5 | }, 6 | "inputs": [ 7 | { 8 | "name": "label", 9 | "type": "double" 10 | }, 11 | { 12 | "name": "label_t+1", 13 | "type": "double" 14 | } 15 | ], 16 | "outputs": [ 17 | { 18 | "name": "label_quantile_inf", 19 | "type": "double" 20 | }, 21 | { 22 | "name": "label_quantile_sup", 23 | "type": "double" 24 | }, 25 | { 26 | "name": "label_t+1_quantile_sup", 27 | "type": "double" 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/SwiftConfigurationException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | public class SwiftConfigurationException extends Exception { 4 | 5 | public SwiftConfigurationException(String message) { 6 | super(message); 7 | } 8 | 9 | public SwiftConfigurationException(Exception e) { 10 | super(e); 11 | } 12 | 13 | public static SwiftConfigurationException keyNotFoundException(String key) { 14 | String message = String.format("Swift key '%s' was not found in configurations collection", key); 15 | return new SwiftConfigurationException(message); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the official list of people who can contribute 2 | # (and typically have contributed) code to the OVH Serving Runtime repository. 3 | # 4 | # Names should be added to this file only after verifying that 5 | # the individual or the individual's organization has agreed to 6 | # the appropriate CONTRIBUTING.md file. 7 | # 8 | # Names should be added to this file like so: 9 | # Individual's name 10 | # Individual's name qqqqhtop 11 | 12 | # 13 | # Please keep the list sorted. 14 | # 15 | 16 | Adrien Carreira 17 | Christophe Rannou 18 | Maël Le Gal -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/IncludeAsEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import java.lang.annotation.Retention; 4 | import java.lang.annotation.RetentionPolicy; 5 | 6 | /** 7 | * Class implementing EvaluatorManifest need to add this annotation and declare a unique type property 8 | * use by the ObjectMapper to deserialize the JSON manifest. All classes with this annotation will be registered 9 | * at runtime as EvaluatorManifest subtypes. 10 | * 11 | * @see EvaluatorUtil 12 | * @see EvaluatorManifest 13 | */ 14 | @Retention(RetentionPolicy.RUNTIME) 15 | public @interface IncludeAsEvaluatorManifest { 16 | String type(); 17 | } 18 | -------------------------------------------------------------------------------- /api/src/test/resources/torch/multiple_input_output_model/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "torch_script", 3 | "saved_model_uri": "model.ts", 4 | "inputs": [ 5 | { 6 | "name": "input_0", 7 | "shape": [ 8 | 4 9 | ], 10 | "type": "float" 11 | }, 12 | { 13 | "name": "input_1", 14 | "shape": [ 15 | 2 16 | ], 17 | "type": "float" 18 | } 19 | ], 20 | "outputs": [ 21 | { 22 | "name": "output_0", 23 | "shape": [ 24 | 1 25 | ], 26 | "type": "float" 27 | }, 28 | { 29 | "name": "output_1", 30 | "shape": [ 31 | 1 32 | ], 33 | "type": "float" 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/EvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 4 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 5 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 6 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 7 | 8 | import java.io.IOException; 9 | 10 | /** 11 | * @see Evaluator 12 | */ 13 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") 14 | @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) 15 | public interface EvaluatorManifest { 16 | 17 | Evaluator create(String path) throws EvaluatorException, IOException; 18 | 19 | String getType(); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/Interval.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | 6 | public class Interval { 7 | 8 | private final Double lowerBound; 9 | private final Double upperBound; 10 | 11 | @JsonCreator 12 | public Interval(@JsonProperty("lower_bound") Double lowerBound, @JsonProperty("upper_bound") Double upperBound) { 13 | this.lowerBound = lowerBound; 14 | this.upperBound = upperBound; 15 | } 16 | 17 | public Double getLowerBound() { 18 | return lowerBound; 19 | } 20 | 21 | public Double getUpperBound() { 22 | return upperBound; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/exceptions/ErrorMessage.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | import org.eclipse.jetty.http.HttpStatus; 4 | 5 | /** 6 | * Error Message Model 7 | */ 8 | public class ErrorMessage { 9 | private final int status; 10 | private final String message; 11 | 12 | public ErrorMessage(String message) { 13 | this.message = message; 14 | this.status = HttpStatus.INTERNAL_SERVER_ERROR_500; 15 | } 16 | 17 | public ErrorMessage(String message, int status) { 18 | this.message = message; 19 | this.status = status; 20 | } 21 | 22 | public int getStatus() { 23 | return status; 24 | } 25 | 26 | public String getMessage() { 27 | return message; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/exceptions/deserialization/UnexpectedValueForColumnException.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions.deserialization; 2 | 3 | import com.ovh.mls.serving.runtime.core.Field; 4 | 5 | public class UnexpectedValueForColumnException extends TableDeserializationException { 6 | 7 | public UnexpectedValueForColumnException(Field expectedField, Object object) { 8 | super( 9 | String.format( 10 | "Expected column '%s' to be of type '%s', found '%s' with value '%s'", 11 | expectedField.getName(), 12 | expectedField.getType().toString(), 13 | object.getClass().getSimpleName(), 14 | object.toString() 15 | ) 16 | ); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/utils/img/BinaryContent.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.utils.img; 2 | 3 | import org.apache.http.entity.ContentType; 4 | 5 | public class BinaryContent { 6 | private String fileExtension; 7 | private ContentType contentType; 8 | private byte[] bytes; 9 | 10 | public BinaryContent(String extension, ContentType contentType, byte[] bytes) { 11 | this.fileExtension = extension; 12 | this.contentType = contentType; 13 | this.bytes = bytes; 14 | } 15 | 16 | public String getFileExtension() { 17 | return fileExtension; 18 | } 19 | 20 | public ContentType getContentType() { 21 | return contentType; 22 | } 23 | 24 | public byte[] getBytes() { 25 | return bytes; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/outputs/0.multipart: -------------------------------------------------------------------------------- 1 | --jetty1735298322k66b8jwv 2 | Content-Type: application/json 3 | Content-Disposition: form-data; name="class_ids"; filename="class_ids.json" 4 | 5 | 0 6 | --jetty1735298322k66b8jwv 7 | Content-Type: application/json 8 | Content-Disposition: form-data; name="logits"; filename="logits.json" 9 | 10 | [3435.8083,-64.110596,107.91369,-479.41333,-688.7862,-592.7844,-576.5078,-386.3663,-845.04346,-607.1198] 11 | --jetty1735298322k66b8jwv 12 | Content-Type: application/json 13 | Content-Disposition: form-data; name="classes"; filename="classes.json" 14 | 15 | "0" 16 | --jetty1735298322k66b8jwv 17 | Content-Type: application/json 18 | Content-Disposition: form-data; name="probabilities"; filename="probabilities.json" 19 | 20 | [1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0] 21 | --jetty1735298322k66b8jwv-- 22 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/AbstractEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public abstract class AbstractEvaluatorManifest implements EvaluatorManifest { 7 | 8 | private List inputs = new ArrayList<>(); 9 | private List outputs = new ArrayList<>(); 10 | 11 | public List getInputs() { 12 | return inputs; 13 | } 14 | 15 | public AbstractEvaluatorManifest setInputs(List inputs) { 16 | this.inputs = inputs; 17 | return this; 18 | } 19 | 20 | public List getOutputs() { 21 | return outputs; 22 | } 23 | 24 | public AbstractEvaluatorManifest setOutputs(List outputs) { 25 | this.outputs = outputs; 26 | return this; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /evaluator-timeseries/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | evaluator-timeseries 13 | 14 | 15 | 16 | com.ovh.mls.serving.runtime 17 | commons 18 | 1.0.1-SNAPSHOT 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /evaluator-processors/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | evaluator-processors 13 | 14 | 15 | 16 | com.ovh.mls.serving.runtime 17 | commons 18 | 1.0.1-SNAPSHOT 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: TUs & TIs 2 | on: 3 | push: 4 | branches: [ '**', '*/*' ] 5 | jobs: 6 | test-build-jar: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout project 10 | uses: actions/checkout@v2 11 | - name: Set up JDK 1.11 12 | uses: actions/setup-java@v1 13 | with: 14 | java-version: 1.11 15 | - name: Set up Python 3.7 16 | uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.7 19 | - name: Cache maven modules 20 | uses: actions/cache@v1 21 | env: 22 | cache-name: serving-runtime-maven-deps 23 | with: 24 | path: ~/.m2/repository 25 | key: cache-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }} 26 | 27 | - name: TUs & TIs 28 | run: | 29 | make initialize-tensorflow initialize-huggingface initialize-torch 30 | make test 31 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/exceptions/JsonParseExceptionMapper.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | 4 | import com.fasterxml.jackson.core.JsonParseException; 5 | import com.google.inject.Singleton; 6 | 7 | import javax.ws.rs.core.MediaType; 8 | import javax.ws.rs.core.Response; 9 | import javax.ws.rs.ext.ExceptionMapper; 10 | import javax.ws.rs.ext.Provider; 11 | 12 | import static org.eclipse.jetty.http.HttpStatus.BAD_REQUEST_400; 13 | 14 | @Provider 15 | @Singleton 16 | public class JsonParseExceptionMapper implements ExceptionMapper { 17 | 18 | @Override 19 | public Response toResponse(JsonParseException exception) { 20 | 21 | return Response 22 | .status(BAD_REQUEST_400) 23 | .entity(new ErrorMessage("Bad json provided", BAD_REQUEST_400)) 24 | .type(MediaType.APPLICATION_JSON) 25 | .build(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/exceptions/JsonMappingExceptionMapper.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | 4 | import com.fasterxml.jackson.databind.JsonMappingException; 5 | import com.google.inject.Singleton; 6 | 7 | import javax.ws.rs.core.MediaType; 8 | import javax.ws.rs.core.Response; 9 | import javax.ws.rs.ext.ExceptionMapper; 10 | import javax.ws.rs.ext.Provider; 11 | 12 | import static org.eclipse.jetty.http.HttpStatus.BAD_REQUEST_400; 13 | 14 | @Provider 15 | @Singleton 16 | public class JsonMappingExceptionMapper implements ExceptionMapper { 17 | 18 | @Override 19 | public Response toResponse(JsonMappingException exception) { 20 | 21 | return Response 22 | .status(BAD_REQUEST_400) 23 | .entity(new ErrorMessage("Bad json provided", BAD_REQUEST_400)) 24 | .type(MediaType.APPLICATION_JSON) 25 | .build(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "saved_model_uri": "src/test/resources/tensorflow/2d_savedmodel/2d_savedmodel.zip", 3 | "batch_size": 2, 4 | "inputs": [ 5 | { 6 | "name": "input", 7 | "shape": [-1, 8], 8 | "type": "float", 9 | "fields": [ 10 | { 11 | "name": "scaled_imputed_t", 12 | "index": 0 13 | }, 14 | { 15 | "name": "scaled_imputed_label", 16 | "index": 1 17 | }, 18 | { 19 | "name": "scaled_imputed_feature1", 20 | "index": 2 21 | }, 22 | { 23 | "name": "scaled_imputed_feature2", 24 | "index": 3 25 | } 26 | ] 27 | } 28 | ], 29 | "outputs": [ 30 | { 31 | "name": "output", 32 | "shape": [-1, 1], 33 | "type": "float", 34 | "fields": [ 35 | { 36 | "name": "scaled_imputed_label_predicted", 37 | "index": 0 38 | } 39 | ] 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /examples/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "tensorflow", 3 | "saved_model_uri": "2d_savedmodel.zip", 4 | "batch_size": 2, 5 | "inputs": { 6 | "dense_1_input:0": { 7 | "shape": "(?, 8)", 8 | "fields": [ 9 | { 10 | "name": "scaled_imputed_t", 11 | "type": "float", 12 | "index": 0 13 | }, 14 | { 15 | "name": "scaled_imputed_label", 16 | "type": "float", 17 | "index": 1 18 | }, 19 | { 20 | "name": "scaled_imputed_feature1", 21 | "type": "float", 22 | "index": 2 23 | }, 24 | { 25 | "name": "scaled_imputed_feature2", 26 | "type": "float", 27 | "index": 3 28 | } 29 | ] 30 | } 31 | }, 32 | "outputs": { 33 | "dense_2/BiasAdd:0": { 34 | "shape": "(?, 1)", 35 | "fields": [ 36 | { 37 | "name": "scaled_imputed_label_predicted", 38 | "type": "double", 39 | "index": 0 40 | } 41 | ] 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest-same-output.json: -------------------------------------------------------------------------------- 1 | { 2 | "saved_model_uri": "src/test/resources/tensorflow/2d_savedmodel/2d_savedmodel.zip", 3 | "batch_size": 2, 4 | "inputs": [ 5 | { 6 | "name": "input", 7 | "shape": [-1, 8], 8 | "type": "float", 9 | "fields": [ 10 | { 11 | "name": "scaled_imputed_t", 12 | "index": 0 13 | }, 14 | { 15 | "name": "scaled_imputed_label", 16 | "index": 1 17 | }, 18 | { 19 | "name": "scaled_imputed_feature1", 20 | "index": 2 21 | }, 22 | { 23 | "name": "scaled_imputed_feature2", 24 | "index": 3 25 | } 26 | ] 27 | } 28 | ], 29 | "outputs": [ 30 | { 31 | "name": "output", 32 | "shape": [-1, 1], 33 | "type": "float", 34 | "fields": [ 35 | { 36 | "name": "scaled_imputed_label", 37 | "index": 0 38 | } 39 | ] 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/resources/tensorflow/3d_savedmodel/3d-tensorflow-model-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "saved_model_uri": "src/test/resources/tensorflow/3d_savedmodel/3d_savedmodel.zip", 3 | "batch_size": 2, 4 | "inputs": [ 5 | { 6 | "name": "input", 7 | "shape": [-1, 2, 4], 8 | "type": "float", 9 | "fields": [ 10 | { 11 | "name": "scaled_imputed_t", 12 | "index": 0 13 | }, 14 | { 15 | "name": "scaled_imputed_label", 16 | "index": 1 17 | }, 18 | { 19 | "name": "scaled_imputed_feature1", 20 | "index": 2 21 | }, 22 | { 23 | "name": "scaled_imputed_feature2", 24 | "index": 3 25 | } 26 | ] 27 | } 28 | ], 29 | "outputs": [ 30 | { 31 | "name": "output", 32 | "shape": [-1, 1], 33 | "type": "float", 34 | "fields": [ 35 | { 36 | "name": "scaled_imputed_label_predicted", 37 | "index": 0 38 | } 39 | ] 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /evaluator-timeseries/src/main/java/com/ovh/mls/serving/runtime/timeseries/DatetimeEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.timeseries; 2 | 3 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 4 | import com.ovh.mls.serving.runtime.core.AbstractEvaluatorManifest; 5 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 6 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 7 | 8 | // In case of direct deserialization we override parent JsonTypeInfo 9 | @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) 10 | @IncludeAsEvaluatorManifest(type = DatetimeEvaluatorManifest.TYPE) 11 | public class DatetimeEvaluatorManifest extends AbstractEvaluatorManifest { 12 | 13 | public static final String TYPE = "datetime_encoder"; 14 | 15 | @Override 16 | public DatetimeEvaluator create(String path) throws EvaluatorException { 17 | return new DatetimeEvaluator(this.getInputs(), this.getOutputs()); 18 | } 19 | 20 | @Override 21 | public String getType() { 22 | return TYPE; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/main/java/com/ovh/mls/serving/runtime/tensorflow/TensorflowPbGenerator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.tensorflow; 2 | 3 | import com.ovh.mls.serving.runtime.core.Evaluator; 4 | import com.ovh.mls.serving.runtime.core.EvaluatorGenerator; 5 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorGenerator; 6 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 7 | import com.typesafe.config.Config; 8 | import org.tensorflow.SavedModelBundle; 9 | 10 | import java.io.File; 11 | 12 | import static com.ovh.mls.serving.runtime.tensorflow.TensorflowEvaluator.DEFAULT_TAG_TENSORFLOW; 13 | 14 | @IncludeAsEvaluatorGenerator(extension = "pb") 15 | public class TensorflowPbGenerator implements EvaluatorGenerator { 16 | 17 | @Override 18 | public Evaluator generate(File filename, Config evaluatorConfig) throws EvaluatorException { 19 | 20 | SavedModelBundle savedModel = SavedModelBundle.load( 21 | filename.getParent(), 22 | DEFAULT_TAG_TENSORFLOW 23 | ); 24 | return TensorflowEvaluator.create(savedModel); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/exceptions/WebApplicationExceptionMapper.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import javax.ws.rs.WebApplicationException; 8 | import javax.ws.rs.core.MediaType; 9 | import javax.ws.rs.core.Response; 10 | import javax.ws.rs.ext.ExceptionMapper; 11 | import javax.ws.rs.ext.Provider; 12 | 13 | 14 | /** 15 | * Wrap all resteasy exception with clear error messages and status 16 | */ 17 | @Provider 18 | public class WebApplicationExceptionMapper implements ExceptionMapper { 19 | private static final Logger LOGGER = LoggerFactory.getLogger(WebApplicationExceptionMapper.class); 20 | 21 | @Override 22 | public Response toResponse(WebApplicationException e) { 23 | LOGGER.error("Error {}", e.getMessage()); 24 | 25 | return Response 26 | .status(e.getResponse().getStatus()) 27 | .entity(new ErrorMessage(e.getMessage(), e.getResponse().getStatus())) 28 | .type(MediaType.APPLICATION_JSON) 29 | .build(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /commons/src/test/java/com/ovh/mls/serving/runtime/core/io/TensorIOTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.io; 2 | 3 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 4 | import org.junit.jupiter.api.Test; 5 | import tech.tablesaw.api.Table; 6 | 7 | import java.util.Map; 8 | 9 | import static org.junit.jupiter.api.Assertions.assertEquals; 10 | 11 | public class TensorIOTest { 12 | 13 | @Test 14 | public void testIntoTable() { 15 | Tensor tensor1 = Tensor.fromLongData(new long[]{1, 2, 3}); 16 | Tensor tensor2 = Tensor.fromStringData(new String[]{"1", "2", "3"}); 17 | 18 | Table table = new TensorIO(Map.of("col1", tensor1, "col2", tensor2)).intoTable(); 19 | assertEquals(2, table.columnCount()); 20 | assertEquals(3, table.rowCount()); 21 | 22 | assertEquals(1L, table.column("col1").get(0)); 23 | assertEquals(2L, table.column("col1").get(1)); 24 | assertEquals(3L, table.column("col1").get(2)); 25 | assertEquals("1", table.column("col2").get(0)); 26 | assertEquals("2", table.column("col2").get(1)); 27 | assertEquals("3", table.column("col2").get(2)); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /evaluator-huggingface/src/main/java/com/ovh/mls/serving/runtime/huggingface/tokenizer/Encoding.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.huggingface.tokenizer; 2 | 3 | import java.util.Optional; 4 | 5 | /** 6 | * Represents the output of a Tokenizer 7 | */ 8 | public class Encoding { 9 | 10 | // Pointer to the Rust structure 11 | private long handle = -1; 12 | 13 | private Encoding() { 14 | } 15 | 16 | public native boolean isEmpty(); 17 | 18 | public native long size(); 19 | 20 | public native String[] getTokens(); 21 | 22 | public native Optional[] getWords(); 23 | 24 | public native int[] getIds(); 25 | 26 | public native int[] getTypeIds(); 27 | 28 | public native Offset[] getOffsets(); 29 | 30 | public native int[] getSpecialTokensMask(); 31 | 32 | public native int[] getAttentionMask(); 33 | 34 | /** 35 | * Give back the Rust pointer to be freed 36 | */ 37 | private native void releaseHandle(); 38 | 39 | @Override 40 | protected void finalize() throws Throwable { 41 | try { 42 | releaseHandle(); 43 | } finally { 44 | super.finalize(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /evaluator-huggingface/src/main/java/com/ovh/mls/serving/runtime/huggingface/tokenizer/Offset.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.huggingface.tokenizer; 2 | 3 | import java.util.Objects; 4 | 5 | public class Offset { 6 | 7 | private final long start; 8 | private final long end; 9 | 10 | public Offset(long start, long end) { 11 | this.start = start; 12 | this.end = end; 13 | } 14 | 15 | public long getStart() { 16 | return start; 17 | } 18 | 19 | public long getEnd() { 20 | return end; 21 | } 22 | 23 | @Override 24 | public boolean equals(Object o) { 25 | if (this == o) { 26 | return true; 27 | } 28 | if (o == null || getClass() != o.getClass()) { 29 | return false; 30 | } 31 | Offset offset = (Offset) o; 32 | return start == offset.start && end == offset.end; 33 | } 34 | 35 | @Override 36 | public int hashCode() { 37 | return Objects.hash(start, end); 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return "Offset{" + 43 | "start=" + start + 44 | ", end=" + end + 45 | '}'; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/exceptions/EvaluationExceptionMapper.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | import com.google.inject.Singleton; 4 | 5 | import javax.servlet.http.HttpServletRequest; 6 | import javax.ws.rs.core.Context; 7 | import javax.ws.rs.core.MediaType; 8 | import javax.ws.rs.core.Response; 9 | import javax.ws.rs.ext.ExceptionMapper; 10 | import javax.ws.rs.ext.Provider; 11 | 12 | 13 | /** 14 | * Return a json error message in case of any exception not handle by the default Exception mapper 15 | */ 16 | @Provider 17 | @Singleton 18 | public class EvaluationExceptionMapper implements ExceptionMapper { 19 | 20 | @Context 21 | private HttpServletRequest request; 22 | 23 | /** 24 | * Create a Json response from an exception 25 | */ 26 | @Override 27 | public Response toResponse(EvaluationException exception) { 28 | return Response 29 | // Return 500 status and Json message 30 | .status(Response.Status.BAD_REQUEST) 31 | .entity(new ErrorMessage(exception.getMessage(), Response.Status.BAD_REQUEST.getStatusCode())) 32 | .type(MediaType.APPLICATION_JSON) 33 | .build(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /evaluator-onnx/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | evaluator-onnx 13 | 14 | 15 | 16 | com.ovh.mls.serving.runtime 17 | commons 18 | 1.0.1-SNAPSHOT 19 | 20 | 21 | org.bytedeco 22 | onnxruntime 23 | 1.2.0-1.5.3 24 | 25 | 26 | org.bytedeco 27 | onnxruntime-platform 28 | 1.2.0-1.5.3 29 | 30 | 31 | -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/2d_savedmodel/2d-tensorflow-model-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "flow", 3 | "evaluator_manifests": [ 4 | { 5 | "type": "tensorflow", 6 | "saved_model_uri": "2d_savedmodel.zip", 7 | "batch_size": 2, 8 | "inputs": [ 9 | { 10 | "name": "input", 11 | "shape": "(?, 8)", 12 | "type": "float", 13 | "fields": [ 14 | { 15 | "name": "scaled_imputed_t", 16 | "index": 0 17 | }, 18 | { 19 | "name": "scaled_imputed_label", 20 | "index": 1 21 | }, 22 | { 23 | "name": "scaled_imputed_feature1", 24 | "index": 2 25 | }, 26 | { 27 | "name": "scaled_imputed_feature2", 28 | "index": 3 29 | } 30 | ] 31 | } 32 | ], 33 | "outputs": [ 34 | { 35 | "name": "output", 36 | "shape": "(?, 1)", 37 | "type": "float", 38 | "fields": [ 39 | { 40 | "name": "scaled_imputed_label_predicted", 41 | "index": 0 42 | } 43 | ] 44 | } 45 | ] 46 | } 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /evaluator-torch/README.md: -------------------------------------------------------------------------------- 1 | # Get libtorch 2 | 3 | ```bash 4 | make initialize 5 | ``` 6 | 7 | # Serve a TorchScript model 8 | 9 | The model MUST be a TorchScript model, to convert pyTorch models see below. 10 | 11 | Example of a `manifest.json`: 12 | ```json 13 | { 14 | "type": "torch_script", 15 | "saved_model_uri": "model.ts", 16 | "inputs": [ 17 | { 18 | "name": "input", 19 | "shape": [ 20 | -1, 21 | 10 22 | ], 23 | "type": "long" 24 | } 25 | ], 26 | "outputs": [ 27 | { 28 | "name": "output", 29 | "shape": [ 30 | 1, 31 | 2 32 | ], 33 | "type": "float" 34 | } 35 | ] 36 | } 37 | ``` 38 | 39 | # Convert pyTorch to TorchScript 40 | 41 | Use `converter/convert.py`. 42 | You may have to modify the generated manifest.json to fit your needs. 43 | 44 | ``` 45 | usage: convert.py [-h] model output_folder input_examples 46 | 47 | Convert pyTorch models into TorchScript models 48 | 49 | positional arguments: 50 | model pyTorch model (e.g. model.pt) 51 | output_folder TorchScript model and manifest will be saved there 52 | input_example JSON used as input for the model (e.g. [[0.5, 0.2]]) 53 | 54 | optional arguments: 55 | -h, --help show this help message and exit 56 | ``` 57 | -------------------------------------------------------------------------------- /evaluator-processors/src/main/java/com/ovh/mls/serving/runtime/processors/StandardScalerManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.processors; 2 | 3 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 4 | import com.ovh.mls.serving.runtime.core.AbstractEvaluatorManifest; 5 | import com.ovh.mls.serving.runtime.core.Field; 6 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 7 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 8 | 9 | import java.util.Map; 10 | 11 | @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) 12 | @IncludeAsEvaluatorManifest(type = "standard_scaler") 13 | public class StandardScalerManifest extends AbstractEvaluatorManifest { 14 | 15 | private static final String type = "standard_scaler"; 16 | 17 | private Map meanStdMap; 18 | 19 | public Map getMeanStdMap() { 20 | return meanStdMap; 21 | } 22 | 23 | public StandardScalerManifest setMeanStdMap(Map meanStdMap) { 24 | this.meanStdMap = meanStdMap; 25 | return this; 26 | } 27 | 28 | @Override 29 | public StandardScaler create(String path) throws EvaluatorException { 30 | return StandardScaler.create(this, path); 31 | } 32 | 33 | @Override 34 | public String getType() { 35 | return type; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/Evaluator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 5 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 6 | 7 | import java.util.List; 8 | 9 | public interface Evaluator { 10 | /** 11 | * Applies the evaluator operations over a Table and returns the initial 12 | * Table enriched with generated outputs. 13 | * 14 | * @param io 'EvaluatorIO' implementation over which the operations are applied 15 | * @return 'EvaluatorIO' implementation with generated outputs 16 | */ 17 | TensorIO evaluate(TensorIO io, EvaluationContext evaluationContext) throws EvaluationException; 18 | 19 | /** 20 | * @return The list of inputs required by the evaluator 21 | */ 22 | @JsonProperty("inputs") 23 | List getInputs(); 24 | 25 | /** 26 | * @return The list of outputs added to the Table after applying the evaluator 27 | */ 28 | @JsonProperty("outputs") 29 | List getOutputs(); 30 | 31 | /** 32 | * @return The required number of rows in the Table for the evaluator 33 | */ 34 | @JsonProperty("rolling_windows_size") 35 | default int getRollingWindowSize() { 36 | return 1; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /evaluator-huggingface/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | evaluator-huggingface 13 | 14 | 15 | 16 | com.ovh.mls.serving.runtime 17 | commons 18 | ${project.version} 19 | 20 | 21 | 22 | 23 | 24 | 25 | huggingface-tokenizer-jni/target/release 26 | 27 | *huggingface_tokenizer_jni.dylib 28 | *huggingface_tokenizer_jni.so 29 | *huggingface_tokenizer_jni.dll 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /evaluator-huggingface/src/main/java/com/ovh/mls/serving/runtime/huggingface/tokenizer/HuggingFaceTokenizerEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.huggingface.tokenizer; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 5 | import com.ovh.mls.serving.runtime.core.AbstractEvaluatorManifest; 6 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 7 | import com.ovh.mls.serving.runtime.core.tensor.TensorField; 8 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 9 | 10 | import java.nio.file.Paths; 11 | 12 | // In case of direct deserialization we override parent JsonTypeInfo 13 | @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) 14 | @IncludeAsEvaluatorManifest(type = HuggingFaceTokenizerEvaluatorManifest.TYPE) 15 | public class HuggingFaceTokenizerEvaluatorManifest extends AbstractEvaluatorManifest { 16 | 17 | public static final String TYPE = "huggingface_tokenizer"; 18 | 19 | @JsonProperty("saved_model_uri") 20 | private String savedModelUri; 21 | 22 | @Override 23 | public HuggingFaceTokenizerEvaluator create(String path) throws EvaluatorException { 24 | Tokenizer tokenizer = Tokenizer.fromFile(Paths.get(savedModelUri)); 25 | return new HuggingFaceTokenizerEvaluator(tokenizer); 26 | } 27 | 28 | @Override 29 | public String getType() { 30 | return TYPE; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "onnx_model_uri": "src/test/resources/onnx/iris/iris.onnx", 3 | "batch_size": 1, 4 | "inputs": [ 5 | { 6 | "name": "float_input", 7 | "shape": [-1, 4], 8 | "type": "float", 9 | "fields": [ 10 | { 11 | "name": "sepal_length", 12 | "index": 0 13 | }, 14 | { 15 | "name": "sepal_width", 16 | "index": 1 17 | }, 18 | { 19 | "name": "petal_length", 20 | "index": 2 21 | }, 22 | { 23 | "name": "petal_width", 24 | "index": 3 25 | } 26 | ] 27 | } 28 | ], 29 | "outputs": [ 30 | { 31 | "name": "output_label", 32 | "shape": [-1], 33 | "type": "long", 34 | "fields": [ 35 | { 36 | "name": "classification", 37 | "type": "long" 38 | } 39 | ] 40 | }, 41 | { 42 | "name": "output_probability", 43 | "shape": [-1, 3], 44 | "type": "float", 45 | "fields": [ 46 | { 47 | "name": "probability(0)", 48 | "index": 0, 49 | "key": 0 50 | }, 51 | { 52 | "name": "probability(1)", 53 | "index": 1, 54 | "key": 1 55 | }, 56 | { 57 | "name": "probability(2)", 58 | "index": 2, 59 | "key": 2 60 | } 61 | ] 62 | } 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/iris/manifest-same-output.json: -------------------------------------------------------------------------------- 1 | { 2 | "onnx_model_uri": "src/test/resources/onnx/iris/iris.onnx", 3 | "batch_size": 1, 4 | "inputs": [ 5 | { 6 | "name": "float_input", 7 | "shape": [-1, 4], 8 | "type": "float", 9 | "fields": [ 10 | { 11 | "name": "sepal_length", 12 | "index": 0 13 | }, 14 | { 15 | "name": "sepal_width", 16 | "index": 1 17 | }, 18 | { 19 | "name": "petal_length", 20 | "index": 2 21 | }, 22 | { 23 | "name": "petal_width", 24 | "index": 3 25 | } 26 | ] 27 | } 28 | ], 29 | "outputs": [ 30 | { 31 | "name": "output_label", 32 | "shape": [-1], 33 | "type": "long", 34 | "fields": [ 35 | { 36 | "name": "sepal_length", 37 | "type": "long" 38 | } 39 | ] 40 | }, 41 | { 42 | "name": "output_probability", 43 | "shape": [-1, 3], 44 | "type": "float", 45 | "fields": [ 46 | { 47 | "name": "sepal_width", 48 | "index": 0, 49 | "key": 0 50 | }, 51 | { 52 | "name": "petal_length", 53 | "index": 1, 54 | "key": 1 55 | }, 56 | { 57 | "name": "petal_width", 58 | "index": 2, 59 | "key": 2 60 | } 61 | ] 62 | } 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /evaluator-timeseries/src/main/java/com/ovh/mls/serving/runtime/timeseries/PredictionIntervalEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.timeseries; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 5 | import com.ovh.mls.serving.runtime.core.AbstractEvaluatorManifest; 6 | import com.ovh.mls.serving.runtime.core.Field; 7 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 8 | 9 | import java.util.Collections; 10 | import java.util.Map; 11 | 12 | // In case of direct deserialization we override parent JsonTypeInfo 13 | @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) 14 | @IncludeAsEvaluatorManifest(type = "pi") 15 | public class PredictionIntervalEvaluatorManifest extends AbstractEvaluatorManifest { 16 | 17 | private static final String type = "pi"; 18 | 19 | @JsonProperty("residuals_std") 20 | private Map residualsStd = Collections.emptyMap(); 21 | 22 | public Map getResidualsStd() { 23 | return residualsStd; 24 | } 25 | 26 | public PredictionIntervalEvaluatorManifest setResidualsStd(Map residualsStd) { 27 | this.residualsStd = residualsStd; 28 | return this; 29 | } 30 | 31 | public PredictionIntervalEvaluator create(String path) { 32 | return PredictionIntervalEvaluator.create(this, path); 33 | } 34 | 35 | @Override 36 | public String getType() { 37 | return type; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/core/builder/from/TensorIOIntoJsonBinary.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder.from; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.ovh.mls.serving.runtime.core.builder.Builder; 6 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 7 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 8 | import com.ovh.mls.serving.runtime.utils.img.BinaryContent; 9 | import org.apache.http.entity.ContentType; 10 | 11 | /** 12 | * Convert a TensorIO into a Json Binary content 13 | */ 14 | public class TensorIOIntoJsonBinary implements Builder { 15 | 16 | private final ObjectMapper mapper; 17 | private final boolean shouldSimplify; 18 | 19 | public TensorIOIntoJsonBinary(ObjectMapper mapper, boolean shouldSimplify) { 20 | this.shouldSimplify = shouldSimplify; 21 | this.mapper = mapper; 22 | } 23 | 24 | @Override 25 | public BinaryContent build(TensorIO input) throws EvaluationException { 26 | try { 27 | return new BinaryContent( 28 | "json", 29 | ContentType.APPLICATION_JSON, 30 | this.mapper.writeValueAsBytes( 31 | input.intoMap(shouldSimplify) 32 | ) 33 | ); 34 | } catch (JsonProcessingException e) { 35 | throw new EvaluationException(e); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/FlowEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 5 | 6 | import java.io.IOException; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | @IncludeAsEvaluatorManifest(type = "flow") 11 | public class FlowEvaluatorManifest implements EvaluatorManifest { 12 | 13 | private static final String type = "flow"; 14 | 15 | private List outputs = new ArrayList<>(); 16 | 17 | @JsonProperty("evaluator_manifests") 18 | private List evaluatorManifests; 19 | 20 | public List getEvaluatorManifests() { 21 | return evaluatorManifests; 22 | } 23 | 24 | public FlowEvaluatorManifest setEvaluatorManifests(List evaluatorManifests) { 25 | this.evaluatorManifests = evaluatorManifests; 26 | return this; 27 | } 28 | 29 | public List getOutputs() { 30 | return outputs; 31 | } 32 | 33 | public FlowEvaluatorManifest setOutputs(List outputs) { 34 | this.outputs = outputs; 35 | return this; 36 | } 37 | 38 | @Override 39 | public FlowEvaluator create(String path) throws EvaluatorException, IOException { 40 | return FlowEvaluator.create(this, path); 41 | } 42 | 43 | @Override 44 | public String getType() { 45 | return type; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /evaluator-onnx/python/iris.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import load_iris 2 | 3 | iris = load_iris() 4 | X, y = iris.data, iris.target 5 | 6 | from sklearn.model_selection import train_test_split 7 | 8 | X_train, X_test, y_train, y_test = train_test_split(X, y) 9 | 10 | from sklearn.linear_model import LogisticRegression 11 | 12 | clr = LogisticRegression() 13 | clr.fit(X_train, y_train) 14 | 15 | from skl2onnx import convert_sklearn 16 | from skl2onnx.common.data_types import FloatTensorType 17 | 18 | initial_type = [('float_input', FloatTensorType([None, 4]))] 19 | onx = convert_sklearn(clr, initial_types=initial_type) 20 | with open("iris.onnx", "wb") as f: 21 | f.write(onx.SerializeToString()) 22 | 23 | import onnxruntime as rt 24 | 25 | sess = rt.InferenceSession("iris.onnx") 26 | 27 | print("input name='{}' and shape={}".format( 28 | sess.get_inputs()[0].name, sess.get_inputs()[0].shape)) 29 | print("output name='{}' and shape={}".format( 30 | sess.get_outputs()[0].name, sess.get_outputs()[0].shape)) 31 | 32 | input_name = sess.get_inputs()[0].name 33 | label_name = sess.get_outputs()[0].name 34 | 35 | import numpy 36 | 37 | pred_onx = sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[:5] 38 | print(X_test.astype(numpy.float32)[:5]) 39 | print("Classification:") 40 | print(pred_onx[:5]) 41 | 42 | prob_name = sess.get_outputs()[1].name 43 | prob_rt = sess.run([prob_name], {input_name: X_test.astype(numpy.float32)})[:5] 44 | 45 | import pprint 46 | 47 | print("Proba") 48 | pprint.pprint(prob_rt[0][:5]) 49 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/exceptions/RestExceptionMapper.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.exceptions; 2 | 3 | import com.google.inject.Singleton; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import javax.servlet.http.HttpServletRequest; 8 | import javax.ws.rs.core.Context; 9 | import javax.ws.rs.core.MediaType; 10 | import javax.ws.rs.core.Response; 11 | import javax.ws.rs.ext.ExceptionMapper; 12 | import javax.ws.rs.ext.Provider; 13 | 14 | import static org.eclipse.jetty.http.HttpStatus.INTERNAL_SERVER_ERROR_500; 15 | 16 | /** 17 | * Return a json error message in case of any exception not handle by the default Exception mapper 18 | */ 19 | @Provider 20 | @Singleton 21 | public class RestExceptionMapper implements ExceptionMapper { 22 | 23 | private static final Logger LOGGER = LoggerFactory.getLogger(RestExceptionMapper.class); 24 | 25 | @Context 26 | private HttpServletRequest request; 27 | 28 | /** 29 | * Create a Json response from an exception 30 | */ 31 | @Override 32 | public Response toResponse(Exception exception) { 33 | 34 | LOGGER.error("Error during Request {}, Exception {}", exception.getMessage()); 35 | LOGGER.error("Error", exception); 36 | 37 | return Response 38 | // Return 500 status and Json message 39 | .status(INTERNAL_SERVER_ERROR_500) 40 | .entity(new ErrorMessage("Internal error")) 41 | .type(MediaType.APPLICATION_JSON) 42 | .build(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/swagger/SwaggerHomeResource.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.swagger; 2 | 3 | 4 | import io.swagger.v3.oas.annotations.Hidden; 5 | 6 | import javax.ws.rs.GET; 7 | import javax.ws.rs.Path; 8 | import javax.ws.rs.PathParam; 9 | import javax.ws.rs.Produces; 10 | import javax.ws.rs.core.Context; 11 | import javax.ws.rs.core.MediaType; 12 | import javax.ws.rs.core.Response; 13 | import javax.ws.rs.core.UriInfo; 14 | import java.io.InputStream; 15 | import java.net.URI; 16 | import java.net.URISyntaxException; 17 | 18 | /** 19 | * Swagger UI 20 | */ 21 | @Path("/") 22 | @Hidden 23 | @Produces(MediaType.TEXT_HTML) 24 | public class SwaggerHomeResource { 25 | private static final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); 26 | 27 | @Context 28 | private UriInfo uriInfo; 29 | 30 | @GET 31 | public Response viewHome() throws URISyntaxException { 32 | if (!uriInfo.getAbsolutePath().toString().endsWith("/")) { 33 | return Response.temporaryRedirect(new URI(uriInfo.getAbsolutePath().toString() + "/")).build(); 34 | } 35 | 36 | return Response 37 | .ok(contextClassLoader.getResourceAsStream("swagger/swagger.html")) 38 | .build(); 39 | } 40 | 41 | @GET 42 | @Path("{file}{ext:(.js|.css)}") 43 | @Produces("text/css") 44 | public InputStream renderFiler(@PathParam("file") String file, @PathParam("ext") String ext) { 45 | return contextClassLoader.getResourceAsStream("swagger/" + file + ext); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2013-2019, OVH SAS 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /evaluator-torch/Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := initialize 2 | 3 | WORKDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) 4 | 5 | OS := 6 | ifeq ($(OS),Windows_NT) 7 | OS = windows 8 | else 9 | UNAME_S := $(shell uname -s) 10 | ifeq ($(UNAME_S),Linux) 11 | OS = linux 12 | endif 13 | ifeq ($(UNAME_S),Darwin) 14 | OS = darwin 15 | endif 16 | endif 17 | 18 | .PHONY: initialize 19 | initialize: initialize-${OS} 20 | 21 | .PHONY: initialize-darwin 22 | initialize-darwin: 23 | curl "https://download.pytorch.org/libtorch/cpu/libtorch-macos-1.5.1.zip" -o libtorch-darwin-cpu.zip && \ 24 | unzip -q libtorch-darwin-cpu.zip && \ 25 | rm -rf libtorch-darwin-cpu && mv -vf libtorch libtorch-darwin-cpu && rm -v libtorch-darwin-cpu.zip 26 | # Fix OpenMP https://github.com/pytorch/pytorch/issues/38607 27 | install_name_tool -id @rpath/libiomp5.dylib libtorch-darwin-cpu/lib/libiomp5.dylib 28 | 29 | .PHONY: initialize-linux 30 | initialize-linux: 31 | curl "https://download.pytorch.org/libtorch/cpu/libtorch-shared-with-deps-1.5.1%2Bcpu.zip" -o libtorch-linux-cpu.zip && \ 32 | unzip -q libtorch-linux-cpu.zip && \ 33 | rm -rf libtorch-linux-cpu && mv -vf libtorch libtorch-linux-cpu && rm -v libtorch-linux-cpu.zip 34 | # Fix OpenMP 35 | patchelf --set-soname libgomp-7c85b1e2.so.1 libtorch-linux-cpu/lib/libgomp-7c85b1e2.so.1 36 | 37 | .PHONY: initialize-windows 38 | initialize-windows: 39 | curl "https://download.pytorch.org/libtorch/cpu/libtorch-win-shared-with-deps-1.5.1%2Bcpu.zip" -o libtorch-windows-cpu.zip && \ 40 | unzip -q libtorch-windows-cpu.zip && \ 41 | rm -rf libtorch-windows-cpu && mv -vf libtorch libtorch-windows-cpu && rm -v libtorch-windows-cpu.zip 42 | -------------------------------------------------------------------------------- /evaluator-tensorflow/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | evaluator-tensorflow 13 | 14 | 15 | 1.15.0 16 | 3.5.1 17 | 18 | 19 | 20 | 21 | com.ovh.mls.serving.runtime 22 | commons 23 | 1.0.1-SNAPSHOT 24 | 25 | 26 | org.tensorflow 27 | tensorflow 28 | ${tensorflow.version} 29 | 30 | 31 | org.tensorflow 32 | proto 33 | ${tensorflow.version} 34 | 35 | 36 | com.google.protobuf 37 | protobuf-java 38 | ${protobuf.version} 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /evaluator-timeseries/src/test/resources/timeseries/datetime-string-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputs": [ 3 | { 4 | "name": "date", 5 | "type": "string", 6 | "shape": [-1, -1], 7 | "format": "yyyy-MM-dd HH:mm:ss" 8 | } 9 | ], 10 | "outputs": [ 11 | { 12 | "name": "year", 13 | "type": "integer", 14 | "shape": [-1, -1], 15 | "format": "YEAR" 16 | }, 17 | { 18 | "name": "month", 19 | "type": "integer", 20 | "shape": [-1, -1], 21 | "format": "MONTH" 22 | }, 23 | { 24 | "name": "dayofmonth", 25 | "type": "integer", 26 | "shape": [-1, -1], 27 | "format": "DAYOFMONTH" 28 | }, 29 | { 30 | "name": "hourofday", 31 | "type": "integer", 32 | "shape": [-1, -1], 33 | "format": "HOUROFDAY" 34 | }, 35 | { 36 | "name": "minuteofhour", 37 | "type": "integer", 38 | "shape": [-1, -1], 39 | "format": "MINUTEOFHOUR" 40 | }, 41 | { 42 | "name": "dayofweek", 43 | "type": "integer", 44 | "shape": [-1, -1], 45 | "format": "DAYOFWEEK" 46 | }, 47 | { 48 | "name": "year-string", 49 | "type": "string", 50 | "shape": [-1, -1], 51 | "format": "yyyy" 52 | }, 53 | { 54 | "name": "year-float", 55 | "type": "float", 56 | "shape": [-1, -1], 57 | "format": "YEAR" 58 | }, 59 | { 60 | "name": "timestamp-seconds", 61 | "type": "long", 62 | "shape": [-1, -1], 63 | "format": "TS_S" 64 | }, 65 | { 66 | "name": "timestamp-years", 67 | "type": "long", 68 | "shape": [-1, -1], 69 | "format": "TS_Y" 70 | } 71 | ] 72 | } 73 | -------------------------------------------------------------------------------- /evaluator-torch/src/test/java/com/ovh/mls/serving/runtime/torch/TorchScriptEvaluatorManifestTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.torch; 2 | 3 | import com.ovh.mls.serving.runtime.core.DataType; 4 | import com.ovh.mls.serving.runtime.core.EvaluatorManifest; 5 | import com.ovh.mls.serving.runtime.core.EvaluatorUtil; 6 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 7 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 8 | import com.typesafe.config.ConfigFactory; 9 | import org.junit.jupiter.api.Assertions; 10 | import org.junit.jupiter.api.Test; 11 | 12 | import java.io.IOException; 13 | import java.util.Map; 14 | 15 | class TorchScriptEvaluatorManifestTest { 16 | 17 | @Test 18 | public void testCreate() throws IOException { 19 | // Load evaluator 20 | EvaluatorUtil evaluatorUtil = new EvaluatorUtil(ConfigFactory.load()); 21 | TorchScriptEvaluatorManifest torchScriptEvaluatorManifest = (TorchScriptEvaluatorManifest) evaluatorUtil 22 | .getObjectMapper() 23 | .readValue( 24 | getClass().getResourceAsStream("/manifest.json"), 25 | EvaluatorManifest.class 26 | ); 27 | TorchScriptEvaluator evaluator = torchScriptEvaluatorManifest.create("src/test/resources"); 28 | 29 | // Evaluate 30 | TensorIO input = new TensorIO(Map.of( 31 | "input_0", new Tensor(DataType.FLOAT, new int[]{2}, new float[]{0.5f, 0.2f}) 32 | )); 33 | TensorIO output = evaluator.evaluateTensor(input); 34 | float[] output0 = (float[]) output.getTensors().get("output_0").getData(); 35 | 36 | Assertions.assertEquals(output0[0], 0.120716706f, 1e-6); 37 | Assertions.assertEquals(output0[1], 0.054772355f, 1e-6); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | WORKDIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) 2 | NAME := serving-runtime-base 3 | REGISTRY := 4 | REPOSITORY := infaas 5 | .DEFAULT_GOAL := build 6 | TAG := $(lastword $(subst /, ,$(shell git rev-parse --abbrev-ref HEAD))) 7 | M2 := '$(HOME)/.m2' 8 | 9 | H5_CONVERTER := h5_converter/dist/h5_converter 10 | MAVEN_PROFILE=full 11 | 12 | .PHONY: docker-base 13 | docker-base: 14 | docker build --target base -t $(NAME) -f dockerfiles/$(MAVEN_PROFILE).Dockerfile . 15 | 16 | .PHONY: docker-test 17 | docker-test: docker-base 18 | docker run --rm -v $(WORKDIR):/usr/src/app -v $(M2):/root/.m2 $(NAME) make test H5_CONVERTER=/usr/src/bin/h5_converter 19 | 20 | .PHONY: docker-test 21 | docker-build: docker-base 22 | docker run --rm -v $(WORKDIR):/usr/src/app -v $(M2):/root/.m2 $(NAME) make build H5_CONVERTER=/usr/src/bin/h5_converter 23 | 24 | .PHONY: docker-build-api 25 | docker-build-api: 26 | docker build --build-arg MAVEN_PROFILE=$(MAVEN_PROFILE) -t $(NAME) -f dockerfiles/$(MAVEN_PROFILE).Dockerfile . 27 | 28 | .PHONY: docker-push-api 29 | docker-push-api: 30 | docker tag $(NAME) $(REGISTRY)/$(REPOSITORY)/$(NAME):$(TAG) 31 | docker push $(REGISTRY)/$(REPOSITORY)/$(NAME):$(TAG) 32 | 33 | .PHONY: build 34 | build: 35 | mvn package -DskipTests -B -P$(MAVEN_PROFILE) 36 | 37 | .PHONY: test 38 | test: 39 | mvn -B verify -DtrimStackTrace=false -Devaluator.tensorflow.h5_converter.path=$(H5_CONVERTER) -P$(MAVEN_PROFILE) 40 | 41 | .PHONY: deploy 42 | deploy: 43 | mvn -B deploy -DskipTests -P$(MAVEN_PROFILE) 44 | 45 | .PHONY: initialize-tensorflow 46 | initialize-tensorflow: 47 | make -C evaluator-tensorflow/h5_converter build 48 | 49 | .PHONY: initialize-huggingface 50 | initialize-huggingface: 51 | make -C evaluator-huggingface/huggingface-tokenizer-jni 52 | 53 | .PHONY: initialize-torch 54 | initialize-torch: 55 | make -C evaluator-torch 56 | -------------------------------------------------------------------------------- /api/src/test/java/com/ovh/mls/serving/runtime/IsCloseTo.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime; 2 | 3 | import org.hamcrest.Description; 4 | import org.hamcrest.Factory; 5 | import org.hamcrest.Matcher; 6 | import org.hamcrest.TypeSafeMatcher; 7 | 8 | public class IsCloseTo extends TypeSafeMatcher { 9 | 10 | private final float delta; 11 | private final float value; 12 | 13 | public IsCloseTo(float value, float error) { 14 | this.delta = error; 15 | this.value = value; 16 | } 17 | 18 | /** 19 | * Creates a matcher of {@link Double}s that matches when an examined double is equal 20 | * to the specified operand, within a range of +/- error. 21 | *

22 | * For example: 23 | * 

assertThat(1.03, is(closeTo(1.0, 0.03)))
24 | * 25 | * @param operand the expected value of matching doubles 26 | * @param error the delta (+/-) within which matches will be allowed 27 | */ 28 | @Factory 29 | public static Matcher closeTo(float operand, float error) { 30 | return new IsCloseTo(operand, error); 31 | } 32 | 33 | @Override 34 | public boolean matchesSafely(Float item) { 35 | return actualDelta(item) <= 0.0; 36 | } 37 | 38 | @Override 39 | public void describeMismatchSafely(Float item, Description mismatchDescription) { 40 | mismatchDescription.appendValue(item) 41 | .appendText(" differed by ") 42 | .appendValue(actualDelta(item)); 43 | } 44 | 45 | @Override 46 | public void describeTo(Description description) { 47 | description.appendText("a numeric value within ") 48 | .appendValue(delta) 49 | .appendText(" of ") 50 | .appendValue(value); 51 | } 52 | 53 | private double actualDelta(Float item) { 54 | return (Math.abs((item - value)) - delta); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/builder/InputStreamJsonIntoTensorIO.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 6 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 7 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 8 | 9 | import java.io.InputStream; 10 | import java.util.AbstractMap; 11 | import java.util.Map; 12 | import java.util.stream.Collectors; 13 | 14 | public class InputStreamJsonIntoTensorIO implements Builder { 15 | 16 | private final ObjectMapper mapper; 17 | 18 | public InputStreamJsonIntoTensorIO(ObjectMapper mapper) { 19 | this.mapper = mapper; 20 | } 21 | 22 | @Override 23 | public TensorIO build(InputStream inputStream) throws EvaluationException { 24 | try { 25 | ObjectIntoTensor tensorBuilder = new ObjectIntoTensor(); 26 | // The InputStream will be converted into Map 27 | TypeReference> typeRef = new TypeReference<>() {}; 28 | Map tensors = mapper.readValue(inputStream, typeRef); 29 | // Convert the map values (Object) into Tensors 30 | Map tensorsIO = tensors 31 | .entrySet() 32 | .stream() 33 | .map(x -> new AbstractMap.SimpleEntry<>(x.getKey(), tensorBuilder.build(x.getValue()))) 34 | .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); 35 | 36 | return new TensorIO(tensorsIO); 37 | } catch (Exception e) { 38 | throw new EvaluationException( 39 | "Unable to parse the given bytes into a correct json map of (name -> tensor)", e); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /evaluator-onnx/src/test/resources/onnx/titanic/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "onnx_model_uri": "src/test/resources/onnx/titanic/pipeline_titanic.onnx", 3 | "batch_size": 1, 4 | "inputs": [ 5 | { 6 | "name": "pclass", 7 | "shape": [-1, 1], 8 | "type": "string", 9 | "fields": [ 10 | { 11 | "name": "pclass", 12 | "index": 0 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "sex", 18 | "shape": [-1, 1], 19 | "type": "string", 20 | "fields": [ 21 | { 22 | "name": "sex", 23 | "index": 0 24 | } 25 | ] 26 | }, 27 | { 28 | "name": "age", 29 | "shape": [-1, 1], 30 | "type": "float", 31 | "fields": [ 32 | { 33 | "name": "age", 34 | "index": 0 35 | } 36 | ] 37 | }, 38 | { 39 | "name": "fare", 40 | "shape": [-1, 1], 41 | "type": "float", 42 | "fields": [ 43 | { 44 | "name": "fare", 45 | "index": 0 46 | } 47 | ] 48 | }, 49 | { 50 | "name": "embarked", 51 | "shape": [-1, 1], 52 | "type": "string", 53 | "fields": [ 54 | { 55 | "name": "embarked", 56 | "index": 0 57 | } 58 | ] 59 | } 60 | ], 61 | "outputs": [ 62 | { 63 | "name": "output_label", 64 | "shape": [-1], 65 | "type": "long", 66 | "fields": [ 67 | { 68 | "name": "classification" 69 | } 70 | ] 71 | }, 72 | { 73 | "name": "output_probability", 74 | "shape": [-1, 2], 75 | "type": "float", 76 | "fields": [ 77 | { 78 | "name": "probability(0)", 79 | "index": 0, 80 | "key": 0 81 | }, 82 | { 83 | "name": "probability(1)", 84 | "index": 1, 85 | "key": 1 86 | } 87 | ] 88 | } 89 | ] 90 | } 91 | -------------------------------------------------------------------------------- /dockerfiles/onnx.Dockerfile: -------------------------------------------------------------------------------- 1 | ## Stage 1 : Build project for onnx 2 | FROM maven:3.6.1-jdk-11-slim AS base 3 | 4 | RUN apt-get update && \ 5 | apt-get install make libgomp1 && \ 6 | rm -rf /var/lib/apt/lists/* 7 | 8 | WORKDIR /usr/src/app 9 | 10 | COPY . /usr/src/app/ 11 | 12 | RUN make build MAVEN_PROFILE=onnx 13 | 14 | ## Stage 2 : create the docker final image 15 | FROM centos:centos8 16 | 17 | USER root 18 | 19 | RUN mkdir -p /deployments 20 | 21 | # JAVA_APP_DIR is used by run-java.sh for finding the binaries 22 | ENV JAVA_APP_DIR=/deployments \ 23 | JAVA_MAJOR_VERSION=11 24 | 25 | # /dev/urandom is used as random source, which is prefectly safe 26 | # according to http://www.2uo.de/myths-about-urandom/ 27 | RUN yum install -y \ 28 | java-11-openjdk.x86_64 \ 29 | java-11-openjdk-devel.x86_64 \ 30 | && echo "securerandom.source=file:/dev/urandom" >> /usr/lib/jvm/jre/lib/security/java.security \ 31 | && yum clean all 32 | 33 | RUN yum install -y libgomp && \ 34 | rm -rf /var/cache/apk/* 35 | 36 | ENV JAVA_HOME /etc/alternatives/jre 37 | 38 | # Add run script as /deployments/run-java.sh and make it executable 39 | COPY run-java.sh /deployments/ 40 | RUN chmod 755 /deployments/run-java.sh 41 | 42 | # Run under user "jboss" and prepare for be running 43 | # under OpenShift, too 44 | RUN groupadd -r jboss -g 1000 \ 45 | && useradd -u 1000 -r -g jboss -m -d /opt/jboss -s /sbin/nologin jboss \ 46 | && chmod 755 /opt/jboss \ 47 | && chown -R jboss /deployments \ 48 | && usermod -g root -G `id -g jboss` jboss \ 49 | && chmod -R "g+rwX" /deployments \ 50 | && chown -R jboss:root /deployments 51 | 52 | USER jboss 53 | 54 | ENV AB_OFF=true 55 | 56 | ENV JAVA_OPTIONS="-Dfiles.path=./models/ -Dconfig.override_with_env_vars=true" 57 | 58 | COPY --from=base /usr/src/app/api/target/lib/* /deployments/lib/ 59 | COPY --from=base /usr/src/app/api/target/api-1.0.1-SNAPSHOT.jar /deployments/app.jar 60 | 61 | WORKDIR /deployments 62 | 63 | ENTRYPOINT [ "/deployments/run-java.sh" ] -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/validation/Validator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.validation; 2 | 3 | import com.ovh.mls.serving.runtime.core.DataType; 4 | import com.ovh.mls.serving.runtime.core.Evaluator; 5 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 6 | import org.apache.commons.lang3.StringUtils; 7 | 8 | import java.util.stream.Collectors; 9 | 10 | public class Validator { 11 | 12 | /** 13 | * Applies validation check over an evaluator instance based on its implementation class available 14 | * annotations. 15 | * 16 | * @param evaluator instance to check 17 | * @throws EvaluatorException if one check fails an exception is thrown 18 | */ 19 | public static void validate(Evaluator evaluator) throws EvaluatorException { 20 | if (evaluator.getClass().isAnnotationPresent(NumberOnly.class)) { 21 | checkNumbers(evaluator); 22 | } 23 | } 24 | 25 | /** 26 | * For evaluator that only support Numbers as input check that all input fields are of Number type. 27 | * @see DataType 28 | * 29 | * @param evaluator evaluator only supporting Number input 30 | * @throws EvaluatorException throws an exception if one field is not a Number type 31 | */ 32 | private static void checkNumbers(Evaluator evaluator) throws EvaluatorException { 33 | 34 | String nonNumberFields = evaluator.getInputs().stream() 35 | .filter(field -> !DataType.isNumberType(field.getType())) 36 | .map(field -> String.format("(%s,%s)", field.getName(), field.getType())) 37 | .collect(Collectors.joining(",")); 38 | 39 | if (StringUtils.isNotEmpty(nonNumberFields)) { 40 | throw new EvaluatorException(String.format( 41 | "Only Number fields are supported for evaluator %s. Unsupported fields: %s", 42 | evaluator.getClass().getSimpleName(), 43 | nonNumberFields 44 | )); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/io/Part.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.io; 2 | 3 | import org.apache.http.entity.ContentType; 4 | import org.apache.tika.Tika; 5 | 6 | import java.io.ByteArrayInputStream; 7 | import java.io.InputStream; 8 | 9 | public class Part { 10 | public String name; 11 | public String filename; 12 | public ContentType contentType; 13 | public byte[] content; 14 | 15 | private Part(String name, String filename, ContentType contentType, byte[] content) { 16 | this.name = name; 17 | this.contentType = contentType; 18 | this.content = content; 19 | } 20 | 21 | public InputStream getContentAsInputStream() { 22 | return new ByteArrayInputStream(this.content); 23 | } 24 | 25 | public static Part from(String name, String filename, String contentTypeString, byte[] content) { 26 | ContentType contentType = null; 27 | if (contentTypeString != null) { 28 | contentType = ContentType.parse(contentTypeString); 29 | } 30 | return from(name, filename, contentType, content); 31 | } 32 | 33 | public static Part from(String name, String filename, ContentType contentType, byte[] content) { 34 | // If content type null or application/octet-stream (default), detect it 35 | if ( 36 | contentType == null || 37 | ContentType.APPLICATION_OCTET_STREAM.getMimeType().equals(contentType.getMimeType()) 38 | ) { 39 | return from(name, filename, content); 40 | } 41 | return new Part(name, filename, contentType, content); 42 | } 43 | 44 | /** 45 | * Create a Part by detecting the content-type 46 | */ 47 | public static Part from(String name, String filename, byte[] content) { 48 | String mimetype = new Tika().detect(content, filename); 49 | ContentType detectedContentType = ContentType.parse(mimetype); 50 | return new Part(name, filename, detectedContentType, content); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /api/src/main/resources/swagger/swagger.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Serving Runtime Swagger 7 | 8 | 30 | 31 | 32 | 33 |
34 | 35 | 36 | 37 | 75 | 76 | -------------------------------------------------------------------------------- /commons/src/test/java/com/ovh/mls/serving/runtime/validation/ValidatorTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.validation; 2 | 3 | import com.ovh.mls.serving.runtime.core.DataType; 4 | import com.ovh.mls.serving.runtime.core.EvaluationContext; 5 | import com.ovh.mls.serving.runtime.core.Evaluator; 6 | import com.ovh.mls.serving.runtime.core.Field; 7 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 8 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 9 | import org.junit.jupiter.api.Assertions; 10 | import org.junit.jupiter.api.Test; 11 | 12 | import java.util.Collections; 13 | import java.util.List; 14 | 15 | class ValidatorTest { 16 | 17 | @Test 18 | void validate() throws EvaluatorException { 19 | List inputs = Collections.singletonList( 20 | new Field("testinput", DataType.DOUBLE) 21 | ); 22 | 23 | EvaluatorTest evaluatorTest = new EvaluatorTest(inputs); 24 | Validator.validate(evaluatorTest); 25 | } 26 | 27 | @Test 28 | void failValidate() { 29 | List inputs = Collections.singletonList( 30 | new Field("testinput", DataType.STRING) 31 | ); 32 | 33 | EvaluatorTest evaluatorTest = new EvaluatorTest(inputs); 34 | Assertions.assertThrows(EvaluatorException.class, () -> 35 | Validator.validate(evaluatorTest) 36 | ); 37 | } 38 | 39 | @NumberOnly 40 | public static class EvaluatorTest implements Evaluator { 41 | 42 | List inputs; 43 | 44 | public EvaluatorTest(List inputs) { 45 | this.inputs = inputs; 46 | } 47 | 48 | @Override 49 | public TensorIO evaluate(TensorIO io, EvaluationContext evaluationContext) { 50 | return null; 51 | } 52 | 53 | @Override 54 | public List getInputs() { 55 | return this.inputs; 56 | } 57 | 58 | @Override 59 | public List getOutputs() { 60 | return null; 61 | } 62 | 63 | @Override 64 | public int getRollingWindowSize() { 65 | return 0; 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/utils/MultipartUtils.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.utils; 2 | 3 | import com.google.common.io.Files; 4 | import com.ovh.mls.serving.runtime.core.io.Part; 5 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 6 | import org.apache.http.entity.ContentType; 7 | import org.eclipse.jetty.http.MultiPartFormInputStream; 8 | 9 | import javax.servlet.MultipartConfigElement; 10 | import java.io.IOException; 11 | import java.io.InputStream; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | public class MultipartUtils { 16 | 17 | /** 18 | * Read a multipart body and create parts from it 19 | * @param contentType The global Content-Type of the payload 20 | * @param inputStream The global input stream 21 | * @return The List of Parts 22 | * @throws EvaluationException 23 | */ 24 | public static List readParts(ContentType contentType, InputStream inputStream) 25 | throws EvaluationException { 26 | 27 | try { 28 | 29 | MultiPartFormInputStream multipartIS = new MultiPartFormInputStream( 30 | inputStream, 31 | contentType.toString(), 32 | new MultipartConfigElement(""), 33 | Files.createTempDir() 34 | ); 35 | 36 | List result = new ArrayList<>(); 37 | for (var part : multipartIS.getParts()) { 38 | try (InputStream partIs = part.getInputStream()) { 39 | String name = part.getName(); 40 | String contentTypeStr = part.getContentType(); 41 | String filename = part.getSubmittedFileName(); 42 | result.add(Part.from(name, filename, contentTypeStr, partIs.readAllBytes())); 43 | } catch (IOException e) { 44 | throw new EvaluationException("Error while reading multipart body", e); 45 | } 46 | } 47 | 48 | return result; 49 | 50 | } catch (IOException e) { 51 | throw new EvaluationException("Error while reading multipart body", e); 52 | } 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/tensor/TensorIndex.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.tensor; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import com.google.common.base.Objects; 5 | import com.ovh.mls.serving.runtime.core.Field; 6 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 7 | import org.apache.commons.lang3.StringUtils; 8 | 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | @JsonIgnoreProperties(ignoreUnknown = true) 13 | public class TensorIndex extends Field { 14 | 15 | private Integer index; 16 | private Object key; 17 | 18 | public TensorIndex() { 19 | 20 | } 21 | 22 | public TensorIndex(String name, Integer index) { 23 | super(name); 24 | this.index = index; 25 | } 26 | 27 | public Integer getIndex() { 28 | return index; 29 | } 30 | 31 | public void setIndex(Integer index) { 32 | this.index = index; 33 | } 34 | 35 | public static void checkFields(List fields) throws EvaluatorException { 36 | String emptyIndexes = fields.stream() 37 | .filter(field -> field.getIndex() == null) 38 | .map(Field::getName) 39 | .collect(Collectors.joining(",")); 40 | 41 | if (!StringUtils.isEmpty(emptyIndexes)) { 42 | throw new EvaluatorException("Indexes are empty for: " + emptyIndexes); 43 | } 44 | } 45 | 46 | public Object getKey() { 47 | return key; 48 | } 49 | 50 | public void setKey(Object key) { 51 | this.key = key; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | return super.hashCode() + Objects.hashCode(index); 57 | } 58 | 59 | @Override 60 | public boolean equals(Object o) { 61 | if (this == o) { 62 | return true; 63 | } 64 | if (o == null || getClass() != o.getClass()) { 65 | return false; 66 | } 67 | TensorIndex field = (TensorIndex) o; 68 | return Objects.equal(index, field.index) && 69 | Objects.equal(key, field.key) && 70 | super.equals(field); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /commons/src/test/java/com/ovh/mls/serving/runtime/core/EvaluatorUtilTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.typesafe.config.ConfigFactory; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.io.IOException; 7 | import java.util.List; 8 | 9 | import static org.junit.jupiter.api.Assertions.assertEquals; 10 | import static org.junit.jupiter.api.Assertions.assertTrue; 11 | 12 | @SuppressWarnings("ALL") 13 | public class EvaluatorUtilTest { 14 | private static final ClassLoader LOADER = EvaluatorUtilTest.class.getClassLoader(); 15 | 16 | @Test 17 | public void reflectiveDeserializationManifest() throws IOException { 18 | EvaluatorUtil evaluatorUtil = new EvaluatorUtil(ConfigFactory.load()); 19 | EvaluatorManifest evaluatorManifest = evaluatorUtil.getObjectMapper() 20 | .readValue(LOADER.getResourceAsStream("core/test-manifest.json"), EvaluatorManifest.class); 21 | assertTrue(evaluatorManifest instanceof TestEvaluatorManifest); 22 | 23 | TestEvaluatorManifest testEvaluatorManifest = (TestEvaluatorManifest) evaluatorManifest; 24 | 25 | assertEquals("someTestString", testEvaluatorManifest.getTestValue()); 26 | 27 | List inputs = testEvaluatorManifest.getInputs(); 28 | assertEquals(1, inputs.size()); 29 | assertEquals("int", inputs.get(0).getName()); 30 | assertEquals(DataType.INTEGER, inputs.get(0).getType()); 31 | 32 | assertEquals(0, testEvaluatorManifest.getOutputs().size()); 33 | } 34 | 35 | @IncludeAsEvaluatorManifest(type = "test") 36 | public static class TestEvaluatorManifest extends AbstractEvaluatorManifest { 37 | private String testValue; 38 | 39 | public String getTestValue() { 40 | return testValue; 41 | } 42 | 43 | public TestEvaluatorManifest setTestValue(String testValue) { 44 | this.testValue = testValue; 45 | return this; 46 | } 47 | 48 | @Override 49 | public Evaluator create(String path) { 50 | return null; 51 | } 52 | 53 | @Override 54 | public String getType() { 55 | return "test"; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /evaluator-huggingface/huggingface-tokenizer-jni/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::fmt::{Display, Formatter}; 3 | 4 | use jni::sys::{jobjectArray, jsize}; 5 | use jni::JNIEnv; 6 | 7 | // Re-export 8 | pub use encoding::*; 9 | pub use tokenizer::*; 10 | 11 | mod encoding; 12 | mod tokenizer; 13 | 14 | /// Wrap JNI and Tokenizers errors 15 | enum Error { 16 | Jni(jni::errors::Error), 17 | Tokenizers(tokenizers::Error), 18 | } 19 | 20 | impl Display for Error { 21 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 22 | match self { 23 | Error::Jni(error) => f.write_fmt(format_args!("{}", error)), 24 | Error::Tokenizers(error) => f.write_fmt(format_args!("{}", error)), 25 | } 26 | } 27 | } 28 | 29 | impl From for Error { 30 | fn from(error: jni::errors::Error) -> Self { 31 | Error::Jni(error) 32 | } 33 | } 34 | 35 | impl From for Error { 36 | fn from(error: tokenizers::Error) -> Self { 37 | Error::Tokenizers(error) 38 | } 39 | } 40 | 41 | /// Throw an exception if an error occurred 42 | fn unwrap_or_throw(env: &JNIEnv, result: Result, default: T) -> T { 43 | // Check if an exception is already thrown 44 | if env.exception_check().unwrap() { 45 | return default; 46 | } 47 | 48 | match result { 49 | Ok(tokenizer) => tokenizer, 50 | Err(error) => { 51 | let exception_class = env 52 | .find_class("com/ovh/mls/serving/runtime/exceptions/EvaluatorException") 53 | .unwrap(); 54 | env.throw_new(exception_class, format!("{}", error)) 55 | .unwrap(); 56 | default 57 | } 58 | } 59 | } 60 | 61 | /// Convert a Java string array to a vec of string 62 | fn jstring_array_to_vec(env: &JNIEnv, array: jobjectArray) -> Result, Error> { 63 | let array_len = env.get_array_length(array)?; 64 | let mut vec = Vec::new(); 65 | for index in 0..array_len { 66 | let item = env.get_object_array_element(array, index as jsize)?; 67 | let item: String = env.get_string(item.into())?.into(); 68 | vec.push(item); 69 | } 70 | Ok(vec) 71 | } 72 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/core/LogFilter.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import io.prometheus.client.Summary; 4 | 5 | import javax.annotation.Priority; 6 | import javax.servlet.http.HttpServletRequest; 7 | import javax.ws.rs.container.ContainerRequestContext; 8 | import javax.ws.rs.container.ContainerRequestFilter; 9 | import javax.ws.rs.container.ContainerResponseContext; 10 | import javax.ws.rs.container.ContainerResponseFilter; 11 | import javax.ws.rs.container.ResourceInfo; 12 | import javax.ws.rs.core.Context; 13 | import javax.ws.rs.ext.Provider; 14 | 15 | 16 | @Provider 17 | @Priority(1) 18 | public class LogFilter implements ContainerRequestFilter, ContainerResponseFilter { 19 | private static final ThreadLocal threadLocal = new ThreadLocal<>(); 20 | 21 | private static final Summary LATENCY = Summary.build() 22 | .name("evaluator_api_request_latency_ms") 23 | .help("Request latency in ms.") 24 | .labelNames("class", "method", "status") 25 | .quantile(0.5, 0.05) 26 | .quantile(0.9, 0.01) 27 | .quantile(0.99, 0.001) 28 | .register(); 29 | 30 | @Context 31 | private ResourceInfo resourceInfo; 32 | 33 | @Context 34 | private HttpServletRequest request; 35 | 36 | @Override 37 | public void filter(ContainerRequestContext requestContext) { 38 | LogFilter.threadLocal.set(System.currentTimeMillis()); 39 | } 40 | 41 | @Override 42 | public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) { 43 | Long start = LogFilter.threadLocal.get(); 44 | 45 | if (start == null) { 46 | // In case of error, we don't enter on the first filter 47 | start = System.currentTimeMillis(); 48 | } 49 | long time = System.currentTimeMillis() - start; 50 | 51 | if (resourceInfo.getResourceClass() != null) { 52 | LATENCY.labels( 53 | resourceInfo.getResourceClass().getSimpleName(), 54 | resourceInfo.getResourceMethod().getName(), 55 | String.valueOf(responseContext.getStatus()) 56 | ).observe(time); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /commons/src/test/java/com/ovh/mls/serving/runtime/core/tensor/TensorShapeTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.tensor; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 6 | 7 | public class TensorShapeTest { 8 | 9 | @Test 10 | public void testResolveNewShape4() { 11 | TensorShape shape = new TensorShape(new int[]{4, 28, 28}); 12 | 13 | assertArrayEquals( 14 | new int[]{4, 28, 28, 1}, 15 | shape.resolveNewShape(new int[]{-1, -1, -1, 1}) 16 | ); 17 | assertArrayEquals( 18 | new int[]{4, 28, 28}, 19 | shape.resolveNewShape(new int[]{-1, -1, -1}) 20 | ); 21 | } 22 | 23 | @Test 24 | public void testResolveNewShape3() { 25 | TensorShape shape = new TensorShape(new int[]{4, 28, 28, 1}); 26 | 27 | assertArrayEquals( 28 | new int[]{4, 28, 28, 1}, 29 | shape.resolveNewShape(new int[]{-1, -1, -1, 1}) 30 | ); 31 | assertArrayEquals( 32 | new int[]{4, 28, 28}, 33 | shape.resolveNewShape(new int[]{-1, -1, -1}) 34 | ); 35 | } 36 | 37 | @Test 38 | public void testResolveNewShape2() { 39 | TensorShape shape = new TensorShape(new int[]{4, 28, 28, 3}); 40 | 41 | assertArrayEquals( 42 | new int[]{4, 28, 28, 3}, 43 | shape.resolveNewShape(new int[]{-1, -1, -1, 3}) 44 | ); 45 | } 46 | 47 | @Test 48 | public void testResolveNewShape1() { 49 | TensorShape shape = new TensorShape(new int[]{3, 2}); 50 | 51 | assertArrayEquals( 52 | new int[]{3, 2}, 53 | shape.resolveNewShape(new int[]{-1, 2}) 54 | ); 55 | assertArrayEquals( 56 | new int[]{3, 2}, 57 | shape.resolveNewShape(new int[]{3, -1}) 58 | ); 59 | assertArrayEquals( 60 | new int[]{3, 2, 1}, 61 | shape.resolveNewShape(new int[]{3, 2, -1}) 62 | ); 63 | assertArrayEquals( 64 | new int[]{6, 1}, 65 | shape.resolveNewShape(new int[]{-1, 1}) 66 | ); 67 | assertArrayEquals( 68 | new int[]{6}, 69 | shape.resolveNewShape(new int[]{-1}) 70 | ); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /evaluator-huggingface/src/main/java/com/ovh/mls/serving/runtime/huggingface/tokenizer/Tokenizer.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.huggingface.tokenizer; 2 | 3 | import com.ovh.mls.serving.runtime.utils.NativeUtils; 4 | 5 | import java.io.IOException; 6 | import java.nio.file.Path; 7 | 8 | /** 9 | * A `Tokenizer` is capable of encoding/decoding any text 10 | */ 11 | public class Tokenizer { 12 | 13 | private static final String NATIVE_LIBRARY_NAME = "huggingface_tokenizer_jni"; 14 | 15 | static { 16 | try { 17 | // Look for library in classpath 18 | System.loadLibrary(NATIVE_LIBRARY_NAME); 19 | } catch (UnsatisfiedLinkError ignored) { 20 | try { 21 | // Look for library in JAR 22 | NativeUtils.loadLibraryFromJar("/" + System.mapLibraryName(NATIVE_LIBRARY_NAME)); 23 | } catch (IOException e) { 24 | throw new IllegalStateException("Cannot load native library", e); 25 | } 26 | } 27 | } 28 | 29 | // Pointer to the Rust structure 30 | private long handle = -1; 31 | 32 | private Tokenizer() { 33 | } 34 | 35 | /** 36 | * Instantiate a new Tokenizer from the given file 37 | * 38 | * May throw an EvaluatorException 39 | */ 40 | public static native Tokenizer fromFile(Path file); 41 | 42 | /** 43 | * Encode the given input 44 | * 45 | * May throw an EvaluatorException 46 | */ 47 | public native Encoding encode(String input); 48 | 49 | /** 50 | * Encode the given input 51 | * 52 | * May throw an EvaluatorException 53 | */ 54 | public native Encoding encode(String input1, String input2); 55 | 56 | /** 57 | * Encode the given input 58 | * 59 | * May throw an EvaluatorException 60 | */ 61 | public native Encoding encode(String[] tokens); 62 | 63 | /** 64 | * Encode the given input 65 | * 66 | * May throw an EvaluatorException 67 | */ 68 | public native Encoding encode(String[] tokens1, String[] tokens2); 69 | 70 | /** 71 | * Give back the Rust pointer to be freed 72 | */ 73 | private native void releaseHandle(); 74 | 75 | @Override 76 | protected void finalize() throws Throwable { 77 | try { 78 | releaseHandle(); 79 | } finally { 80 | super.finalize(); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/main/java/com/ovh/mls/serving/runtime/tensorflow/TensorflowEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.tensorflow; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 5 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 6 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 7 | import com.ovh.mls.serving.runtime.core.EvaluatorManifest; 8 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 9 | import com.ovh.mls.serving.runtime.core.tensor.TensorField; 10 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 11 | 12 | import java.io.IOException; 13 | import java.util.List; 14 | 15 | @IncludeAsEvaluatorManifest(type = "tensorflow") 16 | @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) 17 | @JsonIgnoreProperties(ignoreUnknown = true) 18 | @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) 19 | public class TensorflowEvaluatorManifest implements EvaluatorManifest { 20 | 21 | private static final String type = "tensorflow"; 22 | 23 | private String savedModelUri; 24 | 25 | private Integer batchSize = 1; 26 | 27 | private List inputs; 28 | 29 | private List outputs; 30 | 31 | public TensorflowEvaluatorManifest() { 32 | } 33 | 34 | public String getSavedModelUri() { 35 | return savedModelUri; 36 | } 37 | 38 | public void setSavedModelUri(String savedModelUri) { 39 | this.savedModelUri = savedModelUri; 40 | } 41 | 42 | public Integer getBatchSize() { 43 | return batchSize; 44 | } 45 | 46 | public void setBatchSize(Integer batchSize) { 47 | this.batchSize = batchSize; 48 | } 49 | 50 | @Override 51 | public String getType() { 52 | return type; 53 | } 54 | 55 | public List getInputs() { 56 | return inputs; 57 | } 58 | 59 | public void setInputs(List inputs) { 60 | this.inputs = inputs; 61 | } 62 | 63 | public List getOutputs() { 64 | return outputs; 65 | } 66 | 67 | public void setOutputs(List outputs) { 68 | this.outputs = outputs; 69 | } 70 | 71 | @Override 72 | public TensorflowEvaluator create(String path1) throws EvaluatorException, IOException { 73 | return TensorflowEvaluator.create(this, path1); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /dockerfiles/full.Dockerfile: -------------------------------------------------------------------------------- 1 | ## Stage 1: Build the TF Binary H5 Converter 2 | FROM python:3.7.5-buster as python-builder 3 | 4 | WORKDIR /usr/src/app 5 | 6 | COPY evaluator-tensorflow/h5_converter /usr/src/app 7 | 8 | RUN make 9 | 10 | ## Stage 2: Build the full project 11 | FROM maven:3.6.1-jdk-11-slim AS base 12 | 13 | RUN apt-get update && \ 14 | apt-get install make libgomp1 && \ 15 | rm -rf /var/lib/apt/lists/* 16 | 17 | WORKDIR /usr/src/app 18 | 19 | COPY --from=python-builder /usr/src/app/dist/h5_converter /usr/src/bin/h5_converter 20 | COPY . /usr/src/app/ 21 | 22 | RUN make build MAVEN_PROFILE=full 23 | 24 | ## Stage 3 : create the docker final image 25 | FROM centos:centos8 26 | 27 | USER root 28 | 29 | RUN mkdir -p /deployments 30 | 31 | # JAVA_APP_DIR is used by run-java.sh for finding the binaries 32 | ENV JAVA_APP_DIR=/deployments \ 33 | JAVA_MAJOR_VERSION=11 34 | 35 | # /dev/urandom is used as random source, which is prefectly safe 36 | # according to http://www.2uo.de/myths-about-urandom/ 37 | RUN yum install -y \ 38 | java-11-openjdk.x86_64 \ 39 | java-11-openjdk-devel.x86_64 \ 40 | && echo "securerandom.source=file:/dev/urandom" >> /usr/lib/jvm/jre/lib/security/java.security \ 41 | && yum clean all 42 | 43 | RUN yum install -y libgomp libstdc++ && \ 44 | rm -rf /var/cache/apk/* 45 | 46 | ENV JAVA_HOME /etc/alternatives/jre 47 | 48 | # Add run script as /deployments/run-java.sh and make it executable 49 | COPY run-java.sh /deployments/ 50 | RUN chmod 755 /deployments/run-java.sh 51 | 52 | # Run under user "jboss" and prepare for be running 53 | # under OpenShift, too 54 | RUN groupadd -r jboss -g 1000 \ 55 | && useradd -u 1000 -r -g jboss -m -d /opt/jboss -s /sbin/nologin jboss \ 56 | && chmod 755 /opt/jboss \ 57 | && chown -R jboss /deployments \ 58 | && usermod -g root -G `id -g jboss` jboss \ 59 | && chmod -R "g+rwX" /deployments \ 60 | && chown -R jboss:root /deployments 61 | 62 | USER jboss 63 | 64 | ENV AB_OFF=true 65 | 66 | ENV JAVA_OPTIONS="-Dfiles.path=./models/ -Dconfig.override_with_env_vars=true -Devaluator.tensorflow.h5_converter.path=/deployments/h5_converter" 67 | 68 | COPY --from=base /usr/src/app/api/target/lib/* /deployments/lib/ 69 | COPY --from=base /usr/src/app/api/target/api-1.0.1-SNAPSHOT.jar /deployments/app.jar 70 | COPY --from=python-builder /usr/src/app/dist/h5_converter /deployments/h5_converter 71 | 72 | WORKDIR /deployments 73 | 74 | ENTRYPOINT [ "/deployments/run-java.sh" ] 75 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/core/builder/from/TensorIOIntoImageBinary.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder.from; 2 | 3 | import com.ovh.mls.serving.runtime.core.Field; 4 | import com.ovh.mls.serving.runtime.core.builder.Builder; 5 | import com.ovh.mls.serving.runtime.core.builder.TensorIntoImages; 6 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 7 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 8 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 9 | import com.ovh.mls.serving.runtime.utils.img.BinaryContent; 10 | import com.ovh.mls.serving.runtime.utils.img.ImageDefaults; 11 | import org.apache.http.entity.ContentType; 12 | 13 | import java.awt.image.BufferedImage; 14 | import java.util.Arrays; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | /** 19 | * Convert a TensorIO into a Image Binary content 20 | */ 21 | public class TensorIOIntoImageBinary implements Builder { 22 | 23 | private final ContentType imageContentType; 24 | private final List outputFields; 25 | 26 | public TensorIOIntoImageBinary(List outputFields, ContentType imageContentType) { 27 | this.outputFields = outputFields; 28 | this.imageContentType = imageContentType; 29 | } 30 | 31 | @Override 32 | public BinaryContent build(TensorIO input) throws EvaluationException { 33 | if (input.getTensors().size() != 1) { 34 | throw new EvaluationException( 35 | String.format( 36 | "Unable to convert several tensors (%s) into a single image...", 37 | Arrays.toString(input.tensorsNames().toArray()) 38 | ) 39 | ); 40 | } 41 | Map.Entry firstEntry = input.getTensors().entrySet().iterator().next(); 42 | String tensorName = firstEntry.getKey(); 43 | Tensor tensor = firstEntry.getValue(); 44 | TensorIntoImages imageBuilder = ImageDefaults.getImageBuilderOrFail(tensorName, this.outputFields, tensor); 45 | List images = imageBuilder.build(tensor); 46 | if (images.size() != 1) { 47 | throw new EvaluationException( 48 | String.format( 49 | "Unable to convert a batch of %s images into a single image...", 50 | images.size() 51 | ) 52 | ); 53 | } 54 | return ImageDefaults.buildImage(images.get(0), this.imageContentType); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dockerfiles/tensorflow.Dockerfile: -------------------------------------------------------------------------------- 1 | ## Stage 1: Build the TF Binary H5 Converter 2 | FROM python:3.7.5-buster as python-builder 3 | 4 | WORKDIR /usr/src/app 5 | 6 | RUN pip install --upgrade pip 7 | 8 | USER root 9 | 10 | COPY evaluator-tensorflow/h5_converter /usr/src/app 11 | 12 | RUN make 13 | 14 | FROM maven:3.6.1-jdk-11-slim AS base 15 | 16 | RUN apt-get update && \ 17 | apt-get install make libgomp1 && \ 18 | rm -rf /var/lib/apt/lists/* 19 | 20 | WORKDIR /usr/src/app 21 | 22 | COPY --from=python-builder /usr/src/app/dist/h5_converter /usr/src/bin/h5_converter 23 | 24 | ## Stage 2 : build with maven builder image 25 | FROM base AS build 26 | 27 | COPY . /usr/src/app/ 28 | 29 | RUN make build MAVEN_PROFILE=tensorflow 30 | 31 | ## Stage 3 : create the docker final image 32 | FROM centos:centos8 33 | 34 | USER root 35 | 36 | RUN mkdir -p /deployments 37 | 38 | # JAVA_APP_DIR is used by run-java.sh for finding the binaries 39 | ENV JAVA_APP_DIR=/deployments \ 40 | JAVA_MAJOR_VERSION=11 41 | 42 | # /dev/urandom is used as random source, which is prefectly safe 43 | # according to http://www.2uo.de/myths-about-urandom/ 44 | RUN yum install -y \ 45 | java-11-openjdk.x86_64 \ 46 | java-11-openjdk-devel.x86_64 \ 47 | && echo "securerandom.source=file:/dev/urandom" >> /usr/lib/jvm/jre/lib/security/java.security \ 48 | && yum clean all 49 | 50 | RUN yum install -y libstdc++ && \ 51 | rm -rf /var/cache/apk/* 52 | 53 | ENV JAVA_HOME /etc/alternatives/jre 54 | 55 | # Add run script as /deployments/run-java.sh and make it executable 56 | COPY run-java.sh /deployments/ 57 | RUN chmod 755 /deployments/run-java.sh 58 | 59 | # Run under user "jboss" and prepare for be running 60 | # under OpenShift, too 61 | RUN groupadd -r jboss -g 1000 \ 62 | && useradd -u 1000 -r -g jboss -m -d /opt/jboss -s /sbin/nologin jboss \ 63 | && chmod 755 /opt/jboss \ 64 | && chown -R jboss /deployments \ 65 | && usermod -g root -G `id -g jboss` jboss \ 66 | && chmod -R "g+rwX" /deployments \ 67 | && chown -R jboss:root /deployments 68 | 69 | USER jboss 70 | 71 | ENV AB_OFF=true 72 | 73 | ENV JAVA_OPTIONS="-Dfiles.path=./models/ -Dconfig.override_with_env_vars=true -Devaluator.tensorflow.h5_converter.path=/deployments/h5_converter" 74 | 75 | COPY --from=build /usr/src/app/api/target/lib/* /deployments/lib/ 76 | COPY --from=build /usr/src/app/api/target/api-1.0.1-SNAPSHOT.jar /deployments/app.jar 77 | COPY --from=python-builder /usr/src/app/dist/h5_converter /deployments/h5_converter 78 | 79 | WORKDIR /deployments 80 | 81 | ENTRYPOINT [ "/deployments/run-java.sh" ] 82 | -------------------------------------------------------------------------------- /api/src/test/java/com/ovh/mls/serving/runtime/torch/TorchSimpleIT.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.torch; 2 | 3 | import com.jayway.restassured.RestAssured; 4 | import com.jayway.restassured.response.Header; 5 | import com.ovh.mls.serving.runtime.IsCloseTo; 6 | import com.ovh.mls.serving.runtime.core.ApiServer; 7 | import com.typesafe.config.Config; 8 | import com.typesafe.config.ConfigFactory; 9 | import io.swagger.v3.oas.integration.OpenApiContextLocator; 10 | import org.junit.jupiter.api.BeforeAll; 11 | import org.junit.jupiter.api.Test; 12 | 13 | import java.lang.reflect.Field; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | import static com.jayway.restassured.RestAssured.given; 19 | import static org.hamcrest.Matchers.equalTo; 20 | 21 | public class TorchSimpleIT { 22 | 23 | @BeforeAll 24 | public static void startServer() throws NoSuchFieldException, IllegalAccessException { 25 | // Reset swagger context 26 | Field field = OpenApiContextLocator.class.getDeclaredField("instance"); 27 | field.setAccessible(true); 28 | field.set(null, null); 29 | 30 | Config config = ConfigFactory.load("torch/simple_model/api.conf"); 31 | ApiServer apiServer = new ApiServer(config); 32 | apiServer.start(); 33 | RestAssured.port = 8093; 34 | } 35 | 36 | @Test 37 | public void testDescribe() { 38 | given() 39 | .when() 40 | .get("/describe") 41 | .then() 42 | .statusCode(200) 43 | .body( 44 | "inputs.get(0).name", equalTo("input_0"), 45 | "inputs.get(0).shape", equalTo(List.of(2)), 46 | "inputs.get(0).type", equalTo("float"), 47 | "outputs.get(0).name", equalTo("output_0"), 48 | "outputs.get(0).shape", equalTo(List.of(2)), 49 | "outputs.get(0).type", equalTo("float") 50 | ); 51 | } 52 | 53 | @Test 54 | public void testEvalString() { 55 | Map body = new HashMap<>(); 56 | body.put("input_0", List.of(0.5, 0.2)); 57 | given() 58 | .body(body) 59 | .header(new Header("Content-Type", "application/json")) 60 | .when() 61 | .post("/eval") 62 | .then() 63 | .statusCode(200) 64 | .body( 65 | "output_0.get(0)", IsCloseTo.closeTo(0.120716706f, 1e-6f), 66 | "output_0.get(1)", IsCloseTo.closeTo(0.054772355f, 1e-6f) 67 | ); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /evaluator-onnx/src/main/java/com/ovh/mls/serving/runtime/onnx/OnnxEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.onnx; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 6 | import com.ovh.mls.serving.runtime.core.Evaluator; 7 | import com.ovh.mls.serving.runtime.core.EvaluatorManifest; 8 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 9 | import com.ovh.mls.serving.runtime.core.tensor.TensorField; 10 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 11 | 12 | import java.io.IOException; 13 | import java.util.List; 14 | 15 | // In case of direct deserialization we override parent JsonTypeInfo 16 | @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) 17 | @IncludeAsEvaluatorManifest(type = OnnxEvaluatorManifest.TYPE) 18 | @JsonIgnoreProperties(ignoreUnknown = true) 19 | public class OnnxEvaluatorManifest implements EvaluatorManifest { 20 | public static final String TYPE = "onnx"; 21 | 22 | @JsonProperty("onnx_model_uri") 23 | private String onnxModelUri; 24 | 25 | @JsonProperty("binary") 26 | private String binary; 27 | 28 | private Integer batchSize = 1; 29 | 30 | private List inputs; 31 | private List outputs; 32 | 33 | @Override 34 | public Evaluator create(String path) throws IOException, EvaluatorException { 35 | return OnnxEvaluator.create(this, path); 36 | } 37 | 38 | @Override 39 | public String getType() { 40 | return TYPE; 41 | } 42 | 43 | public String getOnnxModelUri() { 44 | return onnxModelUri; 45 | } 46 | 47 | public void setOnnxModelUri(String onnxModelUri) { 48 | this.onnxModelUri = onnxModelUri; 49 | } 50 | 51 | public Integer getBatchSize() { 52 | return batchSize; 53 | } 54 | 55 | public void setBatchSize(Integer batchSize) { 56 | this.batchSize = batchSize; 57 | } 58 | 59 | public List getInputs() { 60 | return inputs; 61 | } 62 | 63 | public void setInputs(List inputs) { 64 | this.inputs = inputs; 65 | } 66 | 67 | public List getOutputs() { 68 | return outputs; 69 | } 70 | 71 | public void setOutputs(List outputs) { 72 | this.outputs = outputs; 73 | } 74 | 75 | public String getBinary() { 76 | return binary; 77 | } 78 | 79 | public void setBinary(String binary) { 80 | this.binary = binary; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /api/src/test/java/com/ovh/mls/serving/runtime/tensorflow/Tensorflow2DIT.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.tensorflow; 2 | 3 | import com.jayway.restassured.RestAssured; 4 | import com.jayway.restassured.response.Header; 5 | import com.ovh.mls.serving.runtime.IsCloseTo; 6 | import com.ovh.mls.serving.runtime.core.ApiServer; 7 | import com.ovh.mls.serving.runtime.onnx.OnnxIT; 8 | import com.typesafe.config.Config; 9 | import com.typesafe.config.ConfigFactory; 10 | import io.swagger.v3.oas.integration.OpenApiContextLocator; 11 | import org.apache.commons.io.IOUtils; 12 | import org.junit.jupiter.api.BeforeAll; 13 | import org.junit.jupiter.api.Test; 14 | 15 | import java.io.IOException; 16 | import java.io.InputStream; 17 | import java.lang.reflect.Field; 18 | 19 | import static com.jayway.restassured.RestAssured.given; 20 | import static org.hamcrest.Matchers.equalTo; 21 | 22 | public class Tensorflow2DIT { 23 | private static final ClassLoader LOADER = OnnxIT.class.getClassLoader(); 24 | 25 | @BeforeAll 26 | public static void startServer() throws NoSuchFieldException, IllegalAccessException { 27 | // Reset swagger context 28 | Field field = OpenApiContextLocator.class.getDeclaredField("instance"); 29 | field.setAccessible(true); 30 | field.set(null, null); 31 | 32 | Config config = ConfigFactory.load("tensorflow/2d_savedmodel/api.conf"); 33 | ApiServer apiServer = new ApiServer(config); 34 | apiServer.start(); 35 | RestAssured.port = 8085; 36 | } 37 | 38 | @Test 39 | public void testDescribe() { 40 | given() 41 | .when() 42 | .get("/describe") 43 | .then() 44 | .statusCode(200) 45 | .body( 46 | "inputs.get(0).name", equalTo("input"), 47 | "inputs.get(0).type", equalTo("float"), 48 | "outputs.get(0).name", equalTo("output"), 49 | "outputs.get(0).type", equalTo("float") 50 | ); 51 | } 52 | 53 | @Test 54 | public void testPost() throws IOException { 55 | final InputStream resourceAsStream = LOADER.getResourceAsStream("tensorflow/2d_savedmodel/2d_input.json"); 56 | 57 | given() 58 | .body(IOUtils.toByteArray(resourceAsStream)) 59 | .header(new Header("Content-Type", "application/json")) 60 | .when() 61 | .post("/eval") 62 | .then() 63 | .statusCode(200) 64 | .body( 65 | "scaled_imputed_label_predicted", 66 | IsCloseTo.closeTo(-0.1796778291463852F, 0.001F) 67 | ); 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /evaluator-timeseries/src/main/java/com/ovh/mls/serving/runtime/timeseries/DatetimeEvaluator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.timeseries; 2 | 3 | import com.ovh.mls.serving.runtime.core.AbstractTensorEvaluator; 4 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 5 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 6 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 7 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 8 | 9 | import java.time.LocalDateTime; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.function.Function; 14 | 15 | /** 16 | * Evaluator used to encode and decode datetimes, it takes only one input which is the date to decode 17 | * It can answer with several output, each one representing a configured encoded value for the input date 18 | */ 19 | public class DatetimeEvaluator extends AbstractTensorEvaluator { 20 | 21 | private final DateTensorField singleInput; 22 | 23 | public DatetimeEvaluator( 24 | List inputs, 25 | List outputs 26 | ) { 27 | super(inputs, outputs, 1); 28 | if (inputs.size() != 1) { 29 | throw new EvaluatorException("There should be only 1 input for a datetime evaluator"); 30 | } 31 | this.singleInput = this.getInputs().get(0); 32 | } 33 | 34 | @Override 35 | protected TensorIO evaluateTensor(TensorIO tensorIO) throws EvaluationException { 36 | String inputTensorName = this.singleInput.getName(); 37 | Tensor inputTensor = tensorIO.getTensor(inputTensorName); 38 | if (inputTensor == null) { 39 | throw new EvaluationException(String.format("Impossible to find a tensor with name '%s'", inputTensorName)); 40 | } 41 | if (this.singleInput.getType() != inputTensor.getType()) { 42 | throw new EvaluationException( 43 | String.format("Input type for tensor %s is different from given one", inputTensorName)); 44 | } 45 | 46 | Function decodeFunction = this.singleInput.decodeDatetimeFunction(); 47 | Map outputsTensors = new HashMap<>(); 48 | for (DateTensorField outputsField : this.getOutputs()) { 49 | Function encodeFunction = outputsField.encodeDatetimeFunction(); 50 | Function fullTransfo = encodeFunction.compose(decodeFunction); 51 | Tensor outputTensor = inputTensor.apply(fullTransfo, outputsField.getType()); 52 | outputsTensors.put(outputsField.getName(), outputTensor); 53 | } 54 | return new TensorIO(outputsTensors); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/Field.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | import com.google.common.base.Objects; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | @JsonIgnoreProperties(ignoreUnknown = true) 11 | public class Field implements Cloneable { 12 | 13 | private String name; 14 | private DataType type; 15 | 16 | // The possible values for a Field value when a Field is categorical 17 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 18 | private List values = new ArrayList<>(); 19 | 20 | // The optional interval value when a Field is continuous 21 | @JsonInclude(JsonInclude.Include.NON_NULL) 22 | private List continuousDomain; 23 | 24 | public Field() { 25 | 26 | } 27 | 28 | public Field(String name) { 29 | this.name = name; 30 | } 31 | 32 | public Field(String name, DataType type) { 33 | this.name = name; 34 | this.type = type; 35 | } 36 | 37 | @Override 38 | public boolean equals(Object o) { 39 | if (this == o) { 40 | return true; 41 | } 42 | if (o == null || getClass() != o.getClass()) { 43 | return false; 44 | } 45 | Field field = (Field) o; 46 | return Objects.equal(name, field.name) && 47 | type == field.type && 48 | Objects.equal(values, field.values); 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return Objects.hashCode(name, type, values); 54 | } 55 | 56 | public List getValues() { 57 | return values; 58 | } 59 | 60 | public Field setValues(List values) { 61 | this.values = values; 62 | return this; 63 | } 64 | 65 | public DataType getType() { 66 | return type; 67 | } 68 | 69 | public Field setType(DataType type) { 70 | this.type = type; 71 | return this; 72 | } 73 | 74 | public String getName() { 75 | return name; 76 | } 77 | 78 | public Field setName(String name) { 79 | this.name = name; 80 | return this; 81 | } 82 | 83 | public List getContinuousDomain() { 84 | return continuousDomain; 85 | } 86 | 87 | public Field setContinuousDomain(List continuousDomain) { 88 | this.continuousDomain = continuousDomain; 89 | return this; 90 | } 91 | 92 | public Field clone() { 93 | var cloned = new Field(this.name, this.type); 94 | cloned.setValues(this.values); 95 | cloned.setContinuousDomain(this.continuousDomain); 96 | return cloned; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /api/src/test/java/com/ovh/mls/serving/runtime/onnx/OnnxIT.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.onnx; 2 | 3 | import com.jayway.restassured.RestAssured; 4 | import com.jayway.restassured.response.Header; 5 | import com.ovh.mls.serving.runtime.IsCloseTo; 6 | import com.ovh.mls.serving.runtime.core.ApiServer; 7 | import com.typesafe.config.Config; 8 | import com.typesafe.config.ConfigFactory; 9 | import io.swagger.v3.oas.integration.OpenApiContextLocator; 10 | import org.apache.commons.io.IOUtils; 11 | import org.junit.jupiter.api.BeforeAll; 12 | import org.junit.jupiter.api.Test; 13 | 14 | import java.io.IOException; 15 | import java.io.InputStream; 16 | import java.lang.reflect.Field; 17 | 18 | import static com.jayway.restassured.RestAssured.given; 19 | import static org.hamcrest.Matchers.equalTo; 20 | 21 | public class OnnxIT { 22 | private static final ClassLoader LOADER = OnnxIT.class.getClassLoader(); 23 | 24 | @BeforeAll 25 | public static void startServer() throws NoSuchFieldException, IllegalAccessException { 26 | // Reset swagger context 27 | Field field = OpenApiContextLocator.class.getDeclaredField("instance"); 28 | field.setAccessible(true); 29 | field.set(null, null); 30 | 31 | Config config = ConfigFactory.load("onnx/api.conf"); 32 | ApiServer apiServer = new ApiServer(config); 33 | apiServer.start(); 34 | RestAssured.port = 8083; 35 | } 36 | 37 | @Test 38 | public void testDescribe() { 39 | given() 40 | .when() 41 | .get("/describe") 42 | .then() 43 | .statusCode(200) 44 | .log().body() 45 | .body( 46 | "inputs.size()", equalTo(5), 47 | "outputs.size()", equalTo(2) 48 | ); 49 | } 50 | 51 | @Test 52 | public void testPost() throws IOException { 53 | final InputStream resourceAsStream = LOADER.getResourceAsStream("onnx/batch_gen.json"); 54 | byte[] bytes = IOUtils.toByteArray(resourceAsStream); 55 | 56 | given() 57 | .body(bytes) 58 | .header(new Header("Content-Type", "application/json")) 59 | .when() 60 | .post("/eval") 61 | .then() 62 | .statusCode(200) 63 | .log().body() 64 | .body( 65 | "output_label.get(0)", equalTo(0), 66 | "output_label.get(1)", equalTo(1), 67 | "output_probability.get(0).get(0)", IsCloseTo.closeTo(0.80328506F, 0.001F), 68 | "output_probability.get(1).get(0)", IsCloseTo.closeTo(0.39885646F, 0.001F), 69 | "output_probability.get(0).get(1)", IsCloseTo.closeTo(0.19671494F, 0.001F), 70 | "output_probability.get(1).get(1)", IsCloseTo.closeTo(0.60114354F, 0.001F) 71 | ); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/EvaluationContext.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core; 2 | 3 | 4 | public class EvaluationContext { 5 | private static final String INPUT_DEBUG_KEY = "input"; 6 | private static final String OUTPUT_DEBUG_KEY = "output"; 7 | 8 | 9 | private long evaluation = 0L; 10 | 11 | /** 12 | * Current number of the evaluator beeing evaluated 13 | */ 14 | private int currentEvaluator = 0; 15 | 16 | /** 17 | * Attribute indicating if user asked for debugging of an evaluator (i.e. stop and display tensors) 18 | * If null: no debugging asked 19 | * If not null: debugging of the evaluator of that number asked 20 | */ 21 | private Integer debugStep; 22 | 23 | /** 24 | * Boolean indicating : 25 | * If true: should debug the input of the wanted debug step 26 | * If false: should debug the output of the wanted debug step 27 | * If null: doesn't debug anything 28 | */ 29 | private Boolean debugInput; 30 | 31 | /** 32 | * Indicates if the output should be simplified at the end 33 | */ 34 | private final boolean shouldSimplify; 35 | 36 | public EvaluationContext() { 37 | this(null); 38 | } 39 | 40 | public EvaluationContext(String debugStep) { 41 | this.evaluation = 0L; 42 | this.currentEvaluator = 0; 43 | this.debugStep = null; 44 | this.debugInput = null; 45 | 46 | if (debugStep != null) { 47 | shouldSimplify = false; 48 | String[] split = debugStep.split(":"); 49 | if (split.length == 2) { 50 | String key = split[0]; 51 | Integer value = Integer.valueOf(split[1]); 52 | if (INPUT_DEBUG_KEY.equals(key)) { 53 | this.debugInput = true; 54 | } else if (OUTPUT_DEBUG_KEY.equals(key)) { 55 | this.debugInput = false; 56 | } 57 | this.debugStep = value; 58 | } 59 | } else { 60 | shouldSimplify = true; 61 | } 62 | } 63 | 64 | public void incCurrentEvaluator() { 65 | this.currentEvaluator++; 66 | } 67 | 68 | public void incEvaluation() { 69 | this.evaluation++; 70 | } 71 | 72 | public void incEvaluationBy(int val) { 73 | this.evaluation += val; 74 | } 75 | 76 | public Long totalEvaluation() { 77 | return this.evaluation; 78 | } 79 | 80 | public boolean shouldStop(boolean isCurrentStepInput) { 81 | return 82 | this.debugInput != null && this.debugInput == isCurrentStepInput && 83 | this.debugStep != null && this.debugStep == this.currentEvaluator; 84 | } 85 | 86 | public boolean shouldSimplify() { 87 | return shouldSimplify; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /evaluator-torch/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | evaluator-torch 13 | 14 | 15 | 1.5.0-SNAPSHOT 16 | 0.0.3 17 | 0.8.0 18 | 19 | 20 | 21 | 22 | oss.sonatype.org 23 | https://oss.sonatype.org/content/repositories/snapshots 24 | 25 | 26 | jcenter.bintray.com 27 | https://jcenter.bintray.com/ 28 | 29 | 30 | 31 | 32 | 33 | com.ovh.mls.serving.runtime 34 | commons 35 | ${project.version} 36 | 37 | 38 | org.pytorch 39 | pytorch_java_only 40 | ${pytorch.version} 41 | 42 | 43 | com.facebook.fbjni 44 | fbjni-java-only 45 | ${fbjni.version} 46 | 47 | 48 | com.facebook.soloader 49 | nativeloader 50 | ${nativeloader.version} 51 | 52 | 53 | 54 | 55 | 56 | 57 | libtorch-linux-cpu/lib 58 | 59 | *.so* 60 | 61 | 62 | 63 | libtorch-mac-cpu/lib 64 | 65 | *.dylib 66 | 67 | 68 | 69 | libtorch-windows-cpu/lib 70 | 71 | *.dll 72 | 73 | 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /.github/workflows/deploy-packages.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Jar & Docker packages 2 | on: 3 | push: 4 | branches: [ master ] 5 | jobs: 6 | test-build-deploy-jar: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout project 10 | uses: actions/checkout@v2 11 | - name: Set up JDK 1.11 12 | uses: actions/setup-java@v1 13 | with: 14 | java-version: 1.11 15 | - name: Set up Python 3.7 16 | uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.7 19 | - name: Cache maven modules 20 | uses: actions/cache@v1 21 | env: 22 | cache-name: serving-runtime-maven-deps 23 | with: 24 | path: ~/.m2/repository 25 | key: cache-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }} 26 | 27 | - name: Build & Deploy JARs 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | run: | 31 | echo "githubovh${GITHUB_TOKEN}" > ~/.m2/settings.xml 32 | make initialize-tensorflow initialize-huggingface initialize-torch deploy 33 | 34 | build-docker-tensorflow: 35 | needs: test-build-deploy-jar 36 | runs-on: ubuntu-latest 37 | steps: 38 | - name: Checkout project 39 | uses: actions/checkout@v2 40 | - name: Build Docker image for Tensorflow only 41 | env: 42 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 43 | run: | 44 | echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u ovh --password-stdin 45 | make docker-build-api docker-push-api MAVEN_PROFILE=tensorflow NAME=api-tf TAG=latest REGISTRY=docker.pkg.github.com/ovh REPOSITORY=serving-runtime 46 | 47 | build-docker-onnx: 48 | needs: test-build-deploy-jar 49 | runs-on: ubuntu-latest 50 | steps: 51 | - name: Checkout project 52 | uses: actions/checkout@v2 53 | - name: Build Docker image for ONNX only 54 | env: 55 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | run: | 57 | echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u ovh --password-stdin 58 | make docker-build-api docker-push-api MAVEN_PROFILE=onnx NAME=api-onnx TAG=latest REGISTRY=docker.pkg.github.com/ovh REPOSITORY=serving-runtime 59 | 60 | build-docker-full: 61 | needs: test-build-deploy-jar 62 | runs-on: ubuntu-latest 63 | steps: 64 | - name: Checkout project 65 | uses: actions/checkout@v2 66 | - name: Build Docker image for full features 67 | env: 68 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 69 | run: | 70 | echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u ovh --password-stdin 71 | make docker-build-api docker-push-api MAVEN_PROFILE=full NAME=api-full TAG=latest REGISTRY=docker.pkg.github.com/ovh REPOSITORY=serving-runtime 72 | -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/tensor/TensorField.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.tensor; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonIgnore; 5 | import com.fasterxml.jackson.annotation.JsonInclude; 6 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 7 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 8 | import com.ovh.mls.serving.runtime.core.DataType; 9 | import com.ovh.mls.serving.runtime.core.Field; 10 | import com.ovh.mls.serving.runtime.core.transformer.ImageTransformerInfo; 11 | 12 | import java.util.List; 13 | import java.util.Optional; 14 | 15 | @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) 16 | public class TensorField extends Field { 17 | 18 | private int[] shape; 19 | 20 | @JsonInclude(JsonInclude.Include.NON_EMPTY) 21 | private List fields; 22 | 23 | @JsonInclude(JsonInclude.Include.NON_NULL) 24 | private ImageTransformerInfo imageTransformer; 25 | 26 | public TensorField() { 27 | } 28 | 29 | public TensorField(String name, DataType type, int[] shape, List tensorIndexList) { 30 | this(name, type, new TensorShape(shape), tensorIndexList); 31 | } 32 | 33 | public TensorField(String name, DataType type, TensorShape shape, List tensorIndexList) { 34 | super(name, type); 35 | this.shape = shape.getArrayShape(); 36 | this.fields = tensorIndexList; 37 | this.imageTransformer = ImageTransformerInfo.fromShape(this.shape).orElse(null); 38 | this.initFieldsTypesIfNeeded(); 39 | } 40 | 41 | 42 | public ImageTransformerInfo getImageTransformer() { 43 | return imageTransformer; 44 | } 45 | 46 | @JsonIgnore 47 | public Optional getMaybeImageTransformer() { 48 | return Optional.ofNullable(imageTransformer); 49 | } 50 | 51 | public int[] getShape() { 52 | return this.shape; 53 | } 54 | 55 | @JsonIgnore 56 | public TensorShape getTensorShape() { 57 | return new TensorShape(shape); 58 | } 59 | 60 | public void setShape(int[] shape) { 61 | this.shape = shape; 62 | } 63 | 64 | public List getFields() { 65 | return fields; 66 | } 67 | 68 | public void setFields(List fields) { 69 | this.fields = fields; 70 | this.initFieldsTypesIfNeeded(); 71 | } 72 | 73 | /** 74 | * Indexes field should always be of the same data type than their parents 75 | */ 76 | private void initFieldsTypesIfNeeded() { 77 | if (this.fields != null) { 78 | // Setting the same type than the current 79 | for (TensorIndex index: this.fields) { 80 | if (index.getType() == null) { 81 | index.setType(this.getType()); 82 | } 83 | } 84 | } 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /commons/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | serving-runtime 7 | com.ovh.mls.serving.runtime 8 | 1.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | commons 13 | 14 | 15 | 0.36.0 16 | 28.1-jre 17 | 1.3.4 18 | 19 | 20 | 21 | 22 | com.typesafe 23 | config 24 | ${config.version} 25 | 26 | 27 | org.apache.tika 28 | tika-core 29 | 1.23 30 | 31 | 32 | io.github.classgraph 33 | classgraph 34 | 4.8.41 35 | 36 | 37 | org.apache.httpcomponents 38 | httpclient 39 | 4.5.3 40 | 41 | 42 | commons-io 43 | commons-io 44 | 2.3 45 | 46 | 47 | org.apache.commons 48 | commons-lang3 49 | 3.10 50 | 51 | 52 | tech.tablesaw 53 | tablesaw-core 54 | ${tablesaw.version} 55 | 56 | 57 | com.fasterxml.jackson.core 58 | jackson-databind 59 | 2.10.0.pr1 60 | 61 | 62 | com.google.guava 63 | guava 64 | ${guava.version} 65 | 66 | 67 | 68 | javax.activation 69 | activation 70 | 1.1.1 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/main/java/com/ovh/mls/serving/runtime/tensorflow/TensorflowH5Generator.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.tensorflow; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.ovh.mls.serving.runtime.core.Evaluator; 5 | import com.ovh.mls.serving.runtime.core.EvaluatorGenerator; 6 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorGenerator; 7 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 8 | import com.typesafe.config.Config; 9 | import org.apache.commons.io.FileUtils; 10 | import org.apache.commons.io.IOUtils; 11 | import org.apache.commons.lang3.RandomStringUtils; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.tensorflow.SavedModelBundle; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | import java.nio.charset.StandardCharsets; 19 | import java.util.List; 20 | 21 | import static com.ovh.mls.serving.runtime.tensorflow.TensorflowEvaluator.DEFAULT_TAG_TENSORFLOW; 22 | 23 | @IncludeAsEvaluatorGenerator(extension = "h5") 24 | public class TensorflowH5Generator implements EvaluatorGenerator { 25 | private static final Logger LOGGER = LoggerFactory.getLogger(TensorflowH5Generator.class); 26 | 27 | private static void convertH5(String binary, String input, String output) 28 | throws InterruptedException, IOException, EvaluatorException { 29 | 30 | List commands = Lists.newArrayList(binary, input, output); 31 | 32 | ProcessBuilder processBuilder = new ProcessBuilder(commands); 33 | 34 | Process process = processBuilder.start(); 35 | 36 | process.waitFor(); 37 | 38 | if (process.exitValue() != 0) { 39 | LOGGER.error(IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8)); 40 | LOGGER.error(IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8)); 41 | 42 | throw new EvaluatorException("Error during h5 conversion"); 43 | } 44 | } 45 | 46 | @Override 47 | public Evaluator generate(File file, Config evaluatorConfig) throws EvaluatorException { 48 | var path = String.format("tmp/%s/", RandomStringUtils.randomAlphabetic(20)); 49 | 50 | try { 51 | convertH5(evaluatorConfig.getString("tensorflow.h5_converter.path"), file.getAbsolutePath(), path); 52 | SavedModelBundle savedModel = SavedModelBundle.load( 53 | String.format("%s/savedmodel/", path), 54 | DEFAULT_TAG_TENSORFLOW 55 | ); 56 | return TensorflowEvaluator.create(savedModel); 57 | 58 | } catch (IOException | InterruptedException e) { 59 | throw new EvaluatorException("Error during read manifest", e); 60 | } finally { 61 | try { 62 | FileUtils.deleteDirectory(new File(path)); 63 | } catch (IOException e) { 64 | LOGGER.error("Error during delete", e); 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /api/src/test/java/com/ovh/mls/serving/runtime/torch/TorchMultipleInputOutputIT.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.torch; 2 | 3 | import com.jayway.restassured.RestAssured; 4 | import com.jayway.restassured.response.Header; 5 | import com.ovh.mls.serving.runtime.IsCloseTo; 6 | import com.ovh.mls.serving.runtime.core.ApiServer; 7 | import com.typesafe.config.Config; 8 | import com.typesafe.config.ConfigFactory; 9 | import io.swagger.v3.oas.integration.OpenApiContextLocator; 10 | import org.junit.jupiter.api.BeforeAll; 11 | import org.junit.jupiter.api.Test; 12 | 13 | import java.lang.reflect.Field; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | import static com.jayway.restassured.RestAssured.given; 19 | import static org.hamcrest.Matchers.equalTo; 20 | 21 | public class TorchMultipleInputOutputIT { 22 | 23 | @BeforeAll 24 | public static void startServer() throws NoSuchFieldException, IllegalAccessException { 25 | // Reset swagger context 26 | Field field = OpenApiContextLocator.class.getDeclaredField("instance"); 27 | field.setAccessible(true); 28 | field.set(null, null); 29 | 30 | Config config = ConfigFactory.load("torch/multiple_input_output_model/api.conf"); 31 | ApiServer apiServer = new ApiServer(config); 32 | apiServer.start(); 33 | RestAssured.port = 8093; 34 | } 35 | 36 | @Test 37 | public void testDescribe() { 38 | given() 39 | .when() 40 | .get("/describe") 41 | .then() 42 | .statusCode(200) 43 | .body( 44 | "inputs.get(0).name", equalTo("input_0"), 45 | "inputs.get(0).shape", equalTo(List.of(4)), 46 | "inputs.get(0).type", equalTo("float"), 47 | "inputs.get(1).name", equalTo("input_1"), 48 | "inputs.get(1).shape", equalTo(List.of(2)), 49 | "inputs.get(1).type", equalTo("float"), 50 | "outputs.get(0).name", equalTo("output_0"), 51 | "outputs.get(0).shape", equalTo(List.of(1)), 52 | "outputs.get(0).type", equalTo("float"), 53 | "outputs.get(1).name", equalTo("output_1"), 54 | "outputs.get(1).shape", equalTo(List.of(1)), 55 | "outputs.get(1).type", equalTo("float") 56 | ); 57 | } 58 | 59 | @Test 60 | public void testEvalString() { 61 | Map body = new HashMap<>(); 62 | body.put("input_0", List.of(2.0, 5.0, 0.2, -0.2)); 63 | body.put("input_1", List.of(0.5, 1.0)); 64 | given() 65 | .body(body) 66 | .header(new Header("Content-Type", "application/json")) 67 | .when() 68 | .post("/eval") 69 | .then() 70 | .statusCode(200) 71 | .body( 72 | "output_0", IsCloseTo.closeTo(0.67084324f, 1e-6f), 73 | "output_1", IsCloseTo.closeTo(-0.4381053f, 1e-6f) 74 | ); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to OVH Serving Runtime 2 | 3 | This project accepts contributions. In order to contribute, you should 4 | pay attention to a few things: 5 | 6 | 1. your code must follow the coding style rules 7 | 2. your code must be unit-tested 8 | 3. your code must be documented 9 | 4. your work must be signed (see below) 10 | 5. you may contribute through GitHub Pull Requests 11 | 12 | # Coding and documentation Style 13 | 14 | - Code must be indented with spaces 15 | - Code must pass `mvn checkstyle:checkstyle` 16 | 17 | # Submitting Modifications 18 | 19 | The contributions should be submitted through Github Pull Requests 20 | and follow the DCO which is defined below. 21 | 22 | # Licensing for new files 23 | 24 | OVH Serving Runtime is licensed under a BSD 3-Clause license. Anything 25 | contributed to OVH Serving Runtime must be released under this license. 26 | 27 | When introducing a new file into the project, please make sure it has a 28 | copyright header making clear under which license it's being released. 29 | 30 | # Developer Certificate of Origin (DCO) 31 | 32 | To improve tracking of contributions to this project we will use a 33 | process modeled on the modified DCO 1.1 and use a "sign-off" procedure 34 | on patches that are being emailed around or contributed in any other 35 | way. 36 | 37 | The sign-off is a simple line at the end of the explanation for the 38 | patch, which certifies that you wrote it or otherwise have the right 39 | to pass it on as an open-source patch. The rules are pretty simple: 40 | if you can certify the below: 41 | 42 | By making a contribution to this project, I certify that: 43 | 44 | (a) The contribution was created in whole or in part by me and I have 45 | the right to submit it under the open source license indicated in 46 | the file; or 47 | 48 | (b) The contribution is based upon previous work that, to the best of 49 | my knowledge, is covered under an appropriate open source License 50 | and I have the right under that license to submit that work with 51 | modifications, whether created in whole or in part by me, under 52 | the same open source license (unless I am permitted to submit 53 | under a different license), as indicated in the file; or 54 | 55 | (c) The contribution was provided directly to me by some other person 56 | who certified (a), (b) or (c) and I have not modified it. 57 | 58 | (d) The contribution is made free of any other party's intellectual 59 | property claims or rights. 60 | 61 | (e) I understand and agree that this project and the contribution are 62 | public and that a record of the contribution (including all 63 | personal information I submit with it, including my sign-off) is 64 | maintained indefinitely and may be redistributed consistent with 65 | this project or the open source license(s) involved. 66 | 67 | 68 | then you just add a line saying 69 | 70 | Signed-off-by: Random J Developer 71 | 72 | using your real name (sorry, no pseudonyms or anonymous contributions.) -------------------------------------------------------------------------------- /evaluator-processors/src/test/java/com/ovh/mls/serving/runtime/processors/StandardScalerTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.processors; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.ovh.mls.serving.runtime.core.DataType; 5 | import com.ovh.mls.serving.runtime.core.EvaluationContext; 6 | import com.ovh.mls.serving.runtime.core.Field; 7 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 8 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 9 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 10 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 11 | import org.junit.jupiter.api.Assertions; 12 | import org.junit.jupiter.api.BeforeEach; 13 | import org.junit.jupiter.api.Test; 14 | 15 | import java.io.IOException; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.Set; 19 | 20 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | 23 | class StandardScalerTest { 24 | 25 | private static final ClassLoader LOADER = StandardScalerTest.class.getClassLoader(); 26 | 27 | private StandardScaler standardScaler; 28 | 29 | @BeforeEach 30 | void setUp() throws IOException, EvaluatorException { 31 | ObjectMapper objectMapper = new ObjectMapper(); 32 | StandardScalerManifest standardScalerManifest = objectMapper.readValue( 33 | LOADER.getResourceAsStream("processors/standard-scaler-manifest.json"), 34 | StandardScalerManifest.class); 35 | standardScaler = standardScalerManifest.create(""); 36 | } 37 | 38 | @Test 39 | void evaluate() throws EvaluationException { 40 | Tensor tensor = Tensor.fromData(DataType.FLOAT, new Float[]{null, 17.0F, 27.5F}); 41 | TensorIO input = new TensorIO(Map.of("value", tensor)); 42 | 43 | TensorIO io = standardScaler.evaluate(input, new EvaluationContext()); 44 | 45 | assertEquals(3, io.getBatchSize()); 46 | assertEquals(Set.of("scaled_value"), io.tensorsNames()); 47 | assertArrayEquals(new Double[]{null, 5.0, 10.25}, 48 | (Double[]) io.getTensor("scaled_value").getData()); 49 | 50 | tensor = Tensor.fromData(DataType.INTEGER, new Integer[]{null, 17, 27}); 51 | input = new TensorIO(Map.of("value", tensor)); 52 | io = standardScaler.evaluate(input, new EvaluationContext()); 53 | 54 | assertEquals(3, io.getBatchSize()); 55 | assertEquals(Set.of("scaled_value"), io.tensorsNames()); 56 | assertArrayEquals(new Double[]{null, 5.0, 10.0}, 57 | (Double[]) io.getTensor("scaled_value").getData()); 58 | } 59 | 60 | @Test 61 | void getInputs() { 62 | List inputs = standardScaler.getInputs(); 63 | assertEquals(1, inputs.size()); 64 | assertEquals("value", inputs.get(0).getName()); 65 | Assertions.assertEquals(DataType.DOUBLE, inputs.get(0).getType()); 66 | } 67 | 68 | @Test 69 | void getOutputs() { 70 | List outputs = standardScaler.getOutputs(); 71 | assertEquals(1, outputs.size()); 72 | assertEquals("scaled_value", outputs.get(0).getName()); 73 | assertEquals(DataType.DOUBLE, outputs.get(0).getType()); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /evaluator-processors/src/test/java/com/ovh/mls/serving/runtime/processors/StandardScalerWithSameOutputTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.processors; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.ovh.mls.serving.runtime.core.DataType; 5 | import com.ovh.mls.serving.runtime.core.EvaluationContext; 6 | import com.ovh.mls.serving.runtime.core.Field; 7 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 8 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 9 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 10 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 11 | import org.junit.jupiter.api.Assertions; 12 | import org.junit.jupiter.api.BeforeEach; 13 | import org.junit.jupiter.api.Test; 14 | 15 | import java.io.IOException; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.Set; 19 | 20 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | 23 | class StandardScalerWithSameOutputTest { 24 | 25 | private static final ClassLoader LOADER = StandardScalerWithSameOutputTest.class.getClassLoader(); 26 | 27 | private StandardScaler standardScaler; 28 | 29 | @BeforeEach 30 | void setUp() throws IOException, EvaluatorException { 31 | ObjectMapper objectMapper = new ObjectMapper(); 32 | StandardScalerManifest standardScalerManifest = objectMapper.readValue( 33 | LOADER.getResourceAsStream("processors/standard-scaler-same-output-manifest.json"), 34 | StandardScalerManifest.class); 35 | standardScaler = standardScalerManifest.create(""); 36 | } 37 | 38 | @Test 39 | void evaluate() throws EvaluationException { 40 | 41 | Tensor tensorValue = Tensor.fromDoubleData(new Double[]{null, 25.8, 47.0}); 42 | Tensor tensorLabel = Tensor.fromDoubleData(new Double[]{null, 50.0, 90.0}); 43 | 44 | TensorIO input = new TensorIO(Map.of("value", tensorValue, "label", tensorLabel)); 45 | 46 | TensorIO output = standardScaler.evaluate(input, new EvaluationContext()); 47 | 48 | assertEquals(3, output.getBatchSize()); 49 | assertEquals(Set.of("label", "value"), output.tensorsNames()); 50 | assertArrayEquals(new Double[]{null, 10.4, 21.0}, (Double[]) output.getTensor("value").getData()); 51 | assertArrayEquals(new Double[]{null, 10.0, 20.0}, (Double[]) output.getTensor("label").getData()); 52 | } 53 | 54 | @Test 55 | void getInputs() { 56 | List inputs = standardScaler.getInputs(); 57 | assertEquals(2, inputs.size()); 58 | assertEquals("value", inputs.get(0).getName()); 59 | Assertions.assertEquals(DataType.DOUBLE, inputs.get(0).getType()); 60 | assertEquals("label", inputs.get(1).getName()); 61 | Assertions.assertEquals(DataType.DOUBLE, inputs.get(1).getType()); 62 | } 63 | 64 | @Test 65 | void getOutputs() { 66 | List outputs = standardScaler.getOutputs(); 67 | assertEquals(2, outputs.size()); 68 | assertEquals("value", outputs.get(0).getName()); 69 | Assertions.assertEquals(DataType.DOUBLE, outputs.get(0).getType()); 70 | assertEquals("label", outputs.get(1).getName()); 71 | Assertions.assertEquals(DataType.DOUBLE, outputs.get(1).getType()); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /commons/src/test/java/com/ovh/mls/serving/runtime/utils/img/ImagesTensorConversionTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.utils.img; 2 | 3 | import com.ovh.mls.serving.runtime.core.builder.Builder; 4 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 5 | import com.ovh.mls.serving.runtime.core.transformer.ImageTransformerInfo; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import javax.imageio.ImageIO; 9 | import java.awt.image.BufferedImage; 10 | import java.io.IOException; 11 | import java.util.List; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | 15 | public class ImagesTensorConversionTest { 16 | 17 | private static final ClassLoader LOADER = ImagesTensorConversionTest.class.getClassLoader(); 18 | 19 | @Test 20 | void testImageConversionBackAndForward() throws IOException { 21 | BufferedImage input = ImageIO.read(LOADER.getResourceAsStream("./utils/img/amber.jpg")); 22 | int[] shape = new int[]{1, 3, 224, 224}; 23 | Builder, Tensor> builder = ImageTransformerInfo.fromShape(shape).get().tensorBuilder(); 24 | Builder> builderRevert = ImageTransformerInfo.fromShape(shape).get().imageBuilder(); 25 | Tensor tensor = builder.build(List.of(input)); 26 | List output = builderRevert.build(tensor); 27 | assertTrue(compareImages(input, output.get(0))); 28 | } 29 | 30 | @Test 31 | void testImageConversionBackAndForwardWithResize() throws IOException { 32 | BufferedImage input = ImageIO.read(LOADER.getResourceAsStream("./utils/img/amber.jpg")); 33 | BufferedImage expectedOutput = ImageIO.read( 34 | LOADER.getResourceAsStream("./utils/img/amber_100px_100px.png")); 35 | int[] shape = new int[]{1, 3, 100, 100}; 36 | Builder, Tensor> builder = ImageTransformerInfo.fromShape(shape).get().tensorBuilder(); 37 | Builder> builderRevert = ImageTransformerInfo.fromShape(shape).get().imageBuilder(); 38 | Tensor tensor = builder.build(List.of(input)); 39 | List output = builderRevert.build(tensor); 40 | assertTrue(compareImages(expectedOutput, output.get(0))); 41 | } 42 | 43 | /** 44 | * From https://stackoverflow.com/questions/11006394/is-there-a-simple-way-to-compare-bufferedimage-instances 45 | * Compares two images pixel by pixel. 46 | * 47 | * @param imgA the first image. 48 | * @param imgB the second image. 49 | * @return whether the images are both the same or not. 50 | */ 51 | public static boolean compareImages(BufferedImage imgA, BufferedImage imgB) { 52 | // The images must be the same size. 53 | if (imgA.getWidth() != imgB.getWidth() || imgA.getHeight() != imgB.getHeight()) { 54 | return false; 55 | } 56 | 57 | int width = imgA.getWidth(); 58 | int height = imgA.getHeight(); 59 | 60 | // Loop over every pixel. 61 | for (int y = 0; y < height; y++) { 62 | for (int x = 0; x < width; x++) { 63 | // Compare the pixels for equality. 64 | if (imgA.getRGB(x, y) != imgB.getRGB(x, y)) { 65 | return false; 66 | } 67 | } 68 | } 69 | 70 | return true; 71 | } 72 | 73 | 74 | 75 | } 76 | -------------------------------------------------------------------------------- /commons/src/test/java/com/ovh/mls/serving/runtime/core/builder/TensorIntoImagesTest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder; 2 | 3 | import com.ovh.mls.serving.runtime.core.DataType; 4 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 5 | import com.ovh.mls.serving.runtime.core.transformer.ImageTransformerInfo; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.awt.*; 9 | import java.awt.image.BufferedImage; 10 | import java.util.List; 11 | 12 | import static org.junit.jupiter.api.Assertions.*; 13 | 14 | public class TensorIntoImagesTest { 15 | 16 | @Test 17 | void testBuildDefaultImage() { 18 | Tensor tensor = new Tensor( 19 | DataType.INTEGER, 20 | new int[]{1, 2, 2, 3}, 21 | new int[][][][]{ 22 | new int[][][]{ 23 | new int[][]{ 24 | new int[]{0, 0, 255}, 25 | new int[]{0, 255, 0} 26 | }, 27 | new int[][]{ 28 | new int[]{255, 0, 0}, 29 | new int[]{0, 0, 0} 30 | } 31 | } 32 | } 33 | ); 34 | 35 | List images = ImageTransformerInfo.fromShape(tensor.getShapeAsArray()) 36 | .get() 37 | .imageBuilder() 38 | .build(tensor); 39 | 40 | assertEquals(1, images.size()); 41 | 42 | BufferedImage image = images.get(0); 43 | 44 | assertEquals(2, image.getWidth()); 45 | assertEquals(2, image.getHeight()); 46 | assertEquals(new Color(0, 0, 255), new Color(image.getRGB(0, 0))); 47 | assertEquals(new Color(255, 0, 0), new Color(image.getRGB(0, 1))); 48 | assertEquals(new Color(0, 255, 0), new Color(image.getRGB(1, 0))); 49 | assertEquals(new Color(0, 0, 0), new Color(image.getRGB(1, 1))); 50 | } 51 | 52 | @Test 53 | void testBuildDefaultImage2() { 54 | Tensor tensor = new Tensor( 55 | DataType.INTEGER, 56 | new int[]{1, 3, 2, 2}, 57 | new int[][][][]{ 58 | new int[][][]{ 59 | new int[][]{ 60 | new int[]{200, 201}, 61 | new int[]{202, 203} 62 | }, 63 | new int[][]{ 64 | new int[]{0, 1}, 65 | new int[]{2, 3} 66 | }, 67 | new int[][]{ 68 | new int[]{100, 101}, 69 | new int[]{102, 103} 70 | } 71 | } 72 | } 73 | ); 74 | 75 | List images = ImageTransformerInfo.fromShape(tensor.getShapeAsArray()) 76 | .get() 77 | .imageBuilder() 78 | .build(tensor); 79 | assertEquals(1, images.size()); 80 | 81 | BufferedImage image = images.get(0); 82 | 83 | assertEquals(2, image.getWidth()); 84 | assertEquals(2, image.getHeight()); 85 | assertEquals(new Color(200, 0, 100), new Color(image.getRGB(0, 0))); 86 | assertEquals(new Color(201, 1, 101), new Color(image.getRGB(1, 0))); 87 | assertEquals(new Color(202, 2, 102), new Color(image.getRGB(0, 1))); 88 | assertEquals(new Color(203, 3, 103), new Color(image.getRGB(1, 1))); 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/core/builder/from/TensorIOIntoMultipartBinary.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder.from; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.ovh.mls.serving.runtime.core.Field; 5 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 6 | import com.ovh.mls.serving.runtime.utils.img.BinaryContent; 7 | import org.apache.http.entity.ContentType; 8 | import org.apache.http.message.BasicNameValuePair; 9 | import org.eclipse.jetty.util.MultiPartOutputStream; 10 | 11 | import java.io.ByteArrayOutputStream; 12 | import java.io.IOException; 13 | import java.nio.charset.StandardCharsets; 14 | import java.util.List; 15 | 16 | /** 17 | * Convert a TensorIO into a Multipart Binary content 18 | */ 19 | public class TensorIOIntoMultipartBinary extends TensorIOIntoMultipleBinary { 20 | 21 | private static final String APPLICATION_JSON = "application/json"; 22 | private final ByteArrayOutputStream os; 23 | private final ObjectMapper mapper; 24 | 25 | public TensorIOIntoMultipartBinary( 26 | ObjectMapper mapper, 27 | ContentType contentType, 28 | List fields, 29 | boolean shouldSimplify 30 | ) { 31 | super(contentType, fields, shouldSimplify); 32 | this.mapper = mapper; 33 | this.os = new ByteArrayOutputStream(); 34 | } 35 | 36 | @Override 37 | protected MultiPartOutputStream buildOutputStream() throws IOException { 38 | return new MultiPartOutputStream(this.os); 39 | } 40 | 41 | @Override 42 | protected void buildImagePart( 43 | String tensorName, 44 | BinaryContent image, 45 | MultiPartOutputStream outputStream 46 | ) throws IOException { 47 | String imageFilename = String.format("%s.%s", tensorName, image.getFileExtension()); 48 | outputStream.startPart( 49 | image.getContentType().getMimeType(), 50 | new String[]{ 51 | buildContentDispositionHeader(tensorName, imageFilename) 52 | }); 53 | outputStream.write(image.getBytes()); 54 | } 55 | 56 | @Override 57 | protected void buildDefaultPart( 58 | String tensorName, 59 | Tensor tensor, 60 | MultiPartOutputStream outputStream 61 | ) throws IOException { 62 | String jsonFilename = String.format("%s.json", tensorName); 63 | outputStream.startPart( 64 | APPLICATION_JSON, 65 | new String[]{ 66 | buildContentDispositionHeader(tensorName, jsonFilename) 67 | }); 68 | outputStream.write(mapper.writeValueAsBytes(tensor.jsonData(this.shouldSimplify))); 69 | } 70 | 71 | @Override 72 | protected BinaryContent buildBinaryContent(MultiPartOutputStream outputStream) { 73 | BasicNameValuePair param = new BasicNameValuePair("boundary", outputStream.getBoundary()); 74 | return new BinaryContent( 75 | null, 76 | ContentType.MULTIPART_FORM_DATA.withCharset(StandardCharsets.UTF_8).withParameters(param), 77 | this.os.toByteArray() 78 | ); 79 | } 80 | 81 | protected String buildContentDispositionHeader(String name, String filename) { 82 | return String.format( 83 | "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"", 84 | name, 85 | filename 86 | ); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /api/src/test/resources/tensorflow/mnist/inputs/0.json: -------------------------------------------------------------------------------- 1 | {"image":[[[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[44],[140],[254],[183],[12],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[125],[229],[253],[253],[253],[65],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[68],[164],[217],[254],[253],[253],[253],[227],[164],[78],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[64],[246],[253],[253],[254],[253],[182],[253],[253],[253],[182],[78],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[72],[240],[253],[253],[253],[254],[68],[21],[178],[245],[253],[253],[243],[52],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[72],[232],[253],[253],[248],[223],[74],[13],[0],[0],[119],[253],[253],[253],[77],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[224],[253],[253],[253],[128],[0],[0],[0],[0],[0],[36],[214],[253],[253],[232],[56],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[6],[161],[251],[253],[253],[116],[4],[0],[0],[0],[0],[0],[0],[38],[253],[253],[253],[104],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[130],[253],[253],[248],[128],[4],[0],[0],[0],[0],[0],[0],[0],[7],[162],[253],[253],[104],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[36],[217],[253],[253],[223],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[134],[253],[253],[192],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[211],[254],[254],[223],[44],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[135],[255],[254],[255],[0],[0],[0],[0]],[[0],[0],[0],[0],[87],[246],[253],[253],[55],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[134],[253],[253],[253],[0],[0],[0],[0]],[[0],[0],[0],[0],[114],[253],[253],[246],[27],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[134],[253],[253],[253],[0],[0],[0],[0]],[[0],[0],[0],[0],[254],[253],[253],[133],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[86],[191],[253],[253],[182],[0],[0],[0],[0]],[[0],[0],[0],[0],[254],[253],[240],[16],[0],[0],[0],[0],[0],[0],[0],[0],[5],[75],[120],[245],[253],[253],[240],[73],[0],[0],[0],[0]],[[0],[0],[0],[0],[148],[253],[249],[109],[22],[0],[0],[0],[0],[18],[31],[154],[184],[253],[253],[253],[253],[253],[190],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[105],[253],[253],[253],[218],[134],[134],[134],[134],[205],[254],[253],[253],[253],[253],[253],[253],[126],[28],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[6],[212],[253],[253],[253],[253],[253],[253],[253],[253],[255],[253],[253],[253],[253],[242],[84],[1],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[49],[158],[253],[253],[253],[253],[253],[253],[253],[236],[208],[199],[129],[164],[52],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[7],[104],[236],[253],[253],[164],[226],[104],[62],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]],[[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0],[0]]]]} -------------------------------------------------------------------------------- /commons/src/main/java/com/ovh/mls/serving/runtime/core/transformer/ImageTransformerInfo.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.transformer; 2 | 3 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 4 | import com.fasterxml.jackson.databind.annotation.JsonNaming; 5 | import com.ovh.mls.serving.runtime.core.builder.ImagesIntoTensor; 6 | import com.ovh.mls.serving.runtime.core.builder.TensorIntoImages; 7 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 8 | import com.ovh.mls.serving.runtime.utils.img.ImgChanelProperties; 9 | import com.ovh.mls.serving.runtime.utils.img.ImgProperties; 10 | 11 | import java.beans.Transient; 12 | import java.util.List; 13 | import java.util.Optional; 14 | 15 | import static com.ovh.mls.serving.runtime.utils.img.ImageDefaults.getImgChanelProperties; 16 | import static com.ovh.mls.serving.runtime.utils.img.ImageDefaults.getImgProperties; 17 | 18 | @JsonNaming(PropertyNamingStrategy.SnakeCaseStrategy.class) 19 | public class ImageTransformerInfo { 20 | 21 | private List imageShape; 22 | private List propertyChanel; 23 | private int[] tensorShape; 24 | 25 | public ImageTransformerInfo() { 26 | 27 | } 28 | 29 | public ImageTransformerInfo( 30 | List propertyChanel, 31 | List imageShape, 32 | int[] tensorShape 33 | ) { 34 | this.propertyChanel = propertyChanel; 35 | this.imageShape = imageShape; 36 | this.tensorShape = tensorShape; 37 | } 38 | 39 | public List getImageShape() { 40 | return imageShape; 41 | } 42 | 43 | public List getPropertyChanel() { 44 | return propertyChanel; 45 | } 46 | 47 | @Transient 48 | public boolean getSupportBatch() { 49 | return getExpectedBatchSize() != 1; 50 | } 51 | 52 | @Transient 53 | public int getExpectedBatchSize() { 54 | if (!this.imageShape.contains(ImgProperties.BATCH_SIZE)) { 55 | return 1; 56 | } 57 | return this.tensorShape[imageShape.indexOf(ImgProperties.BATCH_SIZE)]; 58 | } 59 | 60 | @Transient 61 | public int getExpectedWidth() { 62 | return this.tensorShape[imageShape.indexOf(ImgProperties.WIDTH)]; 63 | } 64 | 65 | @Transient 66 | public int getExpectedHeigth() { 67 | return this.tensorShape[imageShape.indexOf(ImgProperties.HEIGHT)]; 68 | } 69 | 70 | public static Optional fromShape(int[] expectedShape) { 71 | try { 72 | List shapeAttributes = getImgProperties(expectedShape); 73 | List chanelProperties = getImgChanelProperties(expectedShape); 74 | if ( 75 | shapeAttributes.contains(ImgProperties.HEIGHT) && 76 | shapeAttributes.contains(ImgProperties.WIDTH) && 77 | shapeAttributes.contains(ImgProperties.CHANEL) 78 | ) { 79 | return Optional.of(new ImageTransformerInfo(chanelProperties, shapeAttributes, expectedShape)); 80 | } else { 81 | return Optional.empty(); 82 | } 83 | } catch (EvaluationException e) { 84 | return Optional.empty(); 85 | } 86 | } 87 | 88 | public ImagesIntoTensor tensorBuilder() { 89 | return new ImagesIntoTensor(this); 90 | } 91 | 92 | public TensorIntoImages imageBuilder() { 93 | return new TensorIntoImages(this); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/core/builder/from/TensorIOIntoHTMLBinary.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder.from; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.ovh.mls.serving.runtime.core.Field; 5 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 6 | import com.ovh.mls.serving.runtime.utils.img.BinaryContent; 7 | import org.apache.http.entity.ContentType; 8 | 9 | import java.io.ByteArrayOutputStream; 10 | import java.io.IOException; 11 | import java.nio.charset.StandardCharsets; 12 | import java.util.Base64; 13 | import java.util.List; 14 | 15 | /** 16 | * Convert a TensorIO into an HTML binary 17 | * 18 | * Tensor's name are converted in 'h1' tag 19 | * Scalars are printed inside 'p' tag 20 | * Vectors are printed inside 'ul' and 'li' tag (listing) 21 | * Images are printed as png or jpeg inside 'img' tag 22 | * Other are printed inside 'code' tag 23 | */ 24 | public class TensorIOIntoHTMLBinary extends TensorIOIntoMultipleBinary { 25 | 26 | private final ObjectMapper mapper; 27 | 28 | public TensorIOIntoHTMLBinary( 29 | ContentType contentType, 30 | ObjectMapper mapper, 31 | List fields, 32 | boolean shouldSimplify 33 | ) { 34 | super(contentType, fields, shouldSimplify); 35 | this.mapper = mapper; 36 | } 37 | 38 | @Override 39 | protected ByteArrayOutputStream buildOutputStream() throws IOException { 40 | return new ByteArrayOutputStream(); 41 | } 42 | 43 | @Override 44 | protected void buildImagePart( 45 | String tensorName, 46 | BinaryContent image, 47 | ByteArrayOutputStream outputStream 48 | ) throws IOException { 49 | String imageAsBase64 = Base64.getEncoder().encodeToString(image.getBytes()); 50 | String content = String.format( 51 | "

%s

", 52 | tensorName, 53 | image.getContentType().getMimeType(), 54 | imageAsBase64 55 | ); 56 | outputStream.write(content.getBytes()); 57 | } 58 | 59 | @Override 60 | protected void buildDefaultPart( 61 | String tensorName, 62 | Tensor tensor, 63 | ByteArrayOutputStream outputStream 64 | ) throws IOException { 65 | String content = String.format("

%s

", tensorName); 66 | outputStream.write(content.getBytes()); 67 | if (this.shouldSimplify) { 68 | tensor = tensor.simplifyShape(); 69 | } 70 | if (tensor.isScalar()) { 71 | outputStream.write(String.format("

%s

", tensor.getData().toString()).getBytes()); 72 | } else if (tensor.isVector()) { 73 | outputStream.write("
    ".getBytes()); 74 | for (int i = 0; i < tensor.getShapeAsArray()[0]; i++) { 75 | Object value = tensor.getCoord(i); 76 | outputStream.write(String.format("
  • %s
    • ", value.toString()).getBytes()); 77 | } 78 | outputStream.write("
".getBytes()); 79 | } else { 80 | String value = mapper.writeValueAsString(tensor.jsonData(this.shouldSimplify)); 81 | outputStream.write(String.format("%s", value).getBytes()); 82 | } 83 | } 84 | 85 | @Override 86 | protected BinaryContent buildBinaryContent(ByteArrayOutputStream outputStream) { 87 | return new BinaryContent( 88 | "html", 89 | ContentType.TEXT_HTML.withCharset(StandardCharsets.UTF_8), 90 | outputStream.toByteArray() 91 | ); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/core/builder/into/InputStreamIntoTensorIO.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.core.builder.into; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.ovh.mls.serving.runtime.core.Field; 5 | import com.ovh.mls.serving.runtime.core.builder.Builder; 6 | import com.ovh.mls.serving.runtime.core.builder.ImagesIntoTensor; 7 | import com.ovh.mls.serving.runtime.core.builder.InputStreamJsonIntoTensorIO; 8 | import com.ovh.mls.serving.runtime.core.builder.PartsIntoTensorIO; 9 | import com.ovh.mls.serving.runtime.core.io.Part; 10 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 11 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 12 | import com.ovh.mls.serving.runtime.core.tensor.TensorField; 13 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 14 | import com.ovh.mls.serving.runtime.utils.MultipartUtils; 15 | import com.ovh.mls.serving.runtime.utils.img.ImageDefaults; 16 | import org.apache.http.entity.ContentType; 17 | 18 | import java.awt.image.BufferedImage; 19 | import java.io.InputStream; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | /** 25 | * Builder of TensorIO from an InputStream 26 | */ 27 | public class InputStreamIntoTensorIO implements Builder { 28 | 29 | private final ContentType contentType; 30 | private final ObjectMapper mapper; 31 | private final List fields; 32 | 33 | public InputStreamIntoTensorIO(ObjectMapper mapper) { 34 | this(mapper, ContentType.APPLICATION_JSON, new ArrayList<>()); 35 | } 36 | 37 | public InputStreamIntoTensorIO(ObjectMapper mapper, ContentType contentType, List fields) { 38 | this.contentType = contentType; 39 | this.mapper = mapper; 40 | this.fields = fields; 41 | } 42 | 43 | @Override 44 | public TensorIO build(InputStream inputStream) throws EvaluationException { 45 | String mimeType = this.contentType.getMimeType(); 46 | if (ContentType.APPLICATION_JSON.getMimeType().equals(mimeType)) { 47 | return new InputStreamJsonIntoTensorIO(this.mapper).build(inputStream); 48 | } else if (ContentType.MULTIPART_FORM_DATA.getMimeType().equals(mimeType)) { 49 | final PartsIntoTensorIO builder = new PartsIntoTensorIO(this.mapper, this.fields); 50 | List parts = MultipartUtils.readParts(contentType, inputStream); 51 | return builder.build(parts); 52 | } else if (ImageDefaults.SUPPORTED_IMG_CONTENT_TYPE.contains(mimeType)) { 53 | if (this.fields.size() != 1) { 54 | throw new EvaluationException( 55 | "Your model takes several tensors as parameters but you provided only one" 56 | ); 57 | } 58 | Field field = this.fields.get(0); 59 | if (!(field instanceof TensorField)) { 60 | throw new EvaluationException("Unable to feed an image as tensor on a non-tensor model"); 61 | } 62 | return buildSingleImage((TensorField) field, inputStream); 63 | } else { 64 | throw new EvaluationException(String.format("Unsupported Content-Type: %s", mimeType)); 65 | } 66 | } 67 | 68 | private TensorIO buildSingleImage(TensorField field, InputStream inputStream) throws EvaluationException { 69 | ImagesIntoTensor builder = field.getImageTransformer().tensorBuilder(); 70 | BufferedImage image = ImageDefaults.readImage(inputStream); 71 | Tensor tensor = builder.build(List.of(image)); 72 | return new TensorIO(Map.of(field.getName(), tensor)); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /evaluator-torch/src/main/java/com/ovh/mls/serving/runtime/torch/TorchScriptEvaluatorManifest.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.torch; 2 | 3 | import com.facebook.jni.CppException; 4 | import com.facebook.soloader.nativeloader.NativeLoader; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.ovh.mls.serving.runtime.core.EvaluatorManifest; 7 | import com.ovh.mls.serving.runtime.core.IncludeAsEvaluatorManifest; 8 | import com.ovh.mls.serving.runtime.core.tensor.TensorField; 9 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 10 | import com.ovh.mls.serving.runtime.utils.NativeUtils; 11 | import org.pytorch.Module; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.io.IOException; 16 | import java.nio.file.Paths; 17 | import java.util.List; 18 | 19 | @IncludeAsEvaluatorManifest(type = TorchScriptEvaluatorManifest.TYPE) 20 | public class TorchScriptEvaluatorManifest implements EvaluatorManifest { 21 | 22 | // Load pytorch_jni from JAR 23 | static { 24 | NativeLoader.init(libname -> { 25 | try { 26 | // Load library from filesystem (java.library.path) 27 | System.loadLibrary(libname); 28 | return true; 29 | } catch (UnsatisfiedLinkError e) { 30 | try { 31 | // Load library from JAR 32 | NativeUtils.loadLibraryFromJar("/" + System.mapLibraryName(libname)); 33 | } catch (UnsatisfiedLinkError | IOException ignored) { 34 | throw new EvaluatorException("Cannot load library " + libname, e); 35 | } 36 | } 37 | 38 | return true; 39 | }); 40 | // Load OpenMP (platform dependant) 41 | // Linux 42 | try { 43 | NativeUtils.loadLibraryFromJar("/libgomp-7c85b1e2.so.1"); 44 | } catch (Exception ignored) { 45 | } 46 | // Darwin 47 | try { 48 | NativeLoader.loadLibrary("iomp5"); 49 | } catch (Exception ignored) { 50 | } 51 | NativeLoader.loadLibrary("fbjni"); 52 | NativeLoader.loadLibrary("c10"); 53 | NativeLoader.loadLibrary("torch_cpu"); 54 | NativeLoader.loadLibrary("torch"); 55 | NativeLoader.loadLibrary("pytorch_jni"); 56 | } 57 | 58 | private static final Logger LOGGER = LoggerFactory.getLogger(TorchScriptEvaluatorManifest.class); 59 | 60 | public static final String TYPE = "torch_script"; 61 | 62 | @JsonProperty 63 | private String savedModelUri; 64 | 65 | @JsonProperty 66 | private List inputs; 67 | 68 | @JsonProperty 69 | private List outputs; 70 | 71 | @Override 72 | public TorchScriptEvaluator create(String path) throws EvaluatorException { 73 | Module module; 74 | try { 75 | module = Module.load(savedModelUri); 76 | } catch (CppException e1) { 77 | String localSavedModelUri = Paths.get(path, savedModelUri).toString(); 78 | try { 79 | module = Module.load(localSavedModelUri); 80 | } catch (CppException e2) { 81 | LOGGER.error("Cannot load TorchScript {}", savedModelUri, e1); 82 | LOGGER.error("Cannot load TorchScript {}", localSavedModelUri, e2); 83 | throw new EvaluatorException( 84 | String.format("Cannot load TorchScript %s or %s", savedModelUri, localSavedModelUri) 85 | ); 86 | } 87 | } 88 | return new TorchScriptEvaluator(module, inputs, outputs); 89 | } 90 | 91 | @Override 92 | public String getType() { 93 | return TYPE; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /api/src/main/java/com/ovh/mls/serving/runtime/EvaluationService.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime; 2 | 3 | import com.github.racc.tscg.TypesafeConfig; 4 | import com.ovh.mls.serving.runtime.core.EvaluationContext; 5 | import com.ovh.mls.serving.runtime.core.Evaluator; 6 | import com.ovh.mls.serving.runtime.core.EvaluatorUtil; 7 | import com.ovh.mls.serving.runtime.core.builder.Builder; 8 | import com.ovh.mls.serving.runtime.core.builder.into.InputStreamIntoTensorIO; 9 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 10 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 11 | import com.ovh.mls.serving.runtime.swagger.SwaggerBuilder; 12 | import com.typesafe.config.Config; 13 | import io.prometheus.client.Counter; 14 | import io.swagger.v3.oas.integration.OpenApiConfigurationException; 15 | import org.apache.commons.lang3.StringUtils; 16 | import org.apache.http.entity.ContentType; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | import javax.inject.Inject; 21 | import javax.inject.Singleton; 22 | import java.io.IOException; 23 | import java.io.InputStream; 24 | import java.util.Optional; 25 | 26 | @Singleton 27 | public class EvaluationService { 28 | private static final Logger LOGGER = LoggerFactory.getLogger(EvaluationService.class); 29 | 30 | private static final Counter EVALUATOR_COUNTER = Counter.build() 31 | .name("evaluator_evaluation_count") 32 | .help("Evaluator Counter") 33 | .labelNames() 34 | .register(); 35 | 36 | private final Evaluator evaluator; 37 | private final EvaluatorUtil evaluatorUtil; 38 | 39 | @Inject 40 | EvaluationService( 41 | @TypesafeConfig("files.path") String filePath, 42 | @TypesafeConfig("swagger") Config config, 43 | @TypesafeConfig("evaluator") Config evaluatorConfig 44 | ) { 45 | if (StringUtils.isEmpty(filePath)) { 46 | throw new RuntimeException("Missing Manifest Path"); 47 | } 48 | 49 | this.evaluatorUtil = new EvaluatorUtil(evaluatorConfig); 50 | 51 | final Optional optionalEvaluator = evaluatorUtil.findEvaluator(filePath); 52 | 53 | if (optionalEvaluator.isEmpty()) { 54 | throw new RuntimeException("No manifest or direct binary found or valid from files folder"); 55 | } 56 | 57 | this.evaluator = optionalEvaluator.get(); 58 | 59 | try { 60 | new SwaggerBuilder(config, this.evaluator).build(); 61 | } catch (IOException | OpenApiConfigurationException e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | 66 | Evaluator getEvaluator() { 67 | return evaluator; 68 | } 69 | 70 | TensorIO evaluate(ContentType contentType, InputStream inputStream, EvaluationContext context) 71 | throws EvaluationException { 72 | 73 | // Create builder for input 74 | final Builder inputBuilder = new InputStreamIntoTensorIO( 75 | evaluatorUtil.getObjectMapper(), 76 | contentType, 77 | this.evaluator.getInputs() 78 | ); 79 | // Convert an InputStream into TensorIO 80 | final TensorIO inputIO = inputBuilder.build(inputStream); 81 | return this.evaluate(inputIO, context); 82 | } 83 | 84 | private TensorIO evaluate(TensorIO tensorIO, EvaluationContext context) throws EvaluationException { 85 | // Get output Tensors from the model by feeding input Tensors 86 | final TensorIO outputIO = evaluator.evaluate(tensorIO, context); 87 | EVALUATOR_COUNTER.inc(context.totalEvaluation()); 88 | return outputIO; 89 | } 90 | 91 | EvaluatorUtil getEvaluatorUtil() { 92 | return this.evaluatorUtil; 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /evaluator-tensorflow/src/test/java/com/ovh/mls/serving/runtime/tensorflow/H5Test.java: -------------------------------------------------------------------------------- 1 | package com.ovh.mls.serving.runtime.tensorflow; 2 | 3 | import com.ovh.mls.serving.runtime.core.DataType; 4 | import com.ovh.mls.serving.runtime.core.EvaluationContext; 5 | import com.ovh.mls.serving.runtime.core.EvaluatorUtil; 6 | import com.ovh.mls.serving.runtime.core.Field; 7 | import com.ovh.mls.serving.runtime.core.builder.InputStreamJsonIntoTensorIO; 8 | import com.ovh.mls.serving.runtime.core.io.TensorIO; 9 | import com.ovh.mls.serving.runtime.core.tensor.Tensor; 10 | import com.ovh.mls.serving.runtime.core.tensor.TensorField; 11 | import com.ovh.mls.serving.runtime.core.tensor.TensorShape; 12 | import com.ovh.mls.serving.runtime.exceptions.EvaluationException; 13 | import com.ovh.mls.serving.runtime.exceptions.EvaluatorException; 14 | import com.typesafe.config.Config; 15 | import com.typesafe.config.ConfigFactory; 16 | import org.junit.jupiter.api.BeforeAll; 17 | import org.junit.jupiter.api.Test; 18 | 19 | import java.io.File; 20 | import java.util.List; 21 | import java.util.Set; 22 | 23 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 24 | import static org.junit.jupiter.api.Assertions.assertEquals; 25 | 26 | public class H5Test { 27 | private static final ClassLoader LOADER = H5Test.class.getClassLoader(); 28 | private static final Config config = ConfigFactory.load().getConfig("evaluator"); 29 | private static final EvaluatorUtil MAPPER = new EvaluatorUtil(config); 30 | 31 | private static TensorflowEvaluator tensorFlowEvaluator; 32 | 33 | @BeforeAll 34 | public static void create() throws EvaluatorException { 35 | tensorFlowEvaluator = (TensorflowEvaluator) new TensorflowH5Generator() 36 | .generate(new File(LOADER.getResource("tensorflow/test_h5/test.h5").getFile()), config); 37 | } 38 | 39 | @Test 40 | public void evaluate() throws EvaluationException { 41 | InputStreamJsonIntoTensorIO builder = new InputStreamJsonIntoTensorIO(MAPPER.getObjectMapper()); 42 | TensorIO input = builder.build(LOADER.getResourceAsStream("tensorflow/test_h5/single_gen.json")); 43 | 44 | final EvaluationContext evaluationContext = new EvaluationContext(); 45 | TensorIO output = tensorFlowEvaluator.evaluate(input, evaluationContext); 46 | 47 | assertEquals(1, evaluationContext.totalEvaluation()); 48 | assertEquals(Set.of("prediction"), output.tensorsNames()); 49 | 50 | Tensor tensor = output.getTensor("prediction"); 51 | TensorShape shape = tensor.getShape(); 52 | assertArrayEquals(new int[]{1, 3}, shape.getArrayShape()); 53 | assertEquals(DataType.FLOAT, tensor.getType()); 54 | assertEquals(5.613032953988295E-7, ((float[][]) tensor.getData())[0][0], 0.00000001); 55 | assertEquals(7.821146864444017E-4, ((float[][]) tensor.getData())[0][1], 0.00000001); 56 | assertEquals(0.9992172718048096, ((float[][]) tensor.getData())[0][2], 0.00000001); 57 | } 58 | 59 | @Test 60 | public void getInputs() { 61 | List inputs = tensorFlowEvaluator.getInputs(); 62 | assertEquals(1, inputs.size()); 63 | assertEquals("inputs", inputs.get(0).getName()); 64 | assertEquals(DataType.FLOAT, inputs.get(0).getType()); 65 | } 66 | 67 | @Test 68 | public void getOutputs() { 69 | List outputs = tensorFlowEvaluator.getOutputs(); 70 | 71 | assertEquals(1, outputs.size()); 72 | 73 | assertEquals("prediction", outputs.get(0).getName()); 74 | assertEquals(DataType.FLOAT, outputs.get(0).getType()); 75 | } 76 | 77 | @Test 78 | public void getBatchSize() { 79 | assertEquals(1, tensorFlowEvaluator.getRollingWindowSize()); 80 | } 81 | } 82 | --------------------------------------------------------------------------------