├── .gitignore ├── .gitmodules ├── AUTHORS ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── RELEASE.md ├── WORKSPACE ├── caffe.BUILD ├── tensorflow_serving ├── BUILD ├── apis │ ├── BUILD │ ├── model.proto │ ├── predict.proto │ ├── prediction_service.proto │ └── prediction_service_pb2.py ├── batching │ ├── BUILD │ ├── README.md │ ├── basic_batch_scheduler.h │ ├── basic_batch_scheduler_benchmark.cc │ ├── basic_batch_scheduler_test.cc │ ├── batch_scheduler.h │ ├── batch_scheduler_retrier.h │ ├── batch_scheduler_retrier_test.cc │ ├── batch_scheduler_test.cc │ ├── batching_session.cc │ ├── batching_session.h │ ├── batching_session_test.cc │ ├── shared_batch_scheduler.h │ ├── shared_batch_scheduler_test.cc │ ├── streaming_batch_scheduler.cc │ ├── streaming_batch_scheduler.h │ ├── streaming_batch_scheduler_test.cc │ └── test_util │ │ ├── BUILD │ │ ├── puppet_batch_scheduler.h │ │ └── puppet_batch_scheduler_test.cc ├── config │ ├── BUILD │ └── model_server_config.proto ├── core │ ├── BUILD │ ├── README.md │ ├── aspired_version_policy.h │ ├── aspired_versions_manager.cc │ ├── aspired_versions_manager.h │ ├── aspired_versions_manager_benchmark.cc │ ├── aspired_versions_manager_builder.cc │ ├── aspired_versions_manager_builder.h │ ├── aspired_versions_manager_builder_test.cc │ ├── aspired_versions_manager_test.cc │ ├── basic_manager.cc │ ├── basic_manager.h │ ├── basic_manager_test.cc │ ├── caching_manager.cc │ ├── caching_manager.h │ ├── caching_manager_test.cc │ ├── eager_load_policy.cc │ ├── eager_load_policy.h │ ├── eager_load_policy_test.cc │ ├── eager_unload_policy.cc │ ├── eager_unload_policy.h │ ├── eager_unload_policy_test.cc │ ├── load_servables_fast.cc │ ├── load_servables_fast.h │ ├── loader.h │ ├── loader_harness.cc │ ├── loader_harness.h │ ├── loader_harness_test.cc │ ├── log_collector.cc │ ├── log_collector.h │ ├── log_collector_test.cc │ ├── manager.h │ ├── manager_test.cc │ ├── manager_wrapper.cc │ ├── manager_wrapper.h │ ├── servable_data.h │ ├── servable_data_test.cc │ ├── servable_handle.h │ ├── servable_id.h │ ├── servable_id_test.cc │ ├── servable_state.h │ ├── servable_state_monitor.cc │ ├── servable_state_monitor.h │ ├── servable_state_monitor_test.cc │ ├── simple_loader.h │ ├── simple_loader_test.cc │ ├── source.h │ ├── source_adapter.h │ ├── source_adapter_test.cc │ ├── source_router.h │ ├── source_router_test.cc │ ├── static_manager.cc │ ├── static_manager.h │ ├── static_manager_test.cc │ ├── static_source_router.h │ ├── static_source_router_test.cc │ ├── storage_path.h │ ├── storage_path_test.cc │ ├── target.h │ └── test_util │ │ ├── BUILD │ │ ├── availability_test_util.cc │ │ ├── availability_test_util.h │ │ ├── fake_loader.cc │ │ ├── fake_loader.h │ │ ├── fake_loader_source_adapter.cc │ │ ├── fake_loader_source_adapter.h │ │ ├── fake_storage_path_source_adapter.cc │ │ ├── fake_storage_path_source_adapter.h │ │ ├── manager_test_util.cc │ │ ├── manager_test_util.h │ │ ├── mock_loader.h │ │ ├── mock_storage_path_target.h │ │ ├── source_adapter_test_util.h │ │ └── test_main.cc ├── example │ ├── BUILD │ ├── client_util.py │ ├── imagenet_lsvrc_2015_synsets.txt │ ├── imagenet_metadata.txt │ ├── inception_client.py │ ├── inception_export.py │ ├── inception_k8s.yaml │ ├── mnist_client.py │ ├── mnist_export.py │ ├── mnist_input_data.py │ ├── obj_detector.cc │ ├── obj_detector.proto │ ├── obj_detector_client.py │ ├── obj_detector_fetch.py │ ├── obj_detector_pb2.py │ ├── obj_detector_utils.cc │ ├── obj_detector_utils.h │ └── rpc_utils.h ├── g3doc │ ├── METADATA │ ├── _config.yml │ ├── _includes │ │ ├── footer.html │ │ ├── head.html │ │ ├── header.html │ │ ├── icon-github.html │ │ ├── icon-github.svg │ │ ├── icon-twitter.html │ │ ├── icon-twitter.svg │ │ └── nav.md │ ├── _layouts │ │ └── default.html │ ├── _sass │ │ ├── _base.scss │ │ ├── _layout.scss │ │ └── _syntax-highlighting.scss │ ├── architecture_overview.md │ ├── css │ │ └── main.scss │ ├── custom_servable.md │ ├── custom_source.md │ ├── docker.md │ ├── images │ │ ├── serving_architecture.svg │ │ └── tf_diagram.svg │ ├── index.md │ ├── serving_advanced.md │ ├── serving_basic.md │ ├── serving_inception.md │ └── setup.md ├── model_servers │ ├── BUILD │ ├── main.cc │ ├── model_platform_caffe.h │ ├── model_platform_tensorflow.h │ ├── model_platform_types.h │ ├── server_core.cc │ ├── server_core.h │ ├── server_core_test.cc │ ├── tensorflow_model_server_test.py │ ├── tensorflow_model_server_test_client.py │ └── test_util │ │ ├── BUILD │ │ ├── mock_server_core.h │ │ ├── server_core_test_util.cc │ │ └── server_core_test_util.h ├── resources │ ├── BUILD │ ├── resource_tracker.cc │ ├── resource_tracker.h │ ├── resource_tracker_test.cc │ ├── resource_util.cc │ ├── resource_util.h │ ├── resource_util_test.cc │ ├── resource_values.cc │ ├── resource_values.h │ └── resources.proto ├── servables │ ├── caffe │ │ ├── BUILD │ │ ├── caffe_py_util.cc │ │ ├── caffe_py_util.h │ │ ├── caffe_serving_session.cc │ │ ├── caffe_serving_session.h │ │ ├── caffe_session_bundle.cc │ │ ├── caffe_session_bundle.h │ │ ├── caffe_session_bundle_config.proto │ │ ├── caffe_session_bundle_factory.cc │ │ ├── caffe_session_bundle_factory.h │ │ ├── caffe_session_bundle_factory_test.cc │ │ ├── caffe_signature.cc │ │ ├── caffe_signature.h │ │ ├── caffe_simple_servers.cc │ │ ├── caffe_simple_servers.h │ │ ├── caffe_simple_servers_test.cc │ │ ├── caffe_source_adapter.cc │ │ ├── caffe_source_adapter.h │ │ ├── caffe_source_adapter.proto │ │ ├── caffe_source_adapter_test.cc │ │ ├── predict_impl.cc │ │ ├── predict_impl.h │ │ ├── simple_thread_sink.cc │ │ ├── simple_thread_sink.h │ │ └── test_data │ │ │ ├── BUILD │ │ │ ├── mnist_caffe_fetch.py │ │ │ ├── mnist_sample.h │ │ │ └── py_layers │ │ │ └── 00000001 │ │ │ ├── deploy.prototxt │ │ │ ├── test_python_layer.py │ │ │ └── weights.caffemodel │ ├── hashmap │ │ ├── BUILD │ │ ├── hashmap_source_adapter.cc │ │ ├── hashmap_source_adapter.h │ │ ├── hashmap_source_adapter.proto │ │ └── hashmap_source_adapter_test.cc │ └── tensorflow │ │ ├── BUILD │ │ ├── bundle_factory_test.h │ │ ├── bundle_factory_test_util.cc │ │ ├── bundle_factory_test_util.h │ │ ├── bundle_factory_util.cc │ │ ├── bundle_factory_util.h │ │ ├── bundle_factory_util_test.cc │ │ ├── predict_impl.cc │ │ ├── predict_impl.h │ │ ├── saved_model_bundle_factory.cc │ │ ├── saved_model_bundle_factory.h │ │ ├── saved_model_bundle_factory_test.cc │ │ ├── saved_model_bundle_source_adapter.cc │ │ ├── saved_model_bundle_source_adapter.h │ │ ├── saved_model_bundle_source_adapter_test.cc │ │ ├── serving_session.cc │ │ ├── serving_session.h │ │ ├── session_bundle_config.proto │ │ ├── session_bundle_factory.cc │ │ ├── session_bundle_factory.h │ │ ├── session_bundle_factory_test.cc │ │ ├── session_bundle_source_adapter.cc │ │ ├── session_bundle_source_adapter.h │ │ ├── session_bundle_source_adapter.proto │ │ ├── session_bundle_source_adapter_test.cc │ │ ├── simple_servers.cc │ │ ├── simple_servers.h │ │ ├── simple_servers_test.cc │ │ └── testdata │ │ ├── BUILD │ │ ├── bad_half_plus_two │ │ └── 00000123 │ │ │ ├── checkpoint │ │ │ ├── export │ │ │ └── export.meta │ │ ├── export_bad_half_plus_two.py │ │ ├── export_half_plus_two.py │ │ └── half_plus_two │ │ └── 00000123 │ │ ├── export │ │ └── export.meta ├── serving.bzl ├── sources │ └── storage_path │ │ ├── BUILD │ │ ├── file_system_storage_path_source.cc │ │ ├── file_system_storage_path_source.h │ │ ├── file_system_storage_path_source.proto │ │ ├── file_system_storage_path_source_test.cc │ │ ├── static_storage_path_source.cc │ │ ├── static_storage_path_source.h │ │ ├── static_storage_path_source.proto │ │ └── static_storage_path_source_test.cc ├── test_util │ ├── BUILD │ ├── fake_clock_env.cc │ ├── fake_clock_env.h │ ├── test_util.cc │ └── test_util.h ├── tools │ └── docker │ │ ├── Dockerfile.caffe-devel │ │ ├── Dockerfile.devel │ │ └── README.md ├── util │ ├── BUILD │ ├── any_ptr.h │ ├── any_ptr_test.cc │ ├── cleanup.h │ ├── cleanup_test.cc │ ├── event_bus.h │ ├── event_bus_test.cc │ ├── executor.h │ ├── fast_read_dynamic_ptr.h │ ├── fast_read_dynamic_ptr_benchmark.cc │ ├── fast_read_dynamic_ptr_test.cc │ ├── hash.cc │ ├── hash.h │ ├── inline_executor.cc │ ├── inline_executor.h │ ├── inline_executor_test.cc │ ├── observer.h │ ├── observer_test.cc │ ├── optional.cc │ ├── optional.h │ ├── optional_test.cc │ ├── periodic_function.cc │ ├── periodic_function.h │ ├── periodic_function_test.cc │ ├── threadpool_executor.cc │ ├── threadpool_executor.h │ ├── threadpool_executor_test.cc │ ├── unique_ptr_with_deps.h │ └── unique_ptr_with_deps_test.cc └── workspace.bzl ├── third_party └── caffe │ ├── BUILD │ ├── WORKSPACE │ ├── __init__.py │ ├── config.bzl │ ├── openblas_prelude.h │ └── python_prelude.h └── tools └── bazel.rc /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | /bazel-bin 3 | /bazel-ci_build-cache 4 | /bazel-genfiles 5 | /bazel-out 6 | /bazel-serving 7 | /bazel-tensorflow 8 | /bazel-tensorflow_serving 9 | /bazel-testlogs 10 | /bazel-tf 11 | /bazel-workspace 12 | /third_party/py/numpy/numpy_include 13 | /util/python/python_include 14 | /util/python/python_lib 15 | /.vscode -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tensorflow"] 2 | path = tensorflow 3 | url = https://github.com/tensorflow/tensorflow.git 4 | [submodule "tf_models"] 5 | path = tf_models 6 | url = https://github.com/tensorflow/models.git 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of TensorFlow Serving authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | Google Inc. 10 | IBM Corp. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guidelines 2 | 3 | ## How to become a contributor and submit your own code 4 | 5 | ### Contributor License Agreements 6 | 7 | We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. 8 | 9 | Please fill out either the individual or corporate Contributor License Agreement (CLA). 10 | 11 | * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). 12 | * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). 13 | 14 | Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. 15 | 16 | ***NOTE***: Only original source code from you and other people that have signed the CLA can be accepted into the main repository. 17 | 18 | ### Contributing code 19 | 20 | If you have improvements to TensorFlow Serving, send us your pull requests! 21 | For those just getting started, Github has a [howto](https://help.github.com/articles/using-pull-requests/). 22 | 23 | If you want to contribute but you're not sure where to start, take a look at the 24 | [issues with the "contributions welcome" label](https://github.com/tensorflow/serving/labels/contributions%20welcome). 25 | These are issues that we believe are particularly well suited for outside 26 | contributions, often because we probably won't get to them right now. If you 27 | decide to start on an issue, leave a comment so that other people know that 28 | you're working on it. If you want to help out, but not alone, use the issue 29 | comment thread to coordinate. 30 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/RELEASE.md -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "tf_serving") 2 | 3 | local_repository( 4 | name = "org_tensorflow", 5 | path = "tensorflow", 6 | ) 7 | 8 | # TensorFlow depends on "io_bazel_rules_closure" so we need this here. 9 | # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. 10 | http_archive( 11 | name = "io_bazel_rules_closure", 12 | sha256 = "60fc6977908f999b23ca65698c2bb70213403824a84f7904310b6000d78be9ce", 13 | strip_prefix = "rules_closure-5ca1dab6df9ad02050f7ba4e816407f88690cf7d", 14 | urls = [ 15 | "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/5ca1dab6df9ad02050f7ba4e816407f88690cf7d.tar.gz", # 2017-02-03 16 | "https://github.com/bazelbuild/rules_closure/archive/5ca1dab6df9ad02050f7ba4e816407f88690cf7d.tar.gz", 17 | ], 18 | ) 19 | 20 | # Please add all new TensorFlow Serving dependencies in workspace.bzl. 21 | load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') 22 | tf_serving_workspace(__workspace_dir__) 23 | 24 | # Specify the minimum required bazel version. 25 | load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") 26 | check_version("0.4.2") 27 | -------------------------------------------------------------------------------- /tensorflow_serving/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Tensorflow Serving. 2 | 3 | package( 4 | default_visibility = ["//tensorflow_serving:internal"], 5 | ) 6 | 7 | licenses(["notice"]) # Apache 2.0 8 | 9 | exports_files(["LICENSE"]) 10 | 11 | # open source marker; do not remove 12 | 13 | package_group( 14 | name = "internal", 15 | packages = [ 16 | "//tensorflow_serving/...", 17 | ], 18 | ) 19 | 20 | filegroup( 21 | name = "all_files", 22 | srcs = glob( 23 | ["**/*"], 24 | exclude = [ 25 | "**/METADATA", 26 | "**/OWNERS", 27 | "g3doc/sitemap.md", 28 | ], 29 | ), 30 | ) 31 | -------------------------------------------------------------------------------- /tensorflow_serving/apis/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Tensorflow Serving APIs. 2 | 3 | package( 4 | default_visibility = ["//visibility:public"], 5 | features = ["-layering_check"], 6 | ) 7 | 8 | licenses(["notice"]) # Apache 2.0 9 | 10 | exports_files(["LICENSE"]) 11 | 12 | filegroup( 13 | name = "all_files", 14 | srcs = glob( 15 | ["**/*"], 16 | exclude = [ 17 | "**/METADATA", 18 | "**/OWNERS", 19 | ], 20 | ), 21 | ) 22 | 23 | load("//tensorflow_serving:serving.bzl", "serving_proto_library") 24 | load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") 25 | 26 | serving_proto_library( 27 | name = "model_proto", 28 | srcs = ["model.proto"], 29 | cc_api_version = 2, 30 | go_api_version = 2, 31 | java_api_version = 2, 32 | deps = [ 33 | "@protobuf//:cc_wkt_protos", 34 | ], 35 | ) 36 | 37 | serving_proto_library_py( 38 | name = "model_proto_py_pb2", 39 | srcs = ["model.proto"], 40 | proto_library = "model_proto", 41 | deps = [], 42 | ) 43 | 44 | serving_proto_library( 45 | name = "predict_proto", 46 | srcs = ["predict.proto"], 47 | cc_api_version = 2, 48 | go_api_version = 2, 49 | java_api_version = 2, 50 | deps = [ 51 | ":model_proto", 52 | "@org_tensorflow//tensorflow/core:protos_all_cc", 53 | ], 54 | ) 55 | 56 | serving_proto_library_py( 57 | name = "predict_proto_py_pb2", 58 | srcs = ["predict.proto"], 59 | proto_library = "predict_proto", 60 | deps = [ 61 | ":model_proto_py_pb2", 62 | "@org_tensorflow//tensorflow/core:protos_all_py", 63 | ], 64 | ) 65 | 66 | serving_proto_library( 67 | name = "prediction_service_proto", 68 | srcs = ["prediction_service.proto"], 69 | has_services = 1, 70 | cc_api_version = 2, 71 | cc_grpc_version = 1, 72 | go_api_version = 2, 73 | java_api_version = 2, 74 | deps = [ 75 | ":predict_proto", 76 | ], 77 | ) 78 | 79 | py_library( 80 | name = "prediction_service_proto_py_pb2", 81 | srcs = ["prediction_service_pb2.py"], 82 | deps = [":predict_proto_py_pb2"], 83 | ) 84 | -------------------------------------------------------------------------------- /tensorflow_serving/apis/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional version. If unspecified, will use the latest (numerical) version. 14 | // Typically not needed unless coordinating across multiple models that were 15 | // co-trained and/or have inter-dependencies on the versions used at inference 16 | // time. 17 | google.protobuf.Int64Value version = 2; 18 | 19 | // A named signature to evaluate. If unspecified, the default signature will 20 | // be used. Note that only MultiInference will initially support this. 21 | string signature_name = 3; 22 | } 23 | -------------------------------------------------------------------------------- /tensorflow_serving/apis/predict.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow/core/framework/tensor.proto"; 7 | import "tensorflow_serving/apis/model.proto"; 8 | 9 | // PredictRequest specifies which TensorFlow model to run, as well as 10 | // how inputs are mapped to tensors and how outputs are filtered before 11 | // returning to user. 12 | message PredictRequest { 13 | // Model Specification. 14 | ModelSpec model_spec = 1; 15 | 16 | // Input tensors. 17 | // Names of input tensor are alias names. The mapping from aliases to real 18 | // input tensor names is expected to be stored as named generic signature 19 | // under the key "inputs" in the model export. 20 | // Each alias listed in a generic signature named "inputs" should be provided 21 | // exactly once in order to run the prediction. 22 | map inputs = 2; 23 | 24 | // Output filter. 25 | // Names specified are alias names. The mapping from aliases to real output 26 | // tensor names is expected to be stored as named generic signature under 27 | // the key "outputs" in the model export. 28 | // Only tensors specified here will be run/fetched and returned, with the 29 | // exception that when none is specified, all tensors specified in the 30 | // named signature will be run/fetched and returned. 31 | repeated string output_filter = 3; 32 | } 33 | 34 | // Response for PredictRequest on successful run. 35 | message PredictResponse { 36 | // Output tensors. 37 | map outputs = 1; 38 | } 39 | -------------------------------------------------------------------------------- /tensorflow_serving/apis/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow_serving/apis/predict.proto"; 7 | 8 | // PredictionService provides access to machine-learned models loaded by 9 | // model_servers. 10 | service PredictionService { 11 | // Predict -- provides access to loaded TensorFlow model. 12 | rpc Predict(PredictRequest) returns (PredictResponse); 13 | } -------------------------------------------------------------------------------- /tensorflow_serving/batching/test_util/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Tensorflow Serving batching test utilities. 2 | 3 | package( 4 | default_visibility = ["//tensorflow_serving:internal"], 5 | features = ["-layering_check"], 6 | ) 7 | 8 | licenses(["notice"]) # Apache 2.0 9 | 10 | exports_files(["LICENSE"]) 11 | 12 | filegroup( 13 | name = "all_files", 14 | srcs = glob( 15 | ["**/*"], 16 | exclude = [ 17 | "**/METADATA", 18 | "**/OWNERS", 19 | ], 20 | ), 21 | ) 22 | 23 | cc_library( 24 | name = "puppet_batch_scheduler", 25 | testonly = 1, 26 | hdrs = ["puppet_batch_scheduler.h"], 27 | visibility = ["//visibility:private"], 28 | deps = [ 29 | "//tensorflow_serving/batching:batch_scheduler", 30 | "@org_tensorflow//tensorflow/core:tensorflow", 31 | ], 32 | ) 33 | 34 | cc_test( 35 | name = "puppet_batch_scheduler_test", 36 | srcs = [ 37 | "puppet_batch_scheduler_test.cc", 38 | ], 39 | deps = [ 40 | ":puppet_batch_scheduler", 41 | "//tensorflow_serving/core/test_util:test_main", 42 | "@org_tensorflow//tensorflow/core:lib", 43 | "@org_tensorflow//tensorflow/core:test", 44 | ], 45 | ) 46 | -------------------------------------------------------------------------------- /tensorflow_serving/batching/test_util/puppet_batch_scheduler_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/batching/test_util/puppet_batch_scheduler.h" 17 | 18 | #include 19 | #include "tensorflow/core/lib/core/status.h" 20 | #include "tensorflow/core/lib/core/status_test_util.h" 21 | #include "tensorflow/core/platform/macros.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | namespace test_util { 26 | namespace { 27 | 28 | class FakeTask : public BatchTask { 29 | public: 30 | FakeTask() = default; 31 | ~FakeTask() override = default; 32 | 33 | size_t size() const override { return 1; } 34 | 35 | private: 36 | TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); 37 | }; 38 | 39 | // Creates a FakeTask and calls 'scheduler->Schedule()' on that task, and 40 | // expects the call to succeed. 41 | void ScheduleTask(BatchScheduler* scheduler) { 42 | std::unique_ptr task(new FakeTask); 43 | TF_ASSERT_OK(scheduler->Schedule(&task)); 44 | } 45 | 46 | TEST(PuppetBatchSchedulerTest, Basic) { 47 | int num_tasks_processed = 0; 48 | auto callback = 49 | [&num_tasks_processed](std::unique_ptr> batch) { 50 | ASSERT_TRUE(batch->IsClosed()); 51 | num_tasks_processed += batch->size(); 52 | }; 53 | PuppetBatchScheduler scheduler(callback); 54 | 55 | for (int i = 0; i < 3; ++i) { 56 | EXPECT_EQ(0, num_tasks_processed); 57 | EXPECT_EQ(i, scheduler.NumEnqueuedTasks()); 58 | ScheduleTask(&scheduler); 59 | } 60 | EXPECT_EQ(3, scheduler.NumEnqueuedTasks()); 61 | 62 | scheduler.ProcessTasks(2); 63 | EXPECT_EQ(2, num_tasks_processed); 64 | EXPECT_EQ(1, scheduler.NumEnqueuedTasks()); 65 | 66 | ScheduleTask(&scheduler); 67 | EXPECT_EQ(2, num_tasks_processed); 68 | EXPECT_EQ(2, scheduler.NumEnqueuedTasks()); 69 | 70 | scheduler.ProcessAllTasks(); 71 | EXPECT_EQ(4, num_tasks_processed); 72 | EXPECT_EQ(0, scheduler.NumEnqueuedTasks()); 73 | } 74 | 75 | } // namespace 76 | } // namespace test_util 77 | } // namespace serving 78 | } // namespace tensorflow 79 | -------------------------------------------------------------------------------- /tensorflow_serving/config/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Tensorflow Serving configs. 2 | 3 | package( 4 | default_visibility = ["//visibility:public"], 5 | features = ["-layering_check"], 6 | ) 7 | 8 | licenses(["notice"]) # Apache 2.0 9 | 10 | exports_files(["LICENSE"]) 11 | 12 | filegroup( 13 | name = "all_files", 14 | srcs = glob( 15 | ["**/*"], 16 | exclude = [ 17 | "**/METADATA", 18 | "**/OWNERS", 19 | ], 20 | ), 21 | ) 22 | 23 | load("//tensorflow_serving:serving.bzl", "serving_proto_library") 24 | 25 | serving_proto_library( 26 | name = "model_server_config_proto", 27 | srcs = ["model_server_config.proto"], 28 | cc_api_version = 2, 29 | deps = [ 30 | "@protobuf//:cc_wkt_protos", 31 | "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /tensorflow_serving/config/model_server_config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/any.proto"; 7 | import "tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto"; 8 | 9 | // The type of model. 10 | // TODO(b/31336131): DEPRECATED. 11 | enum ModelType { 12 | MODEL_TYPE_UNSPECIFIED = 0 [deprecated = true]; 13 | TENSORFLOW = 1 [deprecated = true]; 14 | OTHER = 2 [deprecated = true]; 15 | }; 16 | 17 | // Common configuration for loading a model being served. 18 | message ModelConfig { 19 | // Name of the model. 20 | string name = 1; 21 | 22 | // Base path to the model, excluding the version directory. 23 | // E.g> for a model at /foo/bar/my_model/123, where 123 is the version, the 24 | // base path is /foo/bar/my_model. 25 | string base_path = 2; 26 | 27 | // Type of model. 28 | // TODO(b/31336131): DEPRECATED. Please use 'model_platform' instead. 29 | ModelType model_type = 3 [deprecated = true]; 30 | 31 | // Type of model (e.g. "tensorflow"). 32 | string model_platform = 4; 33 | 34 | // Version policy for the model indicating how many versions of the model to be served at the same time. 35 | // The default option is to serve only the latest version of the model. 36 | FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5; 37 | } 38 | 39 | // Static list of models to be loaded for serving. 40 | message ModelConfigList { 41 | repeated ModelConfig config = 1; 42 | } 43 | 44 | // ModelServer config. 45 | message ModelServerConfig { 46 | // ModelServer takes either a static file-based model config list or an Any 47 | // proto representing custom model config that is fetched dynamically at 48 | // runtime (through network RPC, custom service, etc.). 49 | oneof config { 50 | ModelConfigList model_config_list = 1; 51 | google.protobuf.Any custom_model_config = 2; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tensorflow_serving/core/README.md: -------------------------------------------------------------------------------- 1 | Directory for non-application-specific modules. 2 | -------------------------------------------------------------------------------- /tensorflow_serving/core/aspired_versions_manager_builder.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/aspired_versions_manager_builder.h" 17 | 18 | #include "tensorflow_serving/core/manager_wrapper.h" 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | 23 | Status AspiredVersionsManagerBuilder::Create( 24 | Options options, std::unique_ptr* builder) { 25 | std::unique_ptr aspired_versions_manager; 26 | TF_RETURN_IF_ERROR(AspiredVersionsManager::Create(std::move(options), 27 | &aspired_versions_manager)); 28 | builder->reset( 29 | new AspiredVersionsManagerBuilder(std::move(aspired_versions_manager))); 30 | return Status::OK(); 31 | } 32 | 33 | AspiredVersionsManagerBuilder::AspiredVersionsManagerBuilder( 34 | std::unique_ptr manager) 35 | : aspired_versions_manager_(manager.get()) { 36 | manager_with_sources_.SetOwned(std::move(manager)); 37 | } 38 | 39 | std::unique_ptr AspiredVersionsManagerBuilder::Build() { 40 | return std::unique_ptr( 41 | new ManagerWrapper(std::move(manager_with_sources_))); 42 | } 43 | 44 | } // namespace serving 45 | } // namespace tensorflow 46 | -------------------------------------------------------------------------------- /tensorflow_serving/core/eager_load_policy.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/eager_load_policy.h" 17 | #include "tensorflow_serving/core/loader_harness.h" 18 | 19 | namespace tensorflow { 20 | namespace serving { 21 | 22 | optional EagerLoadPolicy::GetNextAction( 23 | const std::vector& all_versions) const { 24 | // If there is a new aspired version, load it. 25 | for (const auto& version : all_versions) { 26 | if (version.is_aspired && version.state == LoaderHarness::State::kNew) { 27 | VLOG(1) << "EagerLoadPolicy requesting to load servable " << version.id; 28 | return {{Action::kLoad, version.id}}; 29 | } 30 | } 31 | 32 | // Second, check if there are any aspired versions that are not ready. In that 33 | // case we can't unload any versions. 34 | const bool aspired_not_serving = 35 | std::any_of(all_versions.begin(), all_versions.end(), 36 | [](const AspiredServableStateSnapshot& version) { 37 | return version.is_aspired && 38 | version.state != LoaderHarness::State::kReady; 39 | }); 40 | if (aspired_not_serving) { 41 | return nullopt; 42 | } 43 | 44 | // If there is no new aspired version, but a not-aspired version, unload the 45 | // latter. 46 | for (const auto& version : all_versions) { 47 | if (!version.is_aspired && version.state == LoaderHarness::State::kReady) { 48 | VLOG(1) << "EagerLoadPolicy requesting to unload servable " << version.id; 49 | return {{Action::kUnload, version.id}}; 50 | } 51 | } 52 | return nullopt; 53 | } 54 | 55 | } // namespace serving 56 | } // namespace tensorflow 57 | -------------------------------------------------------------------------------- /tensorflow_serving/core/eager_load_policy.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ 17 | #define TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ 18 | 19 | #include 20 | 21 | #include "tensorflow_serving/core/aspired_version_policy.h" 22 | #include "tensorflow_serving/core/loader_harness.h" 23 | #include "tensorflow_serving/util/optional.h" 24 | 25 | namespace tensorflow { 26 | namespace serving { 27 | 28 | // AspiredVersionPolicy that loads any aspired versions of a servable before 29 | // unloading any no-longer-aspired versions. 30 | // 31 | // This policy provides servable availability with the trade-off of temporary 32 | // increased resource consumption while the new version loads followed by the 33 | // old versions unloading. 34 | class EagerLoadPolicy final : public AspiredVersionPolicy { 35 | public: 36 | optional GetNextAction( 37 | const std::vector& all_versions) 38 | const override; 39 | }; 40 | 41 | } // namespace serving 42 | } // namespace tensorflow 43 | 44 | #endif // TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ 45 | -------------------------------------------------------------------------------- /tensorflow_serving/core/eager_unload_policy.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/eager_unload_policy.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | optional EagerUnloadPolicy::GetNextAction( 22 | const std::vector& all_versions) const { 23 | // First iterate over all_versions and find any in kReady that are no longer 24 | // aspired. Unload the first if any. 25 | for (const auto& version : all_versions) { 26 | if (version.state == LoaderHarness::State::kReady && !version.is_aspired) { 27 | VLOG(1) << "EagerUnloadPolicy requesting to unload servable " 28 | << version.id; 29 | return AspiredVersionPolicy::ServableAction( 30 | {Action::kUnload, version.id}); 31 | } 32 | } 33 | 34 | // Second, see if there are any not-aspired versions that aren't in an end 35 | // state (kDisabled or kError). If so, do nothing for now. 36 | const bool not_aspired_not_finished = 37 | std::any_of(all_versions.begin(), all_versions.end(), 38 | [](const AspiredServableStateSnapshot& version) { 39 | return !version.is_aspired && 40 | version.state != LoaderHarness::State::kDisabled && 41 | version.state != LoaderHarness::State::kError; 42 | }); 43 | if (not_aspired_not_finished) { 44 | return nullopt; 45 | } 46 | 47 | // Third and only if no action was found earlier, iterate over all 48 | // versions and find any in kNew that are aspired. Load the first if any. 49 | for (const auto& version : all_versions) { 50 | if (version.state == LoaderHarness::State::kNew && version.is_aspired) { 51 | VLOG(1) << "EagerUnloadPolicy requesting to load servable " << version.id; 52 | return AspiredVersionPolicy::ServableAction({Action::kLoad, version.id}); 53 | } 54 | } 55 | 56 | return nullopt; 57 | } 58 | 59 | } // namespace serving 60 | } // namespace tensorflow 61 | -------------------------------------------------------------------------------- /tensorflow_serving/core/eager_unload_policy.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ 17 | #define TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ 18 | 19 | #include 20 | 21 | #include "tensorflow_serving/core/aspired_version_policy.h" 22 | #include "tensorflow_serving/core/loader_harness.h" 23 | #include "tensorflow_serving/util/optional.h" 24 | 25 | namespace tensorflow { 26 | namespace serving { 27 | 28 | // ServablePolicy that eagerly unloads any no-longer-aspired versions of a 29 | // servable stream and only after done unloading, loads newly aspired versions. 30 | // 31 | // This policy minimizes resource consumption with the trade-off of temporary 32 | // servable unavailability while all old versions unload followed by the new 33 | // versions loading. 34 | // 35 | // Servables with a single version consuming the majority of their host's 36 | // resources must use this policy to prevent deadlock. Other typical use-cases 37 | // will be for multi-servable environments where clients can tolerate brief 38 | // interruptions to a single servable's availability on a replica. 39 | // 40 | // NB: This policy does not in any way solve cross-replica availability. 41 | class EagerUnloadPolicy final : public AspiredVersionPolicy { 42 | public: 43 | optional GetNextAction( 44 | const std::vector& all_versions) 45 | const override; 46 | }; 47 | 48 | } // namespace serving 49 | } // namespace tensorflow 50 | 51 | #endif // TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ 52 | -------------------------------------------------------------------------------- /tensorflow_serving/core/load_servables_fast.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ 17 | #define TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "tensorflow/core/lib/core/status.h" 23 | #include "tensorflow_serving/core/aspired_versions_manager.h" 24 | #include "tensorflow_serving/core/loader.h" 25 | #include "tensorflow_serving/core/manager.h" 26 | #include "tensorflow_serving/core/servable_state_monitor.h" 27 | 28 | namespace tensorflow { 29 | namespace serving { 30 | 31 | // Connects 'source' to 'manager', and speeds up loading of the servables 32 | // matching 'initial_servables'. The speeding up is accomplished by boosting the 33 | // number of threads used for loading until the initial servables have been 34 | // loaded, and then resetting it to the manager's originally configured value. 35 | Status ConnectSourceWithFastInitialLoad( 36 | AspiredVersionsManager* manager, Source>* source, 37 | ServableStateMonitor* servable_state_monitor, 38 | const std::vector& initial_servables, 39 | uint32 num_threads = 4 * port::NumSchedulableCPUs()); 40 | 41 | //// 42 | // Implementation detail. API readers may skip. 43 | /// 44 | 45 | namespace internal { 46 | 47 | Status ConnectSourceWithFastInitialLoad( 48 | AspiredVersionsManager* manager, Source>* source, 49 | const std::function& wait_until_loaded_fn, uint32 num_threads); 50 | 51 | } // namespace internal 52 | 53 | } // namespace serving 54 | } // namespace tensorflow 55 | 56 | #endif // TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ 57 | -------------------------------------------------------------------------------- /tensorflow_serving/core/log_collector.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ 17 | #define TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "google/protobuf/message.h" 24 | #include "tensorflow/core/lib/core/status.h" 25 | 26 | namespace tensorflow { 27 | namespace serving { 28 | 29 | // LogCollector defines an abstract interface to use for collecting logs. 30 | // 31 | // Each LogCollector implementation is registered along with a 'type', and if a 32 | // LogCollector is created using this API, we create the LogCollector 33 | // corresponding to the 'type' specified. 34 | class LogCollector { 35 | public: 36 | virtual ~LogCollector() = default; 37 | 38 | // Creates a log-collector for a given 'logname_prefix' and 'id'. The 39 | // 'logname_prefix' should be of the form '/type/', so 40 | // that the factory registered for the type can then be used to create the 41 | // log-collector. The 'id' argument helps in disambiguating logs from 42 | // replicated servers (processes), so it could be a combination of 43 | // task-id and replica-id or process-id and timestamp, etc. 44 | static Status Create(const string& logname_prefix, const uint32 id, 45 | std::unique_ptr* log_collector); 46 | 47 | using Factory = std::function; 48 | // Registers a factory for creating log-collectors for a particular 'type'. 49 | // Returns an error status if a factory is already registered for the 50 | // particular 'type'. 51 | static Status RegisterFactory(const string& type, const Factory& factory); 52 | 53 | // Collects the log as a protocol buffer. 54 | virtual Status CollectMessage(const google::protobuf::Message& message) = 0; 55 | 56 | // Flushes buffered data so that the data can survive an application crash 57 | // (but not an OS crash). 58 | virtual Status Flush() = 0; 59 | }; 60 | 61 | } // namespace serving 62 | } // namespace tensorflow 63 | 64 | #endif // TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ 65 | -------------------------------------------------------------------------------- /tensorflow_serving/core/manager_wrapper.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/manager_wrapper.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | ManagerWrapper::ManagerWrapper(UniquePtrWithDeps wrapped) 22 | : wrapped_(std::move(wrapped)) {} 23 | 24 | std::vector ManagerWrapper::ListAvailableServableIds() const { 25 | return wrapped_->ListAvailableServableIds(); 26 | } 27 | 28 | Status ManagerWrapper::GetUntypedServableHandle( 29 | const ServableRequest& request, 30 | std::unique_ptr* const untyped_handle) { 31 | return wrapped_->GetUntypedServableHandle(request, untyped_handle); 32 | } 33 | 34 | std::map> 35 | ManagerWrapper::GetAvailableUntypedServableHandles() const { 36 | return wrapped_->GetAvailableUntypedServableHandles(); 37 | } 38 | 39 | } // namespace serving 40 | } // namespace tensorflow 41 | -------------------------------------------------------------------------------- /tensorflow_serving/core/manager_wrapper.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_MANAGER_WRAPPER_H_ 17 | #define TENSORFLOW_SERVING_CORE_MANAGER_WRAPPER_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "tensorflow_serving/core/manager.h" 24 | #include "tensorflow_serving/core/servable_id.h" 25 | #include "tensorflow_serving/util/unique_ptr_with_deps.h" 26 | 27 | namespace tensorflow { 28 | namespace serving { 29 | 30 | // An implementation of Manager that delegates all calls to another Manager. 31 | // 32 | // May be useful to override just part of the functionality of another Manager 33 | // or storing a Manager with its dependencies. 34 | class ManagerWrapper : public Manager { 35 | public: 36 | explicit ManagerWrapper(UniquePtrWithDeps wrapped); 37 | ~ManagerWrapper() override = default; 38 | 39 | std::vector ListAvailableServableIds() const override; 40 | 41 | private: 42 | Status GetUntypedServableHandle( 43 | const ServableRequest& request, 44 | std::unique_ptr* untyped_handle) override; 45 | 46 | std::map> 47 | GetAvailableUntypedServableHandles() const override; 48 | 49 | const UniquePtrWithDeps wrapped_; 50 | }; 51 | 52 | } // namespace serving 53 | } // namespace tensorflow 54 | 55 | #endif // TENSORFLOW_SERVING_CORE_MANAGER_WRAPPER_H_ 56 | -------------------------------------------------------------------------------- /tensorflow_serving/core/servable_data_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/servable_data.h" 17 | 18 | #include 19 | 20 | #include 21 | #include "tensorflow/core/lib/core/errors.h" 22 | #include "tensorflow/core/lib/core/status_test_util.h" 23 | #include "tensorflow/core/platform/types.h" 24 | 25 | namespace tensorflow { 26 | namespace serving { 27 | namespace { 28 | 29 | TEST(ServableDataTest, NoError) { 30 | ServableId id = {"name", 42}; 31 | ServableData data(id, "yo"); 32 | EXPECT_EQ(id, data.id()); 33 | TF_EXPECT_OK(data.status()); 34 | EXPECT_EQ("yo", data.DataOrDie()); 35 | EXPECT_EQ("yo", data.ConsumeDataOrDie()); 36 | } 37 | 38 | TEST(ServableDataTest, StaticCreateNoError) { 39 | ServableId id = {"name", 42}; 40 | auto data = CreateServableData(id, "yo"); 41 | EXPECT_EQ(id, data.id()); 42 | TF_EXPECT_OK(data.status()); 43 | EXPECT_EQ("yo", data.DataOrDie()); 44 | EXPECT_EQ("yo", data.ConsumeDataOrDie()); 45 | } 46 | 47 | TEST(ServableDataTest, Error) { 48 | ServableId id = {"name", 42}; 49 | ServableData data(id, errors::Unknown("d'oh")); 50 | EXPECT_EQ(id, data.id()); 51 | EXPECT_EQ(errors::Unknown("d'oh"), data.status()); 52 | } 53 | 54 | } // namespace 55 | } // namespace serving 56 | } // namespace tensorflow 57 | -------------------------------------------------------------------------------- /tensorflow_serving/core/servable_id_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/servable_id.h" 17 | 18 | #include 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | namespace { 23 | 24 | // Note: these tests use EXPECT_TRUE/FALSE in conjunction with the specific 25 | // comparison operator being tested (e.g. ==), instead of e.g. EXPECT_EQ, to 26 | // ensure that the exact operator targeted for testing is being invoked. 27 | 28 | TEST(ServableIdTest, Equals) { 29 | EXPECT_TRUE((ServableId{"a", 1} == ServableId{"a", 1})); 30 | 31 | EXPECT_FALSE((ServableId{"b", 2} == ServableId{"a", 2})); 32 | EXPECT_FALSE((ServableId{"b", 1} == ServableId{"b", 2})); 33 | } 34 | 35 | TEST(ServableIdTest, NotEquals) { 36 | EXPECT_FALSE((ServableId{"a", 1} != ServableId{"a", 1})); 37 | 38 | EXPECT_TRUE((ServableId{"b", 2} != ServableId{"a", 2})); 39 | EXPECT_TRUE((ServableId{"b", 1} != ServableId{"b", 2})); 40 | } 41 | 42 | TEST(ServableIdTest, LessThan) { 43 | EXPECT_TRUE((ServableId{"a", 1} < ServableId{"b", 1})); 44 | EXPECT_TRUE((ServableId{"b", 1} < ServableId{"b", 2})); 45 | EXPECT_TRUE((ServableId{"a", 1} < ServableId{"b", 2})); 46 | 47 | EXPECT_FALSE((ServableId{"a", 1} < ServableId{"a", 1})); 48 | 49 | EXPECT_FALSE((ServableId{"b", 1} < ServableId{"a", 1})); 50 | EXPECT_FALSE((ServableId{"b", 2} < ServableId{"b", 1})); 51 | EXPECT_FALSE((ServableId{"b", 2} < ServableId{"a", 1})); 52 | } 53 | 54 | } // namespace 55 | } // namespace serving 56 | } // namespace tensorflow 57 | -------------------------------------------------------------------------------- /tensorflow_serving/core/static_manager.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/static_manager.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | StaticManagerBuilder::StaticManagerBuilder() { 22 | BasicManager::Options basic_manager_options; 23 | // We don't want multithreading. 24 | basic_manager_options.num_load_unload_threads = 0; 25 | const Status basic_manager_status = 26 | BasicManager::Create(std::move(basic_manager_options), &basic_manager_); 27 | if (!basic_manager_status.ok()) { 28 | LOG(ERROR) << "Error creating BasicManager: " << health_; 29 | health_ = basic_manager_status; 30 | } 31 | } 32 | 33 | std::unique_ptr StaticManagerBuilder::Build() { 34 | if (!health_.ok()) { 35 | LOG(ERROR) << health_; 36 | return nullptr; 37 | } 38 | 39 | // If Build() is called again, we'll produce the following error. 40 | health_ = errors::FailedPrecondition( 41 | "Build() already called on this StaticManagerBuilder."); 42 | 43 | return std::move(basic_manager_); 44 | } 45 | 46 | } // namespace serving 47 | } // namespace tensorflow 48 | -------------------------------------------------------------------------------- /tensorflow_serving/core/storage_path.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // Typedefs and registries pertaining to storage system paths. 17 | 18 | #ifndef TENSORFLOW_SERVING_CORE_STORAGE_PATH_H_ 19 | #define TENSORFLOW_SERVING_CORE_STORAGE_PATH_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include "tensorflow/core/lib/core/status.h" 26 | #include "tensorflow/core/platform/types.h" 27 | #include "tensorflow_serving/core/servable_data.h" 28 | #include "tensorflow_serving/core/servable_id.h" 29 | 30 | namespace tensorflow { 31 | namespace serving { 32 | 33 | // Strings that represent paths in some storage system. 34 | using StoragePath = string; 35 | 36 | inline bool operator==(const ServableData& a, 37 | const ServableData& b) { 38 | if (a.id() != b.id()) { 39 | return false; 40 | } 41 | if (a.status().ok() != b.status().ok()) { 42 | return false; 43 | } 44 | if (a.status().ok()) { 45 | return a.DataOrDie() == b.DataOrDie(); 46 | } else { 47 | return a.status() == b.status(); 48 | } 49 | } 50 | 51 | } // namespace serving 52 | } // namespace tensorflow 53 | 54 | #endif // TENSORFLOW_SERVING_CORE_STORAGE_PATH_H_ 55 | -------------------------------------------------------------------------------- /tensorflow_serving/core/storage_path_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/storage_path.h" 17 | 18 | #include 19 | #include "tensorflow/core/lib/core/errors.h" 20 | 21 | namespace tensorflow { 22 | namespace serving { 23 | namespace { 24 | 25 | TEST(StoragePathTest, ServableDataEquality) { 26 | ServableId id0 = {"0", 0}; 27 | ServableId id1 = {"1", 1}; 28 | 29 | ServableData a(id0, "x"); 30 | ServableData a2(id0, "x"); 31 | EXPECT_TRUE(a == a); 32 | EXPECT_TRUE(a == a2); 33 | EXPECT_TRUE(a2 == a); 34 | 35 | ServableData b(id0, "y"); 36 | ServableData c(id1, "x"); 37 | ServableData d(id0, errors::Unknown("error")); 38 | for (const ServableData& other : {b, c, d}) { 39 | EXPECT_TRUE(other == other); 40 | EXPECT_FALSE(a == other); 41 | EXPECT_FALSE(other == a); 42 | } 43 | } 44 | 45 | } // namespace 46 | } // namespace serving 47 | } // namespace tensorflow 48 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/availability_test_util.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/test_util/availability_test_util.h" 17 | #include "tensorflow/core/platform/env.h" 18 | 19 | namespace tensorflow { 20 | namespace serving { 21 | namespace test_util { 22 | 23 | namespace { 24 | 25 | // Determines whether WaitUntilServableManagerStateIsOneOf()'s condition is 26 | // satisfied. (See that function's documentation.) 27 | bool ServableManagerStateIsOneOf( 28 | const ServableStateMonitor& monitor, const ServableId& servable, 29 | const std::vector& states) { 30 | optional maybe_state = monitor.GetState(servable); 31 | if (!maybe_state) { 32 | return false; 33 | } 34 | const ServableState state = *maybe_state; 35 | 36 | for (const ServableState::ManagerState& desired_manager_state : states) { 37 | if (state.manager_state == desired_manager_state) { 38 | return true; 39 | } 40 | } 41 | return false; 42 | } 43 | 44 | } // namespace 45 | 46 | void WaitUntilServableManagerStateIsOneOf( 47 | const ServableStateMonitor& monitor, const ServableId& servable, 48 | const std::vector& states) { 49 | while (!ServableManagerStateIsOneOf(monitor, servable, states)) { 50 | Env::Default()->SleepForMicroseconds(50 * 1000 /* 50 ms */); 51 | } 52 | } 53 | 54 | } // namespace test_util 55 | } // namespace serving 56 | } // namespace tensorflow 57 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/availability_test_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // Methods related to the availability of servables, that are useful in writing 17 | // tests. (Not intended for production use.) 18 | 19 | #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_AVAILABILITY_TEST_UTIL_H_ 20 | #define TENSORFLOW_SERVING_CORE_TEST_UTIL_AVAILABILITY_TEST_UTIL_H_ 21 | 22 | #include "tensorflow_serving/core/servable_state_monitor.h" 23 | 24 | namespace tensorflow { 25 | namespace serving { 26 | namespace test_util { 27 | 28 | // Waits until 'monitor' shows that the manager state of 'servable' is one of 29 | // 'states'. 30 | void WaitUntilServableManagerStateIsOneOf( 31 | const ServableStateMonitor& monitor, const ServableId& servable, 32 | const std::vector& states); 33 | 34 | } // namespace test_util 35 | } // namespace serving 36 | } // namespace tensorflow 37 | 38 | #endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_AVAILABILITY_TEST_UTIL_H_ 39 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/fake_loader.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/test_util/fake_loader.h" 17 | 18 | #include "tensorflow/core/lib/core/errors.h" 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | namespace test_util { 23 | 24 | thread_local bool FakeLoader::was_deleted_in_this_thread_; 25 | int FakeLoader::num_fake_loaders_ = 0; 26 | mutex FakeLoader::num_fake_loaders_mu_(LINKER_INITIALIZED); 27 | 28 | FakeLoader::FakeLoader(int64 servable, const Status load_status) 29 | : servable_(servable), load_status_(load_status) { 30 | was_deleted_in_this_thread_ = false; 31 | { 32 | mutex_lock l(num_fake_loaders_mu_); 33 | ++num_fake_loaders_; 34 | } 35 | } 36 | 37 | FakeLoader::~FakeLoader() { 38 | { 39 | mutex_lock l(num_fake_loaders_mu_); 40 | --num_fake_loaders_; 41 | } 42 | was_deleted_in_this_thread_ = true; 43 | } 44 | 45 | Status FakeLoader::load_status() { return load_status_; } 46 | 47 | Status FakeLoader::Load() { return load_status_; } 48 | 49 | void FakeLoader::Unload() {} 50 | 51 | AnyPtr FakeLoader::servable() { return AnyPtr(&servable_); } 52 | 53 | bool FakeLoader::was_deleted_in_this_thread() { 54 | return was_deleted_in_this_thread_; 55 | } 56 | 57 | int FakeLoader::num_fake_loaders() { 58 | mutex_lock l(num_fake_loaders_mu_); 59 | return num_fake_loaders_; 60 | } 61 | 62 | } // namespace test_util 63 | } // namespace serving 64 | } // namespace tensorflow 65 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/fake_loader.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_H_ 17 | #define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_H_ 18 | 19 | #include "tensorflow/core/lib/core/status.h" 20 | #include "tensorflow/core/platform/mutex.h" 21 | #include "tensorflow/core/platform/thread_annotations.h" 22 | #include "tensorflow_serving/core/loader.h" 23 | #include "tensorflow_serving/util/any_ptr.h" 24 | 25 | namespace tensorflow { 26 | namespace serving { 27 | namespace test_util { 28 | 29 | // A fake loader to be used in tests. 30 | // 31 | // Useful for - 32 | // 1. Erroring out on load. 33 | // 2. Counting the total number of fake loaders alive. 34 | // 3. Detecting if deletion of the loader happened on a particular thread. 35 | // 36 | // This class is thread-safe. 37 | class FakeLoader : public ResourceUnsafeLoader { 38 | public: 39 | explicit FakeLoader(int64 servable, const Status load_status = Status::OK()); 40 | 41 | ~FakeLoader() override; 42 | 43 | // The status returned during load. 44 | Status load_status(); 45 | 46 | Status Load() override; 47 | 48 | void Unload() override; 49 | 50 | AnyPtr servable() override; 51 | 52 | static int num_fake_loaders(); 53 | 54 | // Returns true if a loader was deleted in this thread. 55 | static bool was_deleted_in_this_thread(); 56 | 57 | private: 58 | // Used to detect the thread in which deletion of a loader occurs. 59 | static thread_local bool was_deleted_in_this_thread_; 60 | 61 | // Counts the number of FakeLoader objects alive. 62 | static int num_fake_loaders_ GUARDED_BY(num_fake_loaders_mu_); 63 | static mutex num_fake_loaders_mu_; 64 | 65 | // The servable returned from this loader. 66 | // 67 | // Don't make const or you'll have to change the handle type to 'const int64'. 68 | int64 servable_; 69 | // The status returned during load. 70 | const Status load_status_; 71 | }; 72 | 73 | } // namespace test_util 74 | } // namespace serving 75 | } // namespace tensorflow 76 | 77 | #endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_H_ 78 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/fake_loader_source_adapter.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | namespace test_util { 21 | 22 | FakeLoaderSourceAdapter::FakeLoaderSourceAdapter( 23 | const string& suffix, std::function call_on_destruct) 24 | : SimpleLoaderSourceAdapter( 25 | [this](const StoragePath& path, 26 | std::unique_ptr* servable_ptr) { 27 | const string servable = suffix_.length() > 0 28 | ? strings::StrCat(path, "/", suffix_) 29 | : path; 30 | servable_ptr->reset(new string(servable)); 31 | return Status::OK(); 32 | }, 33 | SimpleLoaderSourceAdapter::EstimateNoResources()), 35 | suffix_(suffix), 36 | call_on_destruct_(call_on_destruct) {} 37 | 38 | FakeLoaderSourceAdapter::~FakeLoaderSourceAdapter() { 39 | Detach(); 40 | if (call_on_destruct_) { 41 | call_on_destruct_(suffix_); 42 | } 43 | } 44 | 45 | std::function>>*)> 48 | FakeLoaderSourceAdapter::GetCreator() { 49 | return [](const string& model_platform, 50 | std::unique_ptr>>* source) { 52 | source->reset(new FakeLoaderSourceAdapter); 53 | return Status::OK(); 54 | }; 55 | } 56 | } // namespace test_util 57 | } // namespace serving 58 | } // namespace tensorflow 59 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/fake_loader_source_adapter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ 17 | #define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ 18 | 19 | #include "tensorflow/core/lib/core/status.h" 20 | #include "tensorflow_serving/core/simple_loader.h" 21 | #include "tensorflow_serving/core/storage_path.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | namespace test_util { 26 | 27 | // A fake loader source adapter that creates loaders of servable-type string 28 | // from data of type StoragePath. 29 | // 30 | // If path = "/a/simple/path" and suffix = "foo", the servable string becomes 31 | // "a/simple/path/foo". 32 | // 33 | // To help with verifying the order of destruction of these adapters in tests, 34 | // the adapter may take a callback to be invoked upon destruction. The 35 | // suffix provided to the source-adapter is passed to the string argument of the 36 | // callback when it is invoked. 37 | class FakeLoaderSourceAdapter final 38 | : public SimpleLoaderSourceAdapter { 39 | public: 40 | FakeLoaderSourceAdapter( 41 | const string& suffix = "", 42 | std::function call_on_destruct = {}); 43 | 44 | ~FakeLoaderSourceAdapter() override; 45 | 46 | // Returns a function to create a fake source adapter. 47 | static std::function>>*)> 50 | GetCreator(); 51 | 52 | private: 53 | const string suffix_; 54 | std::function call_on_destruct_; 55 | TF_DISALLOW_COPY_AND_ASSIGN(FakeLoaderSourceAdapter); 56 | }; 57 | 58 | } // namespace test_util 59 | } // namespace serving 60 | } // namespace tensorflow 61 | 62 | #endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ 63 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/fake_storage_path_source_adapter.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/test_util/fake_storage_path_source_adapter.h" 17 | 18 | #include "tensorflow/core/lib/core/errors.h" 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | namespace test_util { 23 | 24 | FakeStoragePathSourceAdapter::FakeStoragePathSourceAdapter( 25 | const string& suffix, std::function call_on_destruct) 26 | : suffix_(suffix), call_on_destruct_(call_on_destruct) {} 27 | 28 | FakeStoragePathSourceAdapter::~FakeStoragePathSourceAdapter() { 29 | Detach(); 30 | if (call_on_destruct_) { 31 | call_on_destruct_(suffix_); 32 | } 33 | } 34 | 35 | Status FakeStoragePathSourceAdapter::Convert( 36 | const StoragePath& data, StoragePath* const converted_data) { 37 | if (data == "invalid") { 38 | return errors::InvalidArgument( 39 | "FakeStoragePathSourceAdapter Convert() dutifully failing on " 40 | "\"invalid\" " 41 | "data"); 42 | } 43 | *converted_data = 44 | suffix_.empty() ? data : strings::StrCat(data, "/", suffix_); 45 | return Status::OK(); 46 | } 47 | 48 | } // namespace test_util 49 | } // namespace serving 50 | } // namespace tensorflow 51 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/fake_storage_path_source_adapter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_STORAGE_PATH_SOURCE_ADAPTER_H_ 17 | #define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_STORAGE_PATH_SOURCE_ADAPTER_H_ 18 | 19 | #include "tensorflow/core/lib/core/status.h" 20 | #include "tensorflow_serving/core/source_adapter.h" 21 | #include "tensorflow_serving/core/storage_path.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | namespace test_util { 26 | 27 | // A fake storage-path source-adapter that generates simple string servables 28 | // from paths. If the supplied path is set to "invalid", an invalid-argument 29 | // error is returned upon conversion. 30 | // 31 | // If path = "/a/simple/path" and suffix = "foo", the servable string becomes 32 | // "/a/simple/path/foo". 33 | // 34 | // To help with verifying the order of destruction of these adapters in tests, 35 | // the adapter may take a callback to be invoked upon destruction. The 36 | // suffix provided to the source-adapter is passed to the string argument of the 37 | // callback when it is invoked. 38 | class FakeStoragePathSourceAdapter final 39 | : public UnarySourceAdapter { 40 | public: 41 | FakeStoragePathSourceAdapter( 42 | const string& suffix = "", 43 | std::function call_on_destruct = {}); 44 | 45 | ~FakeStoragePathSourceAdapter(); 46 | 47 | private: 48 | Status Convert(const StoragePath& data, 49 | StoragePath* const converted_data) override; 50 | 51 | const string suffix_; 52 | std::function call_on_destruct_; 53 | }; 54 | 55 | } // namespace test_util 56 | } // namespace serving 57 | } // namespace tensorflow 58 | 59 | #endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_STORAGE_PATH_SOURCE_ADAPTER_H_ 60 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/manager_test_util.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/core/test_util/manager_test_util.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | namespace test_util { 21 | 22 | AspiredVersionsManagerTestAccess::AspiredVersionsManagerTestAccess( 23 | AspiredVersionsManager* manager) 24 | : manager_(manager) {} 25 | 26 | void AspiredVersionsManagerTestAccess::FlushServables() { 27 | manager_->FlushServables(); 28 | } 29 | 30 | void AspiredVersionsManagerTestAccess::HandlePendingAspiredVersionsRequests() { 31 | manager_->HandlePendingAspiredVersionsRequests(); 32 | } 33 | 34 | void AspiredVersionsManagerTestAccess::InvokePolicyAndExecuteAction() { 35 | manager_->InvokePolicyAndExecuteAction(); 36 | } 37 | 38 | void AspiredVersionsManagerTestAccess::SetNumLoadUnloadThreads( 39 | const uint32 num_load_unload_threads) { 40 | manager_->SetNumLoadUnloadThreads(num_load_unload_threads); 41 | } 42 | 43 | uint32 AspiredVersionsManagerTestAccess::num_load_unload_threads() const { 44 | return manager_->num_load_unload_threads(); 45 | } 46 | 47 | BasicManagerTestAccess::BasicManagerTestAccess(BasicManager* manager) 48 | : manager_(manager) {} 49 | 50 | void BasicManagerTestAccess::SetNumLoadUnloadThreads( 51 | const uint32 num_load_unload_threads) { 52 | manager_->SetNumLoadUnloadThreads(num_load_unload_threads); 53 | } 54 | 55 | uint32 BasicManagerTestAccess::num_load_unload_threads() const { 56 | return manager_->num_load_unload_threads(); 57 | } 58 | 59 | CachingManagerTestAccess::CachingManagerTestAccess(CachingManager* manager) 60 | : manager_(manager) {} 61 | 62 | int64 CachingManagerTestAccess::GetLoadMutexMapSize() const { 63 | return manager_->GetLoadMutexMapSize(); 64 | } 65 | 66 | } // namespace test_util 67 | } // namespace serving 68 | } // namespace tensorflow 69 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/mock_loader.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOADER_H_ 17 | #define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOADER_H_ 18 | 19 | #include 20 | #include "tensorflow/core/lib/core/status.h" 21 | #include "tensorflow_serving/core/loader.h" 22 | #include "tensorflow_serving/util/any_ptr.h" 23 | 24 | namespace tensorflow { 25 | namespace serving { 26 | namespace test_util { 27 | 28 | class MockLoader : public Loader { 29 | public: 30 | MOCK_CONST_METHOD1(EstimateResources, Status(ResourceAllocation* estimate)); 31 | MOCK_METHOD1(Load, Status(const ResourceAllocation& available_resources)); 32 | MOCK_METHOD0(Unload, void()); 33 | MOCK_METHOD0(servable, AnyPtr()); 34 | }; 35 | 36 | } // namespace test_util 37 | } // namespace serving 38 | } // namespace tensorflow 39 | 40 | #endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOADER_H_ 41 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/mock_storage_path_target.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_STORAGE_PATH_TARGET_H_ 17 | #define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_STORAGE_PATH_TARGET_H_ 18 | 19 | #include 20 | 21 | #include 22 | #include "tensorflow/core/lib/core/status.h" 23 | #include "tensorflow_serving/core/storage_path.h" 24 | #include "tensorflow_serving/core/target.h" 25 | 26 | namespace tensorflow { 27 | namespace serving { 28 | namespace test_util { 29 | 30 | class MockStoragePathTarget : public TargetBase { 31 | public: 32 | ~MockStoragePathTarget() override { Detach(); } 33 | MOCK_METHOD2(SetAspiredVersions, 34 | void(const StringPiece, std::vector>)); 35 | }; 36 | 37 | } // namespace test_util 38 | } // namespace serving 39 | } // namespace tensorflow 40 | 41 | #endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_STORAGE_PATH_TARGET_H_ 42 | -------------------------------------------------------------------------------- /tensorflow_serving/core/test_util/test_main.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // A program with a main that is suitable for unittests, including those 17 | // that also define microbenchmarks. Based on whether the user specified 18 | // the --benchmark_filter flag which specifies which benchmarks to run, 19 | // we will either run benchmarks or run the gtest tests in the program. 20 | 21 | #include "tensorflow/core/platform/platform.h" 22 | #include "tensorflow/core/platform/types.h" 23 | 24 | #if defined(PLATFORM_GOOGLE) || defined(__ANDROID__) 25 | // main() is supplied by gunit_main 26 | #else 27 | #include "gtest/gtest.h" 28 | #include "tensorflow/core/lib/core/stringpiece.h" 29 | #include "tensorflow/core/platform/test_benchmark.h" 30 | 31 | GTEST_API_ int main(int argc, char** argv) { 32 | std::cout << "Running main() from test_main.cc\n"; 33 | 34 | testing::InitGoogleTest(&argc, argv); 35 | for (int i = 1; i < argc; i++) { 36 | if (tensorflow::StringPiece(argv[i]).starts_with("--benchmarks=")) { 37 | const char* pattern = argv[i] + strlen("--benchmarks="); 38 | tensorflow::testing::Benchmark::Run(pattern); 39 | return 0; 40 | } 41 | } 42 | return RUN_ALL_TESTS(); 43 | } 44 | #endif 45 | -------------------------------------------------------------------------------- /tensorflow_serving/example/client_util.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | class InferenceStats(object): 4 | """Statistics useful for evaluating basic classification and 5 | runtime performance""" 6 | 7 | @staticmethod 8 | def print_summary(stats, percentiles=[50, 90, 99]): 9 | filtered = numpy.ma.masked_invalid(stats.timings).compressed() # remove NaNs 10 | 11 | print '\nInference error rate: %s%%' % ( 12 | stats.classification_error * 100) 13 | 14 | print "Request error rate: %s%%" % ( 15 | (1.0 - float(filtered.size) / stats.timings.size) * 100) 16 | 17 | print "Avg. Throughput: %s reqs/s" % ( 18 | float(stats.num_tests) / stats.total_elapsed_time) 19 | 20 | if filtered.size > 0: 21 | print "Request Latency (percentiles):" 22 | for pc, x in zip(percentiles, numpy.percentile(filtered, percentiles)): 23 | print " %ith ....... %ims" % (pc, x * 1000.0) 24 | 25 | def __init__(self, num_tests, classification_error, 26 | timings, total_elapsed_time): 27 | assert num_tests == timings.size 28 | self.num_tests = num_tests 29 | self.classification_error = classification_error 30 | self.timings = timings 31 | self.total_elapsed_time = total_elapsed_time 32 | -------------------------------------------------------------------------------- /tensorflow_serving/example/inception_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | #!/usr/bin/env python2.7 17 | 18 | """Send JPEG image to tensorflow_model_server loaded with inception model. 19 | """ 20 | 21 | # This is a placeholder for a Google-internal import. 22 | 23 | from grpc.beta import implementations 24 | import tensorflow as tf 25 | 26 | from tensorflow_serving.apis import predict_pb2 27 | from tensorflow_serving.apis import prediction_service_pb2 28 | 29 | 30 | tf.app.flags.DEFINE_string('server', 'localhost:9000', 31 | 'PredictionService host:port') 32 | tf.app.flags.DEFINE_string('image', '', 'path to image in JPEG format') 33 | FLAGS = tf.app.flags.FLAGS 34 | 35 | 36 | def main(_): 37 | host, port = FLAGS.server.split(':') 38 | channel = implementations.insecure_channel(host, int(port)) 39 | stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) 40 | # Send request 41 | with open(FLAGS.image, 'rb') as f: 42 | # See prediction_service.proto for gRPC request/response details. 43 | data = f.read() 44 | request = predict_pb2.PredictRequest() 45 | request.model_spec.name = 'inception' 46 | request.inputs['images'].CopyFrom( 47 | tf.contrib.util.make_tensor_proto(data, shape=[1])) 48 | result = stub.Predict(request, 10.0) # 10 secs timeout 49 | print(result) 50 | 51 | 52 | if __name__ == '__main__': 53 | tf.app.run() 54 | -------------------------------------------------------------------------------- /tensorflow_serving/example/inception_k8s.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: inception-deployment 5 | spec: 6 | replicas: 3 7 | template: 8 | metadata: 9 | labels: 10 | app: inception-server 11 | spec: 12 | containers: 13 | - name: inception-container 14 | image: gcr.io/tensorflow-serving/inception:0.2.0 15 | command: 16 | - /bin/sh 17 | - -c 18 | args: 19 | - /serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server 20 | --port=9000 --model_name=inception --model_base_path=/serving/inception-export 21 | ports: 22 | - containerPort: 9000 23 | --- 24 | apiVersion: v1 25 | kind: Service 26 | metadata: 27 | labels: 28 | run: inception-service 29 | name: inception-service 30 | spec: 31 | ports: 32 | - port: 9000 33 | targetPort: 9000 34 | selector: 35 | run: inception-service 36 | type: LoadBalancer 37 | -------------------------------------------------------------------------------- /tensorflow_serving/example/obj_detector.proto: -------------------------------------------------------------------------------- 1 | // Protobuf definition of obj_detector 2 | // localisation and classification server. 3 | 4 | syntax = "proto3"; 5 | 6 | package tensorflow.serving; 7 | 8 | message Detection { 9 | // bounding box of the object in pixel-space coordinates 10 | int32 roi_x1 = 1; 11 | int32 roi_x2 = 2; 12 | int32 roi_y1 = 3; 13 | int32 roi_y2 = 4; 14 | 15 | // class label for this detection 16 | string class_label = 5; 17 | 18 | // score for this detection 19 | float score = 6; 20 | }; 21 | 22 | message DetectRequest { 23 | // input planar image with shape [c = 3, h, w] and BGR channel order 24 | bytes image_data = 1; 25 | 26 | // minimum score permitted for classification 27 | float min_score_threshold = 2; 28 | }; 29 | 30 | message DetectResponse { 31 | repeated Detection detections = 1; 32 | }; 33 | 34 | message ConfigurationRequest { 35 | }; 36 | 37 | message DetectConfiguration { 38 | // input image shape, [c, h, w] 39 | repeated int32 input_image_shape = 1; 40 | }; 41 | 42 | service DetectService { 43 | rpc GetConfiguration(ConfigurationRequest) returns (DetectConfiguration); 44 | rpc Detect(DetectRequest) returns (DetectResponse); 45 | }; 46 | -------------------------------------------------------------------------------- /tensorflow_serving/example/obj_detector_utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright IBM Corp. All Rights Reserved. */ 2 | 3 | #ifndef TENSORFLOW_SERVING_EXAMPLE_OBJ_DETECTOR_UTILS_H_ 4 | #define TENSORFLOW_SERVING_EXAMPLE_OBJ_DETECTOR_UTILS_H_ 5 | 6 | #include "tensorflow/core/framework/tensor.h" 7 | #include "tensorflow/core/framework/tensor_types.h" 8 | #include "tensorflow/core/framework/types.pb.h" 9 | #include "tensorflow/core/lib/core/status.h" 10 | #include "tensorflow/core/lib/strings/strcat.h" 11 | #include "tensorflow/core/public/session.h" 12 | 13 | #include 14 | #include 15 | 16 | // 3-channel pixel mean representation 17 | using pixel_means_type = 18 | Eigen::TensorFixedSize, Eigen::RowMajor>; 19 | 20 | // apply pixel mean subtraction to a batch of 21 | // images in planar format. 22 | tensorflow::Status BatchMeansSubtract(const pixel_means_type& means, 23 | tensorflow::Tensor* im_batch_blob); 24 | 25 | // canonical representation of an object detection 26 | struct ObjDetection { 27 | std::array roi_rect; 28 | int class_idx; 29 | float score; 30 | 31 | inline std::string DebugString() const { 32 | return tensorflow::strings::StrCat("ObjDetection { class: ", class_idx, 33 | ", score: ", score, ", rect: [", 34 | roi_rect[0], " ", roi_rect[1], " ", 35 | roi_rect[2], " ", roi_rect[3], "] }"); 36 | } 37 | }; 38 | 39 | // faster-rcnn specific utils 40 | namespace rcnn { 41 | tensorflow::Status RunClassification(const tensorflow::Tensor& im_blob, 42 | const tensorflow::Tensor& im_info, 43 | tensorflow::Session* session, 44 | tensorflow::Tensor* pred_boxes, 45 | tensorflow::Tensor* scores, 46 | tensorflow::Tensor* class_labels); 47 | 48 | tensorflow::Status ProcessDetections(const tensorflow::Tensor* pred_boxes, 49 | const tensorflow::Tensor* scores, 50 | const float detection_threshold, 51 | std::vector* dets); 52 | 53 | } // namespace rcnn 54 | 55 | #endif // TENSORFLOW_SERVING_EXAMPLE_OBJ_DETECTOR_UTILS_H_ -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/METADATA: -------------------------------------------------------------------------------- 1 | name: "TensorFlow Serving" 2 | g3doc: { 3 | sitemap_file: "_includes/nav.md" 4 | } 5 | 6 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_config.yml: -------------------------------------------------------------------------------- 1 | # Jekyll configuration file for Github Pages. If you are running a local 2 | # Jekyll server, note that his file is *NOT* reloaded automatically when you change it. If 3 | # you change this file, please restart your local server process. 4 | 5 | # Site settings 6 | title: TensorFlow Serving 7 | email: discuss@tensorflow.org 8 | description: > # this means to ignore newlines until "baseurl:" 9 | TensorFlow Serving is a flexible, high-performance serving system for 10 | machine learning models, designed for production environments. 11 | baseurl: "/serving" # the subpath of your site, e.g. /blog 12 | url: "https://tensorflow.github.io" # the base hostname & protocol for your site 13 | twitter_username: googleresearch 14 | github_username: tensorflow 15 | 16 | # Build settings 17 | markdown: kramdown 18 | defaults: 19 | - 20 | scope: 21 | path: "" 22 | values: 23 | layout: "default" 24 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/footer.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 |

{{ site.title }}

6 | 7 |
8 |
9 | 14 |
15 | 16 |
17 | 30 |
31 | 32 |
33 |

{{ site.description }}

34 |
35 |
36 | 37 |
38 | 44 | 50 |
51 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/head.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {% if page.title %}{{ page.title }}{% else %}{{ site.title }}{% endif %} 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/header.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | {{ site.title }} 5 |
6 | 7 |
8 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/icon-github.html: -------------------------------------------------------------------------------- 1 | {% include icon-github.svg %}{{ include.username }} 2 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/icon-github.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/icon-twitter.html: -------------------------------------------------------------------------------- 1 | {{ include.username }} 2 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/icon-twitter.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_includes/nav.md: -------------------------------------------------------------------------------- 1 | ## Get Started 2 | 3 | * [Architecture Overview](architecture_overview) 4 | * [Installation](setup) 5 | 6 | ## Tutorials 7 | 8 | * [TensorFlow Serving Basics](serving_basic) 9 | * [TensorFlow Serving Advanced](serving_advanced) 10 | * [Serving Inception Model with TensorFlow Serving and Kubernetes](serving_inception) 11 | 12 | ## How To 13 | 14 | * [Create a Custom Servable](custom_servable) 15 | * [Create a Custom Source](custom_source) 16 | 17 | ## Resources 18 | 19 | * [Source code](https://github.com/tensorflow/serving) 20 | * [TensorFlow site](http://tensorflow.org) 21 | * [Docker images](docker) 22 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/_layouts/default.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include head.html %} 5 | 6 | 7 | 8 | {% include header.html %} 9 |
10 |
11 |
12 | {% capture sidebar %}{% include nav.md %}{% endcapture %} 13 | {{ sidebar | markdownify }} 14 |
15 |
16 | 17 |
18 |
19 | {{ content }} 20 |
21 |
22 |
23 | {% include footer.html %} 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/css/main.scss: -------------------------------------------------------------------------------- 1 | --- 2 | # Only the main Sass file needs front matter (the dashes are enough) 3 | --- 4 | @charset "utf-8"; 5 | 6 | 7 | 8 | // Our variables 9 | $base-font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; 10 | $base-font-size: 16px; 11 | $base-font-weight: 400; 12 | $small-font-size: $base-font-size * 0.875; 13 | $base-line-height: 1.5; 14 | 15 | $header-height: 80px; 16 | 17 | $spacing-unit: 30px; 18 | 19 | $text-color: #111; 20 | $background-color: #fdfdfd; 21 | $brand-color: #f6911e; 22 | 23 | $grey-color: #828282; 24 | $grey-color-light: lighten($grey-color, 40%); 25 | $grey-color-dark: darken($grey-color, 25%); 26 | 27 | // Width of the content area 28 | $content-width: 800px; 29 | 30 | $on-palm: 600px; 31 | $on-laptop: 800px; 32 | 33 | 34 | 35 | // Use media queries like this: 36 | // @include media-query($on-palm) { 37 | // .wrapper { 38 | // padding-right: $spacing-unit / 2; 39 | // padding-left: $spacing-unit / 2; 40 | // } 41 | // } 42 | @mixin media-query($device) { 43 | @media screen and (max-width: $device) { 44 | @content; 45 | } 46 | } 47 | 48 | 49 | 50 | // Import partials from `sass_dir` (defaults to `_sass`) 51 | @import 52 | "base", 53 | "layout", 54 | "syntax-highlighting" 55 | ; 56 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/docker.md: -------------------------------------------------------------------------------- 1 | # Using TensorFlow Serving via Docker 2 | 3 | This directory contains Dockerfiles to make it easy to get up and running with 4 | TensorFlow Serving via [Docker](http://www.docker.com/). 5 | 6 | ## Installing Docker 7 | 8 | General installation instructions are 9 | [on the Docker site](https://docs.docker.com/installation/), but we give some 10 | quick links here: 11 | 12 | * [OSX](https://docs.docker.com/installation/mac/): [docker toolbox](https://www.docker.com/toolbox) 13 | * [ubuntu](https://docs.docker.com/installation/ubuntulinux/) 14 | 15 | ## Which containers exist? 16 | 17 | We currently maintain the following Dockerfiles: 18 | 19 | * [`Dockerfile.devel`](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/tools/docker/Dockerfile.devel), 20 | which is a minimal VM with all of the dependencies needed to build TensorFlow 21 | Serving. 22 | 23 | ## Building a container 24 | run; 25 | 26 | ```shell 27 | docker build --pull -t $USER/tensorflow-serving-devel -f Dockerfile.devel . 28 | ``` 29 | 30 | ## Running a container 31 | This assumes you have built the container. 32 | 33 | `Dockerfile.devel`: Use the development container to clone and test the 34 | TensorFlow Serving repository. 35 | 36 | Run the container; 37 | 38 | ```shell 39 | docker run -it $USER/tensorflow-serving-devel 40 | ``` 41 | 42 | Clone, configure and test Tensorflow Serving in the running container; 43 | 44 | ```shell 45 | git clone --recurse-submodules https://github.com/tensorflow/serving 46 | cd serving/tensorflow 47 | ./configure 48 | cd .. 49 | bazel test tensorflow_serving/... 50 | ``` 51 | 52 | -------------------------------------------------------------------------------- /tensorflow_serving/g3doc/index.md: -------------------------------------------------------------------------------- 1 | TensorFlow Serving is a flexible, high-performance serving system for machine 2 | learning models, designed for production environments. TensorFlow Serving 3 | makes it easy to deploy new algorithms and experiments, while keeping the same 4 | server architecture and APIs. TensorFlow Serving provides out-of-the-box 5 | integration with TensorFlow models, but can be easily extended to serve other 6 | types of models and data. 7 | 8 | To get started with TensorFlow Serving: 9 | 10 | * Read the [overview](architecture_overview.md) 11 | * [Set up](setup.md) your environment 12 | * Do the [basic tutorial](serving_basic.md) 13 | 14 | 15 | 16 | ![TensorFlow Serving Diagram](images/tf_diagram.svg){: width="500"} 17 | -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/model_platform_caffe.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CAFFE_H_ 17 | #define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CAFFE_H_ 18 | 19 | #include "tensorflow_serving/model_servers/model_platform_types.h" 20 | 21 | #include "tensorflow_serving/servables/caffe/predict_impl.h" 22 | #include "tensorflow_serving/servables/caffe/caffe_source_adapter.h" 23 | 24 | template <> 25 | struct ModelPlatformTraits { 26 | using SourceAdapter = tensorflow::serving::CaffeSourceAdapter; 27 | using SourceAdapterConfig = tensorflow::serving::CaffeSourceAdapterConfig; 28 | using PredictImpl = tensorflow::serving::CaffePredictImpl; 29 | 30 | static const char* name() { return "caffe"; } 31 | static constexpr bool defined = true; 32 | 33 | static void GlobalInit(int argc, char** argv) { 34 | tensorflow::serving::CaffeGlobalInit(&argc, &argv); 35 | } 36 | 37 | static tensorflow::Status ConfigureSourceAdapter( 38 | bool enable_batching, SourceAdapterConfig* config) { 39 | if (enable_batching) { 40 | tensorflow::serving::BatchingParameters* batching_parameters = 41 | config->mutable_config()->mutable_batching_parameters(); 42 | batching_parameters->mutable_thread_pool_name()->set_value( 43 | "model_server_batch_threads"); 44 | } 45 | return tensorflow::Status::OK(); 46 | } 47 | 48 | template 49 | static tensorflow::Status CreateSourceAdapter( 50 | const SourceAdapterConfig& config, std::unique_ptr* adapter) { 51 | std::unique_ptr typed_adapter; 52 | const auto status = SourceAdapter::Create(config, &typed_adapter); 53 | 54 | if (status.ok()) { 55 | *adapter = std::move(typed_adapter); 56 | } 57 | return status; 58 | } 59 | }; 60 | 61 | #endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CAFFE_H_ -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/model_platform_tensorflow.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TENSORFLOW_H_ 17 | #define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TENSORFLOW_H_ 18 | 19 | #include "tensorflow_serving/model_servers/model_platform_types.h" 20 | 21 | #include "tensorflow/core/platform/init_main.h" 22 | #include "tensorflow_serving/servables/tensorflow/predict_impl.h" 23 | #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" 24 | 25 | template <> 26 | struct ModelPlatformTraits { 27 | static const char* name() { return kTensorFlowModelPlatform; } 28 | static constexpr bool defined = true; 29 | 30 | using SourceAdapter = tensorflow::serving::SessionBundleSourceAdapter; 31 | using SourceAdapterConfig = tensorflow::serving::SessionBundleSourceAdapterConfig; 32 | using PredictImpl = tensorflow::serving::TensorflowPredictImpl; 33 | 34 | static void GlobalInit(int argc, char** argv) { 35 | tensorflow::port::InitMain(argv[0], &argc, &argv); 36 | } 37 | 38 | static tensorflow::Status ConfigureSourceAdapter( 39 | bool enable_batching, SourceAdapterConfig* config) { 40 | if (enable_batching) { 41 | tensorflow::serving::BatchingParameters* batching_parameters = 42 | config->mutable_config()->mutable_batching_parameters(); 43 | batching_parameters->mutable_thread_pool_name()->set_value( 44 | "model_server_batch_threads"); 45 | } 46 | return tensorflow::Status::OK(); 47 | } 48 | 49 | template 50 | static tensorflow::Status CreateSourceAdapter( 51 | const SourceAdapterConfig& config, std::unique_ptr* adapter) { 52 | std::unique_ptr typed_adapter; 53 | const auto status = SourceAdapter::Create(config, &typed_adapter); 54 | 55 | if (status.ok()) { 56 | *adapter = std::move(typed_adapter); 57 | } 58 | return status; 59 | } 60 | }; 61 | 62 | #endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TENSORFLOW_H_ 63 | -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/model_platform_types.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ 17 | #define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ 18 | 19 | // individual model platforms should derive from ModelPlatform 20 | struct ModelPlatform {}; 21 | 22 | // ModelPlatformTraits should be specialized on a per model-platform basis 23 | template 24 | struct ModelPlatformTraits { 25 | static constexpr bool defined = false; 26 | static const char* name() { return "undefined"; } 27 | static void GlobalInit(int argc, char** argv) {} 28 | }; 29 | 30 | // TODO(): merge in to ModelPlatformTraits? 31 | constexpr char kTensorFlowModelPlatform[] = "tensorflow"; 32 | constexpr char kOtherModelPlatform[] = "other"; 33 | 34 | // TensorFlow model platform 35 | struct TensorFlow : ModelPlatform {}; 36 | 37 | // Caffe model platform 38 | struct Caffe : ModelPlatform {}; 39 | 40 | #endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ 41 | -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/tensorflow_model_server_test_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | #!/usr/bin/env python2.7 17 | 18 | """Manual test client for tensorflow_model_server.""" 19 | 20 | # This is a placeholder for a Google-internal import. 21 | 22 | from grpc.beta import implementations 23 | import tensorflow as tf 24 | 25 | from tensorflow.core.framework import types_pb2 26 | from tensorflow.python.platform import flags 27 | from tensorflow_serving.apis import predict_pb2 28 | from tensorflow_serving.apis import prediction_service_pb2 29 | 30 | 31 | tf.app.flags.DEFINE_string('server', 'localhost:8500', 32 | 'inception_inference service host:port') 33 | FLAGS = tf.app.flags.FLAGS 34 | 35 | 36 | def main(_): 37 | # Prepare request 38 | request = predict_pb2.PredictRequest() 39 | request.model_spec.name = 'default' 40 | request.inputs['x'].dtype = types_pb2.DT_FLOAT 41 | request.inputs['x'].float_val.append(2.0) 42 | request.output_filter.append('y') 43 | # Send request 44 | host, port = FLAGS.server.split(':') 45 | channel = implementations.insecure_channel(host, int(port)) 46 | stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) 47 | print stub.Predict(request, 5.0) # 5 secs timeout 48 | 49 | 50 | if __name__ == '__main__': 51 | tf.app.run() 52 | -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/test_util/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Model Server test utility 2 | 3 | package( 4 | default_visibility = [ 5 | "//tensorflow_serving:internal", 6 | ], 7 | features = ["-layering_check"], 8 | ) 9 | 10 | licenses(["notice"]) # Apache 2.0 11 | 12 | exports_files(["LICENSE"]) 13 | 14 | filegroup( 15 | name = "all_files", 16 | srcs = glob( 17 | ["**/*"], 18 | exclude = [ 19 | "**/METADATA", 20 | "**/OWNERS", 21 | ], 22 | ), 23 | ) 24 | 25 | cc_library( 26 | name = "mock_server_core", 27 | testonly = 1, 28 | hdrs = ["mock_server_core.h"], 29 | visibility = [ 30 | "//visibility:public", 31 | ], 32 | deps = [ 33 | "//tensorflow_serving/apis:model_proto", 34 | "//tensorflow_serving/config:model_server_config_proto", 35 | "//tensorflow_serving/core:servable_handle", 36 | "//tensorflow_serving/core:servable_state", 37 | "//tensorflow_serving/model_servers:server_core", 38 | "@org_tensorflow//tensorflow/core:lib", 39 | "@org_tensorflow//tensorflow/core:test", 40 | ], 41 | ) 42 | 43 | cc_library( 44 | name = "server_core_test_util", 45 | testonly = 1, 46 | srcs = ["server_core_test_util.cc"], 47 | hdrs = ["server_core_test_util.h"], 48 | data = [ 49 | "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", 50 | "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", 51 | ], 52 | visibility = [ 53 | "//visibility:public", 54 | ], 55 | deps = [ 56 | "//tensorflow_serving/core:eager_load_policy", 57 | "//tensorflow_serving/core:servable_id", 58 | "//tensorflow_serving/core/test_util:fake_loader_source_adapter", 59 | "//tensorflow_serving/model_servers:model_platform_types", 60 | "//tensorflow_serving/model_servers:server_core", 61 | "//tensorflow_serving/test_util", 62 | "@org_tensorflow//tensorflow/core:test", 63 | ], 64 | ) 65 | -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/test_util/mock_server_core.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // GMock implementation of ServerCore. 17 | #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_MOCK_SERVER_CORE_H_ 18 | #define TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_MOCK_SERVER_CORE_H_ 19 | 20 | #include 21 | #include "tensorflow/core/lib/core/status.h" 22 | #include "tensorflow/core/platform/logging.h" 23 | #include "tensorflow_serving/apis/model.proto.h" 24 | #include "tensorflow_serving/config/model_server_config.proto.h" 25 | #include "tensorflow_serving/core/servable_handle.h" 26 | #include "tensorflow_serving/core/servable_state.h" 27 | #include "tensorflow_serving/model_servers/server_core.h" 28 | 29 | namespace tensorflow { 30 | namespace serving { 31 | namespace test_util { 32 | 33 | class MockServerCore : public ServerCore { 34 | public: 35 | static Options GetOptions( 36 | const SourceAdapterCreator& source_adapter_creator) { 37 | Options options; 38 | options.source_adapter_creator = source_adapter_creator; 39 | options.servable_state_monitor_creator = []( 40 | EventBus* event_bus, 41 | std::unique_ptr* monitor) -> Status { 42 | monitor->reset(new ServableStateMonitor(event_bus)); 43 | return Status::OK(); 44 | }; 45 | options.custom_model_config_loader = []( 46 | const ::google::protobuf::Any& any, EventBus* event_bus, 47 | UniquePtrWithDeps* manager) -> Status { 48 | return Status::OK(); 49 | }; 50 | return options; 51 | } 52 | 53 | explicit MockServerCore(const SourceAdapterCreator& source_adapter_creator) 54 | : ServerCore(GetOptions(source_adapter_creator)) {} 55 | 56 | MOCK_CONST_METHOD0(servable_state_monitor, ServableStateMonitor*()); 57 | MOCK_METHOD1(ReloadConfig, Status(const ModelServerConfig&)); 58 | 59 | template 60 | Status GetServableHandle(const ModelSpec& model_spec, 61 | ServableHandle* const handle) { 62 | LOG(FATAL) << "Not implemented."; 63 | } 64 | }; 65 | 66 | } // namespace test_util 67 | } // namespace serving 68 | } // namespace tensorflow 69 | 70 | #endif // TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_MOCK_SERVER_CORE_H_ 71 | -------------------------------------------------------------------------------- /tensorflow_serving/model_servers/test_util/server_core_test_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ 17 | #define TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ 18 | 19 | #include 20 | #include "tensorflow_serving/core/servable_id.h" 21 | #include "tensorflow_serving/model_servers/server_core.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | namespace test_util { 26 | 27 | constexpr char kTestModelName[] = "test_model"; 28 | constexpr int kTestModelVersion = 123; 29 | 30 | class ServerCoreTest : public ::testing::Test { 31 | protected: 32 | // Returns ModelServerConfig that contains test model. 33 | ModelServerConfig GetTestModelServerConfig(); 34 | 35 | // Create a ServerCore object configured to use FakeLoaderSourceAdapter. 36 | Status CreateServerCore(const ModelServerConfig& config, 37 | std::unique_ptr* server_core); 38 | 39 | // Create a ServerCore object with the supplied SourceAdapterCreator. 40 | Status CreateServerCore( 41 | const ModelServerConfig& config, 42 | const ServerCore::SourceAdapterCreator& source_adapter_creator, 43 | std::unique_ptr* server_core); 44 | 45 | // Create a ServerCore object with the supplied options. The ServerCore uses 46 | // continuous polling to speed up testing. 47 | Status CreateServerCore(ServerCore::Options options, 48 | std::unique_ptr* server_core); 49 | }; 50 | 51 | } // namespace test_util 52 | } // namespace serving 53 | } // namespace tensorflow 54 | #endif // TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ 55 | -------------------------------------------------------------------------------- /tensorflow_serving/resources/BUILD: -------------------------------------------------------------------------------- 1 | package( 2 | default_visibility = [ 3 | "//tensorflow_serving:internal", 4 | ], 5 | features = ["-layering_check"], 6 | ) 7 | 8 | licenses(["notice"]) # Apache 2.0 9 | 10 | exports_files(["LICENSE"]) 11 | 12 | filegroup( 13 | name = "all_files", 14 | srcs = glob( 15 | ["**/*"], 16 | exclude = [ 17 | "**/METADATA", 18 | "**/OWNERS", 19 | "g3doc/sitemap.md", 20 | ], 21 | ), 22 | ) 23 | 24 | load("//tensorflow_serving:serving.bzl", "serving_proto_library") 25 | 26 | serving_proto_library( 27 | name = "resources_proto", 28 | srcs = ["resources.proto"], 29 | cc_api_version = 2, 30 | visibility = ["//visibility:public"], 31 | deps = ["@protobuf//:cc_wkt_protos"], 32 | ) 33 | 34 | cc_library( 35 | name = "resource_values", 36 | srcs = ["resource_values.cc"], 37 | hdrs = ["resource_values.h"], 38 | visibility = ["//visibility:public"], 39 | deps = [ 40 | ], 41 | ) 42 | 43 | cc_library( 44 | name = "resource_util", 45 | srcs = ["resource_util.cc"], 46 | hdrs = ["resource_util.h"], 47 | visibility = ["//visibility:public"], 48 | deps = [ 49 | ":resources_proto", 50 | "@org_tensorflow//tensorflow/core:lib", 51 | ], 52 | ) 53 | 54 | cc_test( 55 | name = "resource_util_test", 56 | size = "small", 57 | srcs = ["resource_util_test.cc"], 58 | deps = [ 59 | ":resource_util", 60 | "//tensorflow_serving/core/test_util:test_main", 61 | "//tensorflow_serving/test_util", 62 | ], 63 | ) 64 | 65 | cc_library( 66 | name = "resource_tracker", 67 | srcs = ["resource_tracker.cc"], 68 | hdrs = ["resource_tracker.h"], 69 | deps = [ 70 | ":resource_util", 71 | ":resources_proto", 72 | "//tensorflow_serving/core:loader", 73 | "@org_tensorflow//tensorflow/core:lib", 74 | ], 75 | ) 76 | 77 | cc_test( 78 | name = "resource_tracker_test", 79 | size = "small", 80 | srcs = ["resource_tracker_test.cc"], 81 | deps = [ 82 | ":resource_tracker", 83 | "//tensorflow_serving/core/test_util:mock_loader", 84 | "//tensorflow_serving/core/test_util:test_main", 85 | "//tensorflow_serving/test_util", 86 | ], 87 | ) 88 | -------------------------------------------------------------------------------- /tensorflow_serving/resources/resource_values.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/resources/resource_values.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | namespace device_types { 22 | const char* const kMain = "main"; 23 | const char* const kGpu = "gpu"; 24 | } // namespace device_types 25 | 26 | namespace resource_kinds { 27 | const char* const kRamBytes = "ram_in_bytes"; 28 | const char* const kProcessingMillis = "processing_in_millicores"; 29 | } // namespace resource_kinds 30 | 31 | } // namespace serving 32 | } // namespace tensorflow 33 | -------------------------------------------------------------------------------- /tensorflow_serving/resources/resource_values.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // Standard values to describe resources, to be used for the string types in 17 | // resources.proto. These values should be used whenever applicable, to avoid 18 | // vocabulary mismatches. 19 | 20 | #ifndef TENSORFLOW_SERVING_RESOURCES_RESOURCE_VALUES_H_ 21 | #define TENSORFLOW_SERVING_RESOURCES_RESOURCE_VALUES_H_ 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | 26 | // Standard device types. 27 | namespace device_types { 28 | 29 | // CPU(s) and main memory. 30 | extern const char* const kMain; 31 | 32 | // Graphics processing unit(s). 33 | extern const char* const kGpu; 34 | 35 | } // namespace device_types 36 | 37 | // Standard resource kinds. 38 | namespace resource_kinds { 39 | 40 | // RAM in bytes. 41 | extern const char* const kRamBytes; 42 | 43 | // Fraction of a processing unit's cycles, in thousandths. 44 | extern const char* const kProcessingMillis; 45 | 46 | } // namespace resource_kinds 47 | 48 | } // namespace serving 49 | } // namespace tensorflow 50 | 51 | #endif // TENSORFLOW_SERVING_RESOURCES_RESOURCE_VALUES_H_ 52 | -------------------------------------------------------------------------------- /tensorflow_serving/resources/resources.proto: -------------------------------------------------------------------------------- 1 | // Representations for resources used by servables, and available in a system. 2 | // 3 | // Each of the string-typed values are free-form, so that they can be extended 4 | // by third parties. However we strongly recommend using the values defined in 5 | // resource_values.h when possible, for standardization. 6 | 7 | syntax = "proto3"; 8 | 9 | import "google/protobuf/wrappers.proto"; 10 | 11 | package tensorflow.serving; 12 | 13 | // One kind of resource on one device (or type of device). 14 | message Resource { 15 | // The type of device on which the resource resides, e.g. CPU or GPU. 16 | string device = 1; 17 | 18 | // A specific instance of the device of type 'device' to which the resources 19 | // are bound (instances are assumed to be numbered 0, 1, ...). 20 | // 21 | // When representing the resources required by a servable that has yet to be 22 | // loaded, this field is optional. If not set, it denotes that the servable's 23 | // resources are not (yet) bound to a specific instance. 24 | google.protobuf.UInt32Value device_instance = 2; 25 | 26 | // The kind of resource on the device (instance), e.g. RAM or compute share. 27 | // 28 | // A given type of resource should have a standard unit that represents the 29 | // smallest useful quantization. We strongly recommend including the unit 30 | // (e.g. bytes or millicores) in this string, as in "ram_bytes". 31 | string kind = 3; 32 | } 33 | 34 | // An allocation of one or more kinds of resources, along with the quantity of 35 | // each. Used to denote the resources that a servable (or collection of 36 | // servables) will use or is currently using. Also used to denote resources 37 | // available to the serving system for loading more servables. 38 | message ResourceAllocation { 39 | // A collection of resources, each with a quantity. Treated as a resource-> 40 | // quantity map, i.e. no resource can repeat and the order is immaterial. 41 | message Entry { 42 | Resource resource = 1; 43 | uint64 quantity = 2; 44 | } 45 | repeated Entry resource_quantities = 1; 46 | } 47 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_py_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_CAFFE_PY_CAFFE_UTIL_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_CAFFE_PY_CAFFE_UTIL_H_ 18 | 19 | #include "tensorflow/core/lib/core/status.h" 20 | 21 | namespace tensorflow { 22 | namespace serving { 23 | 24 | // Returns true if Caffe was built with Python layer. 25 | bool IsPyCaffeAvailable(); 26 | 27 | // Ensure python is loaded and initialized with the caffe 28 | // wrapper module. 29 | tensorflow::Status EnsurePyCaffeInitialized(); 30 | 31 | // frees all memory allocated by the Python interpreter. 32 | // errors during finalization are ignored. 33 | tensorflow::Status FinalizePyCaffe(); 34 | 35 | // Ensure the given path is included in the python 36 | // module search path (sys.path) 37 | tensorflow::Status EnsurePyCaffeSystemPath(const string& path); 38 | 39 | // returns an error if python has an error currently set 40 | tensorflow::Status PythonStatus(); 41 | 42 | } // namespace serving 43 | } // namespaces tensorflow 44 | 45 | #endif // TENSORFLOW_SERVING_SERVABLES_CAFFE_PY_CAFFE_UTIL_H_ -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_session_bundle.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | 17 | #ifndef TENSORFLOW_SERVING_SERVABLES_CAFFE_SESSION_BUNDLE_H_ 18 | #define TENSORFLOW_SERVING_SERVABLES_CAFFE_SESSION_BUNDLE_H_ 19 | 20 | #include 21 | 22 | #include "tensorflow/core/lib/core/status.h" 23 | #include "tensorflow/core/lib/core/stringpiece.h" 24 | #include "tensorflow/core/public/session.h" 25 | #include "tensorflow_serving/servables/caffe/caffe_serving_session.h" 26 | 27 | #include "tensorflow/core/framework/tensor.pb.h" 28 | 29 | namespace tensorflow { 30 | namespace serving { 31 | 32 | // Low-level functionality for setting up a Caffe inference Session. 33 | const char kGraphDefFilename[] = "deploy.prototxt"; 34 | const char kVariablesFilename[] = "weights.caffemodel"; 35 | const char kGraphTxtLabelFilename[] = "classlabels.txt"; 36 | 37 | // A global initialization function that you should call in your main function. 38 | // Currently it just invokes caffe::GlobalInit(..) 39 | void CaffeGlobalInit(int* pargc, char*** pargv); 40 | 41 | // Remarks: Very roughly equivalent to a TF session bundle 42 | struct CaffeSessionBundle { 43 | std::unique_ptr session; 44 | CaffeMetaGraphDef meta_graph_def; 45 | }; 46 | 47 | // Loads a manifest and initialized session using the output of an Exporter 48 | tensorflow::Status LoadSessionBundleFromPath( 49 | const CaffeSessionOptions& options, 50 | const tensorflow::StringPiece export_dir, CaffeSessionBundle* bundle); 51 | 52 | } // namespace serving 53 | } // namespace tensorflow 54 | 55 | #endif // TENSORFLOW_SERVING_SERVABLES_CAFFE_SESSION_BUNDLE_H_ -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_session_bundle_config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | // include TFS session_bundle_config for batching paramaters 4 | import "tensorflow_serving/servables/tensorflow/session_bundle_config.proto"; 5 | import "tensorflow/core/framework/tensor_shape.proto"; 6 | 7 | package tensorflow.serving; 8 | 9 | // Configuration parameters for a Caffe SessionBundle, with optional batching. 10 | // Note; Caffe doesn't have the concept of a session bungle (potentially a 11 | // collection of graphs) but we use the same terminology. 12 | message CaffeSessionBundleConfig { 13 | // Force Caffe to execute in CPU-only mode 14 | bool force_cpu_only = 1; 15 | 16 | // Force Caffe to use a particular GPU, if any. 17 | int32 force_gpu_id = 2; 18 | 19 | // enable python layers for this session bundle. Only has an effect 20 | // when caffe is configured with python layers. 21 | bool enable_py_caffe = 4; 22 | 23 | // Append the python module search paths. Only has an effect when 24 | // caffe is configured with python layers. 25 | repeated string python_path = 5; 26 | 27 | // Initial shape; if set, a loaded network will have its input blob 28 | // reshaped before the first request. If the network has more than one 29 | // input, this will cause an error. 30 | TensorShapeProto initial_shape = 6; 31 | 32 | // same as initial_shape but for specific blobs in the network. Useful for 33 | // exotic network architectures with multiple inputs. 34 | map named_initial_shapes = 7; 35 | 36 | // If set, each emitted session is wrapped with a layer that schedules Run() 37 | // calls in batches. The batching layer is transparent to the client 38 | // (implements the tensorflow::Session API). 39 | // 40 | // IMPORTANT: With batching enabled, client threads will spend most of their 41 | // time blocked on Session::Run() calls, waiting for enough peer threads to 42 | // also call Session::Run() such that a large batch can be formed. For good 43 | // throughput, we recommend setting the number of client threads equal to 44 | // roughly twice the maximum batch size ('max_batch_size' below). 45 | // 46 | // The batching layer uses a SharedBatchScheduler to coordinate batching 47 | // across multiple session servables emitted by this source adapter. A 48 | // BatchSchedulerRetrier is added on top of each batching session. 49 | BatchingParameters batching_parameters = 3; 50 | } -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_signature.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_CAFFE_SIGNATURE_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_CAFFE_SIGNATURE_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "tensorflow/core/framework/tensor.h" 24 | #include "tensorflow/core/lib/core/status.h" 25 | #include "tensorflow/contrib/session_bundle/manifest.pb.h" 26 | 27 | #include "tensorflow_serving/servables/caffe/caffe_serving_session.h" 28 | 29 | namespace tensorflow { 30 | namespace serving { 31 | 32 | // Get Signatures from a MetaGraphDef. 33 | Status GetSignatures(const CaffeMetaGraphDef& meta_graph_def, 34 | Signatures* signatures); 35 | 36 | // Gets the default signature from a MetaGraphDef. 37 | Status GetDefaultSignature(const CaffeMetaGraphDef& meta_graph_def, 38 | Signature* default_signature); 39 | 40 | Status GetNamedSignature(const string& name, 41 | const CaffeMetaGraphDef& meta_graph_def, 42 | Signature* signature); 43 | 44 | // Gets a ClassificationSignature from a MetaGraphDef's default signature. 45 | // Returns an error if the default signature is not a ClassificationSignature, 46 | // or does not exist. 47 | Status GetClassificationSignature(const CaffeMetaGraphDef& meta_graph_def, 48 | ClassificationSignature* signature); 49 | 50 | // Gets a RegressionSignature from a MetaGraphDef's default signature. 51 | // Returns an error if the default signature is not a RegressionSignature, 52 | // or does not exist. 53 | Status GetRegressionSignature(const CaffeMetaGraphDef& meta_graph_def, 54 | RegressionSignature* signature); 55 | 56 | } // namespace 57 | } // namespace 58 | 59 | #endif // TENSORFLOW_SERVING_SERVABLES_CAFFE_SIGNATURE_H_ -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_simple_servers.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // Bootstrapping and configuration utilities for creating simple servers of 17 | // Caffe models. Intended for basic instantiation with default configs. 18 | // 19 | // Note: All methods expect Caffe exports conforming to the export format 20 | // specified at tensorflow_serving/servables/tensorflow/README.md. 21 | 22 | #ifndef TENSORFLOW_SERVING_SERVABLES_CAFFE_CAFFE_SIMPLE_SERVERS_H_ 23 | #define TENSORFLOW_SERVING_SERVABLES_CAFFE_CAFFE_SIMPLE_SERVERS_H_ 24 | 25 | #include 26 | #include 27 | 28 | #include "tensorflow/core/lib/core/status.h" 29 | #include "tensorflow/core/platform/types.h" 30 | #include "tensorflow_serving/core/manager.h" 31 | #include "tensorflow_serving/core/storage_path.h" 32 | #include "tensorflow_serving/core/target.h" 33 | 34 | #include "tensorflow_serving/servables/caffe/caffe_source_adapter.pb.h" 35 | 36 | namespace tensorflow { 37 | namespace serving { 38 | namespace simple_servers { 39 | 40 | // TODO(b/25969594): Add full test coverage. 41 | 42 | // Creates a Manager and associated Source for a single Caffe model that 43 | // automatically loads new versions over time. All versions of the model will be 44 | // loaded from new directories under the specified base path. Uses default 45 | // SessionOptions. 46 | // 47 | // The servables loaded and served from this manager are of type 48 | // tensorflow::serving::SessionBundle. 49 | // 50 | // When new versions arrive the Manager will unload the previous version before 51 | // loading the new version. This is preferable from a resource utilization 52 | // perspective, but has reduced availability. 53 | Status CreateSingleCaffeModelManagerFromBasePath( 54 | const string& base_path, 55 | std::unique_ptr* manager); 56 | 57 | Status CreateSingleCaffeModelManagerFromBasePath( 58 | const string& base_path, 59 | const CaffeSourceAdapterConfig& source_adapter_config, 60 | std::unique_ptr* manager); 61 | 62 | } // namespace simple_servers 63 | } // namespace serving 64 | } // namespace tensorflow 65 | 66 | #endif // TENSORFLOW_SERVING_SERVABLES_CAFFE_CAFFE_SIMPLE_SERVERS_H_ -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_simple_servers_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/servables/caffe/caffe_simple_servers.h" 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include "tensorflow/core/framework/tensor.h" 25 | #include "tensorflow/core/framework/tensor_testutil.h" 26 | #include "tensorflow/core/lib/core/status.h" 27 | #include "tensorflow/core/lib/core/status_test_util.h" 28 | #include "tensorflow/core/lib/core/stringpiece.h" 29 | #include "tensorflow/core/lib/io/path.h" 30 | #include "tensorflow/core/platform/env.h" 31 | #include "tensorflow/core/public/session.h" 32 | #include "tensorflow_serving/core/servable_handle.h" 33 | #include "tensorflow_serving/servables/caffe/caffe_session_bundle.h" 34 | #include "tensorflow_serving/test_util/test_util.h" 35 | 36 | namespace tensorflow { 37 | namespace serving { 38 | namespace { 39 | 40 | class SimpleServersTest : public ::testing::Test { 41 | protected: 42 | SimpleServersTest() 43 | : test_data_path_(test_util::TestSrcDirPath( 44 | "servables/caffe/test_data/mnist_pretrained_caffe")) { 45 | } 46 | 47 | const string test_data_path_; 48 | }; 49 | 50 | TEST_F(SimpleServersTest, Basic) { 51 | std::unique_ptr manager; 52 | const Status status = simple_servers::CreateSingleCaffeModelManagerFromBasePath( 53 | test_data_path_, &manager); 54 | TF_CHECK_OK(status); 55 | // We wait until the manager starts serving the servable. 56 | // TODO(b/25545570): Use the waiter api when it's ready. 57 | while (manager->ListAvailableServableIds().empty()) { 58 | Env::Default()->SleepForMicroseconds(1000); 59 | } 60 | ServableHandle bundle; 61 | const Status handle_status = 62 | manager->GetServableHandle(ServableRequest::Latest("default"), &bundle); 63 | 64 | TF_CHECK_OK(handle_status); 65 | } 66 | 67 | } // namespace 68 | } // namespace serving 69 | } // namespace tensorflow 70 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_source_adapter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_CAFFE_CAFFE_SOURCE_ADAPTER_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_CAFFE_CAFFE_SOURCE_ADAPTER_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "tensorflow_serving/core/simple_loader.h" 23 | #include "tensorflow_serving/core/source_adapter.h" 24 | #include "tensorflow_serving/core/storage_path.h" 25 | #include "tensorflow_serving/servables/caffe/caffe_source_adapter.pb.h" 26 | #include "tensorflow_serving/servables/caffe/caffe_session_bundle_factory.h" 27 | 28 | namespace tensorflow { 29 | namespace serving { 30 | 31 | class CaffeSourceAdapter 32 | : public UnarySourceAdapter> { 33 | public: 34 | static Status Create(const CaffeSourceAdapterConfig& config, 35 | std::unique_ptr* adapter); 36 | 37 | ~CaffeSourceAdapter() override; 38 | 39 | // Returns a function to create a session bundle source adapter. 40 | static std::function>>*)> 42 | GetCreator(const CaffeSourceAdapterConfig& config); 43 | 44 | private: 45 | explicit CaffeSourceAdapter( 46 | std::unique_ptr bundle_factory); 47 | 48 | Status Convert(const StoragePath& path, 49 | std::unique_ptr* loader) override; 50 | 51 | std::unique_ptr bundle_factory_; 52 | 53 | TF_DISALLOW_COPY_AND_ASSIGN(CaffeSourceAdapter); 54 | }; 55 | 56 | } // namespace serving 57 | } // namespace tensorflow 58 | 59 | #endif // TENSORFLOW_SERVING_SERVABLES_CAFFE_CAFFE_SOURCE_ADAPTER_H_ -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/caffe_source_adapter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "tensorflow_serving/servables/caffe/caffe_session_bundle_config.proto"; 4 | 5 | package tensorflow.serving; 6 | 7 | // Config proto for SessionBundleSourceAdapter. 8 | message CaffeSourceAdapterConfig { 9 | CaffeSessionBundleConfig config = 1; 10 | } -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/predict_impl.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_CAFFE_PREDICT_IMPL_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_CAFFE_PREDICT_IMPL_H_ 18 | 19 | #include "tensorflow/core/lib/core/status.h" 20 | #include "tensorflow/core/public/session.h" 21 | 22 | #include "tensorflow_serving/apis/predict.pb.h" 23 | #include "tensorflow_serving/model_servers/server_core.h" 24 | 25 | namespace tensorflow { 26 | namespace serving { 27 | 28 | // Utility methods for implementation of PredictionService::Predict. 29 | // TODO(30682232): add unit test for CaffePredictImpl. 30 | class CaffePredictImpl { 31 | public: 32 | static Status Predict(ServerCore* core, const PredictRequest& request, 33 | PredictResponse* response); 34 | }; 35 | 36 | } // namespace serving 37 | } // namespace tensorflow 38 | 39 | #endif // TENSORFLOW_SERVING_SERVABLES_CAFFE_PREDICT_IMPL_H_ -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/simple_thread_sink.cc: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "simple_thread_sink.h" 25 | 26 | // the constructor just launches some amount of workers 27 | SimpleThreadSink::SimpleThreadSink() : stop_(false) { 28 | worker_ = std::thread([this] { 29 | for (;;) { 30 | std::function task; 31 | { 32 | std::unique_lock lock(this->queue_mutex_); 33 | this->condition_.wait( 34 | lock, [this] { return this->stop_ || !this->tasks_.empty(); }); 35 | 36 | if (this->stop_ && this->tasks_.empty()) return; 37 | 38 | task = std::move(this->tasks_.front()); 39 | this->tasks_.pop(); 40 | } 41 | task(); 42 | } 43 | }); 44 | } 45 | 46 | // stop and join the worker thread 47 | SimpleThreadSink::~SimpleThreadSink() { 48 | { 49 | std::unique_lock lock(queue_mutex_); 50 | stop_ = true; 51 | condition_.notify_all(); 52 | } 53 | worker_.join(); 54 | } 55 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/simple_thread_sink.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef SIMPLE_THREAD_SINK_H_ 17 | #define SIMPLE_THREAD_SINK_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | // SimpleThreadSink is A threadpool with a single worker. All tasks 28 | // issued to the sink are guaranteed to run on the same thread. 29 | // 30 | // Remarks: 31 | // This is useful for interoperating with library code which has 32 | // thread-local state, from within a multi-threaded application. 33 | // 34 | class SimpleThreadSink { 35 | public: 36 | SimpleThreadSink(); 37 | 38 | template 39 | auto run(F&& f, Args&&... args) -> typename std::result_of::type; 40 | 41 | ~SimpleThreadSink(); 42 | 43 | private: 44 | // joinable worker 45 | std::thread worker_; 46 | // the task queue 47 | std::queue> tasks_; 48 | 49 | // synchronization 50 | std::mutex queue_mutex_; 51 | std::condition_variable condition_; 52 | bool stop_; 53 | }; 54 | 55 | ////////// 56 | // Implementation details follow. API users need not read. 57 | 58 | // add new work item to the pool and block until its complete 59 | template 60 | auto SimpleThreadSink::run(F&& f, Args&&... args) -> 61 | typename std::result_of::type { 62 | using return_type = typename std::result_of::type; 63 | 64 | auto task = std::packaged_task( 65 | std::bind(std::forward(f), std::forward(args)...)); 66 | 67 | std::future res = task.get_future(); 68 | { 69 | std::unique_lock lock(queue_mutex_); 70 | tasks_.emplace([&task]() { (task)(); }); 71 | condition_.notify_one(); 72 | } 73 | return res.get(); 74 | } 75 | 76 | #endif -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/test_data/BUILD: -------------------------------------------------------------------------------- 1 | package( 2 | default_visibility = ["//tensorflow_serving:internal"], 3 | ) 4 | 5 | filegroup( 6 | name = "py_layers", 7 | srcs = [ 8 | "py_layers/00000001/deploy.prototxt", 9 | "py_layers/00000001/test_python_layer.py", 10 | "py_layers/00000001/weights.caffemodel", 11 | ], 12 | ) 13 | 14 | py_binary( 15 | name = "mnist_caffe_fetch", 16 | srcs = ["mnist_caffe_fetch.py"], 17 | ) 18 | 19 | genrule( 20 | name = "mnist_pretrained_caffe", 21 | cmd = ''' 22 | rm -rf $(@D)/mnist_pretrained_caffe/00000023; 23 | $(locations :mnist_caffe_fetch) --version=23 $(@D)/mnist_pretrained_caffe; 24 | ''', 25 | outs = [ 26 | "mnist_pretrained_caffe/00000023/deploy.prototxt", 27 | "mnist_pretrained_caffe/00000023/weights.caffemodel", 28 | "mnist_pretrained_caffe/00000023/classlabels.txt", 29 | ], 30 | tools = [ 31 | ":mnist_caffe_fetch", 32 | ], 33 | ) 34 | 35 | cc_library( 36 | name = "mnist_sample", 37 | hdrs = ["mnist_sample.h"] 38 | ) 39 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/test_data/mnist_caffe_fetch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | #!/usr/bin/python2.7 17 | 18 | """Functions for downloading and extracting pretrained MNIST caffe models.""" 19 | from __future__ import print_function 20 | 21 | import argparse 22 | import tarfile 23 | import os 24 | 25 | from six.moves import urllib 26 | 27 | VERSION_FORMAT_SPECIFIER = "%08d" 28 | SOURCE_URL = 'https://github.com/rayglover-ibm/serving-caffe/raw/pretrained-models/mnist_pretrained_caffe.tar' 29 | OUT_FILE = 'mnist_pretrained_caffe.tar' 30 | MODEL_FILES = ['classlabels.txt', 'deploy.prototxt', 'weights.caffemodel'] 31 | 32 | def maybe_download(url, filename, work_directory): 33 | """Download the data""" 34 | if not os.path.exists(work_directory): 35 | os.mkdir(work_directory) 36 | 37 | filepath = os.path.join(work_directory, filename) 38 | if not os.path.exists(filepath): 39 | filepath, _ = urllib.request.urlretrieve(SOURCE_URL, filepath) 40 | statinfo = os.stat(filepath) 41 | print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') 42 | 43 | return filepath 44 | 45 | if __name__ == '__main__': 46 | parser = argparse.ArgumentParser() 47 | parser.add_argument("export_path", help="location to download and extract the model") 48 | parser.add_argument("--version", type=int, default=1, help="model version") 49 | args = parser.parse_args() 50 | 51 | export_dir = os.path.join(args.export_path, VERSION_FORMAT_SPECIFIER % args.version) 52 | if os.path.exists(export_dir): 53 | raise RuntimeError("Overwriting exports can cause corruption and are " 54 | "not allowed. Duplicate export dir: %s" % export_dir) 55 | 56 | os.makedirs(export_dir) 57 | 58 | print('Downloading...', SOURCE_URL) 59 | filename = maybe_download(SOURCE_URL, OUT_FILE, export_dir) 60 | 61 | print('Extracting "%s" to "%s"' % (filename, export_dir)) 62 | with tarfile.open(filename) as tar: 63 | tar.extractall(path=export_dir) 64 | 65 | for p in MODEL_FILES: 66 | if not os.path.isfile(os.path.join(export_dir, p)): 67 | raise FileNotFoundError("Expected model file '%s'" % p) 68 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/test_data/py_layers/00000001/deploy.prototxt: -------------------------------------------------------------------------------- 1 | name: 'pythonnet' 2 | force_backward: true 3 | 4 | input: 'data' 5 | input_shape { dim: 10 dim: 9 dim: 8 } 6 | 7 | layer { type: 'Python' name: 'one' bottom: 'data' top: 'one' 8 | python_param { module: 'test_python_layer' layer: 'TimesTenLayer' } } 9 | 10 | layer { type: 'Python' name: 'two' bottom: 'one' top: 'two' 11 | python_param { module: 'test_python_layer' layer: 'TimesTenLayer' } } 12 | 13 | layer { type: 'Python' name: 'three' bottom: 'two' top: 'three' 14 | python_param { module: 'test_python_layer' layer: 'TimesTenLayer' } } 15 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/test_data/py_layers/00000001/test_python_layer.py: -------------------------------------------------------------------------------- 1 | import caffe 2 | 3 | class TimesTenLayer(caffe.Layer): 4 | """A layer that just multiplies by ten""" 5 | 6 | def setup(self, bottom, top): 7 | pass 8 | 9 | def reshape(self, bottom, top): 10 | top[0].reshape(*bottom[0].data.shape) 11 | 12 | def forward(self, bottom, top): 13 | top[0].data[...] = 10 * bottom[0].data 14 | 15 | def backward(self, top, propagate_down, bottom): 16 | bottom[0].diff[...] = 10 * top[0].diff 17 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/caffe/test_data/py_layers/00000001/weights.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/tensorflow_serving/servables/caffe/test_data/py_layers/00000001/weights.caffemodel -------------------------------------------------------------------------------- /tensorflow_serving/servables/hashmap/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Tensorflow Serving hashmap servable. 2 | 3 | package( 4 | default_visibility = ["//tensorflow_serving:internal"], 5 | features = ["-layering_check"], 6 | ) 7 | 8 | licenses(["notice"]) # Apache 2.0 9 | 10 | exports_files(["LICENSE"]) 11 | 12 | filegroup( 13 | name = "all_files", 14 | srcs = glob( 15 | ["**/*"], 16 | exclude = [ 17 | "**/METADATA", 18 | "**/OWNERS", 19 | "g3doc/sitemap.md", 20 | ], 21 | ), 22 | ) 23 | 24 | cc_library( 25 | name = "hashmap_source_adapter", 26 | srcs = ["hashmap_source_adapter.cc"], 27 | hdrs = ["hashmap_source_adapter.h"], 28 | visibility = [ 29 | "//visibility:public", 30 | ], 31 | deps = [ 32 | ":hashmap_source_adapter_proto", 33 | "//tensorflow_serving/core:simple_loader", 34 | "//tensorflow_serving/core:source_adapter", 35 | "//tensorflow_serving/core:storage_path", 36 | "@org_tensorflow//tensorflow/core:lib", 37 | "@org_tensorflow//tensorflow/core:tensorflow", 38 | ], 39 | ) 40 | 41 | cc_test( 42 | name = "hashmap_source_adapter_test", 43 | size = "medium", 44 | srcs = ["hashmap_source_adapter_test.cc"], 45 | deps = [ 46 | ":hashmap_source_adapter", 47 | ":hashmap_source_adapter_proto", 48 | "//tensorflow_serving/core:loader", 49 | "//tensorflow_serving/core:servable_data", 50 | "//tensorflow_serving/core/test_util:source_adapter_test_util", 51 | "//tensorflow_serving/core/test_util:test_main", 52 | "//tensorflow_serving/util:any_ptr", 53 | "@org_tensorflow//tensorflow/core:lib", 54 | "@org_tensorflow//tensorflow/core:test", 55 | ], 56 | ) 57 | 58 | load("//tensorflow_serving:serving.bzl", "serving_proto_library") 59 | 60 | serving_proto_library( 61 | name = "hashmap_source_adapter_proto", 62 | srcs = ["hashmap_source_adapter.proto"], 63 | cc_api_version = 2, 64 | ) 65 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/hashmap/hashmap_source_adapter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_HASHMAP_HASHMAP_SOURCE_ADAPTER_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_HASHMAP_HASHMAP_SOURCE_ADAPTER_H_ 18 | 19 | #include 20 | #include 21 | 22 | #include "tensorflow_serving/core/simple_loader.h" 23 | #include "tensorflow_serving/core/source_adapter.h" 24 | #include "tensorflow_serving/core/storage_path.h" 25 | #include "tensorflow_serving/servables/hashmap/hashmap_source_adapter.pb.h" 26 | 27 | namespace tensorflow { 28 | namespace serving { 29 | 30 | // A SourceAdapter for string-string hashmaps. It takes storage paths that give 31 | // the locations of serialized hashmaps (in the format indicated in the config) 32 | // and produces loaders for them. 33 | class HashmapSourceAdapter final 34 | : public SimpleLoaderSourceAdapter> { 36 | public: 37 | explicit HashmapSourceAdapter(const HashmapSourceAdapterConfig& config); 38 | ~HashmapSourceAdapter() override; 39 | 40 | private: 41 | TF_DISALLOW_COPY_AND_ASSIGN(HashmapSourceAdapter); 42 | }; 43 | 44 | } // namespace serving 45 | } // namespace tensorflow 46 | 47 | #endif // TENSORFLOW_SERVING_SERVABLES_HASHMAP_HASHMAP_SOURCE_ADAPTER_H_ 48 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/hashmap/hashmap_source_adapter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | // Config proto for HashmapSourceAdapter. 6 | message HashmapSourceAdapterConfig { 7 | // The format used by the file containing a serialized hashmap. 8 | enum Format { 9 | // A simple kind of CSV text file of the form: 10 | // key0,value0\n 11 | // key1,value1\n 12 | // ... 13 | SIMPLE_CSV = 0; 14 | } 15 | Format format = 1; 16 | } 17 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ 18 | 19 | #include 20 | #include "tensorflow/core/public/session.h" 21 | #include "tensorflow_serving/resources/resources.pb.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | namespace test_util { 26 | 27 | // Returns the path of the Saved Model (the pb version) for the half plus two 28 | // model. 29 | string GetTestSavedModelPath(); 30 | 31 | // Returns the Session Bundle export path for the half plus two model. 32 | string GetTestSessionBundleExportPath(); 33 | 34 | // Test that a Session handles a single request for the half plus two 35 | // model properly. The request has size=2, for batching purposes. 36 | void TestSingleRequest(Session* session); 37 | 38 | // Test that a Session handles multiple concurrent requests for the half plus 39 | // two model properly. The request has size=2, for batching purposes. 40 | void TestMultipleRequests(int num_requests, Session* session); 41 | 42 | // Returns the expected resource estimate for the given total file size. 43 | ResourceAllocation GetExpectedResourceEstimate(double total_file_size); 44 | 45 | } // namespace test_util 46 | } // namespace serving 47 | } // namespace tensorflow 48 | 49 | #endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ 50 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/predict_impl.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_IMPL_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_IMPL_H_ 18 | 19 | #include "tensorflow/core/lib/core/status.h" 20 | #include "tensorflow_serving/apis/predict.pb.h" 21 | #include "tensorflow_serving/model_servers/server_core.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | 26 | // Utility methods for implementation of PredictionService::Predict. 27 | // TODO(30682232): add unit test for TensorflowPredictImpl. 28 | class TensorflowPredictImpl { 29 | public: 30 | static Status Predict(ServerCore* core, const PredictRequest& request, 31 | PredictResponse* response); 32 | }; 33 | 34 | } // namespace serving 35 | } // namespace tensorflow 36 | 37 | #endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_IMPL_H_ 38 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/serving_session.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/servables/tensorflow/serving_session.h" 17 | 18 | #include "tensorflow/core/framework/graph.pb.h" 19 | #include "tensorflow/core/lib/core/errors.h" 20 | #include "tensorflow/core/lib/core/status.h" 21 | 22 | namespace tensorflow { 23 | namespace serving { 24 | 25 | Status ServingSession::Create(const GraphDef& graph) { 26 | return errors::PermissionDenied("State changes denied via ServingSession"); 27 | } 28 | 29 | Status ServingSession::Extend(const GraphDef& graph) { 30 | return errors::PermissionDenied("State changes denied via ServingSession"); 31 | } 32 | 33 | Status ServingSession::Close() { 34 | return errors::PermissionDenied("State changes denied via ServingSession"); 35 | } 36 | 37 | } // namespace serving 38 | } // namespace tensorflow 39 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/serving_session.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SERVING_SESSION_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SERVING_SESSION_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "tensorflow/core/public/session.h" 25 | 26 | namespace tensorflow { 27 | namespace serving { 28 | 29 | // A Session that blocks state-changing methods such as Close(), while allowing 30 | // Run() for read-only access (not enforced). Useful for Session implementations 31 | // that intend to be read-only and only implement Run(). 32 | class ServingSession : public Session { 33 | public: 34 | ServingSession() = default; 35 | ~ServingSession() override = default; 36 | 37 | // Methods that return errors. 38 | Status Create(const GraphDef& graph) final; 39 | Status Extend(const GraphDef& graph) final; 40 | Status Close() final; 41 | 42 | // (Subclasses just implement Run().) 43 | }; 44 | 45 | // A ServingSession that wraps a given Session, and blocks all calls other than 46 | // Run(). 47 | class ServingSessionWrapper : public ServingSession { 48 | public: 49 | explicit ServingSessionWrapper(std::unique_ptr wrapped) 50 | : wrapped_(std::move(wrapped)) {} 51 | ~ServingSessionWrapper() override = default; 52 | 53 | Status Run(const std::vector>& inputs, 54 | const std::vector& output_tensor_names, 55 | const std::vector& target_node_names, 56 | std::vector* outputs) override { 57 | return wrapped_->Run(inputs, output_tensor_names, target_node_names, 58 | outputs); 59 | } 60 | 61 | private: 62 | std::unique_ptr wrapped_; 63 | 64 | TF_DISALLOW_COPY_AND_ASSIGN(ServingSessionWrapper); 65 | }; 66 | 67 | } // namespace serving 68 | } // namespace tensorflow 69 | 70 | #endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SERVING_SESSION_H_ 71 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/session_bundle_factory.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/servables/tensorflow/session_bundle_factory.h" 17 | 18 | #include "tensorflow/core/lib/core/errors.h" 19 | #include "tensorflow/core/protobuf/config.pb.h" 20 | #include "tensorflow/core/public/session_options.h" 21 | #include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" 22 | 23 | namespace tensorflow { 24 | namespace serving { 25 | 26 | Status SessionBundleFactory::Create( 27 | const SessionBundleConfig& config, 28 | std::unique_ptr* factory) { 29 | std::shared_ptr batcher; 30 | if (config.has_batching_parameters()) { 31 | TF_RETURN_IF_ERROR( 32 | CreateBatchScheduler(config.batching_parameters(), &batcher)); 33 | } 34 | factory->reset(new SessionBundleFactory(config, batcher)); 35 | return Status::OK(); 36 | } 37 | 38 | Status SessionBundleFactory::EstimateResourceRequirement( 39 | const string& path, ResourceAllocation* estimate) const { 40 | return EstimateResourceFromPath(path, estimate); 41 | } 42 | 43 | Status SessionBundleFactory::CreateSessionBundle( 44 | const string& path, std::unique_ptr* bundle) { 45 | bundle->reset(new SessionBundle); 46 | TF_RETURN_IF_ERROR(LoadSessionBundleFromPathUsingRunOptions( 47 | GetSessionOptions(config_), GetRunOptions(config_), path, bundle->get())); 48 | 49 | if (config_.has_batching_parameters()) { 50 | LOG(INFO) << "Wrapping session to perform batch processing"; 51 | if (batch_scheduler_ == nullptr) { 52 | return errors::Internal("batch_scheduler_ not set"); 53 | } 54 | return WrapSessionForBatching(config_.batching_parameters(), 55 | batch_scheduler_, &(*bundle)->session); 56 | } 57 | return WrapSession(&(*bundle)->session); 58 | } 59 | 60 | SessionBundleFactory::SessionBundleFactory( 61 | const SessionBundleConfig& config, std::shared_ptr batch_scheduler) 62 | : config_(config), batch_scheduler_(batch_scheduler) {} 63 | 64 | } // namespace serving 65 | } // namespace tensorflow 66 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SESSION_BUNDLE_SOURCE_ADAPTER_H_ 17 | #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SESSION_BUNDLE_SOURCE_ADAPTER_H_ 18 | 19 | #include "tensorflow/contrib/session_bundle/session_bundle.h" 20 | #include "tensorflow/core/lib/core/status.h" 21 | #include "tensorflow_serving/core/loader.h" 22 | #include "tensorflow_serving/core/source_adapter.h" 23 | #include "tensorflow_serving/core/storage_path.h" 24 | #include "tensorflow_serving/servables/tensorflow/session_bundle_factory.h" 25 | #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" 26 | 27 | namespace tensorflow { 28 | namespace serving { 29 | 30 | // A SourceAdapter that creates SessionBundle Loaders from export paths. It 31 | // keeps a SessionBundleFactory as its state, which may house a batch scheduler 32 | // that is shared across all of the session bundles it emits. 33 | class SessionBundleSourceAdapter final 34 | : public UnarySourceAdapter> { 35 | public: 36 | static Status Create(const SessionBundleSourceAdapterConfig& config, 37 | std::unique_ptr* adapter); 38 | 39 | ~SessionBundleSourceAdapter() override; 40 | 41 | // Returns a function to create a session bundle source adapter. 42 | static std::function>>*)> 44 | GetCreator(const SessionBundleSourceAdapterConfig& config); 45 | 46 | private: 47 | explicit SessionBundleSourceAdapter( 48 | std::unique_ptr bundle_factory); 49 | 50 | Status Convert(const StoragePath& path, 51 | std::unique_ptr* loader) override; 52 | 53 | // We use a shared ptr to share ownership with Loaders we emit, in case they 54 | // outlive this object. 55 | std::shared_ptr bundle_factory_; 56 | 57 | TF_DISALLOW_COPY_AND_ASSIGN(SessionBundleSourceAdapter); 58 | }; 59 | 60 | } // namespace serving 61 | } // namespace tensorflow 62 | 63 | #endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SESSION_BUNDLE_SOURCE_ADAPTER_H_ 64 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "tensorflow_serving/servables/tensorflow/session_bundle_config.proto"; 4 | 5 | package tensorflow.serving; 6 | 7 | // Config proto for SessionBundleSourceAdapter. 8 | message SessionBundleSourceAdapterConfig { 9 | SessionBundleConfig config = 1; 10 | } 11 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/simple_servers.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | // Bootstrapping and configuration utilities for creating simple servers of 17 | // TensorFlow models. Intended for basic instantiation with default configs. 18 | // 19 | // Note: All methods expect TensorFlow exports conforming to the export format 20 | // specified at tensorflow_serving/servables/tensorflow/README.md. 21 | #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ 22 | #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ 23 | 24 | #include 25 | #include 26 | 27 | #include "tensorflow/contrib/session_bundle/session_bundle.h" 28 | #include "tensorflow/core/lib/core/status.h" 29 | #include "tensorflow/core/platform/types.h" 30 | #include "tensorflow_serving/core/manager.h" 31 | #include "tensorflow_serving/core/storage_path.h" 32 | #include "tensorflow_serving/core/target.h" 33 | 34 | namespace tensorflow { 35 | namespace serving { 36 | namespace simple_servers { 37 | 38 | // TODO(b/25969594): Add full test coverage. 39 | 40 | // Creates a Manager and associated Source for a single TensorFlow model that 41 | // automatically loads new versions over time. All versions of the model will be 42 | // loaded from new directories under the specified base path. Uses default 43 | // SessionOptions. 44 | // 45 | // The servables loaded and served from this manager are of type 46 | // tensorflow::serving::SessionBundle. 47 | // 48 | // When new versions arrive the Manager will unload the previous version before 49 | // loading the new version. This is preferable from a resource utilization 50 | // perspective, but has reduced availability. 51 | Status CreateSingleTFModelManagerFromBasePath( 52 | const string& base_path, std::unique_ptr* manager); 53 | 54 | } // namespace simple_servers 55 | } // namespace serving 56 | } // namespace tensorflow 57 | 58 | #endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ 59 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Example TensorFlow model export. 2 | 3 | package( 4 | default_visibility = [ 5 | "//tensorflow_serving:internal", 6 | ], 7 | features = [ 8 | "-layering_check", 9 | "-parse_headers", 10 | ], 11 | ) 12 | 13 | licenses(["notice"]) # Apache 2.0 14 | 15 | exports_files(["LICENSE"]) 16 | 17 | filegroup( 18 | name = "all_files", 19 | srcs = glob( 20 | ["**/*"], 21 | exclude = [ 22 | "**/METADATA", 23 | "**/OWNERS", 24 | ], 25 | ), 26 | ) 27 | 28 | py_binary( 29 | name = "export_half_plus_two", 30 | srcs = [ 31 | "export_half_plus_two.py", 32 | ], 33 | deps = [ 34 | "@org_tensorflow//tensorflow:tensorflow_py", 35 | "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", 36 | ], 37 | ) 38 | 39 | py_binary( 40 | name = "export_bad_half_plus_two", 41 | srcs = [ 42 | "export_bad_half_plus_two.py", 43 | ], 44 | deps = [ 45 | "@org_tensorflow//tensorflow:tensorflow_py", 46 | ], 47 | ) 48 | 49 | # Note: re-generate these files with :export_half_plus_two whenever model 50 | # changes. 51 | exports_files([ 52 | "half_plus_two/00000123/export", 53 | "half_plus_two/00000123/export.meta", 54 | "bad_half_plus_two/00000123/export", 55 | "bad_half_plus_two/00000123/export.meta", 56 | ]) 57 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/bad_half_plus_two/00000123/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "/tmp/bad_half_plus_two/00000123/export" 2 | all_model_checkpoint_paths: "/tmp/bad_half_plus_two/00000123/export" 3 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/bad_half_plus_two/00000123/export: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/tensorflow_serving/servables/tensorflow/testdata/bad_half_plus_two/00000123/export -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/bad_half_plus_two/00000123/export.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/tensorflow_serving/servables/tensorflow/testdata/bad_half_plus_two/00000123/export.meta -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Exports a toy TensorFlow model without signatures. 16 | 17 | Exports half_plus_two TensorFlow model to /tmp/bad_half_plus_two/ without 18 | signatures. This is used to test the fault-tolerance of tensorflow_model_server. 19 | """ 20 | 21 | import os 22 | 23 | # This is a placeholder for a Google-internal import. 24 | 25 | import tensorflow as tf 26 | 27 | 28 | def Export(): 29 | export_path = "/tmp/bad_half_plus_two/00000123" 30 | with tf.Session() as sess: 31 | # Make model parameters a&b variables instead of constants to 32 | # exercise the variable reloading mechanisms. 33 | a = tf.Variable(0.5) 34 | b = tf.Variable(2.0) 35 | 36 | # Calculate, y = a*x + b 37 | # here we use a placeholder 'x' which is fed at inference time. 38 | x = tf.placeholder(tf.float32) 39 | y = tf.add(tf.mul(a, x), b) 40 | 41 | # Export the model without signatures. 42 | # Note that the model is intentionally exported without using exporter, 43 | # but using the same format. This is to avoid exporter creating default 44 | # empty signatures upon export. 45 | tf.initialize_all_variables().run() 46 | saver = tf.train.Saver() 47 | saver.export_meta_graph( 48 | filename=os.path.join(export_path, "export.meta")) 49 | saver.save(sess, 50 | os.path.join(export_path, "export"), 51 | write_meta_graph=False) 52 | 53 | 54 | def main(_): 55 | Export() 56 | 57 | 58 | if __name__ == "__main__": 59 | tf.app.run() 60 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Exports a toy TensorFlow model. 16 | 17 | Exports a TensorFlow model to /tmp/half_plus_two/. 18 | 19 | This graph calculates, 20 | y = a*x + b 21 | where a and b are variables with a=0.5 and b=2. 22 | """ 23 | 24 | # This is a placeholder for a Google-internal import. 25 | 26 | import tensorflow as tf 27 | from tensorflow.contrib.session_bundle import exporter 28 | 29 | 30 | def Export(): 31 | export_path = "/tmp/half_plus_two" 32 | with tf.Session() as sess: 33 | # Make model parameters a&b variables instead of constants to 34 | # exercise the variable reloading mechanisms. 35 | a = tf.Variable(0.5) 36 | b = tf.Variable(2.0) 37 | 38 | # Calculate, y = a*x + b 39 | # here we use a placeholder 'x' which is fed at inference time. 40 | x = tf.placeholder(tf.float32) 41 | y = tf.add(tf.mul(a, x), b) 42 | 43 | # Run an export. 44 | tf.initialize_all_variables().run() 45 | export = exporter.Exporter(tf.train.Saver()) 46 | export.init(named_graph_signatures={ 47 | "inputs": exporter.generic_signature({"x": x}), 48 | "outputs": exporter.generic_signature({"y": y}) 49 | }) 50 | export.export(export_path, tf.constant(123), sess) 51 | 52 | 53 | def main(_): 54 | Export() 55 | 56 | 57 | if __name__ == "__main__": 58 | tf.app.run() 59 | -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export -------------------------------------------------------------------------------- /tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.meta -------------------------------------------------------------------------------- /tensorflow_serving/serving.bzl: -------------------------------------------------------------------------------- 1 | load("@protobuf//:protobuf.bzl", "cc_proto_library") 2 | load("@protobuf//:protobuf.bzl", "py_proto_library") 3 | 4 | def serving_proto_library(name, srcs=[], has_services=False, 5 | deps=[], visibility=None, testonly=0, # pylint: disable=unused-argument 6 | cc_grpc_version = None, 7 | cc_api_version=2, go_api_version=2, 8 | java_api_version=2, 9 | py_api_version=2): 10 | native.filegroup(name=name + "_proto_srcs", 11 | srcs=srcs, 12 | testonly=testonly,) 13 | 14 | use_grpc_plugin = None 15 | if cc_grpc_version: 16 | use_grpc_plugin = True 17 | cc_proto_library(name=name, 18 | srcs=srcs, 19 | deps=deps, 20 | cc_libs = ["@protobuf//:protobuf"], 21 | protoc="@protobuf//:protoc", 22 | default_runtime="@protobuf//:protobuf", 23 | use_grpc_plugin=use_grpc_plugin, 24 | testonly=testonly, 25 | visibility=visibility,) 26 | 27 | def serving_proto_library_py(name, proto_library, srcs=[], deps=[], visibility=None, testonly=0): # pylint: disable=unused-argument 28 | py_proto_library(name=name, 29 | srcs=srcs, 30 | srcs_version = "PY2AND3", 31 | deps=["@protobuf//:protobuf_python"] + deps, 32 | default_runtime="@protobuf//:protobuf_python", 33 | protoc="@protobuf//:protoc", 34 | visibility=visibility, 35 | testonly=testonly,) 36 | -------------------------------------------------------------------------------- /tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | // Config proto for FileSystemStoragePathSource. 6 | message FileSystemStoragePathSourceConfig { 7 | 8 | // The policy to define how many versions of the servable should be 9 | // served at the same time. 10 | enum VersionPolicy { 11 | // Only serve the latest version that exists in the base path. 12 | // This is the default behavior. 13 | LATEST_VERSION = 0; 14 | // Serves all the versions that exist in the base path. 15 | ALL_VERSIONS = 1; 16 | } 17 | 18 | // A servable name and base path to look for versions of the servable. 19 | message ServableToMonitor { 20 | // The servable name to supply in aspired-versions callback calls. Child 21 | // paths of 'base_path' are considered to be versions of this servable. 22 | string servable_name = 1; 23 | 24 | // The path to monitor, i.e. look for child paths of the form base_path/123. 25 | string base_path = 2; 26 | 27 | // The policy to determines the number of versions of the servable to be 28 | // served at the same time. 29 | VersionPolicy version_policy = 3; 30 | }; 31 | 32 | // The servables to monitor for new versions, and aspire. 33 | repeated ServableToMonitor servables = 5; 34 | 35 | // A single servable name/base_path pair to monitor. 36 | // DEPRECATED: Use 'servables' instead. 37 | // TODO(b/30898016): Stop using these fields, and ultimately remove them here. 38 | string servable_name = 1 [deprecated = true]; 39 | string base_path = 2 [deprecated = true]; 40 | 41 | // How long to wait between file-system polling to look for children of 42 | // 'base_path', in seconds. 43 | // 44 | // For testing use only: a negative value disables the polling thread. 45 | int64 file_system_poll_wait_seconds = 3; 46 | 47 | // If true, then FileSystemStoragePathSource::Create() and ::UpdateConfig() 48 | // fail if, for any configured servables, the file system doesn't currently 49 | // contain at least one version under the base path. 50 | // (Otherwise, it will emit a warning and keep pinging the file system to 51 | // check for a version to appear later.) 52 | bool fail_if_zero_versions_at_startup = 4; 53 | } 54 | -------------------------------------------------------------------------------- /tensorflow_serving/sources/storage_path/static_storage_path_source.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/sources/storage_path/static_storage_path_source.h" 17 | 18 | #include 19 | #include 20 | 21 | #include "tensorflow_serving/core/servable_data.h" 22 | #include "tensorflow_serving/core/servable_id.h" 23 | 24 | namespace tensorflow { 25 | namespace serving { 26 | 27 | Status StaticStoragePathSource::Create( 28 | const StaticStoragePathSourceConfig& config, 29 | std::unique_ptr* result) { 30 | auto raw_result = new StaticStoragePathSource; 31 | raw_result->config_ = config; 32 | result->reset(raw_result); 33 | return Status::OK(); 34 | } 35 | 36 | void StaticStoragePathSource::SetAspiredVersionsCallback( 37 | AspiredVersionsCallback callback) { 38 | const ServableId id = {config_.servable_name(), config_.version_num()}; 39 | LOG(INFO) << "Aspiring servable " << id; 40 | callback(config_.servable_name(), 41 | {CreateServableData(id, config_.version_path())}); 42 | } 43 | 44 | } // namespace serving 45 | } // namespace tensorflow 46 | -------------------------------------------------------------------------------- /tensorflow_serving/sources/storage_path/static_storage_path_source.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_SOURCES_STORAGE_PATH_STATIC_STORAGE_PATH_SOURCE_H_ 17 | #define TENSORFLOW_SERVING_SOURCES_STORAGE_PATH_STATIC_STORAGE_PATH_SOURCE_H_ 18 | 19 | #include 20 | 21 | #include "tensorflow/core/lib/core/status.h" 22 | #include "tensorflow/core/platform/macros.h" 23 | #include "tensorflow/core/platform/types.h" 24 | #include "tensorflow_serving/core/source.h" 25 | #include "tensorflow_serving/core/storage_path.h" 26 | #include "tensorflow_serving/sources/storage_path/static_storage_path_source.pb.h" 27 | 28 | namespace tensorflow { 29 | namespace serving { 30 | 31 | // A StoragePathSource that calls the aspired-versions callback exactly once, 32 | // with a single hard-coded servable and version path. 33 | // Useful for testing and experimental deployments. 34 | class StaticStoragePathSource : public Source { 35 | public: 36 | static Status Create(const StaticStoragePathSourceConfig& config, 37 | std::unique_ptr* result); 38 | ~StaticStoragePathSource() override = default; 39 | 40 | void SetAspiredVersionsCallback(AspiredVersionsCallback callback) override; 41 | 42 | private: 43 | StaticStoragePathSource() = default; 44 | 45 | StaticStoragePathSourceConfig config_; 46 | 47 | TF_DISALLOW_COPY_AND_ASSIGN(StaticStoragePathSource); 48 | }; 49 | 50 | } // namespace serving 51 | } // namespace tensorflow 52 | 53 | #endif // TENSORFLOW_SERVING_SOURCES_STORAGE_PATH_STATIC_STORAGE_PATH_SOURCE_H_ 54 | -------------------------------------------------------------------------------- /tensorflow_serving/sources/storage_path/static_storage_path_source.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | // Config proto for StaticStoragePathSource. 6 | message StaticStoragePathSourceConfig { 7 | // The single servable name, version number and path to supply statically. 8 | string servable_name = 1; 9 | int64 version_num = 2; 10 | string version_path = 3; 11 | } 12 | -------------------------------------------------------------------------------- /tensorflow_serving/sources/storage_path/static_storage_path_source_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/sources/storage_path/static_storage_path_source.h" 17 | 18 | #include 19 | #include 20 | #include "tensorflow/core/lib/core/status_test_util.h" 21 | #include "tensorflow/core/lib/core/stringpiece.h" 22 | #include "tensorflow_serving/core/servable_data.h" 23 | #include "tensorflow_serving/core/target.h" 24 | #include "tensorflow_serving/core/test_util/mock_storage_path_target.h" 25 | #include "tensorflow_serving/test_util/test_util.h" 26 | 27 | using ::testing::ElementsAre; 28 | using ::testing::Eq; 29 | using ::testing::Return; 30 | using ::testing::StrictMock; 31 | 32 | namespace tensorflow { 33 | namespace serving { 34 | namespace { 35 | 36 | TEST(StaticStoragePathSourceTest, Basic) { 37 | auto config = test_util::CreateProto( 38 | "servable_name: 'test_servable_name' " 39 | "version_num: 42 " 40 | "version_path: 'test_version_path' "); 41 | std::unique_ptr source; 42 | TF_ASSERT_OK(StaticStoragePathSource::Create(config, &source)); 43 | 44 | std::unique_ptr target( 45 | new StrictMock); 46 | EXPECT_CALL(*target, SetAspiredVersions(Eq("test_servable_name"), 47 | ElementsAre(ServableData( 48 | {"test_servable_name", 42}, 49 | "test_version_path")))); 50 | ConnectSourceToTarget(source.get(), target.get()); 51 | } 52 | 53 | } // namespace 54 | } // namespace serving 55 | } // namespace tensorflow 56 | -------------------------------------------------------------------------------- /tensorflow_serving/test_util/BUILD: -------------------------------------------------------------------------------- 1 | # Description: Tensorflow Serving test utils. 2 | 3 | package(default_visibility = [ 4 | "//tensorflow_serving:internal", 5 | ]) 6 | 7 | licenses(["notice"]) # Apache 2.0 8 | 9 | exports_files(["LICENSE"]) 10 | 11 | filegroup( 12 | name = "all_files", 13 | srcs = glob( 14 | ["**/*"], 15 | exclude = [ 16 | "**/METADATA", 17 | "**/OWNERS", 18 | ], 19 | ), 20 | ) 21 | 22 | cc_library( 23 | name = "test_util", 24 | testonly = 1, 25 | srcs = ["test_util.cc"], 26 | hdrs = ["test_util.h"], 27 | deps = [ 28 | "//external:gtest", 29 | "@org_tensorflow//tensorflow/core", 30 | "@org_tensorflow//tensorflow/core:lib", 31 | "@org_tensorflow//tensorflow/core:test", 32 | "@protobuf//:protobuf", 33 | ], 34 | ) 35 | 36 | cc_library( 37 | name = "fake_clock_env", 38 | testonly = 1, 39 | srcs = ["fake_clock_env.cc"], 40 | hdrs = ["fake_clock_env.h"], 41 | visibility = [ 42 | "//visibility:public", 43 | ], 44 | deps = [ 45 | "@org_tensorflow//tensorflow/core:lib", 46 | "@org_tensorflow//tensorflow/core:tensorflow", 47 | ], 48 | ) 49 | -------------------------------------------------------------------------------- /tensorflow_serving/test_util/fake_clock_env.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/test_util/fake_clock_env.h" 17 | 18 | #include 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | namespace test_util { 23 | 24 | FakeClockEnv::FakeClockEnv(Env* wrapped) : EnvWrapper(wrapped) {} 25 | 26 | void FakeClockEnv::AdvanceByMicroseconds(int micros) { 27 | { 28 | mutex_lock l(mu_); 29 | current_time_ += micros; 30 | for (auto it = sleeping_threads_.begin(); it != sleeping_threads_.end();) { 31 | if (current_time_ >= it->wake_time) { 32 | it->wake_notification->Notify(); 33 | it = sleeping_threads_.erase(it); 34 | } else { 35 | ++it; 36 | } 37 | } 38 | } 39 | } 40 | 41 | void FakeClockEnv::BlockUntilSleepingThread(uint64 wake_time) { 42 | for (;;) { 43 | { 44 | mutex_lock l(mu_); 45 | for (auto it = sleeping_threads_.begin(); it != sleeping_threads_.end(); 46 | ++it) { 47 | if (it->wake_time == wake_time) { 48 | return; 49 | } 50 | } 51 | } 52 | EnvWrapper::SleepForMicroseconds(100); 53 | } 54 | } 55 | 56 | void FakeClockEnv::BlockUntilThreadsAsleep(int num_threads) { 57 | for (;;) { 58 | { 59 | mutex_lock l(mu_); 60 | if (num_threads <= sleeping_threads_.size()) { 61 | return; 62 | } 63 | } 64 | EnvWrapper::SleepForMicroseconds(100); 65 | } 66 | } 67 | 68 | uint64 FakeClockEnv::NowMicros() { 69 | { 70 | mutex_lock l(mu_); 71 | return current_time_; 72 | } 73 | } 74 | 75 | void FakeClockEnv::SleepForMicroseconds(int64 micros) { 76 | if (micros == 0) { 77 | return; 78 | } 79 | 80 | Notification wake_notification; 81 | { 82 | mutex_lock l(mu_); 83 | sleeping_threads_.push_back({current_time_ + micros, &wake_notification}); 84 | } 85 | wake_notification.WaitForNotification(); 86 | } 87 | 88 | } // namespace test_util 89 | } // namespace serving 90 | } // namespace tensorflow 91 | -------------------------------------------------------------------------------- /tensorflow_serving/test_util/fake_clock_env.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_TEST_UTIL_FAKE_CLOCK_ENV_H_ 17 | #define TENSORFLOW_SERVING_TEST_UTIL_FAKE_CLOCK_ENV_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "tensorflow/core/lib/core/notification.h" 24 | #include "tensorflow/core/lib/core/status.h" 25 | #include "tensorflow/core/platform/env.h" 26 | #include "tensorflow/core/platform/macros.h" 27 | #include "tensorflow/core/platform/mutex.h" 28 | #include "tensorflow/core/platform/thread_annotations.h" 29 | #include "tensorflow/core/platform/types.h" 30 | 31 | namespace tensorflow { 32 | namespace serving { 33 | namespace test_util { 34 | 35 | // An Env implementation with a fake clock for NowMicros() and 36 | // SleepForMicroseconds(). The clock doesn't advance on its own; it advances via 37 | // an explicit Advance() method. 38 | // All other Env virtual methods pass through to a wrapped Env. 39 | class FakeClockEnv : public EnvWrapper { 40 | public: 41 | explicit FakeClockEnv(Env* wrapped); 42 | ~FakeClockEnv() override = default; 43 | 44 | // Advance the clock by a certain number of microseconds. 45 | void AdvanceByMicroseconds(int micros); 46 | 47 | // Blocks until there is a sleeping thread that is scheduled to wake up at 48 | // the given (absolute) time. 49 | void BlockUntilSleepingThread(uint64 wake_time); 50 | 51 | // Blocks until there are at least num_threads sleeping. 52 | void BlockUntilThreadsAsleep(int num_threads); 53 | 54 | // Methods that this class implements. 55 | uint64 NowMicros() override; 56 | void SleepForMicroseconds(int64 micros) override; 57 | 58 | private: 59 | mutex mu_; 60 | 61 | uint64 current_time_ GUARDED_BY(mu_) = 0; 62 | 63 | struct SleepingThread { 64 | uint64 wake_time; 65 | Notification* wake_notification; 66 | }; 67 | std::vector sleeping_threads_ GUARDED_BY(mu_); 68 | 69 | TF_DISALLOW_COPY_AND_ASSIGN(FakeClockEnv); 70 | }; 71 | 72 | } // namespace test_util 73 | } // namespace serving 74 | } // namespace tensorflow 75 | 76 | #endif // TENSORFLOW_SERVING_TEST_UTIL_FAKE_CLOCK_ENV_H_ 77 | -------------------------------------------------------------------------------- /tensorflow_serving/test_util/test_util.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/test_util/test_util.h" 17 | 18 | #include "tensorflow/core/lib/io/path.h" 19 | #include "tensorflow/core/platform/env.h" 20 | #include "tensorflow/core/platform/test.h" 21 | 22 | namespace tensorflow { 23 | namespace serving { 24 | namespace test_util { 25 | 26 | string TensorflowTestSrcDirPath(const string& relative_path) { 27 | const string base_path = tensorflow::io::JoinPath( // 28 | getenv("TEST_SRCDIR"), // 29 | "tf_serving/external/org_tensorflow/tensorflow/"); 30 | return tensorflow::io::JoinPath(base_path, relative_path); 31 | } 32 | 33 | string ContribTestSrcDirPath(const string& relative_path) { 34 | const string base_path = TensorflowTestSrcDirPath("contrib/"); 35 | return tensorflow::io::JoinPath(base_path, relative_path); 36 | } 37 | 38 | string TestSrcDirPath(const string& relative_path) { 39 | const string base_path = tensorflow::io::JoinPath( 40 | getenv("TEST_SRCDIR"), "tf_serving/tensorflow_serving"); 41 | return tensorflow::io::JoinPath(base_path, relative_path); 42 | } 43 | 44 | ProtoStringMatcher::ProtoStringMatcher(const string& expected) 45 | : expected_(expected) {} 46 | ProtoStringMatcher::ProtoStringMatcher(const google::protobuf::Message& expected) 47 | : expected_(expected.DebugString()) {} 48 | 49 | } // namespace test_util 50 | } // namespace serving 51 | } // namespace tensorflow 52 | -------------------------------------------------------------------------------- /tensorflow_serving/tools/docker/Dockerfile.caffe-devel: -------------------------------------------------------------------------------- 1 | FROM tensorflow-serving-devel:latest 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | cmake \ 5 | libboost-thread-dev \ 6 | libboost-system-dev \ 7 | libboost-filesystem-dev \ 8 | libboost-python-dev \ 9 | libpython-dev \ 10 | libgflags-dev \ 11 | libgoogle-glog-dev \ 12 | libhdf5-dev \ 13 | && \ 14 | apt-get clean && \ 15 | rm -rf /var/lib/apt/lists/* 16 | 17 | # OpenBLAS 18 | RUN cd /opt && \ 19 | curl -L -O https://github.com/xianyi/OpenBLAS/archive/v0.2.18.zip && \ 20 | unzip -a v0.2.18.zip && \ 21 | cd OpenBLAS-0.2.18 && \ 22 | make HOSTCC=gcc TARGET=HASWELL USE_OPENMP=0 USE_THREAD=0 NO_LAPACKE=1 ONLY_CBLAS=1 BINARY=64 && \ 23 | make install PREFIX=/opt/OpenBLAS && \ 24 | cd .. && \ 25 | rm v0.2.18.zip && \ 26 | rm -rf OpenBLAS-0.2.18 27 | 28 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /tensorflow_serving/tools/docker/Dockerfile.devel: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | 3 | MAINTAINER Jeremiah Harmsen 4 | 5 | RUN apt-get update && apt-get install -y \ 6 | build-essential \ 7 | curl \ 8 | git \ 9 | libfreetype6-dev \ 10 | libpng12-dev \ 11 | libzmq3-dev \ 12 | pkg-config \ 13 | python-dev \ 14 | python-numpy \ 15 | python-pip \ 16 | software-properties-common \ 17 | swig \ 18 | zip \ 19 | zlib1g-dev \ 20 | libcurl3-dev \ 21 | && \ 22 | apt-get clean && \ 23 | rm -rf /var/lib/apt/lists/* 24 | 25 | RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ 26 | python get-pip.py && \ 27 | rm get-pip.py 28 | 29 | # Set up grpc 30 | 31 | RUN pip install enum34 futures mock six && \ 32 | pip install --pre 'protobuf>=3.0.0a3' && \ 33 | pip install -i https://testpypi.python.org/simple --pre grpcio 34 | 35 | # Set up Bazel. 36 | 37 | # We need to add a custom PPA to pick up JDK8, since trusty doesn't 38 | # have an openjdk8 backport. openjdk-r is maintained by a reliable contributor: 39 | # Matthias Klose (https://launchpad.net/~doko). It will do until 40 | # we either update the base image beyond 14.04 or openjdk-8 is 41 | # finally backported to trusty; see e.g. 42 | # https://bugs.launchpad.net/trusty-backports/+bug/1368094 43 | RUN add-apt-repository -y ppa:openjdk-r/ppa && \ 44 | apt-get update && \ 45 | apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ 46 | apt-get clean && \ 47 | rm -rf /var/lib/apt/lists/* 48 | 49 | # Running bazel inside a `docker build` command causes trouble, cf: 50 | # https://github.com/bazelbuild/bazel/issues/134 51 | # The easiest solution is to set up a bazelrc file forcing --batch. 52 | RUN echo "startup --batch" >>/root/.bazelrc 53 | # Similarly, we need to workaround sandboxing issues: 54 | # https://github.com/bazelbuild/bazel/issues/418 55 | RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ 56 | >>/root/.bazelrc 57 | ENV BAZELRC /root/.bazelrc 58 | # Install the most recent bazel release. 59 | ENV BAZEL_VERSION 0.3.2 60 | WORKDIR / 61 | RUN mkdir /bazel && \ 62 | cd /bazel && \ 63 | curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ 64 | curl -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE.txt && \ 65 | chmod +x bazel-*.sh && \ 66 | ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ 67 | cd / && \ 68 | rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh 69 | 70 | CMD ["/bin/bash"] 71 | -------------------------------------------------------------------------------- /tensorflow_serving/tools/docker/README.md: -------------------------------------------------------------------------------- 1 | Files for using the [Docker](http://www.docker.com) container system. 2 | -------------------------------------------------------------------------------- /tensorflow_serving/util/executor.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_UTIL_EXECUTOR_H_ 17 | #define TENSORFLOW_SERVING_UTIL_EXECUTOR_H_ 18 | 19 | #include 20 | 21 | namespace tensorflow { 22 | namespace serving { 23 | 24 | // An abstract object that can execute closures. 25 | // 26 | // Implementations of executor must be thread-safe. 27 | class Executor { 28 | public: 29 | virtual ~Executor() = default; 30 | 31 | // Schedule the specified 'fn' for execution in this executor. Depending on 32 | // the subclass implementation, this may block in some situations. 33 | virtual void Schedule(std::function fn) = 0; 34 | }; 35 | 36 | } // namespace serving 37 | } // namespace tensorflow 38 | 39 | #endif // TENSORFLOW_SERVING_UTIL_EXECUTOR_H_ 40 | -------------------------------------------------------------------------------- /tensorflow_serving/util/hash.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/util/hash.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | uint64 HashCombine(const uint64 hash1, const uint64 hash2) { 22 | return hash1 ^ (hash2 + 0x9e3779b97f4a7800 + (hash1 << 10) + (hash1 >> 4)); 23 | } 24 | 25 | } // namespace serving 26 | } // namespace tensorflow 27 | -------------------------------------------------------------------------------- /tensorflow_serving/util/hash.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_UTIL_HASH_H_ 17 | #define TENSORFLOW_SERVING_UTIL_HASH_H_ 18 | 19 | #include "tensorflow/core/platform/types.h" 20 | 21 | namespace tensorflow { 22 | namespace serving { 23 | 24 | // Combines 2 hashes and returns a 3rd one. 25 | uint64 HashCombine(uint64 hash1, uint64 hash2); 26 | 27 | } // namespace serving 28 | } // namespace tensorflow 29 | 30 | #endif // TENSORFLOW_SERVING_UTIL_HASH_H_ 31 | -------------------------------------------------------------------------------- /tensorflow_serving/util/inline_executor.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/util/inline_executor.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | InlineExecutor::InlineExecutor() {} 22 | 23 | InlineExecutor::~InlineExecutor() {} 24 | 25 | void InlineExecutor::Schedule(std::function fn) { fn(); } 26 | 27 | } // namespace serving 28 | } // namespace tensorflow 29 | -------------------------------------------------------------------------------- /tensorflow_serving/util/inline_executor.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_UTIL_INLINE_EXECUTOR_H_ 17 | #define TENSORFLOW_SERVING_UTIL_INLINE_EXECUTOR_H_ 18 | 19 | #include 20 | 21 | #include "tensorflow/core/platform/macros.h" 22 | #include "tensorflow_serving/util/executor.h" 23 | 24 | namespace tensorflow { 25 | namespace serving { 26 | 27 | // An InlineExecutor is a trivial executor that immediately executes the closure 28 | // given to it. It's useful as a mock, and in cases where an executor is needed, 29 | // but multi-threadedness is not. 30 | class InlineExecutor : public Executor { 31 | public: 32 | InlineExecutor(); 33 | ~InlineExecutor() override; 34 | void Schedule(std::function fn) override; 35 | }; 36 | 37 | } // namespace serving 38 | } // namespace tensorflow 39 | 40 | #endif // TENSORFLOW_SERVING_UTIL_INLINE_EXECUTOR_H_ 41 | -------------------------------------------------------------------------------- /tensorflow_serving/util/inline_executor_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/util/inline_executor.h" 17 | 18 | #include 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | namespace { 23 | 24 | TEST(InlineExecutorTest, Executes) { 25 | InlineExecutor inline_executor; 26 | 27 | int total_calls = 0; 28 | inline_executor.Schedule([&]() { ++total_calls; }); 29 | EXPECT_EQ(1, total_calls); 30 | } 31 | 32 | } // namespace 33 | } // namespace serving 34 | } // namespace tensorflow 35 | -------------------------------------------------------------------------------- /tensorflow_serving/util/optional.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/util/optional.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | extern const in_place_t in_place{}; 22 | extern const nullopt_t nullopt{{}}; 23 | 24 | } // namespace serving 25 | } // namespace tensorflow 26 | -------------------------------------------------------------------------------- /tensorflow_serving/util/threadpool_executor.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/util/threadpool_executor.h" 17 | 18 | namespace tensorflow { 19 | namespace serving { 20 | 21 | ThreadPoolExecutor::ThreadPoolExecutor(Env* const env, const string& name, 22 | int num_threads) 23 | : thread_pool_(env, name, num_threads) {} 24 | 25 | ThreadPoolExecutor::~ThreadPoolExecutor() {} 26 | 27 | void ThreadPoolExecutor::Schedule(std::function fn) { 28 | thread_pool_.Schedule(fn); 29 | } 30 | 31 | } // namespace serving 32 | } // namespace tensorflow 33 | -------------------------------------------------------------------------------- /tensorflow_serving/util/threadpool_executor.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef TENSORFLOW_SERVING_UTIL_THREADPOOL_EXECUTOR_H_ 17 | #define TENSORFLOW_SERVING_UTIL_THREADPOOL_EXECUTOR_H_ 18 | 19 | #include 20 | 21 | #include "tensorflow/core/lib/core/threadpool.h" 22 | #include "tensorflow/core/platform/env.h" 23 | #include "tensorflow_serving/util/executor.h" 24 | 25 | namespace tensorflow { 26 | namespace serving { 27 | 28 | // An executor which uses a pool of threads to execute the scheduled closures. 29 | class ThreadPoolExecutor : public Executor { 30 | public: 31 | // Constructs a threadpool that has 'num_threads' threads with specified 32 | // 'thread_pool_name'. Env is used to start the thread. 33 | // 34 | // REQUIRES: num_threads > 0. 35 | ThreadPoolExecutor(Env* env, const string& thread_pool_name, int num_threads); 36 | 37 | // Waits until all scheduled work has finished and then destroy the set of 38 | // threads. 39 | ~ThreadPoolExecutor() override; 40 | 41 | void Schedule(std::function fn) override; 42 | 43 | private: 44 | thread::ThreadPool thread_pool_; 45 | 46 | TF_DISALLOW_COPY_AND_ASSIGN(ThreadPoolExecutor); 47 | }; 48 | 49 | } // namespace serving 50 | } // namespace tensorflow 51 | 52 | #endif // TENSORFLOW_SERVING_UTIL_THREADPOOL_EXECUTOR_H_ 53 | -------------------------------------------------------------------------------- /tensorflow_serving/util/threadpool_executor_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow_serving/util/threadpool_executor.h" 17 | 18 | #include 19 | 20 | namespace tensorflow { 21 | namespace serving { 22 | namespace { 23 | 24 | constexpr int kNumThreads = 30; 25 | 26 | TEST(ThreadPoolExecutor, Empty) { 27 | for (int num_threads = 1; num_threads < kNumThreads; num_threads++) { 28 | LOG(INFO) << "Testing with " << num_threads << " threads"; 29 | ThreadPoolExecutor pool(Env::Default(), "test", num_threads); 30 | } 31 | } 32 | 33 | TEST(ThreadPoolExecutor, DoWork) { 34 | for (int num_threads = 1; num_threads < kNumThreads; num_threads++) { 35 | LOG(INFO) << "Testing with " << num_threads << " threads"; 36 | const int kWorkItems = 15; 37 | // Not using std::vector due to its unusual implementation and API - 38 | // http://en.cppreference.com/w/cpp/container/vector_bool 39 | bool work[kWorkItems]; 40 | for (int i = 0; i < kWorkItems; ++i) { 41 | work[i] = false; 42 | } 43 | { 44 | ThreadPoolExecutor executor(Env::Default(), "test", num_threads); 45 | for (int i = 0; i < kWorkItems; i++) { 46 | executor.Schedule([&work, i]() { 47 | ASSERT_FALSE(work[i]); 48 | work[i] = true; 49 | }); 50 | } 51 | } 52 | for (int i = 0; i < kWorkItems; i++) { 53 | ASSERT_TRUE(work[i]); 54 | } 55 | } 56 | } 57 | 58 | } // namespace 59 | } // namespace serving 60 | } // namespace tensorflow 61 | -------------------------------------------------------------------------------- /tensorflow_serving/workspace.bzl: -------------------------------------------------------------------------------- 1 | # TensorFlow Serving external dependencies that can be loaded in WORKSPACE 2 | # files. 3 | 4 | load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') 5 | 6 | # All TensorFlow Serving external dependencies. 7 | # workspace_dir is the absolute path to the TensorFlow Serving repo. If linked 8 | # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' 9 | def tf_serving_workspace(workspace_dir): 10 | native.local_repository( 11 | name = "inception_model", 12 | path = "tf_models/inception", 13 | ) 14 | 15 | tf_workspace(path_prefix = "", tf_repo_name = "org_tensorflow") 16 | 17 | # ===== gRPC dependencies ===== 18 | native.bind( 19 | name = "libssl", 20 | actual = "@boringssl//:ssl", 21 | ) 22 | 23 | native.bind( 24 | name = "zlib", 25 | actual = "@zlib_archive//:zlib", 26 | ) 27 | 28 | # ===== caffe flavors ===== 29 | native.new_git_repository( 30 | name = "caffe", 31 | remote = "https://github.com/BVLC/caffe", 32 | commit = "50c9a0fc8eed0101657e9f8da164d88f66242aeb", 33 | init_submodules = True, 34 | build_file = workspace_dir + "/caffe.BUILD", 35 | ) 36 | 37 | native.new_git_repository( 38 | name = "caffe_rcnn", 39 | remote = "https://github.com/Austriker/caffe", 40 | commit = "3c27a70ae7304feb48c60b268c70adf585879d50", 41 | init_submodules = True, 42 | build_file = workspace_dir + "/caffe.BUILD", 43 | ) 44 | 45 | native.new_git_repository( 46 | name = "caffe_ssd", 47 | remote = "https://github.com/weiliu89/caffe", 48 | commit = "781288500b34a532fd37ee0288710a0f012d2b6c", 49 | init_submodules = True, 50 | build_file = workspace_dir + "/caffe.BUILD", 51 | ) 52 | 53 | # ===== caffe build/integration tools ===== 54 | native.local_repository( 55 | name = "caffe_tools", 56 | path = workspace_dir + "/third_party/caffe" 57 | ) 58 | -------------------------------------------------------------------------------- /third_party/caffe/BUILD: -------------------------------------------------------------------------------- 1 | package( 2 | default_visibility = ["//visibility:public"], 3 | ) 4 | 5 | # 6 | # Caffe config settings 7 | # 8 | 9 | # enable pycaffe 10 | config_setting( 11 | name = "caffe_python_layer", 12 | values = { "define": "caffe_python_layer=ON" }, 13 | ) 14 | # standard caffe 15 | config_setting( 16 | name = "use_caffe_bvlc", 17 | values = { "define": "caffe_flavour=bvlc" }, 18 | ) 19 | # faster-rcnn caffe 20 | config_setting( 21 | name = "use_caffe_rcnn", 22 | values = { "define": "caffe_flavour=rcnn" }, 23 | ) 24 | # ssd 25 | config_setting( 26 | name = "use_caffe_ssd", 27 | values = { "define": "caffe_flavour=ssd" }, 28 | ) 29 | 30 | # 31 | # Shims/preludes 32 | # 33 | 34 | cc_library( 35 | name = "openblas_prelude", 36 | hdrs = ["openblas_prelude.h"], 37 | ) 38 | 39 | cc_library( 40 | name = "python_prelude", 41 | hdrs = ["python_prelude.h"], 42 | ) 43 | 44 | filegroup( 45 | name = "pycaffe_overrides", 46 | srcs = ["__init__.py"] 47 | ) 48 | -------------------------------------------------------------------------------- /third_party/caffe/WORKSPACE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rayglover-ibm/serving-caffe/7dfc79fe42c06397c22a371fe67913a29a27e625/third_party/caffe/WORKSPACE -------------------------------------------------------------------------------- /third_party/caffe/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright IBM Corp. All Rights Reserved. 2 | 3 | ''' 4 | This caffe package __init__ replaces the default caffe package initialization, 5 | and is based on the original: https://github.com/BVLC/caffe/blob/de8ac32a02f3e324b0495f1729bff2446d402c2c/python/caffe/__init__.py 6 | 7 | There are two important differences: 8 | 9 | 1. In this implementation, The '_caffe' module is embedded within the 10 | host process as a built-in module; relative-path imports in the 11 | original pycaffe implementation will fail; the custom 12 | EmbeddedCaffeImporter corrects this behavior. 13 | 14 | 2. This package only exposes a subset of pycaffe to get python layers 15 | working correctly. Other parts of pycaffe, including those that 16 | ultimately import multiprocessing, are not guaranteed to work. ''' 17 | 18 | import sys 19 | 20 | class EmbeddedCaffeImporter(object): 21 | name = 'caffe._caffe' 22 | 23 | def find_module(self, fullname, path): 24 | if fullname == self.name: 25 | return self 26 | 27 | return None 28 | 29 | def load_module(self, fullname): 30 | return __import__('_caffe') 31 | 32 | sys.meta_path.append(EmbeddedCaffeImporter()) 33 | 34 | from .pycaffe import Net, SGDSolver, NesterovSolver, AdaGradSolver, RMSPropSolver, AdaDeltaSolver, AdamSolver 35 | from ._caffe import set_mode_cpu, set_mode_gpu, set_device, Layer, get_solver, layer_type_list, set_random_seed 36 | from ._caffe import __version__ 37 | -------------------------------------------------------------------------------- /third_party/caffe/config.bzl: -------------------------------------------------------------------------------- 1 | def if_pycaffe(if_true, if_false = []): 2 | return select({ 3 | "@caffe_tools//:caffe_python_layer": if_true, 4 | "//conditions:default": if_false 5 | }) 6 | 7 | def caffe_pkg(label): 8 | return select({ 9 | "//conditions:default": ["@caffe//" + label], 10 | "@caffe_tools//:use_caffe_rcnn": ["@caffe_rcnn//" + label], 11 | "@caffe_tools//:use_caffe_ssd": ["@caffe_ssd//" + label], 12 | }) 13 | -------------------------------------------------------------------------------- /third_party/caffe/openblas_prelude.h: -------------------------------------------------------------------------------- 1 | #ifndef OPENBLAS_PRELUDE_H_ 2 | #define OPENBLAS_PRELUDE_H_ 3 | 4 | extern "C" void openblas_set_num_threads(int num_threads); 5 | 6 | #endif -------------------------------------------------------------------------------- /third_party/caffe/python_prelude.h: -------------------------------------------------------------------------------- 1 | #ifndef PYTHON_PRELUDE_H_ 2 | #define PYTHON_PRELUDE_H_ 3 | 4 | extern "C" { 5 | void Py_Initialize(); 6 | void Py_InitializeEx(int initsigs); 7 | void Py_Finalize(); 8 | int Py_IsInitialized(); 9 | 10 | const char* Py_GetVersion(); 11 | char* Py_GetPath(); 12 | int PyImport_AppendInittab(const char *name, void (*initfunc)(void)); 13 | void PySys_SetPath(const char *path); 14 | int PyRun_SimpleString(const char *command); 15 | void PyErr_PrintEx(int set_sys_last_vars); 16 | 17 | typedef struct {} PyObject; 18 | 19 | PyObject* PyErr_Occurred(); 20 | void PyErr_Clear(); 21 | } 22 | 23 | #endif // PYTHON_PRELUDE_H_ -------------------------------------------------------------------------------- /tools/bazel.rc: -------------------------------------------------------------------------------- 1 | build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain 2 | build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true 3 | 4 | build --force_python=py2 5 | build --python2_path=/usr/bin/python 6 | 7 | build --define PYTHON_BIN_PATH=/usr/bin/python 8 | test --define PYTHON_BIN_PATH=/usr/bin/python 9 | run --define PYTHON_BIN_PATH=/usr/bin/python 10 | 11 | build --spawn_strategy=standalone --genrule_strategy=standalone 12 | test --spawn_strategy=standalone --genrule_strategy=standalone 13 | run --spawn_strategy=standalone --genrule_strategy=standalone 14 | --------------------------------------------------------------------------------