├── NOTICE ├── OWNERS ├── tests ├── ut │ ├── CMakeLists.txt │ ├── coverage │ │ ├── cov_config │ │ └── run_coverage.sh │ ├── python │ │ ├── mindspore │ │ │ └── dataset │ │ │ │ └── __init__.py │ │ ├── CMakeLists.txt │ │ └── servable_config │ │ │ └── generate_certs.sh │ ├── cpp │ │ ├── common │ │ │ ├── test_main.cc │ │ │ ├── common_test.h │ │ │ └── common_test.cc │ │ └── runtest.sh │ ├── stub │ │ ├── include │ │ │ ├── api │ │ │ │ ├── format.h │ │ │ │ ├── callback │ │ │ │ │ ├── loss_monitor.h │ │ │ │ │ ├── time_monitor.h │ │ │ │ │ ├── ckpt_saver.h │ │ │ │ │ ├── train_accuracy.h │ │ │ │ │ └── lr_scheduler.h │ │ │ │ ├── visible.h │ │ │ │ ├── metrics │ │ │ │ │ ├── metrics.h │ │ │ │ │ └── accuracy.h │ │ │ │ ├── data_type.h │ │ │ │ ├── graph.h │ │ │ │ └── ops │ │ │ │ │ └── ops.h │ │ │ ├── mindapi │ │ │ │ └── base │ │ │ │ │ └── format.h │ │ │ └── utils │ │ │ │ └── visible.h │ │ └── cxx_api │ │ │ ├── graph │ │ │ ├── graph.cc │ │ │ ├── graph_data.h │ │ │ └── ascend │ │ │ │ ├── ascend_graph_impl.h │ │ │ │ └── ascend_graph_impl.cc │ │ │ └── model │ │ │ ├── model_impl.cc │ │ │ └── ms │ │ │ └── ms_model.h │ └── runtest.sh ├── CMakeLists.txt └── st │ ├── add │ ├── __init__.py │ └── test_serving.py │ ├── resnet │ ├── __init__.py │ └── test_resnet.py │ ├── serving_fault │ ├── __init__.py │ ├── kill_9_master.sh │ ├── kill_9_worker.sh │ ├── kill_15_master.sh │ └── kill_15_worker.sh │ ├── add_sub_pipeline │ ├── __init__.py │ └── test_serving.py │ ├── matmul_distributed │ ├── __init__.py │ └── test_matmul_distribute.py │ ├── distributed_server_fault │ ├── __init__.py │ ├── kill_15_agent.sh │ ├── kill_15_server.sh │ ├── kill_9_server.sh │ └── kill_9_agent.sh │ └── matmul_multi_subgraphs │ ├── __init__.py │ └── test_matmul_multi_subgraphs.py ├── engine └── README.md ├── docs ├── architecture.png └── api │ └── api_python │ ├── client │ ├── mindspore_serving.client.rst │ ├── mindspore_serving.client.SSLConfig.rst │ └── mindspore_serving.client.Client.rst │ ├── server │ ├── mindspore_serving.server.stop.rst │ ├── register │ │ ├── mindspore_serving.server.register.rst │ │ ├── mindspore_serving.server.register.CPUDeviceInfo.rst │ │ ├── mindspore_serving.server.register.GPUDeviceInfo.rst │ │ ├── mindspore_serving.server.register.Model.rst │ │ ├── mindspore_serving.server.register.register_method.rst │ │ ├── mindspore_serving.server.register.Context.rst │ │ ├── mindspore_serving.server.register.add_stage.rst │ │ ├── mindspore_serving.server.register.AscendDeviceInfo.rst │ │ └── mindspore_serving.server.register.declare_model.rst │ ├── distributed │ │ ├── mindspore_serving.server.distributed.rst │ │ ├── mindspore_serving.server.distributed.start_servable.rst │ │ ├── mindspore_serving.server.distributed.declare_servable.rst │ │ └── mindspore_serving.server.distributed.startup_agents.rst │ ├── mindspore_serving.server.rst │ ├── mindspore_serving.server.start_restful_server.rst │ ├── mindspore_serving.server.SSLConfig.rst │ ├── mindspore_serving.server.start_grpc_server.rst │ ├── mindspore_serving.server.start_servables.rst │ └── mindspore_serving.server.ServableStartConfig.rst │ └── mindspore_serving.client.rst ├── example ├── lenet │ ├── test_image │ │ ├── 0-1.png │ │ ├── 1-1.png │ │ ├── 2-1.png │ │ ├── 3-1.png │ │ ├── 4-1.png │ │ ├── 5-1.png │ │ ├── 6-1.png │ │ ├── 7-1.png │ │ ├── 8-1.png │ │ └── 9-1.png │ ├── export_model │ │ ├── lenet │ │ │ ├── lenet_ascend_v111_offical_cv_mnist_bs32_acc98.ckpt │ │ │ └── __init__.py │ │ └── export_lenet.py │ └── serving_server.py ├── resnet │ ├── test_image │ │ ├── car.jpg │ │ └── cat.jpg │ ├── export_model │ │ ├── resnet │ │ │ └── __init__.py │ │ └── export_resnet.py │ └── serving_server.py ├── matmul_distributed │ ├── matmul │ │ └── servable_config.py │ ├── serving_client.py │ ├── rank_table_8pcs.json │ ├── export_model │ │ ├── rank_table_8pcs.json │ │ ├── export_model.sh │ │ ├── net.py │ │ └── distributed_inference.py │ ├── serving_agent.py │ └── serving_server.py ├── matmul_multi_subgraphs │ ├── serving_client.py │ ├── serving_server.py │ └── matmul │ │ └── servable_config.py ├── add_sub_pipeline │ └── serving_server.py └── tensor_add │ └── serving_server.py ├── .gitmodules ├── requirements_test.txt ├── .jenkins └── test │ └── config │ └── dependent_packages.yaml ├── third_party ├── securec │ ├── src │ │ ├── CMakeLists.txt │ │ ├── secureinput_a.c │ │ └── secureinput_w.c │ └── CMakeLists.txt └── patch │ ├── openssl │ ├── CVE-2022-4450.patch │ ├── CVE-2021-3712.patch │ ├── CVE-2022-2097.patch │ ├── CVE-2023-0466.patch │ ├── CVE-2023-0286.patch │ ├── CVE-2023-4807.patch │ └── CVE-2022-0778.patch │ ├── pybind11 │ └── pybind11.patch001 │ ├── libevent │ └── libevent.patch001 │ └── zlib │ └── CVE-2022-37434.patch ├── cmake ├── external_libs │ ├── eigen.cmake │ ├── re2.cmake │ ├── zlib.cmake │ ├── c-ares.cmake │ ├── json.cmake │ ├── glog.cmake │ ├── libevent.cmake │ └── absl.cmake ├── dependency_securec.cmake ├── dependency_ms.cmake ├── dependency_utils.cmake ├── mind_expression.cmake └── check_requirements.cmake ├── mindspore_serving ├── client │ ├── python │ │ └── __init__.py │ └── __init__.py ├── __init__.py ├── ccsrc │ ├── common │ │ ├── heart_beat.cc │ │ ├── grpc_client.cc │ │ ├── serving_common.h │ │ ├── ssl_config.h │ │ ├── instance_data.h │ │ ├── float16.h │ │ ├── utils.h │ │ ├── instance.h │ │ ├── grpc_server.h │ │ └── exit_handle.h │ ├── master │ │ ├── grpc │ │ │ └── grpc_server.cc │ │ ├── master_context.cc │ │ ├── master_context.h │ │ ├── restful │ │ │ └── http_handle.h │ │ └── notify_worker │ │ │ ├── base_notify.h │ │ │ └── grpc_notify.h │ ├── worker │ │ ├── grpc │ │ │ ├── worker_server.cc │ │ │ └── worker_process.h │ │ ├── notfiy_master │ │ │ └── base_notify.h │ │ ├── distributed_worker │ │ │ ├── notify_agent │ │ │ │ ├── base_notify_agent.h │ │ │ │ └── notify_agent.h │ │ │ ├── agent_startup.h │ │ │ └── common.h │ │ └── context.h │ └── python │ │ ├── worker │ │ └── servable_py.h │ │ ├── master │ │ └── master_py.h │ │ └── agent │ │ └── agent_py.h ├── server │ ├── common │ │ ├── __init__.py │ │ ├── utils.py │ │ └── decorator.py │ ├── worker │ │ ├── __init__.py │ │ └── distributed │ │ │ └── __init__.py │ ├── master │ │ ├── __init__.py │ │ └── context.py │ ├── distributed │ │ └── __init__.py │ ├── register │ │ ├── utils.py │ │ └── __init__.py │ └── __init__.py └── proto │ ├── ms_agent.proto │ └── ms_worker.proto ├── .gitee └── PULL_REQUEST_TEMPLATE.md ├── RELEASE_CN.md └── .gitignore /NOTICE: -------------------------------------------------------------------------------- 1 | MindSpore Serving 2 | Copyright 2020 Huawei Technologies Co., Ltd 3 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - zhaizhiqiang 3 | - zhangxuetong 4 | - hangangqiang 5 | -------------------------------------------------------------------------------- /tests/ut/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_subdirectory(python) 3 | add_subdirectory(cpp) 4 | -------------------------------------------------------------------------------- /engine/README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | 3 | An engine supports finetune and inference. 4 | -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/docs/architecture.png -------------------------------------------------------------------------------- /tests/ut/coverage/cov_config: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = */__init__.py,*/*_pb2.py,*/*_pb2_grpc.py,*/tests/* -------------------------------------------------------------------------------- /docs/api/api_python/client/mindspore_serving.client.rst: -------------------------------------------------------------------------------- 1 | MindSpore Serving客户端API，用于通过gRPC访问MindSpore Serving服务器。 2 | -------------------------------------------------------------------------------- /example/lenet/test_image/0-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/0-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/1-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/1-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/2-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/2-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/3-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/3-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/4-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/4-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/5-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/5-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/6-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/6-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/7-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/7-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/8-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/8-1.png -------------------------------------------------------------------------------- /example/lenet/test_image/9-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/test_image/9-1.png -------------------------------------------------------------------------------- /example/resnet/test_image/car.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/resnet/test_image/car.jpg -------------------------------------------------------------------------------- /example/resnet/test_image/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/resnet/test_image/cat.jpg -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/mindspore"] 2 | path = third_party/mindspore 3 | url = https://gitee.com/mindspore/mindspore.git 4 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | numpy >= 1.17.0 2 | protobuf >= 3.13.0 3 | grpcio >= 1.36.0, <= 1.47.0 4 | requests >= 2.22.0 5 | psutil >= 5.9.1 6 | -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.stop.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.stop() 3 | 4 | 停止Serving服务器的运行。 5 | -------------------------------------------------------------------------------- /.jenkins/test/config/dependent_packages.yaml: -------------------------------------------------------------------------------- 1 | mindspore: 2 | 'mindspore/mindspore/version/202310/20231010/master_20231010144855_e5008bcfa07e3e6f3fa50f3ba0ac90175504dfd7/' 3 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #add flags 2 | message("================START BUILD TESTCASES=================") 3 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") 4 | 5 | add_subdirectory("ut") 6 | -------------------------------------------------------------------------------- /example/lenet/export_model/lenet/lenet_ascend_v111_offical_cv_mnist_bs32_acc98.ckpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mindspore-ai/serving/HEAD/example/lenet/export_model/lenet/lenet_ascend_v111_offical_cv_mnist_bs32_acc98.ckpt -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.rst: -------------------------------------------------------------------------------- 1 | 服务注册接口，在服务的servable_config.py配置文件中使用。如何配置servable_config.py文件，请查看 2 | `通过配置模型提供Servable `_ 。 3 | -------------------------------------------------------------------------------- /docs/api/api_python/server/distributed/mindspore_serving.server.distributed.rst: -------------------------------------------------------------------------------- 1 | Serving服务器启动分布式模型服务的接口。如何配置和启动分布式模型，请查看 2 | `基于MindSpore Serving部署分布式推理服务 `_ 。 3 | -------------------------------------------------------------------------------- /third_party/securec/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (CMAKE_SYSTEM_NAME MATCHES "Windows") 2 | list(APPEND SECUREC_SRCS "memset_s.c") 3 | else() 4 | aux_source_directory(. SECUREC_SRCS) 5 | endif() 6 | add_library(securec STATIC ${SECUREC_SRCS}) 7 | -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.rst: -------------------------------------------------------------------------------- 1 | MindSpore Serving是一个轻量级、高性能的服务模块，旨在帮助MindSpore开发者在生产环境中高效部署在线推理服务。 2 | 3 | 用户可通过MindSpore Serving server API启动服务，启动gRPC和RESTful（HTTP）服务器。其中一个服务一般可由一个模型或者一组模型组合提供。客户端通过gRPC和RESTful（HTTP）服务器发送推理任务，接收推理结果。 -------------------------------------------------------------------------------- /docs/api/api_python/mindspore_serving.client.rst: -------------------------------------------------------------------------------- 1 | mindspore_serving.client 2 | ========================== 3 | 4 | .. include:: client/mindspore_serving.client.rst 5 | 6 | .. include:: client/mindspore_serving.client.Client.rst 7 | 8 | .. include:: client/mindspore_serving.client.SSLConfig.rst 9 | 10 | .. automodule:: mindspore_serving.client 11 | :members: 12 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.CPUDeviceInfo.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.register.CPUDeviceInfo(**kwargs) 3 | 4 | 用于CPU设备配置。 5 | 6 | 参数： 7 | - **precision_mode** (str, 可选) - 推理精度选项，值可以是 ``"origin"`` 或 ``"fp16"`` ， ``"origin"`` 表示以模型中指定精度进行推理， ``"fp16"`` 表示以FP16精度进行推理。默认值： ``"origin"`` 。 8 | 9 | 异常： 10 | - **RuntimeError** - 选项无效，或值类型不是字符串。 11 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.GPUDeviceInfo.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.register.GPUDeviceInfo(**kwargs) 3 | 4 | 用于GPU设备配置。 5 | 6 | 参数： 7 | - **precision_mode** (str, 可选) - 推理精度选项，值可以是 ``"origin"`` 或 ``"fp16"`` ， ``"origin"`` 表示以模型中指定精度进行推理， ``"fp16"`` 表示以FP16精度进行推理。默认值： ``"origin"`` 。 8 | 9 | 异常： 10 | - **RuntimeError** - 选项无效，或值类型不是字符串。 11 | -------------------------------------------------------------------------------- /docs/api/api_python/client/mindspore_serving.client.SSLConfig.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.client.SSLConfig(certificate=None, private_key=None, custom_ca=None) 3 | 4 | Serving服务器gRPC SSL使能时，通过SSLConfig封装SSL证书等相关参数。 5 | 6 | 参数： 7 | - **certificate** (str, 可选) - PEM编码的证书链内容，如果为 ``None``，表示不使用证书链。默认值：``None``。 8 | - **private_key** (str, 可选) - PEM编码的私钥内容，如果为 ``None``，表示不使用私钥。默认值：``None``。 9 | - **custom_ca** (str, 可选) - PEM编码的根证书内容，如果为 ``None``，gRPC运行时将从默认位置加载根证书。默认值：``None``。 10 | 11 | 异常： 12 | - **RuntimeError** - 参数的类型或值无效。 13 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2022-4450.patch: -------------------------------------------------------------------------------- 1 | diff --git a/crypto/pem/pem_lib.c b/crypto/pem/pem_lib.c 2 | index d416d939ea..328c30cdbb 100644 3 | --- a/crypto/pem/pem_lib.c 4 | +++ b/crypto/pem/pem_lib.c 5 | @@ -957,7 +957,9 @@ int PEM_read_bio_ex(BIO *bp, char **name_out, char **header, 6 | *data = pem_malloc(len, flags); 7 | if (*header == NULL || *data == NULL) { 8 | pem_free(*header, flags, 0); 9 | + *header = NULL; 10 | pem_free(*data, flags, 0); 11 | + *data = NULL; 12 | goto end; 13 | } 14 | BIO_read(headerB, *header, headerlen); -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.start_restful_server.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.start_restful_server(address, max_msg_mb_size=100, ssl_config=None) 3 | 4 | 启动RESTful服务器，用于Serving客户端和Serving服务器之间的通信。 5 | 6 | 参数： 7 | - **address** (str) - RESTful服务器地址，地址应为Internet domain socket地址。 8 | - **max_msg_mb_size** (int, 可选) - 最大可接收的RESTful消息大小，以MB为单位，取值范围[1, 512]。默认值：``100``。 9 | - **ssl_config** (mindspore_serving.server.SSLConfig, 可选) - 服务器的SSL配置，如果是 ``None``，则禁用SSL。默认值：``None``。 10 | 11 | 异常： 12 | - **RuntimeError** - 启动RESTful服务器失败：参数校验失败，RESTful地址错误或端口重复。 13 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.Model.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.register.Model(model_key) 3 | 4 | 用于表示一个声明的模型。用户不应该直接构造 `Model` 对象，而是来自于 `declare_model` 或 `declare_servable` 的返回。 5 | 6 | 参数： 7 | - **model_key** (str) - 模型的唯一标志。 8 | 9 | .. py:method:: call(*args, subgraph=0) 10 | 11 | 调用模型推理接口。 12 | 13 | 参数： 14 | - **args** - 实例的元组/列表，或一个实例的输入。 15 | - **subgraph** (int, 可选) - 子图索引，当一个模型中存在多个子图时使用。默认值：``0``。 16 | 17 | 返回： 18 | 当输入参数 `args` 为元组/列表时，返回为instances的元组，当前输入 `args` 为一个实例的输入时，输出为这个实例的输出。 19 | 20 | 异常： 21 | - **RuntimeError** - 输入无效。 22 | -------------------------------------------------------------------------------- /third_party/patch/pybind11/pybind11.patch001: -------------------------------------------------------------------------------- 1 | diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h 2 | index 3bffbb28..4a6a9809 100644 3 | --- a/include/pybind11/pybind11.h 4 | +++ b/include/pybind11/pybind11.h 5 | @@ -21,6 +21,7 @@ 6 | # pragma warning disable 1875 // offsetof applied to non-POD (Plain Old Data) types is nonstandard 7 | # pragma warning disable 2196 // warning #2196: routine is both "inline" and "noinline" 8 | #elif defined(_MSC_VER) 9 | +#include 10 | # pragma warning(push) 11 | # pragma warning(disable: 4100) // warning C4100: Unreferenced formal parameter 12 | # pragma warning(disable: 4127) // warning C4127: Conditional expression is constant 13 | 14 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2021-3712.patch: -------------------------------------------------------------------------------- 1 | diff --git a/crypto/ec/ec_asn1.c b/crypto/ec/ec_asn1.c 2 | index 7b7c75ce84..e497a25909 100644 3 | --- a/crypto/ec/ec_asn1.c 4 | +++ b/crypto/ec/ec_asn1.c 5 | @@ -761,7 +761,10 @@ EC_GROUP *EC_GROUP_new_from_ecparameters(const ECPARAMETERS *params) 6 | ret->seed_len = params->curve->seed->length; 7 | } 8 | 9 | - if (!params->order || !params->base || !params->base->data) { 10 | + if (params->order == NULL 11 | + || params->base == NULL 12 | + || params->base->data == NULL 13 | + || params->base->length == 0) { 14 | ECerr(EC_F_EC_GROUP_NEW_FROM_ECPARAMETERS, EC_R_ASN1_ERROR); 15 | goto err; 16 | } 17 | 18 | -------------------------------------------------------------------------------- /cmake/external_libs/eigen.cmake: -------------------------------------------------------------------------------- 1 | set(Eigen3_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") 2 | set(Eigen3_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") 3 | 4 | 5 | set(REQ_URL "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz") 6 | set(SHA256 "8586084f71f9bde545ee7fa6d00288b264a2b7ac3607b974e54d13e7162c1c72") 7 | 8 | 9 | mindspore_add_pkg(Eigen3 10 | VER 3.4.0 11 | URL ${REQ_URL} 12 | SHA256 ${SHA256} 13 | CMAKE_OPTION -DBUILD_TESTING=OFF) 14 | find_package(Eigen3 3.4.0 REQUIRED ${MS_FIND_NO_DEFAULT_PATH}) 15 | include_directories(${Eigen3_INC}) 16 | include_directories(${EIGEN3_INCLUDE_DIR}) 17 | set_property(TARGET Eigen3::Eigen PROPERTY IMPORTED_GLOBAL TRUE) 18 | add_library(mindspore_serving::eigen ALIAS Eigen3::Eigen) 19 | -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.SSLConfig.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.SSLConfig(certificate, private_key, custom_ca=None, verify_client=False) 3 | 4 | Serving服务器中，使能gRPC或RESTful服务器SSL功能时，SSL的参数配置。 5 | 6 | 参数： 7 | - **certificate** (str) - PEM编码的证书链内容，如果值为 ``None``，则表示不使用证书链。 8 | - **private_key** (str) - PEM编码的私钥内容，如果值为 ``None``，则表示不使用私钥。 9 | - **custom_ca** (str, 可选) - PEM编码的根证书内容。当 `verify_client` 为 ``True`` 时， `custom_ca` 必须指定。当 `verify_client` 为 ``False`` 时，将忽略此参数。默认值：``None``。 10 | - **verify_client** (bool, 可选) - 如果 `verify_client` 为 ``True``，则启用客户端服务器双向认证。如果为 ``False``，则仅启用客户端对服务器的单向认证。默认值：``False``。 11 | 12 | 异常： 13 | - **RuntimeError** - 参数的类型或值无效。 14 | -------------------------------------------------------------------------------- /tests/st/add/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /mindspore_serving/client/python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /tests/st/resnet/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /tests/ut/python/mindspore/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /tests/st/serving_fault/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /tests/st/add_sub_pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /tests/st/matmul_distributed/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /mindspore_serving/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """MindSpore Serving.""" 16 | -------------------------------------------------------------------------------- /tests/st/distributed_server_fault/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /tests/st/matmul_multi_subgraphs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | -------------------------------------------------------------------------------- /example/lenet/export_model/lenet/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Resnet export model""" 16 | -------------------------------------------------------------------------------- /example/resnet/export_model/resnet/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Resnet export model""" 16 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/heart_beat.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "common/heart_beat.h" 18 | namespace mindspore::serving {} // namespace mindspore::serving 19 | -------------------------------------------------------------------------------- /cmake/dependency_securec.cmake: -------------------------------------------------------------------------------- 1 | # securec library 2 | # 3 | # 4 | # SECUREC_LIBRARY 5 | # 6 | 7 | if(NOT TARGET securec) 8 | set(_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE ${CMAKE_POSITION_INDEPENDENT_CODE}) 9 | set(_ms_tmp_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) 10 | 11 | set(CMAKE_C_FLAGS "${SECURE_CXX_FLAGS}") 12 | if(CMAKE_SYSTEM_NAME MATCHES "Windows") 13 | add_compile_definitions(SECUREC_ONLY_DECLARE_MEMSET) 14 | endif() 15 | add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/securec ${CMAKE_BINARY_DIR}/securec) 16 | set(CMAKE_POSITION_INDEPENDENT_CODE ${_ms_tmp_CMAKE_POSITION_INDEPENDENT_CODE}) 17 | set(CMAKE_C_FLAGS ${_ms_tmp_CMAKE_C_FLAGS}) 18 | endif() 19 | 20 | include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/securec/include) 21 | 22 | set(SECUREC_LIBRARY securec) 23 | -------------------------------------------------------------------------------- /third_party/securec/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | SET(CMAKE_BUILD_TYPE "Debug") 2 | if (CMAKE_SYSTEM_NAME MATCHES "Windows") 3 | SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all") 4 | else() 5 | SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -fstack-protector-all -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") 6 | endif() 7 | SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -Wno-deprecated-declarations -fstack-protector-all") 8 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 9 | 10 | #add flags 11 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") 12 | 13 | include_directories(./include) 14 | add_subdirectory(src) 15 | -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.start_grpc_server.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.start_grpc_server(address, max_msg_mb_size=100, ssl_config=None) 3 | 4 | 启动gRPC服务器，用于Serving客户端和Serving服务器之间的通信。 5 | 6 | 参数： 7 | - **address** (str) - gRPC服务器地址，地址可以是 `{ip}:{port}` 或 `unix:{unix_domain_file_path}` 。 8 | 9 | - `{ip}:{port}` - Internet domain socket地址。 10 | - `unix:{unix_domain_file_path}` - Unix domain socket地址，用于与同一台计算机上的多个进程通信。 `{unix_domain_file_path}` 可以是相对路径或绝对路径，但文件所在的目录必须已经存在。 11 | 12 | - **max_msg_mb_size** (int, 可选) - 可接收的最大gRPC消息大小（MB），取值范围[1, 512]。默认值：``100``。 13 | - **ssl_config** (mindspore_serving.server.SSLConfig, 可选) - 服务器的SSL配置，如果 ``None``，则禁用SSL。默认值：``None``。 14 | 15 | 异常： 16 | - **RuntimeError** - 启动gRPC服务器失败：参数校验失败，gRPC地址错误或端口重复。 17 | -------------------------------------------------------------------------------- /cmake/external_libs/re2.cmake: -------------------------------------------------------------------------------- 1 | if(ENABLE_GITEE) 2 | set(REQ_URL "https://gitee.com/mirrors/re2/repository/archive/2019-12-01.tar.gz") 3 | set(SHA256 "7268e1b4254d9ffa5ccf010fee954150dbb788fd9705234442e7d9f0ee5a42d3") 4 | else() 5 | set(REQ_URL "https://github.com/google/re2/archive/2019-12-01.tar.gz") 6 | set(SHA256 "7268e1b4254d9ffa5ccf010fee954150dbb788fd9705234442e7d9f0ee5a42d3") 7 | endif() 8 | 9 | if(NOT ENABLE_GLIBCXX) 10 | set(re2_CXXFLAGS "${re2_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") 11 | endif() 12 | 13 | mindspore_add_pkg(re2 14 | VER 20191201 15 | LIBS re2 16 | URL ${REQ_URL} 17 | SHA256 ${SHA256} 18 | CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE) 19 | 20 | include_directories(${re2_INC}) 21 | add_library(mindspore_serving::re2 ALIAS re2::re2) -------------------------------------------------------------------------------- /cmake/external_libs/zlib.cmake: -------------------------------------------------------------------------------- 1 | if(ENABLE_GITEE) 2 | set(REQ_URL "https://gitee.com/mirrors/zlib/repository/archive/v1.2.11.tar.gz") 3 | set(SHA256 "f21b3885cc7732f0ab93dbe06ff1ec58069bb58657b3fda89531d1562d8ad708") 4 | else() 5 | set(REQ_URL "https://github.com/madler/zlib/archive/v1.2.11.tar.gz") 6 | set(SHA256 "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff") 7 | endif() 8 | 9 | mindspore_add_pkg(zlib 10 | VER 1.2.11 11 | LIBS z 12 | URL ${REQ_URL} 13 | SHA256 ${SHA256} 14 | CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release 15 | PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/zlib/CVE-2018-25032.patch 16 | PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/zlib/CVE-2022-37434.patch) 17 | 18 | include_directories(${zlib_INC}) 19 | add_library(mindspore_serving::z ALIAS zlib::z) 20 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.register_method.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.register.register_method(output_names) 3 | 4 | 在服务的servable_config.py配置文件中使用，用于注册服务的方法，一个服务可以包括一个或多个方法，每个方法可基于模型提供不同的功能，客户端访问服务时需要指定服务和方法。MindSpore Serving支持由多个Python函数和多个模型组合串接提供服务。 5 | 6 | .. note:: 本接口需要在Serving服务器导入servable_config.py时生效。因此，建议在servable_config.py中全局使用此接口。 7 | 8 | 此接口将定义方法的签名和处理流程。 9 | 10 | 签名包括方法名称、方法的输入和输出名称。当Serving客户端访问服务时，客户端需要指定服务名称、方法名称，并提供一个或多个推理实例。每个实例通过输入名称指定输入数据，并通过输出名称获取输出结果。 11 | 12 | 处理流程由一个或多个阶段（stage）组成，每个阶段可以是一个Python函数或模型。即，一个方法的处理流程可以包括一个或多个Python函数和一个或多个模型。此外，接口还定义了这些阶段之间的数据流。 13 | 14 | 参数： 15 | - **output_names** (Union[str, tuple[str], list[str]]) - 指定方法的输出名称。输入名称通过注册函数的参数名称指定。 16 | 17 | 异常： 18 | - **RuntimeError** - 参数的类型或值无效，或发生其他错误。 19 | -------------------------------------------------------------------------------- /tests/ut/cpp/common/test_main.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #include "gtest/gtest.h" 17 | 18 | GTEST_API_ int main(int argc, char **argv) { 19 | testing::InitGoogleTest(&argc, argv); 20 | int ret = RUN_ALL_TESTS(); 21 | return ret; 22 | } 23 | -------------------------------------------------------------------------------- /third_party/securec/src/secureinput_a.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #define SECUREC_FORMAT_OUTPUT_INPUT 1 18 | #ifdef SECUREC_FOR_WCHAR 19 | #undef SECUREC_FOR_WCHAR 20 | #endif 21 | 22 | #include "secinput.h" 23 | 24 | #include "input.inl" 25 | 26 | -------------------------------------------------------------------------------- /mindspore_serving/server/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """MindSpore Serving.""" 16 | 17 | from . import check_type 18 | from .utils import get_abs_path 19 | from .decorator import deprecated 20 | -------------------------------------------------------------------------------- /mindspore_serving/server/worker/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """MindSpore worker implement""" 16 | 17 | from ._worker import start_servable, start_extra_servable, stop, get_newest_version_number 18 | -------------------------------------------------------------------------------- /tests/st/serving_fault/kill_9_master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_master() 7 | { 8 | get_master_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_worker_count 16 | if [ $? -eq 0 ] 17 | then 18 | echo "worker start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | ps aux | grep 'serving_server.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 23 | if [ $? -ne 0 ] 24 | then 25 | echo "kill master failed" 26 | fi 27 | wait_worker_exit 28 | } 29 | 30 | test_master_fault_model() 31 | { 32 | start_serving_server 33 | kill_master 34 | clean_pid 35 | } 36 | 37 | echo "-----serving start-----" 38 | init 39 | test_master_fault_model 40 | echo "### end to serving test ###" 41 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2022-2097.patch: -------------------------------------------------------------------------------- 1 | diff --git a/crypto/aes/asm/aesni-x86.pl b/crypto/aes/asm/aesni-x86.pl 2 | index fe2b26542a..812758e02e 100644 3 | --- a/crypto/aes/asm/aesni-x86.pl 4 | +++ b/crypto/aes/asm/aesni-x86.pl 5 | @@ -2027,7 +2027,7 @@ my ($l_,$block,$i1,$i3,$i5) = ($rounds_,$key_,$rounds,$len,$out); 6 | &movdqu (&QWP(-16*2,$out,$inp),$inout4); 7 | &movdqu (&QWP(-16*1,$out,$inp),$inout5); 8 | &cmp ($inp,$len); # done yet? 9 | - &jb (&label("grandloop")); 10 | + &jbe (&label("grandloop")); 11 | 12 | &set_label("short"); 13 | &add ($len,16*6); 14 | @@ -2453,7 +2453,7 @@ my ($l_,$block,$i1,$i3,$i5) = ($rounds_,$key_,$rounds,$len,$out); 15 | &pxor ($rndkey1,$inout5); 16 | &movdqu (&QWP(-16*1,$out,$inp),$inout5); 17 | &cmp ($inp,$len); # done yet? 18 | - &jb (&label("grandloop")); 19 | + &jbe (&label("grandloop")); 20 | 21 | &set_label("short"); 22 | &add ($len,16*6); -------------------------------------------------------------------------------- /tests/st/serving_fault/kill_9_worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_worker() 7 | { 8 | get_master_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_worker_count 16 | if [ $? -eq 0 ] 17 | then 18 | echo "worker start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | 23 | ps aux | grep 'start_worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 24 | if [ $? -ne 0 ] 25 | then 26 | echo "kill worker failed" 27 | fi 28 | wait_master_exit 29 | } 30 | 31 | test_worker_fault_model() 32 | { 33 | start_serving_server 34 | kill_worker 35 | clean_pid 36 | } 37 | 38 | echo "-----serving start-----" 39 | init 40 | test_worker_fault_model 41 | echo "### end to serving test ###" 42 | -------------------------------------------------------------------------------- /tests/st/serving_fault/kill_15_master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_master() 7 | { 8 | get_master_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_worker_count 16 | if [ $? -eq 0 ] 17 | then 18 | echo "worker start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | ps aux | grep 'serving_server.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15 23 | if [ $? -ne 0 ] 24 | then 25 | echo "kill master failed" 26 | fi 27 | wait_master_exit 28 | wait_worker_exit 29 | } 30 | 31 | test_master_fault_model() 32 | { 33 | start_serving_server 34 | kill_master 35 | clean_pid 36 | } 37 | 38 | echo "-----serving start-----" 39 | init 40 | test_master_fault_model 41 | echo "### end to serving test ###" 42 | -------------------------------------------------------------------------------- /tests/st/serving_fault/kill_15_worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_worker() 7 | { 8 | get_master_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_worker_count 16 | if [ $? -eq 0 ] 17 | then 18 | echo "worker start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | 23 | ps aux | grep 'start_worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15 24 | if [ $? -ne 0 ] 25 | then 26 | echo "kill master failed" 27 | fi 28 | wait_master_exit 29 | wait_worker_exit 30 | } 31 | 32 | test_worker_fault_model() 33 | { 34 | start_serving_server 35 | kill_worker 36 | clean_pid 37 | } 38 | 39 | echo "-----serving start-----" 40 | init 41 | test_worker_fault_model 42 | echo "### end to serving test ###" 43 | -------------------------------------------------------------------------------- /cmake/dependency_ms.cmake: -------------------------------------------------------------------------------- 1 | # Compile MindSpore 2 | 3 | message(STATUS "**********begin to compile MindSpore**********") 4 | set(MS_SOURCE_DIR ${CMAKE_SOURCE_DIR}/third_party/mindspore) 5 | message(STATUS "MindSpore dir: ${MS_SOURCE_DIR}") 6 | message(STATUS "MindSpore compile method: -e${MS_BACKEND}") 7 | message(STATUS "MindSpore compile thread num: -j${THREAD_NUM}") 8 | message(STATUS "MindSpore version: -V${MS_VERSION}") 9 | 10 | if(MS_VERSION) 11 | set(MS_VERSION_OPTION -V${MS_VERSION}) 12 | endif() 13 | 14 | set(EXEC_COMMAND bash ${MS_SOURCE_DIR}/build.sh -e${MS_BACKEND} ${MS_VERSION_OPTION} -j${THREAD_NUM}) 15 | execute_process( 16 | COMMAND ${EXEC_COMMAND} 17 | WORKING_DIRECTORY ${MS_SOURCE_DIR} 18 | RESULT_VARIABLE RESULT 19 | ) 20 | if(NOT RESULT EQUAL "0") 21 | message(FATAL_ERROR "error! when ${EXEC_COMMAND} in ${MS_SOURCE_DIR}") 22 | endif() 23 | 24 | message(STATUS "**********end to compile MindSpore**********") -------------------------------------------------------------------------------- /third_party/patch/libevent/libevent.patch001: -------------------------------------------------------------------------------- 1 | diff -Npur libevent/CMakeLists.txt libevent-modify/CMakeLists.txt 2 | --- libevent/CMakeLists.txt 2020-07-05 20:02:46.000000000 +0800 3 | +++ libevent-modify/CMakeLists.txt 2021-04-19 16:36:57.982307500 +0800 4 | @@ -852,7 +852,7 @@ if (NOT EVENT__DISABLE_OPENSSL) 5 | 6 | list(APPEND SRC_OPENSSL bufferevent_openssl.c) 7 | list(APPEND HDR_PUBLIC include/event2/bufferevent_ssl.h) 8 | - list(APPEND LIB_APPS ${OPENSSL_LIBRARIES}) 9 | + list(APPEND LIB_APPS ${OPENSSL_LIBRARIES} -ldl) 10 | endif() 11 | 12 | if (NOT EVENT__DISABLE_THREAD_SUPPORT) 13 | diff -Npur libevent/cmake/AddEventLibrary.cmake libevent-modify/cmake/AddEventLibrary.cmake 14 | --- libevent/cmake/AddEventLibrary.cmake 2020-07-05 20:02:46.000000000 +0800 15 | +++ libevent-modify/cmake/AddEventLibrary.cmake 2021-04-19 16:36:57.982307500 +0800 16 | @@ -153,1 +153,0 @@ 17 | - INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib" 18 | -------------------------------------------------------------------------------- /third_party/patch/zlib/CVE-2022-37434.patch: -------------------------------------------------------------------------------- 1 | diff -Npur zlib-1.2.11/inflate.c zlib-1.2.11-change/inflate.c 2 | --- zlib-1.2.11/inflate.c 2017-01-01 15:37:10.000000000 +0800 3 | +++ zlib-1.2.11-change/inflate.c 2022-08-17 06:25:06.033176873 +0800 4 | @@ -759,8 +759,9 @@ int flush; 5 | if (copy > have) copy = have; 6 | if (copy) { 7 | if (state->head != Z_NULL && 8 | - state->head->extra != Z_NULL) { 9 | - len = state->head->extra_len - state->length; 10 | + state->head->extra != Z_NULL && 11 | + (len = state->head->extra_len - state->length) < 12 | + state->head->extra_max) { 13 | zmemcpy(state->head->extra + len, next, 14 | len + copy > state->head->extra_max ? 15 | state->head->extra_max - len : copy); 16 | -------------------------------------------------------------------------------- /mindspore_serving/server/worker/distributed/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """MindSpore Serving Distributed Worker.""" 16 | 17 | from .agent_startup import startup_agents 18 | from .register import declare_servable 19 | from .distributed_worker import start_servable 20 | -------------------------------------------------------------------------------- /cmake/external_libs/c-ares.cmake: -------------------------------------------------------------------------------- 1 | if(ENABLE_GITEE) 2 | set(REQ_URL "https://gitee.com/mirrors/c-ares/repository/archive/cares-1_15_0.tar.gz") 3 | set(SHA256 "7deb7872cbd876c29036d5f37e30c4cbc3cc068d59d8b749ef85bb0736649f04") 4 | else() 5 | set(REQ_URL "https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz") 6 | set(SHA256 "6cdb97871f2930530c97deb7cf5c8fa4be5a0b02c7cea6e7c7667672a39d6852") 7 | endif() 8 | 9 | mindspore_add_pkg(c-ares 10 | VER 1.15.0 11 | LIBS cares 12 | URL ${REQ_URL} 13 | SHA256 ${SHA256} 14 | CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release 15 | -DCARES_SHARED:BOOL=OFF 16 | -DCARES_STATIC:BOOL=ON 17 | -DCARES_STATIC_PIC:BOOL=ON 18 | -DHAVE_LIBNSL:BOOL=OFF 19 | PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/c-ares/CVE-2021-3672.patch) 20 | 21 | include_directories(${c-ares_INC}) 22 | add_library(mindspore_serving::cares ALIAS c-ares::cares) 23 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/master/grpc/grpc_server.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "master/grpc/grpc_server.h" 18 | #include 19 | #include 20 | #include "common/grpc_async_server.h" 21 | 22 | namespace mindspore { 23 | namespace serving {} // namespace serving 24 | } // namespace mindspore 25 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/grpc/worker_server.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "worker/grpc/worker_server.h" 18 | #include 19 | #include 20 | #include "common/grpc_server.h" 21 | 22 | namespace mindspore { 23 | namespace serving {} // namespace serving 24 | } // namespace mindspore 25 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/grpc_client.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "common/grpc_client.h" 18 | 19 | namespace mindspore { 20 | namespace serving { 21 | std::unique_ptr client_; 22 | std::unique_ptr distributed_client_; 23 | } // namespace serving 24 | } // namespace mindspore 25 | -------------------------------------------------------------------------------- /tests/st/distributed_server_fault/kill_15_agent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_serving_agent() 7 | { 8 | get_serving_server_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_serving_agent_count 16 | if [ $? -ne 9 ] 17 | then 18 | echo "serving agent start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | ps aux | grep 'serving_agent.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15 23 | if [ $? -ne 0 ] 24 | then 25 | echo "kill agent failed" 26 | fi 27 | 28 | wait_agent_exit 29 | wait_server_exit 30 | } 31 | 32 | test_kill_serving_agent() 33 | { 34 | start_serving_server 35 | start_serving_agent 36 | kill_serving_agent 37 | clean_pid 38 | } 39 | 40 | echo "-----serving start-----" 41 | init 42 | test_kill_serving_agent 43 | echo "### end to serving test ###" 44 | -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.start_servables.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.start_servables(servable_configs, enable_lite=False) 3 | 4 | 用于Serving服务器中启动一个或多个服务，一个模型可结合预处理、后处理提供一个服务，多个模型也可串接组合提供一个服务。 5 | 6 | 本接口可以用来启动多个不同的服务。一个服务可以部署在多个设备上，其中每个设备运行一个服务副本。 7 | 8 | 在Ascend 910硬件平台上，每个服务的每个副本都独占一个设备。不同的服务或同一服务的不同版本需要部署在不同的设备上。在Ascend 310P和GPU硬件平台上，一个设备可以被多个服务共享，不同服务或同一服务的不同版本可以部署在同一设备上，实现设备复用。 9 | 10 | 如何配置模型提供服务请查看 11 | `基于MindSpore Serving部署推理服务 `_ 和 12 | `通过配置模型提供Servable `_ 。 13 | 14 | 参数： 15 | - **servable_configs** (Union[ServableStartConfig, list[ServableStartConfig], tuple[ServableStartConfig]]) - 一个或多个服务的启动配置。 16 | - **enable_lite** (bool) - 是否使用MindSpore Lite推理后端。 默认值：``False``。 17 | 18 | 异常： 19 | - **RuntimeError** - 启动一个或多个服务失败。相关日志可查看本Serving服务器启动脚本所在目录的子目录serving_logs。 20 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.Context.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.register.Context(**kwargs) 3 | 4 | Context用于自定义设备配置，如果不指定Context，MindSpore Serving将使用默认设备配置。当使用推理后端为MindSpore Lite，且目标设备为Ascend或Nvidia GPU时，模型部分算子可能运行在CPU设备上，将额外配置 `CPUDeviceInfo` 。 5 | 6 | 参数： 7 | - **thread_num** (int, 可选) - 设置运行时的CPU线程数量，该选项仅当推理后端为MindSpore Lite有效。 8 | - **thread_affinity_core_list** (tuple[int], list[int], 可选) - 设置运行时的CPU绑核列表，该选项仅当推理后端为MindSpore Lite有效。 9 | - **enable_parallel** (bool, 可选) - 设置运行时是否支持并行，该选项仅当推理后端为MindSpore Lite有效。 10 | 11 | 异常： 12 | - **RuntimeError** - 输入参数的类型或值无效。 13 | 14 | .. py:method:: append_device_info(device_info) 15 | 16 | 用于添加一个用户自定义的设备配置。 17 | 18 | 参数： 19 | - **device_info** (Union[CPUDeviceInfo, GPUDeviceInfo, AscendDeviceInfo]) - 用户自定义设备配置，用户不指定设备配置时将使用默认值。可以为每个可能的设备自定义设备配置，系统根据实际的后端设备和推理包选择所需的设备信息。 20 | 21 | 异常： 22 | - **RuntimeError** - 输入参数的类型或值无效。 23 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/serving_common.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_SERVING_COMMON_H 18 | #define MINDSPORE_SERVING_SERVING_COMMON_H 19 | 20 | #include 21 | 22 | #include "common/status.h" 23 | #include "common/log.h" 24 | #include "common/tensor.h" 25 | #include "common/utils.h" 26 | 27 | #endif // MINDSPORE_SERVING_SERVING_COMMON_H 28 | -------------------------------------------------------------------------------- /.gitee/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | **What type of PR is this?** 9 | > Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line: 10 | > 11 | > /kind bug 12 | > /kind task 13 | > /kind feature 14 | 15 | 16 | **What does this PR do / why do we need it**: 17 | 18 | 19 | **Which issue(s) this PR fixes**: 20 | 24 | Fixes # 25 | 26 | **Special notes for your reviewers**: 27 | 28 | 29 | -------------------------------------------------------------------------------- /mindspore_serving/client/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """MindSpore Serving Client API, which can be used to access the Serving Server through gRPC""" 16 | 17 | from .python.client import Client 18 | from .python.client import SSLConfig 19 | 20 | __all__ = [] 21 | __all__.extend([ 22 | "Client", 23 | "SSLConfig" 24 | ]) 25 | -------------------------------------------------------------------------------- /tests/st/distributed_server_fault/kill_15_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_serving_server() 7 | { 8 | get_serving_server_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "master_with_worker start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_serving_agent_count 16 | if [ $? -ne 9 ] 17 | then 18 | echo "agent start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | ps aux | grep 'serving_server.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -15 23 | if [ $? -ne 0 ] 24 | then 25 | echo "kill master_with_worker failed" 26 | fi 27 | 28 | wait_agent_exit 29 | wait_server_exit 30 | } 31 | 32 | test_kill_serving_server() 33 | { 34 | start_serving_server 35 | start_serving_agent 36 | kill_serving_server 37 | clean_pid 38 | } 39 | 40 | echo "-----serving start-----" 41 | init 42 | test_kill_serving_server 43 | echo "### end to serving test ###" 44 | -------------------------------------------------------------------------------- /docs/api/api_python/server/distributed/mindspore_serving.server.distributed.start_servable.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.distributed.start_servable(servable_directory, servable_name, rank_table_json_file, version_number=1, distributed_address='0.0.0.0:6200', wait_agents_time_in_seconds=0) 3 | 4 | 启动在 `servable_directory` 中定义的名为 `servable_name` 的分布式服务。 5 | 6 | 参数： 7 | - **servable_directory** (str) - 服务所在的目录。预期有一个名为 `servable_name` 的目录。详细信息可以查看 `通过配置模型提供Servable `_ 。 8 | - **servable_name** (str) - 服务名称。 9 | - **version_number** (int, 可选) - 要加载的服务版本号。版本号应为正整数，从1开始。默认值：``1``。 10 | - **rank_table_json_file** (str) - rank table json文件名。 11 | - **distributed_address** (str, 可选) - Worker代理（Agent）连接的分布式Worker服务器地址。默认值： ``"0.0.0.0:6200"`` 。 12 | - **wait_agents_time_in_seconds** (int, 可选) - 等待所有Worker代理就绪的最长时间（以秒为单位），``0`` 表示无限时间。默认值：``0``。 13 | 14 | 异常： 15 | - **RuntimeError** - 启动分布式服务失败。 16 | -------------------------------------------------------------------------------- /mindspore_serving/server/common/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """common function utils""" 16 | import os 17 | import sys 18 | 19 | 20 | def get_abs_path(path): 21 | """get the absolute path""" 22 | script_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 23 | abs_path = os.path.realpath(os.path.join(script_dir, path)) 24 | return abs_path 25 | -------------------------------------------------------------------------------- /mindspore_serving/server/master/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """The master process of serving server: used to receive requests and dispatcher them to worker process""" 16 | 17 | from ._master import start_grpc_server, start_restful_server, stop, stop_on_except, SSLConfig 18 | from ._master import start_master_server, at_stop_list, only_model_stage 19 | from . import context 20 | -------------------------------------------------------------------------------- /tests/ut/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(STUB_DIR ../stub) 2 | set(ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) 3 | 4 | file(GLOB_RECURSE UT_SERVING_STUB RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "${STUB_DIR}/cxx_api/*.cc" 5 | "${STUB_DIR}/graph_impl_stub.cc" "${STUB_DIR}/include/utils/*.cc") 6 | 7 | add_library(mindspore SHARED ${UT_SERVING_STUB}) 8 | 9 | set(UT_SERVING_COMMON ${UT_SERVING_CORE_SRC} ${UT_SERVING_STUB}) 10 | 11 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 12 | include_directories(${STUB_DIR}/..) 13 | include_directories(${STUB_DIR}) 14 | include_directories(${STUB_DIR}/include) 15 | 16 | include_directories(${ROOT_DIR}/third_party) 17 | link_directories(${CMKAE_BINARY_DIR}/securec/src) 18 | 19 | target_link_libraries(mindspore PRIVATE ${SECUREC_LIBRARY} pthread) 20 | target_link_libraries(mindspore PRIVATE mindspore_serving::glog) 21 | 22 | set(LIBRARY_OUTPUT_PATH ${ROOT_DIR}/build/package/tests/mindspore/lib/) 23 | 24 | # copy mindspore include 25 | file(COPY ${STUB_DIR}/include/api DESTINATION ${ROOT_DIR}/build/package/tests/mindspore/include) 26 | -------------------------------------------------------------------------------- /cmake/dependency_utils.cmake: -------------------------------------------------------------------------------- 1 | # MS Utils 2 | # 3 | 4 | function(find_python_package out_inc out_lib) 5 | # Use PYTHON_EXECUTABLE if it is defined, otherwise default to python 6 | if("${PYTHON_EXECUTABLE}" STREQUAL "") 7 | set(PYTHON_EXECUTABLE "python3") 8 | else() 9 | set(PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") 10 | endif() 11 | 12 | execute_process( 13 | COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())" 14 | RESULT_VARIABLE result 15 | OUTPUT_VARIABLE inc) 16 | string(STRIP "${inc}" inc) 17 | set(${out_inc} ${inc} PARENT_SCOPE) 18 | 19 | execute_process( 20 | COMMAND "${PYTHON_EXECUTABLE}" -c "import distutils.sysconfig as sysconfig; import os; \ 21 | print(os.path.join(sysconfig.get_config_var('LIBDIR'), sysconfig.get_config_var('LDLIBRARY')))" 22 | RESULT_VARIABLE result 23 | OUTPUT_VARIABLE lib) 24 | string(STRIP "${lib}" lib) 25 | set(${out_lib} ${lib} PARENT_SCOPE) 26 | endfunction() 27 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/format.h: -------------------------------------------------------------------------------- 1 | /** 2 | * This is the C++ adaptation and derivative work of Myia (https://github.com/mila-iqia/myia/). 3 | * 4 | * Copyright 2021 Huawei Technologies Co., Ltd 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef MINDSPORE_INCLUDE_API_FORMAT_H 20 | #define MINDSPORE_INCLUDE_API_FORMAT_H 21 | 22 | #if __has_include("include/mindapi/base/format.h") 23 | #include "include/mindapi/base/format.h" 24 | #else 25 | #include "mindapi/base/format.h" 26 | #endif 27 | 28 | #endif // MINDSPORE_INCLUDE_API_FORMAT_H 29 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.add_stage.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.register.add_stage(stage, *args, outputs_count, batch_size=None, tag=None) 3 | 4 | 在服务的 `servable_config.py` 中，通过 `register_method` 装饰（wrap）Python函数定义服务的一个方法（method），本接口用于定义这个方法中的一个运行步骤（stage），可以是一个Python函数或者模型。 5 | 6 | .. note:: 入参 `args` 的长度应等于函数或模型的输入个数。 7 | 8 | 参数： 9 | - **stage** (Union(function, Model)) - 用户定义的Python函数或由 `declare_model` 返回 `Model` 对象。 10 | - **outputs_count** (int) - 用户定义的Python函数或模型的输出个数。 11 | - **batch_size** (int, 可选) - 仅当stage是Python函数，且函数一次可以处理多实例时，此参数有效。默认值：``None``。 12 | 13 | - ``None``，函数的输入将是一个实例的输入。 14 | - ``0``，函数的输入将是实例的元组对象，实例元组的最大长度由服务器根据模型的batch大小确定。 15 | - int value >= 1，函数的输入将是实例的元组对象，实例元组的最大长度是 `batch_size` 指定的值。 16 | 17 | - **args** - stage输入占位符，可以是 `register_method` 装饰（wrap）的函数的输入或其他 `add_stage` 的输出。 `args` 的长度应等于Python函数或模型的输入数量。 18 | - **tag** (str, 可选) - stage的自定义标签，如 ``"preprocess"``，默认值：``None``。 19 | 20 | 异常： 21 | - **RuntimeError** - 参数的类型或值无效，或发生其他错误。 22 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/ssl_config.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_SSL_CONFIG_H 18 | #define MINDSPORE_SERVING_SSL_CONFIG_H 19 | 20 | #include 21 | 22 | namespace mindspore::serving { 23 | struct SSLConfig { 24 | std::string certificate; 25 | std::string private_key; 26 | std::string custom_ca; 27 | bool verify_client{false}; 28 | bool use_ssl{false}; 29 | }; 30 | 31 | } // namespace mindspore::serving 32 | 33 | #endif // MINDSPORE_SERVING_SSL_CONFIG_H 34 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/instance_data.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_INSTANCE_DATA_H 18 | #define MINDSPORE_SERVING_INSTANCE_DATA_H 19 | 20 | #include 21 | #include "common/serving_common.h" 22 | 23 | namespace mindspore::serving { 24 | using InstanceData = std::vector; 25 | 26 | struct ResultInstance { 27 | InstanceData data; 28 | Status error_msg = SUCCESS; 29 | }; 30 | } // namespace mindspore::serving 31 | 32 | #endif // MINDSPORE_SERVING_INSTANCE_DATA_H 33 | -------------------------------------------------------------------------------- /example/lenet/export_model/export_lenet.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """export Lenet for mnist dataset""" 16 | 17 | import os 18 | from shutil import copyfile 19 | from lenet.export import export_lenet 20 | 21 | if __name__ == '__main__': 22 | export_lenet() 23 | dst_dir = '../lenet/1' 24 | try: 25 | os.mkdir(dst_dir) 26 | except OSError: 27 | pass 28 | dst_file = os.path.join(dst_dir, 'lenet.mindir') 29 | copyfile('lenet.mindir', dst_file) 30 | -------------------------------------------------------------------------------- /example/matmul_distributed/matmul/servable_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Distributed matmul config python file""" 16 | from mindspore_serving.server import distributed 17 | from mindspore_serving.server import register 18 | 19 | model = distributed.declare_servable(rank_size=8, stage_size=1, with_batch_dim=False) 20 | 21 | 22 | @register.register_method(output_names=["y"]) 23 | def predict(x): 24 | y = register.add_stage(model, x, outputs_count=1) 25 | return y 26 | -------------------------------------------------------------------------------- /tests/st/resnet/test_resnet.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | 16 | import os 17 | import pytest 18 | import numpy as np 19 | 20 | 21 | @pytest.mark.level0 22 | @pytest.mark.platform_arm_ascend_training 23 | @pytest.mark.env_single 24 | def test_resnet(): 25 | """test_serving""" 26 | sh_path = os.path.split(os.path.realpath(__file__))[0] 27 | ret = os.system(f"sh {sh_path}/resnet.sh") 28 | assert np.allclose(ret, 0) 29 | 30 | 31 | if __name__ == '__main__': 32 | test_resnet() 33 | -------------------------------------------------------------------------------- /tests/st/add/test_serving.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | 16 | import os 17 | import pytest 18 | import numpy as np 19 | 20 | 21 | @pytest.mark.level0 22 | @pytest.mark.platform_arm_ascend_training 23 | @pytest.mark.env_single 24 | def test_serving_add(): 25 | """test_serving""" 26 | sh_path = os.path.split(os.path.realpath(__file__))[0] 27 | ret = os.system(f"sh {sh_path}/add.sh") 28 | assert np.allclose(ret, 0) 29 | 30 | 31 | if __name__ == '__main__': 32 | test_serving_add() 33 | -------------------------------------------------------------------------------- /tests/st/add_sub_pipeline/test_serving.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | 16 | import os 17 | import pytest 18 | import numpy as np 19 | 20 | 21 | @pytest.mark.level0 22 | @pytest.mark.platform_arm_ascend_training 23 | @pytest.mark.env_single 24 | def test_serving_add(): 25 | """test_serving""" 26 | sh_path = os.path.split(os.path.realpath(__file__))[0] 27 | ret = os.system(f"sh {sh_path}/add_sub.sh") 28 | assert np.allclose(ret, 0) 29 | 30 | 31 | if __name__ == '__main__': 32 | test_serving_add() 33 | -------------------------------------------------------------------------------- /cmake/external_libs/json.cmake: -------------------------------------------------------------------------------- 1 | if(MSVC) 2 | set(flatbuffers_CXXFLAGS "${CMAKE_CXX_FLAGS}") 3 | set(flatbuffers_CFLAGS "${CMAKE_CXX_FLAGS}") 4 | set(flatbuffers_LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS}") 5 | else() 6 | set(nlohmann_json3101_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2") 7 | set(nlohmann_json3101_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") 8 | endif() 9 | 10 | if(ENABLE_GITEE OR ENABLE_GITEE_EULER) # Channel GITEE_EULER is NOT supported now, use GITEE instead. 11 | set(REQ_URL "https://gitee.com/mirrors/JSON-for-Modern-CPP/repository/archive/v3.10.1.zip") 12 | set(SHA256 "5c7d0a0542431fef628f8dc4c34fd022fe8747ccb577012d58f38672d8747e0d") 13 | set(INCLUDE "./include") 14 | else() 15 | 16 | set(REQ_URL "https://github.com/nlohmann/json/releases/download/v3.10.1/include.zip") 17 | set(SHA256 "144268f7f85afb0f0fbea7c796723c849724c975f9108ffdadde9ecedaa5f0b1") 18 | set(INCLUDE "./include") 19 | endif() 20 | 21 | mindspore_add_pkg(nlohmann_json3101 22 | VER 3.10.1 23 | HEAD_ONLY ${INCLUDE} 24 | URL ${REQ_URL} 25 | SHA256 ${SHA256}) 26 | include_directories(${nlohmann_json3101_INC}) 27 | add_library(mindspore_serving::json ALIAS nlohmann_json3101) 28 | -------------------------------------------------------------------------------- /docs/api/api_python/server/distributed/mindspore_serving.server.distributed.declare_servable.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.distributed.declare_servable(rank_size, stage_size, with_batch_dim=True, without_batch_dim_inputs=None, enable_pipeline_infer=False) 3 | 4 | 用于在servable_config.py中声明分布式服务，详细可参考 5 | `基于MindSpore Serving部署分布式推理服务 `_ 。 6 | 7 | 参数： 8 | - **rank_size** (int) - 分布式模型的rank大小。 9 | - **stage_size** (int) - 分布式模型的stage大小。 10 | - **with_batch_dim** (bool, 可选) - 模型输入和输出shape的第一个维度是否是batch维度。默认值：``True``。 11 | - **without_batch_dim_inputs** (Union[int, tuple[int], list[int]], 可选) - 当 `with_batch_dim` 为 ``True`` 时，用于指定shape不包括batch维度的模型输入的索引，比如模型输入0的shape不包括batch维度，则 `without_batch_dim_inputs=(0,)` 。默认值：``None``。 12 | - **enable_pipeline_infer** (bool, 可选) - 是否开启流水线并行推理，流水线并行可有效提升推理性能，详情可参考 `流水线并行 `_ 。默认值：``False``。 13 | 14 | 返回： 15 | `Model` ，此模型的标识，可以用来调用 `Model.call` 或作为 `add_stage` 的输入。 16 | 17 | 异常： 18 | - **RuntimeError** - 参数的类型或值无效。 19 | -------------------------------------------------------------------------------- /tests/ut/cpp/common/common_test.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef TESTS_UT_COMMON_UT_COMMON_H_ 17 | #define TESTS_UT_COMMON_UT_COMMON_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | #include "gtest/gtest.h" 23 | namespace UT { 24 | class Common : public testing::Test { 25 | public: 26 | // TestCase only enter once 27 | static void SetUpTestCase(); 28 | static void TearDownTestCase(); 29 | 30 | // every TEST_F macro will enter one 31 | virtual void SetUp(); 32 | virtual void TearDown(); 33 | }; 34 | } // namespace UT 35 | #endif // TESTS_UT_COMMON_UT_COMMON_H_ 36 | -------------------------------------------------------------------------------- /tests/st/matmul_distributed/test_matmul_distribute.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | 16 | import os 17 | import pytest 18 | import numpy as np 19 | 20 | 21 | @pytest.mark.level0 22 | @pytest.mark.platform_arm_ascend_training 23 | @pytest.mark.env_single 24 | def test_serving_matmul_distributed(): 25 | """test_serving""" 26 | sh_path = os.path.split(os.path.realpath(__file__))[0] 27 | ret = os.system(f"sh {sh_path}/matmul_distribute.sh") 28 | assert np.allclose(ret, 0) 29 | 30 | 31 | if __name__ == '__main__': 32 | test_serving_matmul_distributed() 33 | -------------------------------------------------------------------------------- /tests/st/matmul_multi_subgraphs/test_matmul_multi_subgraphs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | 16 | import os 17 | import pytest 18 | import numpy as np 19 | 20 | 21 | @pytest.mark.level0 22 | @pytest.mark.platform_arm_ascend_training 23 | @pytest.mark.env_single 24 | def test_serving_pipeline_distributed(): 25 | """test_serving""" 26 | sh_path = os.path.split(os.path.realpath(__file__))[0] 27 | ret = os.system(f"sh {sh_path}/matmul_multi_subgraphs.sh") 28 | assert np.allclose(ret, 0) 29 | 30 | 31 | if __name__ == '__main__': 32 | test_serving_pipeline_distributed() 33 | -------------------------------------------------------------------------------- /example/matmul_distributed/serving_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Client for distributed matmul""" 16 | import numpy as np 17 | from mindspore_serving.client import Client 18 | 19 | 20 | def run_matmul(): 21 | """Run client of distributed matmul""" 22 | client = Client("localhost:5500", "matmul", "predict") 23 | instance = {"x": np.ones((128, 96), np.float32)} 24 | result = client.infer(instance) 25 | print("result:\n", result) 26 | assert len(result) == 1 27 | assert "y" in result[0] 28 | 29 | 30 | if __name__ == '__main__': 31 | run_matmul() 32 | -------------------------------------------------------------------------------- /example/lenet/serving_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Start Servable lenet""" 16 | 17 | import os 18 | import sys 19 | from mindspore_serving import server 20 | 21 | 22 | def start(): 23 | servable_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 24 | config = server.ServableStartConfig(servable_directory=servable_dir, servable_name="lenet", device_ids=(0, 1)) 25 | server.start_servables(config) 26 | 27 | server.start_grpc_server("127.0.0.1:5500") 28 | server.start_restful_server("127.0.0.1:1500") 29 | 30 | 31 | if __name__ == "__main__": 32 | start() 33 | -------------------------------------------------------------------------------- /example/matmul_multi_subgraphs/serving_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Client for distributed matmul""" 16 | import numpy as np 17 | from mindspore_serving.client import Client 18 | 19 | 20 | def run_matmul(): 21 | """Run client of distributed matmul""" 22 | client = Client("localhost:5500", "matmul", "predict") 23 | instance = {"x": np.ones((128, 96), np.float32), "y": np.ones((8, 96), np.float32)} 24 | result = client.infer(instance) 25 | print("result:\n", result) 26 | assert "z" in result[0] 27 | 28 | 29 | if __name__ == '__main__': 30 | run_matmul() 31 | -------------------------------------------------------------------------------- /tests/ut/runtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2019 Huawei Technologies Co., Ltd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================ 16 | 17 | set -e 18 | 19 | CURRPATH=$( 20 | cd "$(dirname $0)" || exit 21 | pwd 22 | ) 23 | 24 | if [ $# -gt 0 ]; then 25 | if [ $1 == "python" ]; then 26 | echo "run python ut" 27 | bash ${CURRPATH}/python/runtest.sh $2 28 | elif [ $1 == "cpp" ]; then 29 | echo "run cpp ut" 30 | bash ${CURRPATH}/cpp/runtest.sh 31 | fi 32 | else 33 | echo "run all ut" 34 | # 1.run python testcases 35 | bash ${CURRPATH}/python/runtest.sh $2 36 | 37 | # 2.run c++ ut testcases 38 | bash ${CURRPATH}/cpp/runtest.sh 39 | fi 40 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/callback/loss_monitor.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_CALLBACK_LOSS_MONITOR_H 17 | #define MINDSPORE_INCLUDE_API_CALLBACK_LOSS_MONITOR_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include "include/api/callback/callback.h" 23 | 24 | namespace mindspore { 25 | 26 | class LossMonitor: public TrainCallBack { 27 | public: 28 | explicit LossMonitor(int print_every_n_steps = INT_MAX); 29 | virtual ~LossMonitor(); 30 | const std::vector &GetLossPoints(); 31 | }; 32 | } // namespace mindspore 33 | #endif // MINDSPORE_INCLUDE_API_CALLBACK_LOSS_MONITOR_H 34 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/visible.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_VISIBLE_H 17 | #define MINDSPORE_INCLUDE_API_VISIBLE_H 18 | 19 | #ifndef MS_API 20 | #ifdef _WIN32 21 | #define MS_API __declspec(dllexport) 22 | #else 23 | #define MS_API __attribute__((visibility("default"))) 24 | #endif // _WIN32 25 | #endif 26 | 27 | #ifdef _MSC_VER 28 | #ifdef BUILDING_DATASET_DLL 29 | #define DATASET_API __declspec(dllexport) 30 | #else 31 | #define DATASET_API __declspec(dllimport) 32 | #endif 33 | #else 34 | #define DATASET_API __attribute__((visibility("default"))) 35 | #endif // _MSC_VER 36 | 37 | #endif // MINDSPORE_INCLUDE_API_VISIBLE_H 38 | -------------------------------------------------------------------------------- /tests/ut/stub/include/mindapi/base/format.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_CORE_MINDAPI_BASE_FORMAT_H_ 18 | #define MINDSPORE_CORE_MINDAPI_BASE_FORMAT_H_ 19 | 20 | #include 21 | 22 | namespace mindspore { 23 | enum Format : int64_t { 24 | NCHW = 0, 25 | NHWC = 1, 26 | NHWC4 = 2, 27 | HWKC = 3, 28 | HWCK = 4, 29 | KCHW = 5, 30 | CKHW = 6, 31 | KHWC = 7, 32 | CHWK = 8, 33 | HW = 9, 34 | HW4 = 10, 35 | NC = 11, 36 | NC4 = 12, 37 | NC4HW4 = 13, 38 | NUM_OF_FORMAT = 14, 39 | NCDHW = 15, 40 | NWC = 16, 41 | NCW = 17, 42 | }; 43 | } // namespace mindspore 44 | #endif // MINDSPORE_CORE_MINDAPI_BASE_FORMAT_H_ 45 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.AscendDeviceInfo.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.register.AscendDeviceInfo(**kwargs) 3 | 4 | 用于设置Ascend设备配置。 5 | 6 | 参数： 7 | - **insert_op_cfg_path** (str, 可选) - AIPP配置文件的路径。 8 | - **input_format** (str, 可选) - 模型输入格式，取值可以是 ``"ND"`` 、 ``"NCHW"`` 、 ``"NHWC"`` 、 ``"CHWN"`` 、 ``"NC1HWC0"`` 或 ``"NHWC1C0"`` 。 9 | - **input_shape** (str, 可选) - 模型输入形状，如 ``"input_op_name1: n1,c2,h3,w4;input_op_name2: n4,c3,h2,w1"`` 。 10 | - **output_type** (str, 可选) - 模型输出类型，值可以是 ``"FP16"`` 、 ``"UINT8"`` 或 ``"FP32"`` ，默认值： ``"FP32"`` 。 11 | - **precision_mode** (str, 可选) - 模型精度模式，取值可以是 ``"force_fp16"`` 、 ``"allow_fp32_to_fp16"`` 、 ``"must_keep_origin_dtype"`` 或者 ``"allow_mix_precision"`` 。默认值： ``"force_fp16"`` 。 12 | - **op_select_impl_mode** (str, 可选) - 运算符选择模式，值可以是 ``"high_performance"`` 或 ``"high_precision"`` 。默认值： ``"high_performance"`` 。 13 | - **fusion_switch_config_path** (str, 可选) - 融合配置文件路径，包括图融合和UB融合。系统内置图融合和UB融合规则，默认启用。您可以通过设置此参数禁用指定的融合规则。 14 | - **buffer_optimize_mode** (str, 可选) - 数据缓存优化策略，值可以是 ``"l1_optimize"`` 、 ``"l2_optimize"`` 、 ``"off_optimize"`` 或者 ``"l1_and_l2_optimize"`` 。默认 ``"l2_optimize"`` 。 15 | 16 | 异常： 17 | - **RuntimeError** - Ascend设备配置无效。 18 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/float16.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_SERVING_COMMON_FLOAT16_H_ 17 | #define MINDSPORE_SERVING_COMMON_FLOAT16_H_ 18 | 19 | #if defined(ENABLE_ARM32) || defined(ENABLE_ARM64) 20 | // Built for lite and ARM 21 | #include 22 | 23 | using float16 = float16_t; 24 | inline float half_to_float(float16 h) { return static_cast(h); } 25 | #else 26 | #include 27 | #include "Eigen/Core" 28 | 29 | using float16 = Eigen::half; 30 | using HalfToFloat = std::function; 31 | const inline HalfToFloat half_to_float = Eigen::half_impl::half_to_float; 32 | #endif 33 | #endif // MINDSPORE_SERVING_COMMON_FLOAT16_H_ 34 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/master/master_context.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "master/master_context.h" 18 | 19 | namespace mindspore::serving { 20 | std::shared_ptr MasterContext::Instance() { 21 | static std::shared_ptr instance = nullptr; 22 | if (instance == nullptr) { 23 | instance = std::make_shared(); 24 | } 25 | return instance; 26 | } 27 | 28 | void MasterContext::SetMaxEnqueuedRequests(uint32_t max_enqueued_requests) { 29 | max_enqueued_requests_ = max_enqueued_requests; 30 | } 31 | 32 | uint32_t MasterContext::GetMaxEnqueuedRequests() const { return max_enqueued_requests_; } 33 | } // namespace mindspore::serving 34 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/notfiy_master/base_notify.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_BASE_NOTIFY_H 18 | #define MINDSPORE_SERVING_WORKER_BASE_NOTIFY_H 19 | #include 20 | #include "common/serving_common.h" 21 | #include "common/servable.h" 22 | 23 | namespace mindspore { 24 | namespace serving { 25 | class MS_API BaseNotifyMaster { 26 | public: 27 | BaseNotifyMaster() = default; 28 | virtual ~BaseNotifyMaster() = default; 29 | virtual Status Register(const WorkerRegSpec &worker_spec) = 0; 30 | virtual Status Unregister() = 0; 31 | }; 32 | 33 | } // namespace serving 34 | } // namespace mindspore 35 | 36 | #endif // MINDSPORE_SERVING_WORKER_BASE_NOTIFY_H 37 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/callback/time_monitor.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_CALLBACK_TIME_MONITOR_H 17 | #define MINDSPORE_INCLUDE_API_CALLBACK_TIME_MONITOR_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "include/api/callback/callback.h" 24 | 25 | namespace mindspore { 26 | 27 | class TimeMonitor: public TrainCallBack { 28 | public: 29 | virtual ~TimeMonitor() = default; 30 | void EpochBegin(const TrainCallBackData &cb_data) override; 31 | CallbackRetValue EpochEnd(const TrainCallBackData &cb_data) override; 32 | }; 33 | } // namespace mindspore 34 | #endif // MINDSPORE_INCLUDE_API_CALLBACK_TIME_MONITOR_H 35 | -------------------------------------------------------------------------------- /example/matmul_distributed/rank_table_8pcs.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "server_count": "1", 4 | "server_list": [ 5 | { 6 | "server_id": "127.0.0.1", 7 | "device": [ 8 | { 9 | "device_id": "0", 10 | "device_ip": "192.1.27.6", 11 | "rank_id": "0" 12 | }, 13 | { 14 | "device_id": "1", 15 | "device_ip": "192.2.27.6", 16 | "rank_id": "1" 17 | }, 18 | { 19 | "device_id": "2", 20 | "device_ip": "192.3.27.6", 21 | "rank_id": "2" 22 | }, 23 | { 24 | "device_id": "3", 25 | "device_ip": "192.4.27.6", 26 | "rank_id": "3" 27 | }, 28 | { 29 | "device_id": "4", 30 | "device_ip": "192.1.27.7", 31 | "rank_id": "4" 32 | }, 33 | { 34 | "device_id": "5", 35 | "device_ip": "192.2.27.7", 36 | "rank_id": "5" 37 | }, 38 | { 39 | "device_id": "6", 40 | "device_ip": "192.3.27.7", 41 | "rank_id": "6" 42 | }, 43 | { 44 | "device_id": "7", 45 | "device_ip": "192.4.27.7", 46 | "rank_id": "7" 47 | } 48 | ], 49 | "host_nic_ip": "reserve" 50 | } 51 | ], 52 | "status": "completed" 53 | } 54 | -------------------------------------------------------------------------------- /tests/ut/cpp/runtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2019 Huawei Technologies Co., Ltd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================ 16 | set -e 17 | BASEPATH=$( 18 | cd "$(dirname "$0")" 19 | pwd 20 | ) 21 | PROJECT_PATH=${BASEPATH}/../../.. 22 | if [ $BUILD_PATH ]; then 23 | echo "BUILD_PATH = $BUILD_PATH" 24 | else 25 | BUILD_PATH=${PROJECT_PATH}/build 26 | echo "BUILD_PATH = $BUILD_PATH" 27 | fi 28 | cd ${BUILD_PATH}/mindspore_serving/tests/ut/cpp 29 | export LD_LIBRARY_PATH=${BUILD_PATH}/mindspore_serving/tests/ut/cpp:${LD_LIBRARY_PATH} 30 | echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH" 31 | 32 | if [ $# -gt 0 ]; then 33 | ./serving_ut --gtest_filter=$1 34 | else 35 | ./serving_ut 36 | fi 37 | RET=$? 38 | cd - 39 | 40 | exit ${RET} 41 | -------------------------------------------------------------------------------- /example/matmul_distributed/export_model/rank_table_8pcs.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0", 3 | "server_count": "1", 4 | "server_list": [ 5 | { 6 | "server_id": "127.0.0.1", 7 | "device": [ 8 | { 9 | "device_id": "0", 10 | "device_ip": "192.1.27.6", 11 | "rank_id": "0" 12 | }, 13 | { 14 | "device_id": "1", 15 | "device_ip": "192.2.27.6", 16 | "rank_id": "1" 17 | }, 18 | { 19 | "device_id": "2", 20 | "device_ip": "192.3.27.6", 21 | "rank_id": "2" 22 | }, 23 | { 24 | "device_id": "3", 25 | "device_ip": "192.4.27.6", 26 | "rank_id": "3" 27 | }, 28 | { 29 | "device_id": "4", 30 | "device_ip": "192.1.27.7", 31 | "rank_id": "4" 32 | }, 33 | { 34 | "device_id": "5", 35 | "device_ip": "192.2.27.7", 36 | "rank_id": "5" 37 | }, 38 | { 39 | "device_id": "6", 40 | "device_ip": "192.3.27.7", 41 | "rank_id": "6" 42 | }, 43 | { 44 | "device_id": "7", 45 | "device_ip": "192.4.27.7", 46 | "rank_id": "7" 47 | } 48 | ], 49 | "host_nic_ip": "reserve" 50 | } 51 | ], 52 | "status": "completed" 53 | } 54 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/master/master_context.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_MASTER_CONTEXT_H 18 | #define MINDSPORE_SERVING_MASTER_CONTEXT_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include "common/serving_common.h" 24 | 25 | namespace mindspore::serving { 26 | class MS_API MasterContext { 27 | public: 28 | static std::shared_ptr Instance(); 29 | 30 | void SetMaxEnqueuedRequests(uint32_t max_enqueued_requests); 31 | uint32_t GetMaxEnqueuedRequests() const; 32 | 33 | private: 34 | uint32_t max_enqueued_requests_ = 10000; // default 10000 35 | }; 36 | 37 | } // namespace mindspore::serving 38 | 39 | #endif // MINDSPORE_SERVING_MASTER_CONTEXT_H 40 | -------------------------------------------------------------------------------- /example/matmul_multi_subgraphs/serving_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Start Distributed Servable matmul""" 16 | 17 | import os 18 | import sys 19 | from mindspore_serving import server 20 | 21 | 22 | def start(): 23 | servable_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 24 | 25 | servable_config = server.ServableStartConfig(servable_directory=servable_dir, servable_name="matmul", 26 | device_ids=(0, 1)) 27 | server.start_servables(servable_config) 28 | 29 | server.start_grpc_server("127.0.0.1:5500") 30 | server.start_restful_server("127.0.0.1:1500") 31 | 32 | 33 | if __name__ == "__main__": 34 | start() 35 | -------------------------------------------------------------------------------- /tests/ut/cpp/common/common_test.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #include "common/common_test.h" 17 | 18 | #define private public 19 | #include "mindspore_serving/ccsrc/common/servable.h" 20 | #undef private 21 | #include "mindspore_serving/ccsrc/worker/servable_register.h" 22 | 23 | #ifdef __cplusplus 24 | #if __cplusplus 25 | extern "C" { 26 | #endif 27 | #endif 28 | 29 | namespace UT { 30 | 31 | void Common::SetUpTestCase() {} 32 | 33 | void Common::TearDownTestCase() {} 34 | 35 | void Common::SetUp() {} 36 | 37 | void Common::TearDown() { 38 | mindspore::serving::ServableRegister::Instance() = mindspore::serving::ServableRegister(); 39 | } 40 | 41 | } // namespace UT 42 | 43 | #ifdef __cplusplus 44 | #if __cplusplus 45 | } 46 | #endif 47 | #endif 48 | -------------------------------------------------------------------------------- /example/add_sub_pipeline/serving_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """The server of example add_sub pipeline""" 16 | 17 | import os 18 | from mindspore_serving import server 19 | 20 | 21 | def start(): 22 | servable_dir = os.path.dirname(os.path.realpath(__file__)) 23 | 24 | servable_config = server.ServableStartConfig(servable_directory=servable_dir, servable_name="add_sub", 25 | device_ids=(0, 1)) 26 | server.start_servables(servable_configs=servable_config) 27 | 28 | server.start_grpc_server(address="127.0.0.1:5500") 29 | server.start_restful_server(address="127.0.0.1:1500") 30 | 31 | 32 | if __name__ == "__main__": 33 | start() 34 | -------------------------------------------------------------------------------- /example/tensor_add/serving_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """The server of example add""" 16 | 17 | import os 18 | import sys 19 | from mindspore_serving import server 20 | 21 | 22 | def start(): 23 | servable_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 24 | 25 | servable_config = server.ServableStartConfig(servable_directory=servable_dir, servable_name="add", 26 | device_ids=(0, 1)) 27 | server.start_servables(servable_configs=servable_config) 28 | 29 | server.start_grpc_server(address="127.0.0.1:5500") 30 | server.start_restful_server(address="127.0.0.1:1500") 31 | 32 | 33 | if __name__ == "__main__": 34 | start() 35 | -------------------------------------------------------------------------------- /tests/ut/stub/include/utils/visible.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019-2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_CORE_UTILS_VISIBLE_H_ 18 | #define MINDSPORE_CORE_UTILS_VISIBLE_H_ 19 | 20 | #if (defined(_WIN32) || defined(__WIN32__) || defined(WIN32) || defined(__CYGWIN__)) 21 | #ifdef BUILDING_DLL 22 | #define MS_CORE_API __declspec(dllexport) 23 | #define MS_EXPORT __declspec(dllexport) 24 | #else 25 | #define MS_CORE_API __declspec(dllimport) 26 | #define MS_EXPORT __declspec(dllimport) 27 | #endif 28 | #define MS_LOCAL 29 | #else 30 | #define MS_CORE_API __attribute__((visibility("default"))) 31 | #define MS_EXPORT __attribute__((visibility("default"))) 32 | #define MS_LOCAL __attribute__((visibility("hidden"))) 33 | #endif 34 | 35 | #endif // MINDSPORE_CORE_UTILS_VISIBLE_H_ 36 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2023-0466.patch: -------------------------------------------------------------------------------- 1 | diff --git a/doc/man3/X509_VERIFY_PARAM_set_flags.pod b/doc/man3/X509_VERIFY_PARAM_set_flags.pod 2 | index f6f304bf7b..aa292f9336 100644 3 | --- a/doc/man3/X509_VERIFY_PARAM_set_flags.pod 4 | +++ b/doc/man3/X509_VERIFY_PARAM_set_flags.pod 5 | @@ -92,8 +92,9 @@ B. 6 | X509_VERIFY_PARAM_set_time() sets the verification time in B to 7 | B. Normally the current time is used. 8 | 9 | -X509_VERIFY_PARAM_add0_policy() enables policy checking (it is disabled 10 | -by default) and adds B to the acceptable policy set. 11 | +X509_VERIFY_PARAM_add0_policy() adds B to the acceptable policy set. 12 | +Contrary to preexisting documentation of this function it does not enable 13 | +policy checking. 14 | 15 | X509_VERIFY_PARAM_set1_policies() enables policy checking (it is disabled 16 | by default) and sets the acceptable policy set to B. Any existing 17 | @@ -377,6 +378,10 @@ and has no effect. 18 | 19 | The X509_VERIFY_PARAM_get_hostflags() function was added in OpenSSL 1.1.0i. 20 | 21 | +The function X509_VERIFY_PARAM_add0_policy() was historically documented as 22 | +enabling policy checking however the implementation has never done this. 23 | +The documentation was changed to align with the implementation. 24 | + 25 | =head1 COPYRIGHT 26 | 27 | Copyright 2009-2020 The OpenSSL Project Authors. All Rights Reserved. 28 | -------------------------------------------------------------------------------- /example/matmul_distributed/export_model/export_model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | EXEC_PATH=$(pwd) 4 | 5 | export RANK_TABLE_FILE=${EXEC_PATH}/rank_table_8pcs.json 6 | export RANK_SIZE=8 7 | 8 | rm -rf device* 9 | for ((i = 1; i < ${RANK_SIZE}; i++)); do 10 | mkdir device$i 11 | cp *.py ./device$i 12 | cd ./device$i 13 | export DEVICE_ID=$i 14 | export RANK_ID=$i 15 | echo "start inference for device $i" 16 | pytest -sv ./distributed_inference.py::test_inference >inference.log$i 2>&1 & 17 | cd ../ 18 | done 19 | 20 | mkdir device0 21 | cp *.py ./device0 22 | cd ./device0 23 | export DEVICE_ID=0 24 | export RANK_ID=0 25 | echo "start inference for device 0" 26 | pytest -sv ./distributed_inference.py::test_inference >inference.log0 2>&1 27 | if [ $? -eq 0 ]; then 28 | echo "inference success" 29 | else 30 | echo "inference failed" 31 | cat inference.log0 32 | exit 2 33 | fi 34 | cd ../ 35 | 36 | ls device*/ -l 37 | 38 | num=`ls device*/matmul.mindir -l | wc -l` 39 | if [ ${num} -ne 8 ] 40 | then 41 | echo "export matmul mindir failed" 42 | cat device0/inference.log0 43 | exit 2 44 | fi 45 | 46 | output_dir=../model 47 | rm -rf ${output_dir}/device* 48 | for ((i = 0; i < ${RANK_SIZE}; i++)); do 49 | mkdir -p ${output_dir}/device${i} 50 | cp device${i}/*.mindir ${output_dir}/device${i}/ 51 | cp device${i}/*.pb ${output_dir}/device${i}/ 52 | done 53 | echo "copy models success" 54 | -------------------------------------------------------------------------------- /mindspore_serving/server/distributed/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """The interface to startup serving server with distributed servable. 16 | See how to configure and startup distributed model, please refer to 17 | `MindSpore Serving-based Distributed Inference Service Deployment `_.""" 18 | 19 | from mindspore_serving.server.worker.distributed import startup_agents 20 | from mindspore_serving.server.worker.distributed.register import declare_servable 21 | from ._distributed import start_servable 22 | 23 | __all__ = [] 24 | __all__.extend([ 25 | "start_servable", 26 | 'startup_agents', 27 | 'declare_servable' 28 | ]) 29 | -------------------------------------------------------------------------------- /example/matmul_distributed/serving_agent.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Start agents of Distributed Servable matmul""" 16 | 17 | from mindspore_serving.server import distributed 18 | 19 | 20 | def start_agents(): 21 | """Start all the agents in current machine""" 22 | model_files = [] 23 | group_configs = [] 24 | for i in range(8): 25 | model_files.append(f"model/device{i}/matmul.mindir") 26 | group_configs.append(f"model/device{i}/group_config.pb") 27 | 28 | distributed.startup_agents(distributed_address="127.0.0.1:6200", model_files=model_files, 29 | group_config_files=group_configs) 30 | 31 | 32 | if __name__ == '__main__': 33 | start_agents() 34 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/metrics/metrics.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_METRICS_METRICS_H 17 | #define MINDSPORE_INCLUDE_API_METRICS_METRICS_H 18 | #include 19 | #include "include/api/model.h" 20 | 21 | namespace mindspore { 22 | 23 | class MetricsImpl; 24 | class ModelImpl; 25 | class MSTensor; 26 | 27 | class Metrics { 28 | public: 29 | virtual ~Metrics() = default; 30 | virtual void Clear() {} 31 | virtual float Eval() { return 0.0; } 32 | virtual void Update(std::vector inputs, std::vector outputs) {} 33 | protected: 34 | friend class Model; 35 | friend class ModelImpl; 36 | MetricsImpl* metrics_impl_; 37 | }; 38 | 39 | } // namespace mindspore 40 | #endif // MINDSPORE_INCLUDE_API_METRICS_METRICS_H 41 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/utils.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_COMMON_UTILS_H 18 | #define MINDSPORE_SERVING_COMMON_UTILS_H 19 | 20 | #include 21 | #include "common/status.h" 22 | 23 | namespace mindspore::serving::common { 24 | 25 | static inline std::string GetEnv(const std::string &env_var) { 26 | const char *value = ::getenv(env_var.c_str()); 27 | if (value == nullptr) { 28 | return std::string(); 29 | } 30 | return std::string(value); 31 | } 32 | 33 | Status CheckAddress(const std::string &address, const std::string &server_tag, std::string *ip, uint16_t *port); 34 | 35 | bool DirOrFileExist(const std::string &file_path); 36 | 37 | } // namespace mindspore::serving::common 38 | 39 | #endif // MINDSPORE_SERVING_COMMON_UTILS_H 40 | -------------------------------------------------------------------------------- /example/matmul_multi_subgraphs/matmul/servable_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Distributed matmul config python file""" 16 | from mindspore_serving.server import register 17 | 18 | model = register.declare_model(model_file=["matmul_0.mindir", "matmul_1.mindir"], model_format="MindIR", 19 | with_batch_dim=False) 20 | 21 | 22 | def process(x, y): 23 | z1 = model.call(x, subgraph=0) # 128,96 matmul 16,96 -> reduce sum axis 0-> 16 24 | z2 = model.call(y, subgraph=1) # 8,96 matmul 16,96 -> reduce sum axis 0-> 16 25 | return z1 + z2 26 | 27 | 28 | @register.register_method(output_names=["z"]) 29 | def predict(x, y): 30 | z = register.add_stage(process, x, y, outputs_count=1) 31 | return z 32 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/metrics/accuracy.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_METRICS_ACCURACY_H 17 | #define MINDSPORE_INCLUDE_API_METRICS_ACCURACY_H 18 | #include 19 | #include "include/api/metrics/metrics.h" 20 | 21 | namespace mindspore { 22 | 23 | constexpr int METRICS_CLASSIFICATION = 0; 24 | constexpr int METRICS_MULTILABEL = 1; 25 | 26 | class AccuracyMetrics : public Metrics { 27 | public: 28 | explicit AccuracyMetrics(int accuracy_metrics = METRICS_CLASSIFICATION, const std::vector &input_indexes = {1}, 29 | const std::vector &output_indexes = {0}); 30 | virtual ~AccuracyMetrics(); 31 | void Clear() override; 32 | float Eval() override; 33 | }; 34 | 35 | } // namespace mindspore 36 | #endif // MINDSPORE_INCLUDE_API_METRICS_ACCURACY_H 37 | -------------------------------------------------------------------------------- /docs/api/api_python/server/register/mindspore_serving.server.register.declare_model.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.register.declare_model(model_file, model_format, with_batch_dim=True, options=None, without_batch_dim_inputs=None, context=None, config_file=None) 3 | 4 | 在服务的servable_config.py配置文件中使用，用于声明一个模型。 5 | 6 | .. note:: 本接口需要在Serving服务器导入servable_config.py时生效。因此，建议在servable_config.py中全局使用此接口。 7 | 8 | .. warning:: 参数 `options` 从1.6.0版本中已弃用，并将在未来版本中删除，请改用参数 `context` 。 9 | 10 | 参数： 11 | - **model_file** (Union[str, list[str]]) - 模型文件名。 12 | - **model_format** (str) - 模型格式， ``"MindIR"`` 或 ``"MindIR_Lite"`` ，忽略大小写。 13 | - **with_batch_dim** (bool, 可选) - 模型输入和输出的shape第一个维度是否是batch维度。默认值：``True``。 14 | - **options** (Union[AclOptions, GpuOptions], 可选) - 模型的选项，支持 ``AclOptions`` 或 ``GpuOptions`` 。默认值：``None``。 15 | - **context** (Context) - 用于配置设备环境的上下文信息，值为 ``None`` 时，Serving将依据部署的设备设置默认的设备上下文。默认值：``None``。 16 | - **without_batch_dim_inputs** (Union[int, tuple[int], list[int]], 可选) - 当 `with_batch_dim` 为 ``True`` 时，用于指定shape不包括batch维度的模型输入的索引，比如模型输入0的shape不包括batch维度，则 `without_batch_dim_inputs` 可赋值为 `(0,)` 。默认值：``None``。 17 | - **config_file** (str, 可选) - 用于设置混合精度推理的配置文件。文件路径可以是servable_config.py所在目录的绝对路径或相对路径。默认值：``None``。 18 | 19 | 返回： 20 | `Model` ，此模型的标识，可以用来调用 `Model.call` 或作为 `add_stage` 的输入。 21 | 22 | 异常： 23 | - **RuntimeError** - 参数的类型或值无效。 24 | -------------------------------------------------------------------------------- /example/matmul_distributed/serving_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Start Distributed Servable matmul""" 16 | 17 | import os 18 | import sys 19 | from mindspore_serving import server 20 | from mindspore_serving.server import distributed 21 | 22 | 23 | def start(): 24 | servable_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 25 | distributed.start_servable(servable_dir, "matmul", 26 | rank_table_json_file="rank_table_8pcs.json", 27 | version_number=1, 28 | distributed_address="127.0.0.1:6200") 29 | 30 | server.start_grpc_server("127.0.0.1:5500") 31 | server.start_restful_server("127.0.0.1:1500") 32 | 33 | 34 | if __name__ == "__main__": 35 | start() 36 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/master/restful/http_handle.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_MASTER_HTTP_HANDLE_H 18 | #define MINDSPORE_SERVING_MASTER_HTTP_HANDLE_H 19 | 20 | #include 21 | #include 22 | #include "common/serving_common.h" 23 | #include "master/restful/restful_request.h" 24 | 25 | using nlohmann::json; 26 | namespace mindspore { 27 | namespace serving { 28 | size_t Base64Encode(const uint8_t *input, size_t length, uint8_t *output); 29 | size_t Base64Decode(const uint8_t *target, size_t target_length, uint8_t *origin); 30 | size_t GetB64TargetSize(size_t origin_len); 31 | size_t GetB64OriginSize(size_t target_len, size_t tail_size); 32 | size_t GetTailEqualSize(const std::string &str); 33 | 34 | } // namespace serving 35 | } // namespace mindspore 36 | #endif // MINDSPORE_SERVING_MASTER_HTTP_HANDLE_H 37 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/instance.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_INSTANCE_H 18 | #define MINDSPORE_SERVING_INSTANCE_H 19 | 20 | #include 21 | #include 22 | #include "common/serving_common.h" 23 | #include "common/servable.h" 24 | #include "common/instance_data.h" 25 | 26 | namespace mindspore::serving { 27 | struct Instance { 28 | InstanceData data; // for inputs of function, predict, output 29 | 30 | const MethodSignature *method_def = nullptr; 31 | uint64_t stage_index = 0; 32 | uint64_t stage_max = 0; 33 | std::map stage_data_list; // input: 0, stage: 1-n 34 | 35 | uint64_t user_id = 0; 36 | Status error_msg = SUCCESS; 37 | }; 38 | 39 | using InstancePtr = std::shared_ptr; 40 | } // namespace mindspore::serving 41 | 42 | #endif // MINDSPORE_SERVING_INSTANCE_H 43 | -------------------------------------------------------------------------------- /docs/api/api_python/server/distributed/mindspore_serving.server.distributed.startup_agents.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:function:: mindspore_serving.server.distributed.startup_agents(distributed_address, model_files, group_config_files=None, agent_start_port=7000, agent_ip=None, rank_start=None, dec_key=None, dec_mode='AES-GCM') 3 | 4 | 在当前计算机上启动所有所需的Worker代理（Agent），这组Worker代理进程将负责本机器设备上的推理任务，详细可参考 5 | `基于MindSpore Serving部署分布式推理服务 `_ 。 6 | 7 | 参数： 8 | - **distributed_address** (str) - Worker代理连接分布式Worker服务器地址。 9 | - **model_files** (Union[list[str], tuple[str]]) - 当前计算机中需要的所有模型文件，为绝对路径或相对于此启动Python脚本的路径。 10 | - **group_config_files** (Union[list[str], tuple[str]], 可选) - 当前计算机中需要的所有组配置文件，相对于此启动Python脚本的绝对路径或相对路径，为 ``None`` 时表示没有配置文件。默认值：``None``。 11 | - **agent_start_port** (int, 可选) - Worker代理连接Worker服务器的起始端口号。默认值：``7000``。 12 | - **agent_ip** (str, 可选) - 本地Worker代理ip，如果为无，则代理ip将从rank table文件中获取。参数 `agent_ip` 和参数 `rank_start` 必须同时有值，或者同时是 ``None``。默认值：``None``。 13 | - **rank_start** (int, 可选) - 此计算机的起始rank id，如果为 ``None``，则将从rank table文件中获取rank id。参数 `agent_ip` 和参数 `rank_start` 必须同时有值，或者同时是 ``None``。默认值：``None``。 14 | - **dec_key** (bytes, 可选) - 用于解密的密钥，类型为字节。有效长度为16、24或32。默认值：``None``。 15 | - **dec_mode** (str, 可选) - 指定解密模式，在设置了 `dec_key` 时生效。值可为： ``'AES-GCM'`` 或 ``'AES-CBC'`` 。默认值： ``'AES-GCM'`` 。 16 | 17 | 异常： 18 | - **RuntimeError** - 启动Worker代理失败。 19 | -------------------------------------------------------------------------------- /example/resnet/serving_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Start Servable resnet50""" 16 | 17 | import os 18 | import sys 19 | from mindspore_serving import server 20 | 21 | 22 | def start(): 23 | servable_dir = os.path.dirname(os.path.realpath(sys.argv[0])) 24 | # Total 4 worker, one worker occupy device 0, the model inference tasks of other workers are forwarded to the worker 25 | # that occupies the device. 26 | config = server.ServableStartConfig(servable_directory=servable_dir, servable_name="resnet50", device_ids=0, 27 | num_parallel_workers=4) 28 | server.start_servables(config) 29 | 30 | server.start_grpc_server("127.0.0.1:5500") 31 | server.start_restful_server("127.0.0.1:1500") 32 | 33 | 34 | if __name__ == "__main__": 35 | start() 36 | -------------------------------------------------------------------------------- /tests/ut/stub/cxx_api/graph/graph.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #include "include/api/graph.h" 17 | #include "cxx_api/graph/graph_data.h" 18 | #include "utils/log_adapter.h" 19 | 20 | namespace mindspore { 21 | Graph::Graph() : graph_data_(nullptr) {} 22 | 23 | Graph::Graph(const std::shared_ptr &graph_data) : graph_data_(graph_data) {} 24 | 25 | Graph::Graph(std::shared_ptr &&graph_data) : graph_data_(graph_data) {} 26 | 27 | Graph::~Graph() {} 28 | 29 | Graph::Graph(std::nullptr_t) : graph_data_(nullptr) {} 30 | 31 | bool Graph::operator==(std::nullptr_t) const { return graph_data_ == nullptr; } 32 | 33 | bool Graph::operator!=(std::nullptr_t) const { return graph_data_ != nullptr; } 34 | 35 | ModelType Graph::ModelType() const { 36 | MS_EXCEPTION_IF_NULL(graph_data_); 37 | return graph_data_->ModelType(); 38 | } 39 | } // namespace mindspore 40 | -------------------------------------------------------------------------------- /cmake/external_libs/glog.cmake: -------------------------------------------------------------------------------- 1 | set(glog_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 ${SECURE_CXX_FLAGS} -Dgoogle=mindspore_serving_private") 2 | set(glog_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") 3 | set(glog_patch ${CMAKE_SOURCE_DIR}/third_party/patch/glog/glog.patch001) 4 | set(glog_lib mindspore_serving_glog) 5 | 6 | if(NOT ENABLE_GLIBCXX) 7 | set(glog_CXXFLAGS "${glog_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") 8 | endif() 9 | 10 | if(ENABLE_GITEE OR ENABLE_GITEE_EULER) # Channel GITEE_EULER is NOT supported now, use GITEE instead. 11 | set(REQ_URL "https://gitee.com/mirrors/glog/repository/archive/v0.4.0.tar.gz") 12 | set(SHA256 "e17cd4bb7c06951a12fc9db5130ec63a9f090b84340b8556fa0d530f73c6b634") 13 | else() 14 | set(REQ_URL "https://github.com/google/glog/archive/v0.4.0.tar.gz") 15 | set(SHA256 "f28359aeba12f30d73d9e4711ef356dc842886968112162bc73002645139c39c") 16 | endif() 17 | 18 | set(glog_option -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON -DWITH_GFLAGS=OFF 19 | -DCMAKE_BUILD_TYPE=Release) 20 | 21 | if(WIN32 AND NOT MSVC) 22 | if(CMAKE_SIZEOF_VOID_P EQUAL 4) 23 | set(glog_option ${glog_option} -DHAVE_DBGHELP=ON) 24 | endif() 25 | endif() 26 | 27 | mindspore_add_pkg(glog 28 | VER 0.4.0 29 | LIBS ${glog_lib} 30 | URL ${REQ_URL} 31 | SHA256 ${SHA256} 32 | PATCHES ${glog_patch} 33 | CMAKE_OPTION ${glog_option}) 34 | include_directories(${glog_INC}) 35 | add_library(mindspore_serving::glog ALIAS glog::${glog_lib}) 36 | -------------------------------------------------------------------------------- /docs/api/api_python/client/mindspore_serving.client.Client.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.client.Client(address, servable_name, method_name, version_number=0, ssl_config=None) 3 | 4 | 通过Client访问Serving服务器gRPC接口，可用于创建请求、访问服务和解析结果。 5 | 6 | .. note:: Serving客户端在一个请求中可以发送的最大数据量为512MB，Serving服务器可以接收的最大数据量可以配置为1~512MB，默认为100MB。 7 | 8 | 参数： 9 | - **address** (str) - Serving服务器gRPC接口地址。 10 | - **servable_name** (str) - Serving服务器提供的服务的名称。 11 | - **method_name** (str) - 服务中方法的名称。 12 | - **version_number** (int, optional) - 服务的版本号，``0`` 表示指定所有正在运行的一个或多个版本的服务中最大的版本号。默认值：``0``。 13 | - **ssl_config** (mindspore_serving.client.SSLConfig, optional) - SSL配置，如果 ``None``，则禁用SSL。默认值：``None``。 14 | 15 | 异常： 16 | - **RuntimeError** - 参数的类型或值无效，或发生其他错误。 17 | 18 | .. py:method:: infer(instances) 19 | 20 | 用于创建请求、访问服务、解析和返回结果。 21 | 22 | 参数： 23 | - **instances** (Union[dict, tuple[dict]]) - 一个实例或一组实例的输入，每个实例都是dict。dict的key是输入名称，value是输入值。value的类型可以是Python int、float、bool、str、bytes、numpy scalar或numpy array对象。 24 | 25 | 异常： 26 | - **RuntimeError** - 参数的类型或值无效，或发生其他错误。 27 | 28 | .. py:method:: infer_async(instances) 29 | 30 | 用于创建请求，异步访问服务。 31 | 32 | 参数： 33 | - **instances** (Union[dict, tuple[dict]]) - 一个实例或一组实例的输入，每个实例都是dict。dict的key是输入名称，value是输入值。value的类型可以是Python int、float、bool、str、bytes、numpy scalar或numpy array对象。 34 | 35 | 异常： 36 | - **RuntimeError** - 参数的类型或值无效，或发生其他错误。 37 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/master/notify_worker/base_notify.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_MASTER_BASE_NOTIFY_H 18 | #define MINDSPORE_SERVING_MASTER_BASE_NOTIFY_H 19 | #include 20 | #include 21 | #include 22 | #include "common/serving_common.h" 23 | #include "common/servable.h" 24 | #include "proto/ms_service.pb.h" 25 | #include "common/grpc_client.h" 26 | 27 | namespace mindspore { 28 | namespace serving { 29 | class MS_API BaseNotifyWorker { 30 | public: 31 | BaseNotifyWorker() = default; 32 | virtual ~BaseNotifyWorker() = default; 33 | virtual Status DispatchAsync(const proto::PredictRequest &request, proto::PredictReply *reply, 34 | const PredictOnFinish &on_finish) = 0; 35 | }; 36 | 37 | } // namespace serving 38 | } // namespace mindspore 39 | 40 | #endif // MINDSPORE_SERVING_MASTER_BASE_NOTIFY_H 41 | -------------------------------------------------------------------------------- /cmake/mind_expression.cmake: -------------------------------------------------------------------------------- 1 | set(SECURE_CXX_FLAGS "") 2 | if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") 3 | set(SECURE_CXX_FLAGS "-fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack") 4 | endif() 5 | set(_ms_tmp_CMAKE_CXX_FLAGS_F ${CMAKE_CXX_FLAGS}) 6 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") 7 | 8 | # define third party library download function 9 | include(cmake/utils.cmake) 10 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) 11 | include(${CMAKE_SOURCE_DIR}/cmake/dependency_securec.cmake) 12 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/json.cmake) 13 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) 14 | 15 | # build dependencies of gRPC 16 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/absl.cmake) 17 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/c-ares.cmake) 18 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zlib.cmake) 19 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/openssl.cmake) 20 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/re2.cmake) 21 | # build gRPC 22 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/grpc.cmake) 23 | # build event 24 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/libevent.cmake) 25 | 26 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pybind11.cmake) 27 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/gtest.cmake) 28 | include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake) 29 | 30 | set(CMAKE_CXX_FLAGS ${_ms_tmp_CMAKE_CXX_FLAGS_F}) 31 | 32 | if(MS_BACKEND) 33 | include(${CMAKE_SOURCE_DIR}/cmake/dependency_ms.cmake) 34 | endif() 35 | -------------------------------------------------------------------------------- /mindspore_serving/proto/ms_agent.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // ms_manager.proto 18 | syntax = "proto3"; 19 | 20 | package mindspore.serving.proto; 21 | import "mindspore_serving/proto/ms_service.proto"; 22 | 23 | message DistributedPredictRequest { 24 | repeated Tensor inputs = 1; 25 | bool return_result = 2; 26 | int64 subgraph = 3; 27 | } 28 | 29 | message DistributedPredictReply { 30 | repeated Tensor outputs = 1; 31 | ErrorMsg error_msg = 2; 32 | } 33 | 34 | message DistributedExitRequest { 35 | string address = 1; 36 | } 37 | 38 | message DistributedExitReply { 39 | ErrorMsg error_msg = 1; 40 | } 41 | 42 | service MSAgent { 43 | rpc Predict(DistributedPredictRequest) returns (DistributedPredictReply) {} 44 | rpc Exit(DistributedExitRequest) returns (DistributedExitReply) {} 45 | rpc Ping(PingRequest) returns (PingReply) {} 46 | rpc Pong(PongRequest) returns (PongReply) {} 47 | } 48 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/callback/ckpt_saver.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_CALLBACK_CKPT_SAVER_H 17 | #define MINDSPORE_INCLUDE_API_CALLBACK_CKPT_SAVER_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "include/api/callback/callback.h" 24 | #include "include/api/dual_abi_helper.h" 25 | 26 | namespace mindspore { 27 | 28 | class CkptSaver: public TrainCallBack { 29 | public: 30 | inline CkptSaver(int save_every_n, const std::string &filename_prefix); 31 | virtual ~CkptSaver(); 32 | 33 | private: 34 | CkptSaver(int save_every_n, const std::vector &filename_prefix); 35 | }; 36 | 37 | CkptSaver::CkptSaver(int save_every_n, const std::string &filename_prefix) 38 | : CkptSaver(save_every_n, StringToChar(filename_prefix)) {} 39 | 40 | } // namespace mindspore 41 | #endif // MINDSPORE_INCLUDE_API_CALLBACK_CKPT_SAVER_H 42 | -------------------------------------------------------------------------------- /tests/ut/python/servable_config/generate_certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "[req] 3 | default_bits = 2048 4 | distinguished_name = req_distinguished_name 5 | x509_extensions = v3_req 6 | prompt = no 7 | [req_distinguished_name] 8 | countryName = XX 9 | stateOrProvinceName = Self-signed Cert 10 | commonName = Self-signed Cert 11 | [v3_req] 12 | basicConstraints = CA:TRUE" > ca.cnf 13 | 14 | # generate ca's cert and private key for signing server and client cert 15 | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ca.key -out ca.crt -config ca.cnf 16 | 17 | rm ca.cnf 18 | 19 | # generate server's cert 20 | 21 | IP=$SERVING_IP 22 | DNS=$SERVING_HOSTNAME 23 | CN=$SERVING_COMMON_NAME 24 | 25 | echo " 26 | authorityKeyIdentifier=keyid,issuer 27 | basicConstraints=CA:FALSE 28 | keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment 29 | subjectAltName = @alt_names 30 | [alt_names] 31 | IP.1 = $IP 32 | DNS.1 = $DNS 33 | " > server.cnf 34 | 35 | openssl genrsa -out server.key 2048 36 | 37 | openssl req -new -key server.key -out server.csr -subj "/C=XX/ST=MyST/L=XX/O=HW/OU=gRPC/CN=$CN" 38 | 39 | openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 730 -sha256 -extfile server.cnf 40 | 41 | rm server.cnf 42 | 43 | # generate client's cert 44 | 45 | openssl genrsa -out client.key 2048 46 | 47 | openssl req -new -key client.key -out client.csr -subj "/C=XX/ST=MyST/L=XX/O=HW/OU=gRPC/CN=client" 48 | 49 | openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 730 -sha256 -------------------------------------------------------------------------------- /mindspore_serving/server/register/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Common implement for worker""" 16 | import inspect 17 | import os 18 | 19 | 20 | def get_servable_dir(): 21 | """Get the directory where servable is located. The name of the directory is the name of servable""" 22 | stack = inspect.stack() 23 | for item in stack: 24 | if item.filename.endswith("servable_config.py"): 25 | abs_path = os.path.realpath(item.filename) 26 | last_dir = os.path.split(abs_path)[0] 27 | last_dir = os.path.split(last_dir)[1] 28 | if not last_dir: 29 | continue 30 | return last_dir 31 | raise RuntimeError("Failed to obtain the directory of servable_config.py") 32 | 33 | 34 | def get_func_name(func): 35 | """Get function name for preprocess and postprocess, as the identification name""" 36 | return func.__name__ 37 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/data_type.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_DATA_TYPE_H_ 17 | #define MINDSPORE_INCLUDE_API_DATA_TYPE_H_ 18 | 19 | #include 20 | 21 | namespace mindspore { 22 | enum class DataType : int { 23 | kTypeUnknown = 0, 24 | kObjectTypeString = 12, 25 | kObjectTypeList = 13, 26 | kObjectTypeTuple = 14, 27 | kObjectTypeTensorType = 17, 28 | kNumberTypeBegin = 29, 29 | kNumberTypeBool = 30, 30 | kNumberTypeInt8 = 32, 31 | kNumberTypeInt16 = 33, 32 | kNumberTypeInt32 = 34, 33 | kNumberTypeInt64 = 35, 34 | kNumberTypeUInt8 = 37, 35 | kNumberTypeUInt16 = 38, 36 | kNumberTypeUInt32 = 39, 37 | kNumberTypeUInt64 = 40, 38 | kNumberTypeFloat16 = 42, 39 | kNumberTypeFloat32 = 43, 40 | kNumberTypeFloat64 = 44, 41 | kNumberTypeEnd = 46, 42 | // add new enum here 43 | kInvalidType = INT32_MAX, 44 | }; 45 | } // namespace mindspore 46 | #endif // MINDSPORE_INCLUDE_API_DATA_TYPE_H_ 47 | -------------------------------------------------------------------------------- /mindspore_serving/server/common/decorator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Providing decorators.""" 16 | 17 | from functools import wraps 18 | 19 | from mindspore_serving import log 20 | 21 | 22 | def deprecated(version, substitute): 23 | """deprecated warning 24 | 25 | Args: 26 | version (str): version that the operator or function is deprecated. 27 | substitute (str): the substitute name for deprecated operator or function. 28 | """ 29 | 30 | def decorate(func): 31 | @wraps(func) 32 | def wrapper(*args, **kwargs): 33 | name = func.__name__ 34 | log.warning(f"'{name}' is deprecated from version {version} and " 35 | f"will be removed in a future version, use '{substitute}' instead.") 36 | ret = func(*args, **kwargs) 37 | return ret 38 | 39 | return wrapper 40 | 41 | return decorate 42 | -------------------------------------------------------------------------------- /tests/ut/stub/cxx_api/model/model_impl.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020-2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #include "cxx_api/model/model_impl.h" 17 | 18 | namespace mindspore { 19 | Status ModelImpl::Predict(const std::vector &inputs, std::vector *outputs) { 20 | MS_EXCEPTION_IF_NULL(outputs); 21 | if (graph_ == nullptr) { 22 | MS_LOG(ERROR) << "Invalid data, graph_ is null."; 23 | return kMCFailed; 24 | } 25 | 26 | if (graph_cell_ == nullptr) { 27 | MS_LOG(WARNING) << "Model has not been built, it will be built with default options"; 28 | Status ret = Build(); 29 | if (ret != kSuccess) { 30 | MS_LOG(ERROR) << "Build model failed."; 31 | return ret; 32 | } 33 | } 34 | 35 | MS_EXCEPTION_IF_NULL(graph_cell_); 36 | Status ret = graph_cell_->Run(inputs, outputs); 37 | if (ret != kSuccess) { 38 | MS_LOG(ERROR) << "Run graph failed."; 39 | return ret; 40 | } 41 | 42 | return kSuccess; 43 | } 44 | } // namespace mindspore 45 | -------------------------------------------------------------------------------- /mindspore_serving/proto/ms_worker.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // ms_manager.proto 18 | syntax = "proto3"; 19 | 20 | package mindspore.serving.proto; 21 | import "mindspore_serving/proto/ms_service.proto"; 22 | import "mindspore_serving/proto/ms_master.proto"; 23 | import "mindspore_serving/proto/ms_distributed.proto"; 24 | 25 | service MSWorker { 26 | // for master 27 | rpc Predict(PredictRequest) returns (PredictReply) {} 28 | rpc Exit(ExitRequest) returns (ExitReply) {} 29 | } 30 | 31 | service MSDistributedWorker { 32 | // for worker agent 33 | rpc AgentExit(AgentExitRequest) returns (AgentExitReply) {} 34 | rpc AgentRegister(AgentRegisterRequest) returns (AgentRegisterReply) {} 35 | rpc AgentFailed(AgentFailedRequest) returns (AgentFailedReply) {} 36 | rpc AgentConfigAcquire(AgentConfigAcquireRequest) returns (AgentConfigAcquireReply) {} 37 | rpc Ping(PingRequest) returns (PingReply) {} 38 | rpc Pong(PongRequest) returns (PongReply) {} 39 | } 40 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/callback/train_accuracy.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_CALLBACK_TRAIN_ACCURACY_H 17 | #define MINDSPORE_INCLUDE_API_CALLBACK_TRAIN_ACCURACY_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "include/api/callback/callback.h" 25 | #include "include/api/metrics/accuracy.h" 26 | 27 | namespace mindspore { 28 | 29 | class TrainAccuracy: public TrainCallBack { 30 | public: 31 | explicit TrainAccuracy(int print_every_n = INT_MAX, 32 | int accuracy_metrics = METRICS_CLASSIFICATION, 33 | const std::vector &input_indexes = {1}, 34 | const std::vector &output_indexes = {0}); 35 | virtual ~TrainAccuracy(); 36 | const std::vector &GetAccuracyPoints(); 37 | }; 38 | } // namespace mindspore 39 | #endif // MINDSPORE_INCLUDE_API_CALLBACK_TRAIN_ACCURACY_H 40 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/graph.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_GRAPH_H 17 | #define MINDSPORE_INCLUDE_API_GRAPH_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "include/api/status.h" 24 | #include "include/api/types.h" 25 | 26 | namespace mindspore { 27 | class MS_API Graph { 28 | public: 29 | class GraphData; 30 | Graph(); 31 | explicit Graph(const std::shared_ptr &graph_data); 32 | explicit Graph(std::shared_ptr &&graph_data); 33 | explicit Graph(std::nullptr_t); 34 | ~Graph(); 35 | 36 | enum ModelType ModelType() const; 37 | bool operator==(std::nullptr_t) const; 38 | bool operator!=(std::nullptr_t) const; 39 | 40 | private: 41 | friend class GraphCell; 42 | friend class ModelImpl; 43 | friend class GraphImplStubAdd; 44 | std::shared_ptr graph_data_; 45 | }; 46 | } // namespace mindspore 47 | #endif // MINDSPORE_INCLUDE_API_GRAPH_H 48 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2023-0286.patch: -------------------------------------------------------------------------------- 1 | diff --git a/crypto/x509v3/v3_genn.c b/crypto/x509v3/v3_genn.c 2 | index 87a5eff47c..e54ddc55c9 100644 3 | --- a/crypto/x509v3/v3_genn.c 4 | +++ b/crypto/x509v3/v3_genn.c 5 | @@ -98,7 +98,7 @@ int GENERAL_NAME_cmp(GENERAL_NAME *a, GENERAL_NAME *b) 6 | return -1; 7 | switch (a->type) { 8 | case GEN_X400: 9 | - result = ASN1_TYPE_cmp(a->d.x400Address, b->d.x400Address); 10 | + result = ASN1_STRING_cmp(a->d.x400Address, b->d.x400Address); 11 | break; 12 | 13 | case GEN_EDIPARTY: 14 | diff --git a/include/openssl/x509v3.h b/include/openssl/x509v3.h 15 | index 90fa3592ce..e61c0f29d4 100644 16 | --- a/include/openssl/x509v3.h 17 | +++ b/include/openssl/x509v3.h 18 | @@ -136,7 +136,7 @@ typedef struct GENERAL_NAME_st { 19 | OTHERNAME *otherName; /* otherName */ 20 | ASN1_IA5STRING *rfc822Name; 21 | ASN1_IA5STRING *dNSName; 22 | - ASN1_TYPE *x400Address; 23 | + ASN1_STRING *x400Address; 24 | X509_NAME *directoryName; 25 | EDIPARTYNAME *ediPartyName; 26 | ASN1_IA5STRING *uniformResourceIdentifier; 27 | diff --git a/test/v3nametest.c b/test/v3nametest.c 28 | index d1852190b8..37819da8fd 100644 29 | --- a/test/v3nametest.c 30 | +++ b/test/v3nametest.c 31 | @@ -646,6 +646,14 @@ static struct gennamedata { 32 | 0xb7, 0x09, 0x02, 0x02 33 | }, 34 | 15 35 | + }, { 36 | + /* 37 | + * Regression test for CVE-2023-0286. 38 | + */ 39 | + { 40 | + 0xa3, 0x00 41 | + }, 42 | + 2 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/distributed_worker/notify_agent/base_notify_agent.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_BASE_NOTIFY_AGENT_H 18 | #define MINDSPORE_SERVING_WORKER_BASE_NOTIFY_AGENT_H 19 | #include 20 | #include 21 | #include 22 | #include "common/serving_common.h" 23 | #include "common/servable.h" 24 | #include "proto/ms_agent.pb.h" 25 | #include "common/grpc_client.h" 26 | 27 | namespace mindspore { 28 | namespace serving { 29 | class MS_API BaseNotifyAgent { 30 | public: 31 | BaseNotifyAgent() = default; 32 | virtual ~BaseNotifyAgent() = default; 33 | virtual Status Exit() = 0; 34 | virtual Status DispatchAsync(const proto::DistributedPredictRequest &request, proto::DistributedPredictReply *reply, 35 | AsyncPredictCallback callback) = 0; 36 | }; 37 | 38 | } // namespace serving 39 | } // namespace mindspore 40 | 41 | #endif // MINDSPORE_SERVING_WORKER_BASE_NOTIFY_AGENT_H 42 | -------------------------------------------------------------------------------- /tests/st/distributed_server_fault/kill_9_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_serving_server() 7 | { 8 | get_serving_server_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | num=`ps -ef | grep start_distributed_worker.py | grep -v grep | wc -l` 16 | if [ ${num} -ne 1 ] 17 | then 18 | echo "serving worker start failed" 19 | echo ${num} 20 | clean_pid && exit 1 21 | fi 22 | get_serving_agent_count 23 | if [ $? -ne 9 ] 24 | then 25 | echo "serving agent start failed" 26 | echo $? 27 | clean_pid && exit 1 28 | fi 29 | num=`grep -E 'Recv Ping Time Out from' serving_server.log | wc -l` 30 | if [ $num -ne 0 ] 31 | then 32 | echo "serving agent has exited" 33 | echo $num 34 | clean_pid && exit 1 35 | fi 36 | ps aux | grep 'start_distributed_worker.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 37 | if [ $? -ne 0 ] 38 | then 39 | echo "kill serving worker failed" 40 | fi 41 | sleep 25 42 | num=`grep -E 'Recv Ping Time Out from' serving_agent.log | wc -l` 43 | if [ $num -ne 8 ] 44 | then 45 | echo "catch serving server exit failed" 46 | echo $num 47 | clean_pid && exit 1 48 | fi 49 | } 50 | 51 | test_kill_serving_server() 52 | { 53 | start_serving_server 54 | start_serving_agent 55 | kill_serving_server 56 | clean_pid 57 | } 58 | 59 | echo "-----serving start-----" 60 | init 61 | test_kill_serving_server 62 | echo "### end to serving test ###" 63 | -------------------------------------------------------------------------------- /mindspore_serving/server/master/context.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Set context of serving""" 16 | from mindspore_serving._mindspore_serving import MasterContext_ 17 | from mindspore_serving.server.common import check_type 18 | 19 | __all__ = ["set_max_enqueued_requests"] 20 | 21 | _context = MasterContext_.get_instance() 22 | 23 | 24 | def set_max_enqueued_requests(max_enqueued_requests): 25 | r""" 26 | Set the maximum number of requests waiting to be processed. 27 | 28 | Args: 29 | max_enqueued_requests (int): The maximum acceptable infer message size in number, default ``10000``, 30 | Max infer number should be a positive integer. 31 | 32 | Raises: 33 | RuntimeError: The type or value of the parameters are invalid, or other error happened. 34 | """ 35 | check_type.check_int("max_enqueued_requests", max_enqueued_requests, 1) 36 | _context.set_max_enqueued_requests(max_enqueued_requests) 37 | -------------------------------------------------------------------------------- /example/matmul_distributed/export_model/net.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | '''net 16 | The sample can be run on Ascend 910 AI processor. 17 | ''' 18 | import numpy as np 19 | from mindspore import Tensor, Parameter, ops 20 | from mindspore.nn import Cell 21 | 22 | 23 | class Net(Cell): 24 | """Net""" 25 | 26 | def __init__(self, matmul_size, transpose_a=False, transpose_b=False, strategy=None): 27 | """init""" 28 | super().__init__() 29 | matmul_np = np.full(matmul_size, 0.5, dtype=np.float32) 30 | self.matmul_weight = Parameter(Tensor(matmul_np)) 31 | self.matmul = ops.MatMul(transpose_a=transpose_a, transpose_b=transpose_b) 32 | self.neg = ops.Neg() 33 | if strategy is not None: 34 | self.matmul.shard(strategy) 35 | 36 | def construct(self, inputs): 37 | """construct""" 38 | x = self.matmul(inputs, self.matmul_weight) 39 | x = self.neg(x) 40 | return x 41 | -------------------------------------------------------------------------------- /tests/ut/stub/cxx_api/graph/graph_data.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_CCSRC_CXX_API_GRAPH_GRAPH_DATA_H 17 | #define MINDSPORE_CCSRC_CXX_API_GRAPH_GRAPH_DATA_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "include/api/graph.h" 24 | #include "include/api/types.h" 25 | #include "utils/utils.h" 26 | 27 | namespace mindspore { 28 | class Graph::GraphData { 29 | public: 30 | GraphData(); 31 | 32 | explicit GraphData(const FuncGraphPtr &func_graph, enum ModelType model_type = kMindIR); 33 | 34 | GraphData(const Buffer &om_data, enum ModelType model_type); 35 | 36 | ~GraphData(); 37 | 38 | enum ModelType ModelType() const { return model_type_; } 39 | 40 | FuncGraphPtr GetFuncGraph() const; 41 | 42 | Buffer GetOMData() const; 43 | 44 | private: 45 | FuncGraphPtr func_graph_; 46 | Buffer om_data_; 47 | enum ModelType model_type_; 48 | }; 49 | } // namespace mindspore 50 | #endif // MINDSPORE_CCSRC_CXX_API_GRAPH_GRAPH_DATA_H 51 | -------------------------------------------------------------------------------- /third_party/securec/src/secureinput_w.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* if some platforms don't have wchar.h, dont't include it */ 18 | #if !(defined(SECUREC_VXWORKS_PLATFORM)) 19 | /* This header file is placed below secinput.h, which will cause tool alarm, 20 | * but If there is no macro above, it will cause vs2010 compiling alarm 21 | */ 22 | #if defined(_MSC_VER) && (_MSC_VER >= 1400) 23 | #ifndef __STDC_WANT_SECURE_LIB__ 24 | /* The order of adjustment is to eliminate alarm of Duplicate Block */ 25 | #define __STDC_WANT_SECURE_LIB__ 0 26 | #endif 27 | #ifndef _CRTIMP_ALTERNATIVE 28 | #define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ 29 | #endif 30 | #endif 31 | #include 32 | #endif 33 | #define SECUREC_ENABLE_WCHAR_FUNC 0 34 | #define SECUREC_FORMAT_OUTPUT_INPUT 1 35 | #ifndef SECUREC_FOR_WCHAR 36 | #define SECUREC_FOR_WCHAR 37 | #endif 38 | 39 | #include "secinput.h" 40 | 41 | #ifndef WEOF 42 | #define WEOF ((wchar_t)(-1)) 43 | #endif 44 | 45 | #include "input.inl" 46 | 47 | -------------------------------------------------------------------------------- /mindspore_serving/server/register/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """Servable register interface, used in servable_config.py of one servable. 16 | See how to configure servable_config.py file, please refer to 17 | `Servable Provided Through Model Configuration `_.""" 18 | 19 | from .model import declare_model, Model, Context, AclOptions, GpuOptions 20 | from .model import AscendDeviceInfo, CPUDeviceInfo, GPUDeviceInfo 21 | from .method import register_method, add_stage 22 | 23 | from .model import declare_servable 24 | from .method import call_preprocess, call_servable, call_postprocess 25 | from .method import call_preprocess_pipeline, call_postprocess_pipeline 26 | 27 | __all__ = [] 28 | __all__.extend([ 29 | "declare_model", 30 | "Model", 31 | "AscendDeviceInfo", 32 | "CPUDeviceInfo", 33 | "GPUDeviceInfo", 34 | "Context", 35 | 'register_method', 36 | 'add_stage' 37 | ]) 38 | -------------------------------------------------------------------------------- /docs/api/api_python/server/mindspore_serving.server.ServableStartConfig.rst: -------------------------------------------------------------------------------- 1 | 2 | .. py:class:: mindspore_serving.server.ServableStartConfig(servable_directory, servable_name, device_ids=None, version_number=0, device_type=None, num_parallel_workers=0, dec_key=None, dec_mode='AES-GCM') 3 | 4 | 启动一个服务的配置。详情请查看 5 | `基于MindSpore Serving部署推理服务 `_ 和 6 | `通过配置模型提供Servable `_ 。 7 | 8 | 参数： 9 | - **servable_directory** (str) - 服务所在的目录。预期有一个名为 `servable_name` 的目录。 10 | - **servable_name** (str) - 服务名称。 11 | - **device_ids** (Union[int, list[int], tuple[int]], 可选) - 模型部署和运行的设备列表，列表中的每个会设备将部署和运行一个服务副本。当设备类型为Nvidia GPU、Ascend 310P/910时使用。默认值：``None``。 12 | - **version_number** (int, 可选) - 要加载的服务的版本号。版本号应为正整数，从1开始，``0`` 表示加载最新版本。默认值：``0``。 13 | - **device_type** (str, 可选) - 模型部署的目标设备类型，目前支持 ``"Ascend"``、``"GPU"``、``"CPU"`` 和 ``None``。默认值：``None``。 14 | 15 | - ``"Ascend"``：目标设备为Ascend 310P/910等。 16 | - ``"GPU"``：目标设备为Nvidia GPU。 17 | - ``"CPU"``：目标设备为CPU。 18 | - ``None``：系统根据实际的后端设备和MindSpor推理包决定目标设备，推荐使用默认值 ``None``。 19 | 20 | - **num_parallel_workers** (int, 可选) - 处理Python任务的进程数，用于提高预处理、后处理等Python任务的处理能力。值小于 `device_ids` 的长度时，处理Python任务的进程数为 `device_ids` 的长度。值的范围为[0,64]。默认值：``0``。 21 | - **dec_key** (bytes, 可选) - 用于解密的字节类型密钥。有效长度为16、24或32。默认值：``None``。 22 | - **dec_mode** (str, 可选) - 指定解密模式，设置 `dec_key` 时生效。值可为： ``'AES-GCM'`` 或 ``'AES-CBC'`` 。默认值： ``'AES-GCM'`` 。 23 | 24 | 异常： 25 | - **RuntimeError** - 参数的类型或值无效。 26 | 27 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/context.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_CONTEXT_H 18 | #define MINDSPORE_SERVING_WORKER_CONTEXT_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include "common/serving_common.h" 24 | #include "worker/inference/inference.h" 25 | 26 | namespace mindspore::serving { 27 | class MS_API ServableContext { 28 | public: 29 | static std::shared_ptr Instance(); 30 | 31 | Status SetDeviceTypeStr(const std::string &device_type); 32 | 33 | void SetDeviceType(DeviceType device_type); 34 | DeviceType GetDeviceType() const; 35 | void SetDeviceId(uint32_t device_id); 36 | uint32_t GetDeviceId() const; 37 | 38 | void SetEnableLite(bool enable_lite); 39 | bool EnableLite() const; 40 | 41 | private: 42 | DeviceType device_type_ = kDeviceTypeNotSpecified; 43 | uint32_t device_id_ = 0; 44 | bool enable_lite_ = false; 45 | }; 46 | 47 | } // namespace mindspore::serving 48 | 49 | #endif // MINDSPORE_SERVING_WORKER_CONTEXT_H 50 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2023-4807.patch: -------------------------------------------------------------------------------- 1 | From a632d534c73eeb3e3db8c7540d811194ef7c79ff Mon Sep 17 00:00:00 2001 2 | From: Bernd Edlinger 3 | Date: Tue, 22 Aug 2023 16:07:30 +0200 4 | Subject: [PATCH] Avoid clobbering non-volatile XMM registers 5 | 6 | This affects some Poly1305 assembler functions 7 | which are only used for certain CPU types. 8 | 9 | Remove those functions for Windows targets, 10 | as a simple interim solution. 11 | 12 | Fixes #21522 13 | 14 | Reviewed-by: Tomas Mraz 15 | Reviewed-by: Paul Dale 16 | (Merged from https://github.com/openssl/openssl/pull/21808) 17 | 18 | (cherry picked from commit 7b8e27bc2e02238986d89ef0ece067ec1b48e165) 19 | --- 20 | crypto/poly1305/asm/poly1305-x86_64.pl | 4 ++-- 21 | 1 file changed, 2 insertions(+), 2 deletions(-) 22 | 23 | diff --git a/crypto/poly1305/asm/poly1305-x86_64.pl b/crypto/poly1305/asm/poly1305-x86_64.pl 24 | index 5f834d8faf..801455c639 100755 25 | --- a/crypto/poly1305/asm/poly1305-x86_64.pl 26 | +++ b/crypto/poly1305/asm/poly1305-x86_64.pl 27 | @@ -193,7 +193,7 @@ $code.=<<___ if ($avx>1); 28 | bt \$`5+32`,%r9 # AVX2? 29 | cmovc %rax,%r10 30 | ___ 31 | -$code.=<<___ if ($avx>3); 32 | +$code.=<<___ if ($avx>3 && !$win64); 33 | mov \$`(1<<31|1<<21|1<<16)`,%rax 34 | shr \$32,%r9 35 | and %rax,%r9 36 | @@ -2722,7 +2722,7 @@ $code.=<<___; 37 | .cfi_endproc 38 | .size poly1305_blocks_avx512,.-poly1305_blocks_avx512 39 | ___ 40 | -if ($avx>3) { 41 | +if ($avx>3 && !$win64) { 42 | ######################################################################## 43 | # VPMADD52 version using 2^44 radix. 44 | # 45 | -- 46 | 2.34.1 47 | 48 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/master/notify_worker/grpc_notify.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_MASTER_GRPC_NOTIFY_H 18 | #define MINDSPORE_SERVING_MASTER_GRPC_NOTIFY_H 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "master/notify_worker/base_notify.h" 24 | #include "proto/ms_worker.pb.h" 25 | #include "proto/ms_worker.grpc.pb.h" 26 | 27 | namespace mindspore { 28 | namespace serving { 29 | class MS_API GrpcNotifyWorker : public BaseNotifyWorker { 30 | public: 31 | explicit GrpcNotifyWorker(const std::string &worker_address); 32 | ~GrpcNotifyWorker() override; 33 | 34 | Status DispatchAsync(const proto::PredictRequest &request, proto::PredictReply *reply, 35 | const PredictOnFinish &on_finish) override; 36 | 37 | private: 38 | std::string worker_address_; 39 | std::shared_ptr stub_ = nullptr; 40 | }; 41 | 42 | } // namespace serving 43 | } // namespace mindspore 44 | 45 | #endif // MINDSPORE_SERVING_MASTER_GRPC_NOTIFY_H 46 | -------------------------------------------------------------------------------- /mindspore_serving/server/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """ 16 | MindSpore Serving is a lightweight and high-performance service module that helps MindSpore developers efficiently 17 | deploy online inference services in the production environment. 18 | 19 | MindSpore Serving server API, which can be used to start servables, gRPC and RESTful server. A servable corresponds to 20 | the service provided by a model. The client sends inference tasks and receives inference results through gRPC and 21 | RESTful server. 22 | """ 23 | 24 | from .master import start_grpc_server, start_restful_server, stop, SSLConfig 25 | from ._server import start_servables, ServableStartConfig 26 | from . import register 27 | from . import distributed 28 | 29 | __all__ = [] 30 | __all__.extend([ 31 | "start_grpc_server", 32 | "start_restful_server", 33 | "stop", 34 | "start_servables", 35 | 'ServableStartConfig', 36 | "SSLConfig" 37 | ]) 38 | 39 | __all__.extend(register.__all__) 40 | __all__.extend(distributed.__all__) 41 | -------------------------------------------------------------------------------- /tests/st/distributed_server_fault/kill_9_agent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CURRPATH=$(cd "$(dirname $0)" || exit; pwd) 4 | source ${CURRPATH}/common.sh 5 | 6 | kill_serving_agent() 7 | { 8 | get_serving_server_count 9 | if [ $? -ne 1 ] 10 | then 11 | echo "serving server start failed" 12 | echo $? 13 | clean_pid && exit 1 14 | fi 15 | get_serving_agent_count 16 | if [ $? -ne 9 ] 17 | then 18 | echo "serving agent start failed" 19 | echo $? 20 | clean_pid && exit 1 21 | fi 22 | num=`grep -E 'Recv Pong Time Out from' serving_logs/log_matmul*.log | wc -l` 23 | if [ $num -ne 0 ] 24 | then 25 | echo "serving agent has exited" 26 | echo $num 27 | clean_pid && exit 1 28 | fi 29 | ps aux | grep 'serving_agent.py' | grep ${CURRUSER} | grep -v grep | awk '{print $2}' | xargs kill -9 30 | if [ $? -ne 0 ] 31 | then 32 | echo "kill serving agent failed" 33 | fi 34 | sleep 25 35 | get_serving_agent_count 36 | if [ $? -ne 0 ] 37 | then 38 | echo "agent exit failed" 39 | echo $? 40 | clean_pid && exit 1 41 | fi 42 | get_serving_server_count 43 | if [ $? -ne 1 ] 44 | then 45 | echo "serving server start failed" 46 | echo $? 47 | clean_pid && exit 1 48 | fi 49 | num=`grep -E 'Recv Pong Time Out from' serving_logs/log_matmul*.log | wc -l` 50 | if [ $num -ne 8 ] 51 | then 52 | echo "catch agent exit failed" 53 | echo $num 54 | clean_pid && exit 1 55 | fi 56 | } 57 | 58 | test_kill_serving_agent() 59 | { 60 | start_serving_server 61 | start_serving_agent 62 | kill_serving_agent 63 | clean_pid 64 | } 65 | 66 | echo "-----serving start-----" 67 | init 68 | test_kill_serving_agent 69 | echo "### end to serving test ###" 70 | -------------------------------------------------------------------------------- /cmake/external_libs/libevent.cmake: -------------------------------------------------------------------------------- 1 | set(openssl_USE_STATIC_LIBS ON) 2 | set(libevent_CFLAGS "-fPIC -fvisibility=hidden -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2") 3 | if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") 4 | set(libevent_LDFLAGS "-Wl,-z,now") 5 | endif() 6 | 7 | if(NOT MINDSPORE_PROJECT_DIR) 8 | set(MINDSPORE_PROJECT_DIR ${CMAKE_SOURCE_DIR}) 9 | endif() 10 | 11 | if(ENABLE_GITEE OR ENABLE_GITEE_EULER) # Channel GITEE_EULER is NOT supported now, use GITEE instead. 12 | set(REQ_URL "https://gitee.com/mirrors/libevent/repository/archive/release-2.1.12-stable.tar.gz") 13 | set(SHA256 "7180a979aaa7000e1264da484f712d403fcf7679b1e9212c4e3d09f5c93efc24") 14 | else() 15 | set(REQ_URL 16 | "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz") 17 | set(SHA256 "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb") 18 | endif() 19 | 20 | message("libevent using openssl stub dir: " ${openssl_ROOT}) 21 | 22 | mindspore_add_pkg(libevent 23 | VER 2.1.12 24 | LIBS event event_pthreads event_core event_openssl 25 | URL ${REQ_URL} 26 | SHA256 ${SHA256} 27 | PATCHES ${MINDSPORE_PROJECT_DIR}/third_party/patch/libevent/libevent.patch001 28 | CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_TESTING=OFF -DOPENSSL_ROOT_DIR:PATH=${openssl_ROOT} 29 | -DEVENT__LIBRARY_TYPE:STRING=STATIC) 30 | 31 | include_directories(${libevent_INC}) 32 | 33 | add_library(mindspore_serving::event ALIAS libevent::event) 34 | add_library(mindspore_serving::event_pthreads ALIAS libevent::event_pthreads) 35 | add_library(mindspore_serving::event_core ALIAS libevent::event_core) 36 | add_library(mindspore_serving::event_openssl ALIAS libevent::event_openssl) 37 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/distributed_worker/notify_agent/notify_agent.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_NOTIFY_AGENT_H 18 | #define MINDSPORE_SERVING_WORKER_NOTIFY_AGENT_H 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "worker/distributed_worker/notify_agent/base_notify_agent.h" 24 | #include "proto/ms_agent.pb.h" 25 | #include "proto/ms_agent.grpc.pb.h" 26 | 27 | namespace mindspore { 28 | namespace serving { 29 | class MS_API GrpcNotifyAgent : public BaseNotifyAgent { 30 | public: 31 | explicit GrpcNotifyAgent(const std::string &worker_address); 32 | ~GrpcNotifyAgent() override; 33 | 34 | Status Exit() override; 35 | 36 | Status DispatchAsync(const proto::DistributedPredictRequest &request, proto::DistributedPredictReply *reply, 37 | AsyncPredictCallback callback) override; 38 | 39 | private: 40 | std::string agent_address_; 41 | std::shared_ptr stub_ = nullptr; 42 | }; 43 | 44 | } // namespace serving 45 | } // namespace mindspore 46 | 47 | #endif // MINDSPORE_SERVING_WORKER_NOTIFY_AGENT_H 48 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/python/worker/servable_py.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_SERVABLE_PY_H 18 | #define MINDSPORE_SERVING_WORKER_SERVABLE_PY_H 19 | 20 | #include 21 | #include "common/servable.h" 22 | 23 | #include "pybind11/pybind11.h" 24 | #include "pybind11/numpy.h" 25 | #include "pybind11/stl.h" 26 | #include "python/tensor_py.h" 27 | 28 | namespace py = pybind11; 29 | 30 | namespace mindspore::serving { 31 | class MS_API PyServableRegister { 32 | public: 33 | static void RegisterMethod(const MethodSignature &method); 34 | 35 | static void DeclareModel(const ModelMeta &servable); 36 | static void DeclareDistributedModel(const ModelMeta &servable); 37 | 38 | static void RegisterInputOutputInfo(const std::string &model_key, size_t inputs_count, size_t outputs_count, 39 | uint64_t subgraph = 0); 40 | 41 | // input args: list, output: tuple 42 | static py::tuple Run(const std::string &model_key, const py::tuple &args, uint64_t subgraph); 43 | }; 44 | 45 | } // namespace mindspore::serving 46 | 47 | #endif // MINDSPORE_SERVING_WORKER_SERVABLE_PY_H 48 | -------------------------------------------------------------------------------- /tests/ut/stub/cxx_api/graph/ascend/ascend_graph_impl.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_CCSRC_CXX_API_GRAPH_MS_ASCEND_GRAPH_IMPL_H 17 | #define MINDSPORE_CCSRC_CXX_API_GRAPH_MS_ASCEND_GRAPH_IMPL_H 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "include/api/status.h" 25 | #include "include/api/graph.h" 26 | #include "cxx_api/graph/graph_impl.h" 27 | #include "cxx_api/model/model_impl.h" 28 | 29 | namespace mindspore { 30 | class AscendGraphImpl : public GraphCell::GraphImpl { 31 | public: 32 | AscendGraphImpl(); 33 | ~AscendGraphImpl() override; 34 | 35 | Status Run(const std::vector &inputs, std::vector *outputs) override; 36 | Status Load(uint32_t device_id) override; 37 | std::vector GetInputs() override; 38 | std::vector GetOutputs() override; 39 | bool CheckDeviceSupport(mindspore::DeviceType device_type) override; 40 | private: 41 | std::shared_ptr graph_imp_stub_; 42 | }; 43 | 44 | } // namespace mindspore 45 | #endif // MINDSPORE_CCSRC_CXX_API_GRAPH_MS_ASCEND_GRAPH_IMPL_H 46 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/python/master/master_py.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVER_MASTER_PY_H 18 | #define MINDSPORE_SERVER_MASTER_PY_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include "common/serving_common.h" 26 | #include "common/ssl_config.h" 27 | 28 | namespace py = pybind11; 29 | 30 | namespace mindspore { 31 | namespace serving { 32 | class MS_API PyMaster { 33 | public: 34 | static void StartGrpcServer(const std::string &socket_address, const SSLConfig &ssl_config, 35 | int max_msg_mb_size = 100); 36 | static void StartGrpcMasterServer(const std::string &master_address); 37 | static void StartRestfulServer(const std::string &socket_address, const SSLConfig &ssl_config, 38 | int max_msg_mb_size = 100); 39 | static void WaitAndClear(); 40 | static void StopAndClear(); 41 | 42 | static bool OnlyModelStage(const std::string &servable_name); 43 | }; 44 | 45 | } // namespace serving 46 | } // namespace mindspore 47 | 48 | #endif // MINDSPORE_SERVER_MASTER_PY_H 49 | -------------------------------------------------------------------------------- /tests/ut/coverage/run_coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Huawei Technologies Co., Ltd 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================ 16 | set -e 17 | BASEPATH=$( 18 | cd "$(dirname "$0")" 19 | pwd 20 | ) 21 | 22 | PROJECT_PATH=$( 23 | cd ${BASEPATH}/../../.. 24 | pwd 25 | ) 26 | 27 | BUILD_PKG=${PROJECT_PATH}/build/package 28 | 29 | export PYTHONPATH=${BUILD_PKG}:${PROJECT_PATH}/tests/ut/python:$PYTHONPATH 30 | export LD_LIBRARY_PATH=${BUILD_PKG}/tests/mindspore/lib:${LD_LIBRARY_PATH} 31 | 32 | echo "PYTHONPATH=$PYTHONPATH" 33 | echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" 34 | export GLOG_v=1 35 | 36 | unset http_proxy 37 | unset https_proxy 38 | 39 | rm -rf cov_output htmlcov .coverage 40 | 41 | # run python ut 42 | pytest -v ${PROJECT_PATH}/tests/ut/python/tests/ --cov=${BUILD_PKG}/mindspore_serving --cov-config=${BASEPATH}/cov_config --cov-report=html --cov-branch 43 | # run cpp ut 44 | bash ../cpp/runtest.sh 45 | 46 | mkdir cov_output && cd cov_output 47 | lcov --capture --directory ${PROJECT_PATH}/build/mindspore_serving/ --output-file coverage.info; 48 | lcov --extract coverage.info '*/ccsrc/*' -o coverage.info; 49 | genhtml coverage.info --output-directory ./ --sort --legend 50 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/python/agent/agent_py.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVER_AGENT_PY_H 18 | #define MINDSPORE_SERVER_AGENT_PY_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include "common/serving_common.h" 26 | #include "worker/distributed_worker/common.h" 27 | 28 | namespace py = pybind11; 29 | 30 | namespace mindspore { 31 | namespace serving { 32 | class MS_API PyAgent { 33 | public: 34 | static void StartAgent(const AgentStartUpConfig &start_config, const std::string &dec_key, 35 | const std::string &dec_mode); 36 | 37 | static DistributedServableConfig GetAgentsConfigsFromWorker(const std::string &distributed_address); 38 | static void WaitAndClear(); 39 | static void StopAndClear(); 40 | // from start up, not agent 41 | static void NotifyFailed(const std::string &distributed_address); 42 | static void StartupNotifyExit(const std::string &distributed_address, const std::string &agent_ip); 43 | }; 44 | 45 | } // namespace serving 46 | } // namespace mindspore 47 | 48 | #endif // MINDSPORE_SERVER_AGENT_PY_H 49 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/distributed_worker/agent_startup.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_AGENT_STARTUP_H 18 | #define MINDSPORE_SERVING_WORKER_AGENT_STARTUP_H 19 | #include 20 | #include 21 | #include "common/serving_common.h" 22 | #include "worker/distributed_worker/common.h" 23 | #include "worker/inference/inference.h" 24 | 25 | namespace mindspore { 26 | namespace serving { 27 | class MS_API WorkerAgentStartUp { 28 | public: 29 | static WorkerAgentStartUp &Instance(); 30 | // from python, worker_agent.py 31 | // start_worker_agent 32 | // step1, get agents config from worker 33 | Status GetAgentsConfigsFromWorker(const std::string &distributed_address); 34 | // step2, invoke from python 35 | Status GetDistributedServableConfig(DistributedServableConfig *config); 36 | 37 | Status NotifyFailed(const std::string &distributed_address); 38 | void StartupNotifyExit(const std::string &distributed_address, const std::string &agent_ip); 39 | 40 | private: 41 | DistributedServableConfig config_; 42 | std::string worker_address_; 43 | }; 44 | 45 | } // namespace serving 46 | } // namespace mindspore 47 | 48 | #endif // MINDSPORE_SERVING_WORKER_AGENT_STARTUP_H 49 | -------------------------------------------------------------------------------- /example/matmul_distributed/export_model/distributed_inference.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | '''distributed inference 16 | The sample can be run on Ascend 910 AI processor. 17 | ''' 18 | import numpy as np 19 | from net import Net 20 | from mindspore import context, Model, Tensor, export 21 | from mindspore.communication import init 22 | 23 | 24 | def test_inference(): 25 | """distributed inference after distributed training""" 26 | context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") 27 | init(backend_name="hccl") 28 | context.set_auto_parallel_context(full_batch=True, parallel_mode="semi_auto_parallel", 29 | device_num=8, group_ckpt_save_file="./group_config.pb") 30 | 31 | predict_data = create_predict_data() 32 | network = Net(matmul_size=(96, 16)) 33 | model = Model(network) 34 | model.infer_predict_layout(Tensor(predict_data)) 35 | export(model.predict_network, Tensor(predict_data), file_name="matmul", file_format="MINDIR") 36 | 37 | 38 | def create_predict_data(): 39 | """user-defined predict data""" 40 | inputs_np = np.random.randn(128, 96).astype(np.float32) 41 | return Tensor(inputs_np) 42 | -------------------------------------------------------------------------------- /tests/ut/stub/cxx_api/model/ms/ms_model.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_CCSRC_SESSION_SESSION_H 17 | #define MINDSPORE_CCSRC_SESSION_SESSION_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "include/api/status.h" 27 | #include "cxx_api/model/model_impl.h" 28 | 29 | namespace mindspore { 30 | class MsModel : public ModelImpl { 31 | public: 32 | MsModel() {} 33 | ~MsModel() = default; 34 | 35 | Status Build() override; 36 | Status Resize(const std::vector &inputs, const std::vector> &dims) override; 37 | 38 | std::vector GetInputs() override; 39 | std::vector GetOutputs() override; 40 | 41 | bool CheckDeviceSupport(mindspore::DeviceType device_type) override; 42 | bool CheckModelSupport(enum ModelType model_type) override; 43 | 44 | private: 45 | std::shared_ptr GenerateGraphCell(const std::vector> &dims); 46 | uint32_t GetDeviceID() const; 47 | 48 | std::map> dynamic_size_graph_map_; 49 | }; 50 | } // namespace mindspore 51 | #endif // MINDSPORE_CCSRC_SESSION_SESSION_BASIC_H 52 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/grpc_server.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_GRPC_SERVER_H 18 | #define MINDSPORE_SERVING_GRPC_SERVER_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include "common/serving_common.h" 28 | 29 | namespace mindspore::serving { 30 | constexpr int gRpcDefaultMsgMBSize = 100; 31 | constexpr int gRpcMaxMBMsgSize = 512; // max 512 MB 32 | 33 | class GrpcServer { 34 | public: 35 | GrpcServer() = default; 36 | ~GrpcServer() noexcept { Stop(); } 37 | 38 | Status Start(const std::shared_ptr &service, const std::string &server_address, int max_msg_size, 39 | const std::string &server_tag); 40 | void Stop(); 41 | static std::shared_ptr CreateChannel(const std::string &target_str); 42 | 43 | private: 44 | std::unique_ptr server_; 45 | std::thread grpc_thread_; 46 | bool in_running_ = false; 47 | std::shared_ptr service_; 48 | }; 49 | } // namespace mindspore::serving 50 | 51 | #endif // MINDSPORE_SERVING_GRPC_SERVER_H 52 | -------------------------------------------------------------------------------- /cmake/check_requirements.cmake: -------------------------------------------------------------------------------- 1 | ## define customized find functions, print customized error messages 2 | function(find_required_package pkg_name) 3 | find_package(${pkg_name}) 4 | if(NOT ${pkg_name}_FOUND) 5 | message(FATAL_ERROR "Required package ${pkg_name} not found, please install the package and try" 6 | " building mindspore_serving again.") 7 | endif() 8 | endfunction() 9 | 10 | ## find python, quit if the found python is static 11 | set(Python3_USE_STATIC_LIBS FALSE) 12 | set(Python3_FIND_VIRTUALENV ONLY) 13 | find_package(Python3 COMPONENTS Interpreter Development) 14 | if(Python3_FOUND) 15 | message("Python3 found, version: ${Python3_VERSION}") 16 | message("Python3 library path: ${Python3_LIBRARY}") 17 | message("Python3 interpreter: ${Python3_EXECUTABLE}") 18 | elseif(Python3_LIBRARY AND Python3_EXECUTABLE AND 19 | ${Python3_VERSION} VERSION_GREATER_EQUAL "3.7.0" AND ${Python3_VERSION} VERSION_LESS "3.10.0") 20 | message(WARNING "Maybe python3 environment is broken.") 21 | message("Python3 library path: ${Python3_LIBRARY}") 22 | message("Python3 interpreter: ${Python3_EXECUTABLE}") 23 | else() 24 | message(FATAL_ERROR "Python3 not found, please install Python>=3.7.5, and set --enable-shared " 25 | "if you are building Python locally") 26 | endif() 27 | 28 | ## packages used both on windows and linux 29 | if(DEFINED ENV{MS_PATCH_PATH}) 30 | find_program(Patch_EXECUTABLE patch PATHS $ENV{MS_PATCH_PATH}) 31 | set(Patch_FOUND ${Patch_EXECUTABLE}) 32 | else() 33 | find_package(Patch) 34 | endif() 35 | if(NOT Patch_FOUND) 36 | message(FATAL_ERROR "Patch not found, please set environment variable MS_PATCH_PATH to path where Patch is located," 37 | " usually found in GIT_PATH/usr/bin on Windows") 38 | endif() 39 | message(PATCH_EXECUTABLE = ${Patch_EXECUTABLE}) 40 | 41 | find_required_package(Threads) 42 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/distributed_worker/common.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_DISTRIBUTED_WORKER_COMMON_H 18 | #define MINDSPORE_SERVING_DISTRIBUTED_WORKER_COMMON_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include "common/serving_common.h" 24 | #include "worker/inference/inference.h" 25 | #include "common/servable.h" 26 | 27 | namespace mindspore { 28 | namespace serving { 29 | struct OneRankConfig { 30 | std::string ip; 31 | uint32_t device_id = 0; 32 | }; 33 | 34 | struct DistributedServableConfig { 35 | std::string rank_table_content; 36 | std::vector rank_list; 37 | 38 | CommonModelMeta common_meta; 39 | DistributedModelMeta distributed_meta; 40 | }; 41 | 42 | struct AgentStartUpConfig { 43 | uint32_t rank_id; 44 | uint32_t device_id; 45 | std::vector model_file_names; 46 | std::vector group_file_names; 47 | std::string rank_table_json_file_name; 48 | 49 | std::string agent_address; 50 | std::string distributed_address; 51 | uint32_t worker_port; 52 | 53 | CommonModelMeta common_meta; 54 | }; 55 | 56 | } // namespace serving 57 | } // namespace mindspore 58 | 59 | #endif // MINDSPORE_SERVING_DISTRIBUTED_WORKER_COMMON_H 60 | -------------------------------------------------------------------------------- /tests/ut/stub/cxx_api/graph/ascend/ascend_graph_impl.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #include "cxx_api/graph/ascend/ascend_graph_impl.h" 17 | #include 18 | #include "include/api/context.h" 19 | #include "cxx_api/factory.h" 20 | #include "stub/graph_impl_stub.h" 21 | 22 | namespace mindspore { 23 | API_FACTORY_REG(GraphCell::GraphImpl, AscendGraphImpl); 24 | 25 | AscendGraphImpl::AscendGraphImpl() { graph_imp_stub_ = std::make_shared(); } 26 | 27 | AscendGraphImpl::~AscendGraphImpl() {} 28 | 29 | std::vector AscendGraphImpl::GetInputs() { return graph_imp_stub_->GetInputs(); } 30 | 31 | std::vector AscendGraphImpl::GetOutputs() { return graph_imp_stub_->GetOutputs(); } 32 | 33 | Status AscendGraphImpl::Load(uint32_t device_id) { 34 | graph_imp_stub_->SetGraph(graph_); 35 | graph_imp_stub_->SetContext(graph_context_); 36 | return graph_imp_stub_->Load(device_id); 37 | } 38 | 39 | Status AscendGraphImpl::Run(const std::vector &inputs, std::vector *outputs) { 40 | return graph_imp_stub_->Run(inputs, outputs); 41 | } 42 | 43 | bool AscendGraphImpl::CheckDeviceSupport(mindspore::DeviceType device_type) { 44 | return graph_imp_stub_->CheckDeviceSupport(device_type); 45 | } 46 | 47 | } // namespace mindspore 48 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/worker/grpc/worker_process.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_WORKER_WORKER_PROCESS_H 18 | #define MINDSPORE_SERVING_WORKER_WORKER_PROCESS_H 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include "common/serving_common.h" 26 | #include "common/heart_beat.h" 27 | #include "common/grpc_client.h" 28 | #include "proto/ms_worker.pb.h" 29 | #include "proto/ms_worker.grpc.pb.h" 30 | #include "proto/ms_master.pb.h" 31 | #include "proto/ms_master.grpc.pb.h" 32 | #include "proto/ms_agent.pb.h" 33 | #include "proto/ms_agent.grpc.pb.h" 34 | namespace mindspore { 35 | namespace serving { 36 | // Service Implement 37 | class MSWorkerImpl { 38 | public: 39 | MSWorkerImpl() = default; 40 | ~MSWorkerImpl() = default; 41 | void Exit(const proto::ExitRequest *request, proto::ExitReply *reply); 42 | void PredictAsync(const proto::PredictRequest *request, proto::PredictReply *reply, const PredictOnFinish &on_finish); 43 | }; 44 | 45 | } // namespace serving 46 | } // namespace mindspore 47 | 48 | #endif // MINDSPORE_SERVING_WORKER_WORKER_PROCESS_H 49 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/callback/lr_scheduler.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_CALLBACK_LR_SCHEDULER_H 17 | #define MINDSPORE_INCLUDE_API_CALLBACK_LR_SCHEDULER_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "include/api/callback/callback.h" 24 | 25 | namespace mindspore { 26 | 27 | constexpr int DONT_UPDATE_LR = 0; 28 | constexpr int UPDATE_LR = 1; 29 | 30 | using LR_Lambda = std::function; 31 | 32 | /// \brief Multiply the LR by a factor of gamma every epoch 33 | int MultiplicativeLRLambda(float *lr, int epoch, void *multiplication); 34 | 35 | /// \brief Multiply the LR by a factor of gamma every step_size 36 | int StepLRLambda(float *lr, int epoch, void *step_size); 37 | struct StepLRLambda { 38 | StepLRLambda(int step, float g) : step_size(step), gamma(g) {} 39 | 40 | int step_size; // period of LR decay 41 | float gamma; // LR decay factor 42 | }; 43 | 44 | class LRScheduler: public TrainCallBack { 45 | public: 46 | explicit LRScheduler(LR_Lambda lambda_func, void *lr_cb_data = nullptr, int step = 1); 47 | virtual ~LRScheduler(); 48 | }; 49 | 50 | } // namespace mindspore 51 | #endif // MINDSPORE_INCLUDE_API_CALLBACK_LR_SCHEDULER_H 52 | -------------------------------------------------------------------------------- /example/resnet/export_model/export_resnet.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Huawei Technologies Co., Ltd 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================ 15 | """export resnet50 for cifar10 dataset""" 16 | 17 | import os 18 | import sys 19 | from shutil import copyfile 20 | from resnet.export import export_resnet 21 | 22 | if __name__ == '__main__': 23 | if len(sys.argv) > 1 and sys.argv[1] == 'False': # python export_resnet.py False 24 | ckpt_file = None 25 | else: 26 | ckpt_file = "resnet50_ascend_v111_cifar10_offical_cv_bs32_acc92.ckpt" 27 | if not os.path.exists(ckpt_file): 28 | print("downloading resnet50 cifar10 checkpoint---------------------------------") 29 | os.system(f"wget https://download.mindspore.cn/model_zoo/r1.1/" 30 | f"resnet50_ascend_v111_cifar10_offical_cv_bs32_acc92/{ckpt_file} --no-check-certificate") 31 | print("end downloading resnet50 cifar10 checkpoint---------------------------------") 32 | export_resnet('resnet50_cifar10', ckpt_file, 'resnet50_1b_cifar10') 33 | 34 | dst_dir = '../resnet50/1' 35 | try: 36 | os.mkdir(dst_dir) 37 | except OSError: 38 | pass 39 | dst_file = os.path.join(dst_dir, 'resnet50_1b_cifar10.mindir') 40 | copyfile('resnet50_1b_cifar10.mindir', dst_file) 41 | -------------------------------------------------------------------------------- /mindspore_serving/ccsrc/common/exit_handle.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef MINDSPORE_SERVING_EXIT_HANDLE_H 18 | #define MINDSPORE_SERVING_EXIT_HANDLE_H 19 | #include 20 | #include 21 | #include 22 | #include "common/serving_common.h" 23 | 24 | namespace mindspore { 25 | namespace serving { 26 | // Handle Ctrl+C signal. When the master or worker is waiting for the Ctrl+C signal, 27 | // it can continue to perform subsequent operations, such as cleaning. 28 | class MS_API ExitSignalHandle { 29 | public: 30 | static ExitSignalHandle &Instance(); 31 | void InitSignalHandle(); 32 | void MasterWait(); 33 | void WorkerWait(); 34 | void AgentWait(); 35 | void Start(); 36 | void Stop(); 37 | bool HasStopped() const; 38 | 39 | private: 40 | std::promise master_exit_requested_; 41 | std::promise worker_exit_requested_; 42 | std::promise agent_exit_requested_; 43 | std::atomic_flag has_exited_ = true; 44 | std::atomic_flag has_inited_ = ATOMIC_FLAG_INIT; 45 | std::atomic_bool is_running_ = false; 46 | int exit_signal_ = 0; 47 | 48 | static void HandleSignal(int sig); 49 | void HandleSignalInner(int sig); 50 | }; 51 | } // namespace serving 52 | } // namespace mindspore 53 | 54 | #endif // MINDSPORE_SERVING_EXIT_HANDLE_H 55 | -------------------------------------------------------------------------------- /RELEASE_CN.md: -------------------------------------------------------------------------------- 1 | # MindSpore Serving Release Notes 2 | 3 | [View English](./RELEASE.md) 4 | 5 | ## MindSpore Serving 2.0.2 Release Notes 6 | 7 | ### 主要特性和增强 8 | 9 | - 配套MindSpore 2.2.0版本接口。 10 | - 修复第三方库OpenSSL漏洞CVE-2023-3446、CVE-2023-4807。 11 | 12 | ### 贡献者 13 | 14 | 感谢以下人员做出的贡献: 15 | 16 | qinzheng, xuyongfei, zhangyinxia, zhoufeng. 17 | 18 | 欢迎以任何形式对项目提供贡献！ 19 | 20 | ## MindSpore Serving 2.0.0 Release Notes 21 | 22 | ### 主要特性和增强 23 | 24 | - 配套MindSpore 2.0.0rc1版本接口。 25 | - 修复第三方库OpenSSL漏洞CVE-2022-4304、CVE-2022-4450、CVE-2022-4450、CVE-2023-0286、CVE-2023-0464、CVE-2023-0465、CVE-2023-0466。 26 | 27 | ### 贡献者 28 | 29 | 感谢以下人员做出的贡献: 30 | 31 | qinzheng, xuyongfei, zhangyinxia, zhoufeng. 32 | 33 | 欢迎以任何形式对项目提供贡献！ 34 | 35 | ## MindSpore Serving 1.8.0 Release Notes 36 | 37 | ### 主要特性和增强 38 | 39 | - [STABLE] Serving部署流水线并行的大模型时，支持流水线并行处理多个推理实例。 40 | 41 | ### 贡献者 42 | 43 | 感谢以下人员做出的贡献： 44 | 45 | qinzheng, xuyongfei, zhangyinxia, zhoufeng. 46 | 47 | 欢迎以任何形式对项目提供贡献！ 48 | 49 | ## MindSpore Serving 1.7.0 Release Notes 50 | 51 | ### 主要特性和增强 52 | 53 | - [DEMO] Ascend 310P可以作为MindSpore Serving的硬件后端，详情可参考[MindSpore Serving后端](https://www.mindspore.cn/serving/docs/zh-CN/master/serving_install.html#installation)。 54 | - [DEMO] MindSpore Lite作为MindSpore Serving推理后端时，支持MindIR模型格式，详情可参考[MindSpore Serving后端](https://www.mindspore.cn/serving/docs/zh-CN/master/serving_install.html#installation)。 55 | 56 | #### 不建议使用 57 | 58 | ##### Python API 59 | 60 | - `AclOptions`和 `GpuOptions`从1.7.0版本开始被移除，使用 `AscendDeviceInfo`和 `GPUDeviceInfo`替代。 61 | - `register.declare_sevable`和 `register.call_servable`从1.7.0版本开始被移除，使用 `register.declare_model`和 `register.add_stage`替代。 62 | - `register.call_preprocess`，`register.call_preprocess_pipeline`，`register.call_postprocess`和 `register.call_postprocess_pipeline`从1.7.0版本开始被移除，使用 `register.add_stage`替代。 63 | 64 | ### 贡献者 65 | 66 | 感谢以下人员做出的贡献: 67 | 68 | qinzheng, xuyongfei, zhangyinxia, zhoufeng. 69 | 70 | 欢迎以任何形式对项目提供贡献！ 71 | -------------------------------------------------------------------------------- /tests/ut/stub/include/api/ops/ops.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Huawei Technologies Co., Ltd 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #ifndef MINDSPORE_INCLUDE_API_OPS_OPS_H 17 | #define MINDSPORE_INCLUDE_API_OPS_OPS_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "include/api/status.h" 24 | #include "include/api/types.h" 25 | #include "include/api/cell.h" 26 | 27 | namespace mindspore { 28 | struct MS_API Conv2D : public OpCell { 29 | Conv2D() : OpCell("Conv2D") {} 30 | ~Conv2D() override = default; 31 | std::vector Construct(const std::vector &inputs) override; 32 | Conv2D(int out_channel, const std::vector &kernel_size, int mode = 1, const std::string &pad_mode = "valid", 33 | const std::vector &pad = {0, 0, 0, 0}, const std::vector &stride = {1, 1, 1, 1}, 34 | const std::vector &dilation = {1, 1, 1, 1}, int group = 1); 35 | 36 | Output operator()(const Input &, const Input &) const; 37 | 38 | int out_channel; 39 | std::vector kernel_size; 40 | int mode = 1; 41 | std::string pad_mode = "valid"; 42 | std::vector pad = {0, 0, 0, 0}; 43 | std::vector stride = {1, 1, 1, 1}; 44 | std::vector dilation = {1, 1, 1, 1}; 45 | int group = 1; 46 | }; 47 | } // namespace mindspore 48 | #endif // MINDSPORE_INCLUDE_API_OPS_OPS_H 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # MindSpore Serving 2 | build/ 3 | mindspore_serving/lib 4 | output 5 | *.ir 6 | .coverage* 7 | htmlcov/ 8 | cov_output/ 9 | 10 | # Cmake files 11 | CMakeFiles/ 12 | cmake_install.cmake 13 | CMakeCache.txt 14 | Makefile 15 | cmake-build-debug 16 | 17 | # Prerequisites 18 | *.d 19 | 20 | # Compiled Object files 21 | *.slo 22 | *.lo 23 | *.o 24 | *.obj 25 | 26 | # Precompiled Headers 27 | *.gch 28 | *.pch 29 | 30 | # Compiled Dynamic libraries 31 | *.so 32 | *.dylib 33 | *.dll 34 | *.so.* 35 | 36 | # Fortran module files 37 | *.mod 38 | *.smod 39 | 40 | # Compiled Static libraries 41 | *.lai 42 | *.la 43 | *.a 44 | *.lib 45 | 46 | # Executables 47 | *.exe 48 | *.out 49 | *.app 50 | 51 | # Protocol buffers 52 | *_pb2.py 53 | *.pb.h 54 | *.pb.cc 55 | *.pb 56 | *_grpc.py 57 | 58 | # Editor 59 | .vscode 60 | .idea/ 61 | 62 | # Cquery 63 | .cquery_cached_index/ 64 | compile_commands.json 65 | 66 | # Ctags and cscope 67 | tags 68 | TAGS 69 | CTAGS 70 | GTAGS 71 | GRTAGS 72 | GSYMS 73 | GPATH 74 | cscope.* 75 | 76 | # Python files 77 | *__pycache__* 78 | .pytest_cache 79 | 80 | # Mac files 81 | *.DS_Store 82 | 83 | # Test results 84 | test_temp_summary_event_file/ 85 | *.dot 86 | *.dat 87 | *.svg 88 | *.perf 89 | *.info 90 | *.ckpt 91 | *.shp 92 | *.pkl 93 | .clangd 94 | mindspore_serving/version.py 95 | mindspore_serving/default_config.py 96 | mindspore_serving/.commit_id 97 | tests/ut/python/tests/ca.crt 98 | tests/ut/python/tests/ca.key 99 | tests/ut/python/tests/ca.srl 100 | tests/ut/python/tests/server.crt 101 | tests/ut/python/tests/server.csr 102 | tests/ut/python/tests/server.key 103 | tests/ut/python/tests/client.crt 104 | tests/ut/python/tests/client.csr 105 | tests/ut/python/tests/client.key 106 | tests/ut/python/tests/serving_logs/ 107 | tests/ut/python/tests/unix_socket_files/ 108 | tests/ut/python/tests/serving_python_ut_servables/ 109 | -------------------------------------------------------------------------------- /third_party/patch/openssl/CVE-2022-0778.patch: -------------------------------------------------------------------------------- 1 | diff --git a/crypto/bn/bn_sqrt.c b/crypto/bn/bn_sqrt.c 2 | index 1723d5ded5..53b0f55985 100644 3 | --- a/crypto/bn/bn_sqrt.c 4 | +++ b/crypto/bn/bn_sqrt.c 5 | @@ -14,7 +14,8 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) 6 | /* 7 | * Returns 'ret' such that ret^2 == a (mod p), using the Tonelli/Shanks 8 | * algorithm (cf. Henri Cohen, "A Course in Algebraic Computational Number 9 | - * Theory", algorithm 1.5.1). 'p' must be prime! 10 | + * Theory", algorithm 1.5.1). 'p' must be prime, otherwise an error or 11 | + * an incorrect "result" will be returned. 12 | */ 13 | { 14 | BIGNUM *ret = in; 15 | @@ -301,18 +302,23 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) 16 | goto vrfy; 17 | } 18 | 19 | - /* find smallest i such that b^(2^i) = 1 */ 20 | - i = 1; 21 | - if (!BN_mod_sqr(t, b, p, ctx)) 22 | - goto end; 23 | - while (!BN_is_one(t)) { 24 | - i++; 25 | - if (i == e) { 26 | - BNerr(BN_F_BN_MOD_SQRT, BN_R_NOT_A_SQUARE); 27 | - goto end; 28 | + /* Find the smallest i, 0 < i < e, such that b^(2^i) = 1. */ 29 | + for (i = 1; i < e; i++) { 30 | + if (i == 1) { 31 | + if (!BN_mod_sqr(t, b, p, ctx)) 32 | + goto end; 33 | + 34 | + } else { 35 | + if (!BN_mod_mul(t, t, t, p, ctx)) 36 | + goto end; 37 | } 38 | - if (!BN_mod_mul(t, t, t, p, ctx)) 39 | - goto end; 40 | + if (BN_is_one(t)) 41 | + break; 42 | + } 43 | + /* If not found, a is not a square or p is not prime. */ 44 | + if (i >= e) { 45 | + BNerr(BN_F_BN_MOD_SQRT, BN_R_NOT_A_SQUARE); 46 | + goto end; 47 | } 48 | 49 | /* t := y^2^(e - i - 1) */ -------------------------------------------------------------------------------- /cmake/external_libs/absl.cmake: -------------------------------------------------------------------------------- 1 | if(ENABLE_GITEE_EULER) 2 | set(GIT_REPOSITORY "https://gitee.com/src-openeuler/abseil-cpp.git") 3 | set(GIT_TAG "openEuler-22.03-LTS") 4 | set(SHA256 "365b1ecbbcd81b4c58101808a8a28a3cf9ad7f9d05c08080a35c0d4283a44afa") 5 | set(ABSL_SRC "${CMAKE_SOURCE_DIR}/build/mindspore_serving/_deps/absl-src") 6 | __download_pkg_with_git(absl ${GIT_REPOSITORY} ${GIT_TAG} ${SHA256}) 7 | execute_process(COMMAND tar -xf ${ABSL_SRC}/abseil-cpp-20210324.2.tar.gz --strip-components 1 -C ${ABSL_SRC}) 8 | else() 9 | if(ENABLE_GITEE) 10 | set(REQ_URL "https://gitee.com/mirrors/abseil-cpp/repository/archive/20210324.2.tar.gz") 11 | set(SHA256 "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f") 12 | else() 13 | set(REQ_URL "https://github.com/abseil/abseil-cpp/archive/20210324.2.tar.gz") 14 | set(SHA256 "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f") 15 | endif() 16 | endif() 17 | 18 | if(NOT ENABLE_GLIBCXX) 19 | set(absl_CXXFLAGS "${absl_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") 20 | endif() 21 | 22 | mindspore_add_pkg(absl 23 | VER 20210324.2 24 | LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access 25 | URL ${REQ_URL} 26 | SHA256 ${SHA256} 27 | CMAKE_OPTION 28 | -DCMAKE_BUILD_TYPE:STRING=Release 29 | -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE 30 | -DCMAKE_CXX_STANDARD=11 31 | ) 32 | 33 | include_directories(${absl_INC}) 34 | 35 | add_library(mindspore_serving::absl_strings ALIAS absl::absl_strings) 36 | add_library(mindspore_serving::absl_throw_delegate ALIAS absl::absl_throw_delegate) 37 | add_library(mindspore_serving::absl_raw_logging_internal ALIAS absl::absl_raw_logging_internal) 38 | add_library(mindspore_serving::absl_int128 ALIAS absl::absl_int128) 39 | add_library(mindspore_serving::absl_bad_optional_access ALIAS absl::absl_bad_optional_access) 40 | --------------------------------------------------------------------------------