├── api ├── rest_api │ ├── client │ │ └── scripts │ │ │ └── curl_test.sh │ └── server │ │ ├── rest_server_echo_api.go │ │ └── rest_server_http_api_deprecated.go ├── grpc_api │ ├── client │ │ ├── grpc_api_client_test.go │ │ └── grpc_api_client.go │ └── server │ │ └── grpc_server_api.go ├── dubbo_api │ ├── client │ │ ├── dubbo_api_client_test.go │ │ └── dubbo_api_client.go │ └── server │ │ └── dubbo_server_api.go └── api_factory.go ├── scripts ├── deployments │ ├── faiss │ │ ├── faiss_start.sh │ │ └── faiss_docker_run.sh │ ├── infer │ │ ├── infer_docker_run.sh │ │ └── services_start.sh │ ├── elkb │ │ ├── kibana.sh │ │ ├── filebeat.sh │ │ ├── es.sh │ │ └── logstash.sh │ └── tfserving │ │ └── tfserving_start.sh └── build │ ├── docker_build │ ├── faiss │ │ └── faiss_build.sh │ ├── tfserving │ │ └── tfserving_docker_build.sh │ └── infer │ │ └── infer_docker_build.sh │ ├── go_build │ └── go_build.sh │ └── protoc │ └── protoc.sh ├── test ├── model_live.sh └── http_api_test.sh ├── pkg ├── login │ ├── login_check.go │ └── check.go ├── infer_models │ ├── recall │ │ ├── cf │ │ │ ├── swing.go │ │ │ ├── item_cf.go │ │ │ └── user_cf.go │ │ ├── cold_start │ │ │ ├── clustering_recall.go │ │ │ ├── labels_keywords_recall.go │ │ │ ├── look_alike.go │ │ │ └── u2i_dssm_cold_start.go │ │ ├── i2i │ │ │ └── i2i_dssm.go │ │ ├── simple_recall │ │ │ ├── lastN_recall.go │ │ │ ├── search_recall.go │ │ │ ├── geop_recall.go │ │ │ └── author_recall.go │ │ └── multi_recall.go │ ├── re_rank │ │ ├── rerank_rules.go │ │ ├── dpp.go │ │ └── mmr.go │ ├── ranking │ │ ├── light_ranker │ │ │ ├── fm.go │ │ │ └── lr.go │ │ └── heavy_ranker │ │ │ └── sim.go │ ├── pre_ranking │ │ └── dssm_rank.go │ ├── README.MD │ ├── model_strategy_context.go │ └── model_strategy_factory.go ├── register │ └── user_register.go ├── .DS_Store ├── infer_services │ ├── service_validate.go │ ├── io │ │ ├── item_info.go │ │ ├── response.go │ │ └── request.go │ ├── service_factory.go │ ├── dubbo_service │ │ └── dubbo_service.go │ ├── grpc_service │ │ └── grpc_service.go │ └── rest_service │ │ └── echo_service.go ├── nacos │ ├── nacos_config_parser_test.go │ ├── nacos_factory.go │ └── nacos_config_parser.go ├── infer_samples │ ├── infer_sample_observer.go │ ├── README.MD │ ├── generate_samples.go │ ├── bloom.go │ ├── infer_sample_subject.go │ ├── bloom_filter_warmup.go │ └── ready_samples.go ├── infer_features │ ├── item_profile_features.go │ ├── user_profile_features.go │ ├── last_n.go │ ├── hash.go │ ├── item_embedding.go │ ├── user_embedding.go │ ├── tfrecords_example_features.go │ ├── README.MD │ ├── infer_feature.go │ └── context_features.go ├── config_loader │ ├── README.md │ ├── redis_config │ │ ├── redis_config_loader_test.go │ │ └── redis_config_loader.go │ ├── faiss_config │ │ ├── faiss_config_loader_test.go │ │ └── faiss_config_loader.go │ ├── service_config_builder.go │ ├── service_config_director.go │ ├── model_config │ │ ├── model_config_loader_test.go │ │ └── model_config_loader.go │ ├── config_factory.go │ ├── service_config.go │ └── pipeline_config │ │ └── pipeline_config_loader.go ├── infer_pipeline │ ├── README.MD │ └── interface.go └── ann │ └── faiss │ └── faiss_index_search.go ├── Dockerfile ├── README.md ├── infer │ └── infer_docker_file ├── faiss │ └── faiss_docker_file └── tfserving │ └── tfserving_docker_file ├── .gitignore ├── cmd └── infer │ └── main │ ├── README.md │ └── infer.go ├── tools └── protoc.exe ├── internal ├── utils │ ├── .DS_Store │ ├── float_util.go │ ├── struct_util.go │ └── request_id_util.go ├── flags │ ├── README.MD │ ├── flag_jwt.go │ ├── flag_redis.go │ ├── flag_dubbo.go │ ├── flag_bloom.go │ ├── flag_kafka.go │ ├── flag_tensorflow.go │ ├── flag_viper.go │ ├── flag_logs.go │ ├── flag_start_info.go │ ├── flag_skywalking.go │ ├── flag_nacos.go │ ├── flag_cache.go │ └── flag_hystrix.go ├── tensorflow_gogofaster │ └── core │ │ └── framework │ │ └── proto │ │ ├── dataset_metadata.proto │ │ ├── reader_base.proto │ │ ├── tensor_description.proto │ │ ├── allocation_description.proto │ │ ├── versions.proto │ │ ├── tensor_slice.proto │ │ ├── resource_handle.proto │ │ ├── kernel_def.proto │ │ ├── dataset.proto │ │ ├── tensor_shape.proto │ │ ├── device_attributes.proto │ │ ├── optimized_function_graph.proto │ │ ├── graph_debug_info.proto │ │ ├── graph_transfer_info.proto │ │ ├── graph.proto │ │ ├── log_memory.proto │ │ ├── attr_value.proto │ │ ├── types.proto │ │ ├── step_stats.proto │ │ ├── cost_graph.proto │ │ ├── variable.proto │ │ ├── tensor.proto │ │ └── model.proto ├── skywalking.go ├── logs │ ├── color.go │ ├── README.md │ ├── accesslog.go │ ├── console.go │ ├── conn.go │ └── multifile.go ├── jwt │ ├── jwt_echo.go │ └── jwt_http.go ├── tfserving_gogofaster │ └── proto │ │ ├── model.proto │ │ ├── prediction_service.proto │ │ └── predict.proto ├── hystrix.go ├── kafka.go └── db │ └── redis │ └── redis.go ├── .vscode └── settings.json ├── Protofile ├── grpc_api │ └── recommender.proto └── faiss │ └── faissIndex.proto └── configs ├── dubbo ├── dubbo_server-1_config.yml └── dubbo_server-2_config.yml └── nacos └── service_start_config.json /api/rest_api/client/scripts/curl_test.sh: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/deployments/faiss/faiss_start.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | ./faiss -------------------------------------------------------------------------------- /test/model_live.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | curl http://10.124.10.7:8201/v1/models/models -------------------------------------------------------------------------------- /pkg/login/login_check.go: -------------------------------------------------------------------------------- 1 | package check 2 | 3 | //INFO: web system login check func. 4 | -------------------------------------------------------------------------------- /Dockerfile/README.md: -------------------------------------------------------------------------------- 1 | #deploy faiss service 、tfserving service and infer service with docker. -------------------------------------------------------------------------------- /pkg/infer_models/recall/cf/swing.go: -------------------------------------------------------------------------------- 1 | package cf 2 | 3 | //TODO: ADD 4 | 5 | //INFO: Swing recall. -------------------------------------------------------------------------------- /pkg/register/user_register.go: -------------------------------------------------------------------------------- 1 | package register 2 | 3 | //INFO: user register the web system. 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #ignore 2 | search_recall 3 | test 4 | logs 5 | cache 6 | *.txt 7 | *.log 8 | web 9 | tools -------------------------------------------------------------------------------- /pkg/login/check.go: -------------------------------------------------------------------------------- 1 | package check 2 | 3 | type CheckInterface interface { 4 | Check() bool 5 | } 6 | -------------------------------------------------------------------------------- /cmd/infer/main/README.md: -------------------------------------------------------------------------------- 1 | ## New service 2 | When we add new type service, we need implent ServerStartInterface. -------------------------------------------------------------------------------- /pkg/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solidglue/Recommender_System_Inference_Services/HEAD/pkg/.DS_Store -------------------------------------------------------------------------------- /pkg/infer_services/service_validate.go: -------------------------------------------------------------------------------- 1 | package infer_services 2 | 3 | //INFO: web system service validate func. 4 | -------------------------------------------------------------------------------- /tools/protoc.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solidglue/Recommender_System_Inference_Services/HEAD/tools/protoc.exe -------------------------------------------------------------------------------- /pkg/infer_models/recall/cf/item_cf.go: -------------------------------------------------------------------------------- 1 | package cf 2 | 3 | //TODO: ADD 4 | 5 | //INFO: item Collaborative Filtering recall. -------------------------------------------------------------------------------- /pkg/infer_models/recall/cf/user_cf.go: -------------------------------------------------------------------------------- 1 | package cf 2 | 3 | //TODO: ADD 4 | 5 | //INFO: user Collaborative Filtering recall. -------------------------------------------------------------------------------- /pkg/infer_models/re_rank/rerank_rules.go: -------------------------------------------------------------------------------- 1 | package re_rank 2 | 3 | //TODO: ADD 4 | 5 | //INFO: Some business rules in rerank. -------------------------------------------------------------------------------- /internal/utils/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/solidglue/Recommender_System_Inference_Services/HEAD/internal/utils/.DS_Store -------------------------------------------------------------------------------- /internal/flags/README.MD: -------------------------------------------------------------------------------- 1 | # Stable or universal parameter configurations are here,Flexible or business related parameters placed in nacos -------------------------------------------------------------------------------- /pkg/infer_models/recall/cold_start/clustering_recall.go: -------------------------------------------------------------------------------- 1 | package cold_start 2 | 3 | //TODO:ADD 4 | 5 | //INFO: clustering by items embedding. -------------------------------------------------------------------------------- /pkg/nacos/nacos_config_parser_test.go: -------------------------------------------------------------------------------- 1 | package nacos 2 | 3 | import "testing" 4 | 5 | func TestNacosConfigParser(t *testing.T) { 6 | 7 | } 8 | -------------------------------------------------------------------------------- /pkg/infer_models/recall/i2i/i2i_dssm.go: -------------------------------------------------------------------------------- 1 | package i2i 2 | 3 | //TODO: ADD 4 | 5 | //INFO: item to item DSSM model, used to calculate item similarity. 6 | -------------------------------------------------------------------------------- /scripts/build/docker_build/faiss/faiss_build.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | docker build -f /data/loki/the-infer/Dockerfile/faiss/faiss_docker_file -t faiss:v1.0.0 . -------------------------------------------------------------------------------- /pkg/infer_samples/infer_sample_observer.go: -------------------------------------------------------------------------------- 1 | package infer_samples 2 | 3 | // Observer 4 | type Observer interface { 5 | //notify 6 | notify(sub Subject) 7 | } 8 | -------------------------------------------------------------------------------- /scripts/deployments/faiss/faiss_docker_run.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | docker run -d --name infer -p 21000:5000 --restart=always faiss:v1.0.0 4 | curl localhost:21000 5 | -------------------------------------------------------------------------------- /pkg/infer_models/recall/simple_recall/lastN_recall.go: -------------------------------------------------------------------------------- 1 | package rules_recall 2 | 3 | //TODO: ADD 4 | 5 | //INFO:Recall similar items based on the last n played video items. -------------------------------------------------------------------------------- /pkg/infer_models/recall/simple_recall/search_recall.go: -------------------------------------------------------------------------------- 1 | package rules_recall 2 | 3 | //TODO: ADD 4 | 5 | //INFO:Recall similar items from user searched video items recently. -------------------------------------------------------------------------------- /pkg/infer_models/recall/cold_start/labels_keywords_recall.go: -------------------------------------------------------------------------------- 1 | package cold_start 2 | 3 | //TODO: ADD 4 | 5 | //INFO: using item labels and keywords to recall for new items. 6 | -------------------------------------------------------------------------------- /pkg/infer_models/recall/simple_recall/geop_recall.go: -------------------------------------------------------------------------------- 1 | package rules_recall 2 | 3 | //TODO: ADD 4 | 5 | //INFO:Recall high-quality items from the user's location when using the app. -------------------------------------------------------------------------------- /pkg/infer_models/recall/simple_recall/author_recall.go: -------------------------------------------------------------------------------- 1 | package rules_recall 2 | 3 | //TODO: ADD 4 | 5 | //INFO:User focused authors recall, inclued other high-quality items and new items. -------------------------------------------------------------------------------- /scripts/build/docker_build/tfserving/tfserving_docker_build.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | docker build -f /data/loki/the-infer/Dockerfile/tfserving/tfserving_docker_file -t tfserving:v1.15.0 . -------------------------------------------------------------------------------- /scripts/build/go_build/go_build.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | #build go file 4 | go build -o /data/loki/the-infer/cmd/infer/main/recsys-go-infer /data/loki/the-infer/cmd/infer/main/info.go 5 | -------------------------------------------------------------------------------- /pkg/infer_features/item_profile_features.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 4 | -------------------------------------------------------------------------------- /pkg/infer_features/user_profile_features.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 4 | -------------------------------------------------------------------------------- /pkg/infer_models/ranking/light_ranker/fm.go: -------------------------------------------------------------------------------- 1 | package light_ranker 2 | 3 | //TODO: ADD 4 | 5 | //INFO: Call this when the heavy_ranker services are downgraded. 6 | 7 | type FM struct { 8 | } 9 | -------------------------------------------------------------------------------- /pkg/infer_models/ranking/light_ranker/lr.go: -------------------------------------------------------------------------------- 1 | package light_ranker 2 | 3 | //TODO: ADD 4 | 5 | //INFO: Call this when the heavy_ranker services are downgraded. 6 | 7 | type LR struct { 8 | } 9 | -------------------------------------------------------------------------------- /pkg/infer_models/re_rank/dpp.go: -------------------------------------------------------------------------------- 1 | package re_rank 2 | 3 | //TODO: ADD 4 | 5 | //INFO: Using the DPP(Determinantal Point Process) algorithm to ensure diversity. 6 | 7 | type DPP struct { 8 | } 9 | -------------------------------------------------------------------------------- /pkg/infer_models/re_rank/mmr.go: -------------------------------------------------------------------------------- 1 | package re_rank 2 | 3 | //TODO: ADD 4 | 5 | //INFO: Using the MMR(Maximal Marginal Relevance) algorithm to ensure diversity. 6 | 7 | type MMR struct { 8 | } 9 | -------------------------------------------------------------------------------- /scripts/build/docker_build/infer/infer_docker_build.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | #build docker mir 4 | docker build -f /data/loki/the-infer/Dockerfile/infer/infer_docker_file -t recsys-go-infer:v1.0.0 . -------------------------------------------------------------------------------- /pkg/infer_features/last_n.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 4 | 5 | //last n seq 6 | -------------------------------------------------------------------------------- /pkg/infer_models/recall/cold_start/look_alike.go: -------------------------------------------------------------------------------- 1 | package cold_start 2 | 3 | //TODO: ADD 4 | 5 | //INFO: look-alike recall, using the average embeddings of the seed users as new item embedding to recall. -------------------------------------------------------------------------------- /pkg/infer_models/recall/multi_recall.go: -------------------------------------------------------------------------------- 1 | package recall 2 | 3 | //TODO: ADD 4 | //INFO: Recall multiple models at once, multi-models only one network request. 5 | 6 | type MultiRecall struct { 7 | } 8 | -------------------------------------------------------------------------------- /test/http_api_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | curl http://10.124.20.5:8650/infer -X POST -d 'data={"modelId":"deepfm-1|infer","userId":"112223333444555","itemIdList":["111","222","333","444"]}' 4 | 5 | -------------------------------------------------------------------------------- /scripts/deployments/infer/infer_docker_run.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | #6020 rest port ,6021 grpc port 4 | docker run -d --name infer -p 22000:6020 -p 22001:6021 --restart=always recsys-go-infer:v1.0.0 5 | curl localhost:22000 -------------------------------------------------------------------------------- /pkg/infer_features/hash.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 4 | 5 | //INFO: hash sparse features before embedding. -------------------------------------------------------------------------------- /pkg/infer_features/item_embedding.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 4 | 5 | //INFO: get item embedding from dssm model, used to i2i. -------------------------------------------------------------------------------- /pkg/infer_models/pre_ranking/dssm_rank.go: -------------------------------------------------------------------------------- 1 | package pre_ranking 2 | 3 | //TODO: add dssm rank, 4 | //INFO: thousands items to pre_ranking after recall . this dsmm model is different with dssm recall model. 5 | 6 | type DssmRank struct { 7 | } 8 | -------------------------------------------------------------------------------- /pkg/infer_features/user_embedding.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 4 | 5 | //INFO:get user embedding from dssm model, , used to look-alike recall. -------------------------------------------------------------------------------- /pkg/infer_models/recall/cold_start/u2i_dssm_cold_start.go: -------------------------------------------------------------------------------- 1 | package cold_start 2 | 3 | //TODO: ADD 4 | 5 | //INFO: all the items share a same ID embedding when training model, or using average embedding of topK similar items as new item embedding when ANN search. -------------------------------------------------------------------------------- /pkg/infer_models/ranking/heavy_ranker/sim.go: -------------------------------------------------------------------------------- 1 | package heavy_ranker 2 | 3 | //TODO: ADD 4 | 5 | //INFO: long sequence model, such as thousands videos a user played in the past months. 6 | //https://arxiv.org/abs/2006.05639 7 | 8 | type SIM struct { 9 | } 10 | -------------------------------------------------------------------------------- /pkg/infer_models/README.MD: -------------------------------------------------------------------------------- 1 | ## Demonstration 2 | A real industrial recommender system includes many recall and rank algorithms, such as LR、FM、DSSM、DIN and SIM etc. 3 | But only u2i DSSM recall algorithms and DIN rank algorithm are implemented for demonstration in this project. -------------------------------------------------------------------------------- /internal/utils/float_util.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | func FloatRound(f float32, n int) float64 { 9 | format := "%." + strconv.Itoa(n) + "f" 10 | res, _ := strconv.ParseFloat(fmt.Sprintf(format, f), 64) 11 | return res 12 | } 13 | -------------------------------------------------------------------------------- /pkg/config_loader/README.md: -------------------------------------------------------------------------------- 1 | ## config files 2 | Most configurations required for the inference service are on Nacos. 3 | 4 | ## ConfigLoadInterface 5 | all the nacos config loader should implement ConfigLoadInterface,such as faiss config loader ,model config loader and redis config loader. -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "editor.fontSize": 14, 4 | "javascript.validate.enable": true, 5 | "editor.showUnused": true, 6 | "workbench.colorCustomizations": { 7 | "editorUnnecessaryCode.border": "#ff0000" 8 | }, 9 | "files.autoSave": "afterDelay" 10 | } -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/dataset_metadata.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.data; 4 | 5 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/dataset_metadata_go_proto"; 6 | 7 | // next: 2 8 | message Metadata { 9 | bytes name = 1; 10 | } 11 | -------------------------------------------------------------------------------- /pkg/infer_features/tfrecords_example_features.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | type SeqExampleBuff struct { 4 | Key *string 5 | Buff *[]byte 6 | } 7 | 8 | type ExampleFeatures struct { 9 | UserExampleFeatures *SeqExampleBuff 10 | UserContextExampleFeatures *SeqExampleBuff 11 | ItemSeqExampleFeatures *[]SeqExampleBuff 12 | } 13 | -------------------------------------------------------------------------------- /pkg/infer_samples/README.MD: -------------------------------------------------------------------------------- 1 | ## The current solution - solution-A 2 | All tfrecored format samples have been preprocessed and stored in Redis. 3 | 4 | ## TODO: add new solution-B 5 | Query 、process and build tfrecord samples during inference. 6 | 7 | ## advantages and disadvantages 8 | Solution-A has better inference performance, solution-B more flexible. -------------------------------------------------------------------------------- /scripts/deployments/elkb/kibana.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | 4 | vi config/kibana.yml 5 | server.port: 5601 #kibana port 6 | server.host: "127.0.0.1" #kibana ip 7 | server.name: "infer-kibana" #kibana name 8 | elasticsearch.url: http://127.0.0.1:9200 #es addr 9 | 10 | docker run --name kibana -e ELASTICSEARCH_URL=http://127.0.0.1:9200 -p 5601:5601 -d kibana:5.6.9 11 | -------------------------------------------------------------------------------- /scripts/deployments/tfserving/tfserving_start.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | docker run -p 20888:8500 -p 20881:8501 \ 4 | --mount type=bind,source=/data/tensorflow/models/deepfm,target=/models/deepfm \ 5 | --mount type=bind,source=/data/tensorflow/config/deepfm/model.conf,target=/models/model.conf \ 6 | -t tfserving:v1.15.0 \ 7 | --model_config_file=/models/model.conf 8 | --restart=always 9 | -------------------------------------------------------------------------------- /scripts/build/protoc/protoc.sh: -------------------------------------------------------------------------------- 1 | protoc -I proto/ proto/recommender.proto --gogofaster_out=Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/duration.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,Mgoogle/protobuf/wrappers.proto=github.com/gogo/protobuf/types,plugins=grpc:. -------------------------------------------------------------------------------- /pkg/infer_features/README.MD: -------------------------------------------------------------------------------- 1 | ## The current solution: solution-A 2 | All features have been preprocessed and tfrecored format samples have been generated and stored in Redis. 3 | 4 | ## TODO: add new solution-B 5 | Query and process features during inference，such as features hash、embedding. 6 | 7 | ## advantages and disadvantages 8 | Solution-A has better inference performance, but solution-B more flexible. 9 | -------------------------------------------------------------------------------- /pkg/infer_services/io/item_info.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | type ItemInfo struct { 4 | itemId string 5 | score float32 6 | } 7 | 8 | // itemId 9 | func (i *ItemInfo) SetItemId(itemId string) { 10 | i.itemId = itemId 11 | } 12 | 13 | func (i *ItemInfo) GetItemId() string { 14 | return i.itemId 15 | } 16 | 17 | // score 18 | func (i *ItemInfo) SetScore(score float32) { 19 | i.score = score 20 | } 21 | 22 | func (i *ItemInfo) GetScore() float32 { 23 | return i.score 24 | } 25 | -------------------------------------------------------------------------------- /internal/flags/flag_jwt.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | type FlagJwt struct { 4 | jwtKey *string 5 | } 6 | 7 | var flagJwtInstance *FlagJwt 8 | 9 | // singleton instance 10 | func init() { 11 | flagJwtInstance = new(FlagJwt) 12 | } 13 | 14 | func getFlagJwtInstance() *FlagJwt { 15 | return flagJwtInstance 16 | } 17 | 18 | // jwtKey 19 | func (s *FlagJwt) setJwtKey(jwtKey *string) { 20 | s.jwtKey = jwtKey 21 | } 22 | 23 | func (s *FlagJwt) GetJwtKey() *string { 24 | return s.jwtKey 25 | } 26 | -------------------------------------------------------------------------------- /pkg/infer_features/infer_feature.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | type InferFeature struct { 4 | userId string 5 | userProfiles map[string]string 6 | lastNLong map[string]string //such as last n item ids ,last n item labels. thousands length. 7 | lastNShort map[string]string //such as last n item ids ,last n item labels. hundreds length. 8 | contextFeatures map[string]string //such as geop, time, os 9 | itemId string 10 | itemProfiles map[string]string 11 | } 12 | -------------------------------------------------------------------------------- /internal/flags/flag_redis.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagRedisInstance *FlagRedis 4 | 5 | type FlagRedis struct { 6 | //redis 7 | redisPassword *string 8 | } 9 | 10 | // singleton instance 11 | func init() { 12 | flagRedisInstance = new(FlagRedis) 13 | } 14 | 15 | func getFlagRedisInstance() *FlagRedis { 16 | return flagRedisInstance 17 | } 18 | 19 | // redis_password 20 | func (s *FlagRedis) setRedisPassword(redisPassword *string) { 21 | s.redisPassword = redisPassword 22 | } 23 | 24 | func (s *FlagRedis) GetRedisPassword() *string { 25 | return s.redisPassword 26 | } 27 | -------------------------------------------------------------------------------- /pkg/infer_features/context_features.go: -------------------------------------------------------------------------------- 1 | package infer_features 2 | 3 | import ( 4 | "infer-microservices/internal" 5 | ) 6 | 7 | //TODO: add new solution-B：Query and process features during inference, and then generate samples, More flexible. 8 | 9 | //get runtime feature from kafka and save to redis 10 | func init() { 11 | go internal.KafkaConsumer(runtimeFeature) 12 | } 13 | 14 | func runtimeFeature(msgKey string, msgValue string) { 15 | 16 | //parse user action json data from kafka ,extra runtime sequence feature. 17 | 18 | //feature engine and save features to redis. 19 | 20 | } 21 | -------------------------------------------------------------------------------- /internal/flags/flag_dubbo.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagDubboInstance *flagDubbo 4 | 5 | type flagDubbo struct { 6 | dubboServiceFile *string 7 | } 8 | 9 | // INFO: singleton instance 10 | func init() { 11 | flagDubboInstance = new(flagDubbo) 12 | } 13 | 14 | func getFlagDubboInstance() *flagDubbo { 15 | return flagDubboInstance 16 | } 17 | 18 | // dubbo_serverconf 19 | func (s *flagDubbo) setDubboServiceFile(dubboServiceFile *string) { 20 | s.dubboServiceFile = dubboServiceFile 21 | } 22 | 23 | func (s *flagDubbo) GetDubboServiceFile() *string { 24 | return s.dubboServiceFile 25 | } 26 | -------------------------------------------------------------------------------- /internal/utils/struct_util.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | func ConvertStructToJson(param interface{}) string { 8 | dataType, _ := json.Marshal(param) 9 | dataString := string(dataType) 10 | return dataString 11 | } 12 | 13 | func ConvertJsonToStruct(jsonStr string) map[string]interface{} { 14 | var tempMap map[string]interface{} 15 | var err error 16 | if jsonStr != "" { 17 | err = json.Unmarshal([]byte(jsonStr), &tempMap) 18 | if err != nil { 19 | panic(err) 20 | } 21 | } else { 22 | tempMap = make(map[string]interface{}, 0) 23 | } 24 | 25 | return tempMap 26 | } 27 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/reader_base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "ReaderBaseProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/reader_base_go_proto"; 10 | 11 | // For serializing and restoring the state of ReaderBase, see 12 | // reader_base.h for details. 13 | message ReaderBaseState { 14 | int64 work_started = 1; 15 | int64 work_finished = 2; 16 | int64 num_records_produced = 3; 17 | bytes current_work = 4; 18 | } 19 | -------------------------------------------------------------------------------- /pkg/infer_pipeline/README.MD: -------------------------------------------------------------------------------- 1 | ## Infer Pipeline 2 | Infer pipeline, using pipeline to encapsulate inference steps ,such as build samples、recall 、pre-ranking、ranking and rerank. 3 | 4 | Usage: 5 | 6 | Pipeline(steps=[ 7 | ('recall_sample',recallSample),('dssm_recall',dssmRecall), // recall 8 | ('pre_ranking_sample','preRankingSample'),('pre_ranking','dssmRank'), // pre-ranking 9 | ('ranking_sample','rankingSample'),('ranking','deepfm'), // ranking 10 | ('re_rank','reRank') // rerank 11 | ] 12 | ) -------------------------------------------------------------------------------- /pkg/infer_samples/generate_samples.go: -------------------------------------------------------------------------------- 1 | package infer_samples 2 | 3 | import ( 4 | config_loader "infer-microservices/pkg/config_loader" 5 | feature "infer-microservices/pkg/infer_features" 6 | 7 | "github.com/allegro/bigcache" 8 | bloomv3 "github.com/bits-and-blooms/bloom/v3" 9 | ) 10 | 11 | //TODO: ADD 12 | //INFO: solution-B：Query 、process and build tfrecord samples during inference. 13 | 14 | type InferSampleGenerate struct { 15 | modelName string 16 | serviceConfig *config_loader.ServiceConfig 17 | bigCacheSample *bigcache.BigCache 18 | userBloomFilter *bloomv3.BloomFilter 19 | itemBloomFilter *bloomv3.BloomFilter 20 | inferFeature feature.InferFeature // all the features using to generate tfrecord samples. 21 | } 22 | -------------------------------------------------------------------------------- /internal/skywalking.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "infer-microservices/internal/logs" 5 | 6 | "github.com/SkyAPM/go2sky" 7 | "github.com/SkyAPM/go2sky/reporter" 8 | ) 9 | 10 | // go2sky 11 | var report go2sky.Reporter 12 | var tracer *go2sky.Tracer 13 | var errReport error 14 | var errTracer error 15 | 16 | func GetTracer() *go2sky.Tracer { 17 | return tracer 18 | } 19 | 20 | func SkywalkingTracer(go2skyAddr string, serverName string) { 21 | report, errReport = reporter.NewGRPCReporter(go2skyAddr) 22 | tracer, errTracer = go2sky.NewTracer(serverName, go2sky.WithReporter(report)) 23 | if errReport != nil { 24 | logs.Error("crate grpc reporter error: %v \n", errReport) 25 | } 26 | if errTracer != nil { 27 | logs.Error("crate tracer error: %v \n", errTracer) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/infer_services/io/response.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | type RecResponse struct { 4 | code int 5 | message string 6 | data []string //ItemInfo string 7 | } 8 | 9 | // code 10 | func (r *RecResponse) SetCode(code int) { 11 | r.code = code 12 | } 13 | 14 | func (r *RecResponse) GetCode() int { 15 | return r.code 16 | } 17 | 18 | // message 19 | func (r *RecResponse) SetMessage(message string) { 20 | r.message = message 21 | } 22 | 23 | func (r *RecResponse) GetMessage() string { 24 | return r.message 25 | } 26 | 27 | // data 28 | func (r *RecResponse) SetData(data []string) { 29 | r.data = data 30 | } 31 | 32 | func (r *RecResponse) GetData() []string { 33 | return r.data 34 | } 35 | 36 | func (rsp *RecResponse) JavaClassName() string { 37 | return "com.loki.www.infer.RecResponse" 38 | } 39 | -------------------------------------------------------------------------------- /pkg/config_loader/redis_config/redis_config_loader_test.go: -------------------------------------------------------------------------------- 1 | package redis_config 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestRedisConfigLoader(t *testing.T) { 8 | 9 | testKey := "test_userid:1001" 10 | redisTestStr := ` 11 | { 12 | "redis_conf": { 13 | "redisCluster": { 14 | "addrs": [], 15 | "password": "", 16 | "idleTimeoutMs": 100, 17 | "readTimeoutMs": 100, 18 | "writeTimeoutMs":100, 19 | "dialTimeoutS":600, 20 | "maxRetries":2, 21 | "minIdleConns":50 22 | } 23 | } 24 | } 25 | ` 26 | redisConf := RedisConfig{} 27 | redisConf.ConfigLoad("testId", redisTestStr) 28 | t.Log("redisConf:", redisConf) 29 | 30 | rst, err := redisConf.redisPool.HGetAll(testKey) 31 | if err != nil { 32 | t.Errorf("redis init failed") 33 | } 34 | 35 | t.Log("redis result:", rst) 36 | } 37 | -------------------------------------------------------------------------------- /internal/flags/flag_bloom.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | type FlagBloom struct { 4 | //redis 5 | userCountLevel *uint 6 | itemCountLevel *uint 7 | } 8 | 9 | var flagBloomInstance *FlagBloom 10 | 11 | // singleton instance 12 | func init() { 13 | flagBloomInstance = new(FlagBloom) 14 | } 15 | 16 | func getFlagBloomInstance() *FlagBloom { 17 | return flagBloomInstance 18 | } 19 | 20 | // userCountLevel 21 | func (s *FlagBloom) setUserCountLevel(userCountLevel *uint) { 22 | s.userCountLevel = userCountLevel 23 | } 24 | 25 | func (s *FlagBloom) GetUserCountLevel() *uint { 26 | return s.userCountLevel 27 | } 28 | 29 | // itemCountLevel 30 | func (s *FlagBloom) setItemCountLevel(itemCountLevel *uint) { 31 | s.itemCountLevel = itemCountLevel 32 | } 33 | 34 | func (s *FlagBloom) GetItemCountLevel() *uint { 35 | return s.itemCountLevel 36 | } 37 | -------------------------------------------------------------------------------- /Protofile/grpc_api/recommender.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "./;grpc_service"; 3 | 4 | message StringList { 5 | repeated string value = 1; 6 | } 7 | 8 | message ItemInfo { 9 | string itemid = 1; 10 | float score = 2; 11 | } 12 | 13 | message ItemInfoList { 14 | repeated ItemInfo iteminfo_ = 1; 15 | } 16 | 17 | message RecommendRequest { 18 | string DataId = 1; 19 | string GroupId = 2; 20 | string Namespace = 3; 21 | string ModelType = 4; 22 | string UserId = 5; 23 | int32 RecallNum = 6; 24 | StringList ItemList = 7; 25 | } 26 | 27 | message RecommendResponse { 28 | int32 Code = 1; 29 | string Message = 2; 30 | ItemInfoList Data = 3; 31 | } 32 | 33 | service RecommenderInferService { 34 | rpc RecommenderInfer(RecommendRequest) returns(RecommendResponse); 35 | } 36 | -------------------------------------------------------------------------------- /api/grpc_api/client/grpc_api_client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | grpc_service "infer-microservices/pkg/infer_services/grpc_service" 5 | "testing" 6 | ) 7 | 8 | var grpcAddress string 9 | 10 | func init() { 11 | grpcAddress = "10.10.10.10:8888" 12 | } 13 | 14 | func TestDubboApiClient(t *testing.T) { 15 | itemList := []string{"1001", "1002", "1003", "1004"} 16 | req := &grpc_service.RecommendRequest{ 17 | UserId: "$userid", 18 | ItemList: &grpc_service.StringList{Value: itemList}, 19 | } 20 | 21 | grpcClient := GrpcApiClient{} 22 | grpcClient.setGrpcAddress(grpcAddress) 23 | rsp, err := grpcClient.grpcServiceApiInfer(req) 24 | if err != nil { 25 | t.Fatal("service return err", err) 26 | } 27 | 28 | if rsp.Data.Size() == 0 { 29 | t.Errorf("Expected recall_num or rank_num, but got 0") 30 | } else { 31 | t.Log(rsp.Data.Iteminfo_) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /configs/dubbo/dubbo_server-1_config.yml: -------------------------------------------------------------------------------- 1 | 2 | 3 | dubbo: 4 | application: 5 | name: infer # metadata: application=myApp; name=myApp 6 | module: dubbogo #metadata: module=opensource 7 | group: recsys # no metadata record 8 | organization: dubbo # metadata: organization=dubbo 9 | owner: # metadata: owner=laurence 10 | version: 1.0.0 # metadata: app.version=myversion 11 | registries: 12 | inferENG: 13 | protocol: nacos 14 | address: 10.10.10.10:8822 15 | group: DEFAULT_GROUP # nacos group, default is DEFAULT_GROUP 16 | registry-type: interface 17 | protocols: 18 | dubbo: 19 | name: dubbo # 20 | port: 10000 # 21 | provider: 22 | register: true 23 | registryIDS: 24 | - inferENG 25 | services: 26 | DubbogoInferService: 27 | protocol: dubbo 28 | interface: com.loki.www.infer.DubbogoInferService 29 | -------------------------------------------------------------------------------- /configs/dubbo/dubbo_server-2_config.yml: -------------------------------------------------------------------------------- 1 | 2 | 3 | dubbo: 4 | application: 5 | name: infer # metadata: application=myApp; name=myApp 6 | module: dubbogo #metadata: module=opensource 7 | group: recsys # no metadata record 8 | organization: dubbo # metadata: organization=dubbo 9 | owner: # metadata: owner=laurence 10 | version: 1.0.0 # metadata: app.version=myversion 11 | registries: 12 | inferENG: 13 | protocol: nacos 14 | address: 10.10.10.10:8822 15 | group: DEFAULT_GROUP # nacos group, default is DEFAULT_GROUP 16 | registry-type: interface 17 | protocols: 18 | dubbo: 19 | name: dubbo # 20 | port: 10001 # 21 | provider: 22 | register: true 23 | registryIDS: 24 | - inferENG 25 | services: 26 | DubbogoInferService: 27 | protocol: dubbo 28 | interface: com.loki.www.infer.DubbogoInferService 29 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/tensor_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/allocation_description.proto"; 6 | import "tensorflow/core/framework/tensor_shape.proto"; 7 | import "tensorflow/core/framework/types.proto"; 8 | 9 | option cc_enable_arenas = true; 10 | option java_outer_classname = "TensorDescriptionProtos"; 11 | option java_multiple_files = true; 12 | option java_package = "org.tensorflow.framework"; 13 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_description_go_proto"; 14 | 15 | message TensorDescription { 16 | // Data type of tensor elements 17 | DataType dtype = 1; 18 | 19 | // Shape of the tensor. 20 | TensorShapeProto shape = 2; 21 | 22 | // Information about the size and allocator used for the data 23 | AllocationDescription allocation_description = 4; 24 | } 25 | -------------------------------------------------------------------------------- /internal/logs/color.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build !windows 16 | 17 | package logs 18 | 19 | import "io" 20 | 21 | type ansiColorWriter struct { 22 | w io.Writer 23 | mode outputMode 24 | } 25 | 26 | func (cw *ansiColorWriter) Write(p []byte) (int, error) { 27 | return cw.w.Write(p) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/nacos/nacos_factory.go: -------------------------------------------------------------------------------- 1 | package nacos 2 | 3 | import ( 4 | "infer-microservices/internal/logs" 5 | "infer-microservices/pkg/infer_services/io" 6 | 7 | validator "github.com/go-playground/validator/v10" 8 | ) 9 | 10 | type NacosFactory struct { 11 | } 12 | 13 | func (n *NacosFactory) CreateNacosConfig(nacosIp string, nacosPort uint64, in *io.RecRequest) NacosConnConfig { 14 | //nacos listen need follow parms. 15 | nacosConn := NacosConnConfig{} 16 | dataId := in.GetDataId() 17 | groupId := in.GetGroupId() 18 | namespaceId := in.GetNamespaceId() 19 | 20 | nacosConn.SetDataId(dataId) 21 | nacosConn.SetGroupId(groupId) 22 | nacosConn.SetNamespaceId(namespaceId) 23 | nacosConn.SetIp(nacosIp) 24 | nacosConn.SetPort(uint64(nacosPort)) 25 | 26 | validate := validator.New() 27 | err := validate.Struct(nacosConn) 28 | if err != nil { 29 | logs.Error(err) 30 | return NacosConnConfig{} 31 | } 32 | 33 | return nacosConn 34 | 35 | } 36 | -------------------------------------------------------------------------------- /api/grpc_api/client/grpc_api_client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "infer-microservices/internal/logs" 5 | grpc_service "infer-microservices/pkg/infer_services/grpc_service" 6 | 7 | "golang.org/x/net/context" 8 | "google.golang.org/grpc" 9 | ) 10 | 11 | type GrpcApiClient struct { 12 | grpcAddress string 13 | } 14 | 15 | func (d *GrpcApiClient) setGrpcAddress(grpcAddress string) { 16 | d.grpcAddress = grpcAddress 17 | } 18 | 19 | func (g *GrpcApiClient) grpcServiceApiInfer(req *grpc_service.RecommendRequest) (*grpc_service.RecommendResponse, error) { 20 | conn, err := grpc.Dial(g.grpcAddress, grpc.WithInsecure()) 21 | if err != nil { 22 | logs.Error("GRPC dial failed") 23 | } 24 | defer conn.Close() 25 | 26 | client := grpc_service.NewRecommenderInferServiceClient(conn) 27 | rsp, err := client.RecommenderInfer(context.Background(), req) 28 | if err != nil { 29 | logs.Error(err) 30 | return nil, err 31 | } 32 | 33 | return rsp, nil 34 | } 35 | -------------------------------------------------------------------------------- /scripts/deployments/elkb/filebeat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.4.3-linux-x86_64.tar.gz 5 | tar -zxvf filebeat-5.4.3-linux-x86_64.tar.gz 6 | mv filebeat-5.4.3-linux-x86_64 filebeat 7 | 8 | 9 | #config multi log files 10 | filebeat.inputs: 11 | 12 | - type: log 13 | enabled: true 14 | paths: 15 | - /var/logs/info/* 16 | tags: ["infer-info-1"] 17 | 18 | - type: log 19 | enabled: true 20 | paths: 21 | - /var/logs/error/* 22 | tags: ["infer-info-2"] 23 | 24 | output.logstash: 25 | hosts: ["localhost:5044"] 26 | #output.elasticsearch: 27 | # # Array of hosts to connect to. 28 | # hosts: ["192.168.81.129:9200"] 29 | # indices: 30 | # - index: "infer-info-1-%{+yyyy.MM.dd}" 31 | # when.contains: 32 | # tags: "infer-info-1" 33 | # - index: "infer-info-2-%{+yyyy.MM.dd}" 34 | # when.contains: 35 | # tags: "infer-info-2" 36 | 37 | ./filebeat -e -c client.yml 38 | -------------------------------------------------------------------------------- /api/dubbo_api/client/dubbo_api_client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "infer-microservices/pkg/infer_services/io" 5 | "testing" 6 | ) 7 | 8 | var dubboConfigFile string 9 | 10 | func init() { 11 | dubboConfigFile = "../configs/dubbo/dubbogo_client_1.yml" 12 | } 13 | 14 | func TestDubboApiClient(t *testing.T) { 15 | //itemList := []string{"1001", "1002", "1003", "1004"} 16 | 17 | req := io.RecRequest{} 18 | req.SetDataId("$dataid|$groupid") //nacos dataid|groupid 19 | req.SetUserId("$userid") //userid 20 | //req.SetItemList(itemList) //rank items 21 | 22 | dubboApiClient := DubboApiClient{} 23 | dubboApiClient.setDubboConfigFile(dubboConfigFile) 24 | rsp, err := dubboApiClient.dubboServiceApiInfer(req) 25 | if err != nil { 26 | t.Fatal("service return err", err) 27 | } 28 | 29 | if len(rsp.GetData()) == 0 { 30 | t.Errorf("Expected recall_num or rank_num, but got 0") 31 | } else { 32 | t.Log("service rsp:", rsp) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pkg/infer_pipeline/interface.go: -------------------------------------------------------------------------------- 1 | package infer_pipeline 2 | 3 | import ( 4 | "infer-microservices/pkg/config_loader/model_config" 5 | feature "infer-microservices/pkg/infer_features" 6 | "net/http" 7 | ) 8 | 9 | type InferPipelineInterface interface { 10 | //sample 11 | RecallSampleDirector(userId string, offlineFeature bool, onlineFeature bool, featureList []string) (feature.ExampleFeatures, error) 12 | RankingSampleDirector(userId string, offlineFeature bool, onlineFeature bool, itemIdList []string, featureList []string) (feature.ExampleFeatures, error) 13 | 14 | //infer 15 | ModelInferSkywalking(model model_config.ModelConfig, requestId string, exposureList []string, r *http.Request, inferSample feature.ExampleFeatures, retNum int) (map[string][]map[string]interface{}, error) 16 | ModelInferNoSkywalking(model model_config.ModelConfig, requestId string, exposureList []string, inferSample feature.ExampleFeatures, retNum int) (map[string][]map[string]interface{}, error) 17 | } 18 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/allocation_description.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "AllocationDescriptionProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/allocation_description_go_proto"; 10 | 11 | message AllocationDescription { 12 | // Total number of bytes requested 13 | int64 requested_bytes = 1; 14 | 15 | // Total number of bytes allocated if known 16 | int64 allocated_bytes = 2; 17 | 18 | // Name of the allocator used 19 | string allocator_name = 3; 20 | 21 | // Identifier of the allocated buffer if known 22 | int64 allocation_id = 4; 23 | 24 | // Set if this tensor only has one remaining reference 25 | bool has_single_reference = 5; 26 | 27 | // Address of the allocation. 28 | uint64 ptr = 6; 29 | } 30 | -------------------------------------------------------------------------------- /internal/flags/flag_kafka.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagKafkaInstance *flagKafka 4 | 5 | type flagKafka struct { 6 | // tensorflow 7 | kafkaUrl string 8 | kafkaTopic string 9 | kafkaGroup string 10 | } 11 | 12 | // singleton instance 13 | func init() { 14 | flagKafkaInstance = new(flagKafka) 15 | } 16 | 17 | func getFlagKafkaInstance() *flagKafka { 18 | return flagKafkaInstance 19 | } 20 | 21 | // kafkaUrl 22 | func (s *flagKafka) setKafkaUrl(kafkaUrl string) { 23 | s.kafkaUrl = kafkaUrl 24 | } 25 | 26 | func (s *flagKafka) GetKafkaUrl() string { 27 | return s.kafkaUrl 28 | } 29 | 30 | // kafkaTopic 31 | func (s *flagKafka) setKafkaTopic(kafkaTopic string) { 32 | s.kafkaTopic = kafkaTopic 33 | } 34 | 35 | func (s *flagKafka) GetKafkaTopic() string { 36 | return s.kafkaTopic 37 | } 38 | 39 | // kafkaGroup 40 | func (s *flagKafka) setKafkaGroup(kafkaGroup string) { 41 | s.kafkaGroup = kafkaGroup 42 | } 43 | 44 | func (s *flagKafka) GetKafkaGroup() string { 45 | return s.kafkaGroup 46 | } 47 | -------------------------------------------------------------------------------- /internal/flags/flag_tensorflow.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagTensorflowInstance *flagTensorflow 4 | 5 | type flagTensorflow struct { 6 | // tensorflow 7 | tfservingModelVersion *int64 8 | tfservingTimeoutMs *int64 9 | } 10 | 11 | // singleton instance 12 | func init() { 13 | flagTensorflowInstance = new(flagTensorflow) 14 | } 15 | 16 | func getFlagTensorflowInstance() *flagTensorflow { 17 | return flagTensorflowInstance 18 | } 19 | 20 | // tfserving_model_version 21 | func (s *flagTensorflow) setTfservingModelVersion(tfservingModelVersion *int64) { 22 | s.tfservingModelVersion = tfservingModelVersion 23 | } 24 | 25 | func (s *flagTensorflow) GetTfservingModelVersion() *int64 { 26 | return s.tfservingModelVersion 27 | } 28 | 29 | // tfserving_model_version 30 | func (s *flagTensorflow) setTfservingTimeoutMs(tfservingTimeoutMs *int64) { 31 | s.tfservingTimeoutMs = tfservingTimeoutMs 32 | } 33 | 34 | func (s *flagTensorflow) GetTfservingTimeoutMs() *int64 { 35 | return s.tfservingTimeoutMs 36 | } 37 | -------------------------------------------------------------------------------- /pkg/config_loader/faiss_config/faiss_config_loader_test.go: -------------------------------------------------------------------------------- 1 | package faiss_config 2 | 3 | import "testing" 4 | 5 | func TestFaissConfigLoader(t *testing.T) { 6 | 7 | faissTestStr := ` 8 | { 9 | "index_conf": { 10 | "faissGrpcAddr": { 11 | "addrs": [], 12 | "pool_size": 50, 13 | "initCap":10, 14 | "idleTimeoutMs": 100, 15 | "readTimeoutMs": 100, 16 | "writeTimeoutMs":100, 17 | "dialTimeoutS":600 18 | }, 19 | "indexInfo":{ 20 | "index-001": { 21 | "recallNum": 100, 22 | "indexName": "index-001" 23 | } 24 | } 25 | }, 26 | } 27 | ` 28 | faissConfs := FaissIndexConfigs{} 29 | faissConfs.ConfigLoad("testId", faissTestStr) 30 | 31 | t.Log("faissConf:", faissConfs) 32 | 33 | for _, faissConf := range faissConfs.faissIndexConfigs { 34 | if faissConf.faissIndexs.RecallNum <= 0 { 35 | t.Errorf("recall num must > 0") 36 | } 37 | 38 | if faissConf.faissIndexs.IndexName == "" { 39 | t.Errorf("index name cant be empt") 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /internal/flags/flag_viper.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | type FlagViper struct { 4 | configName *string 5 | configType *string 6 | configPath *string 7 | } 8 | 9 | var flagViperInstance *FlagViper 10 | 11 | // singleton instance 12 | func init() { 13 | flagViperInstance = new(FlagViper) 14 | } 15 | 16 | func getFlagViperInstance() *FlagViper { 17 | return flagViperInstance 18 | } 19 | 20 | // SetConfigName 21 | func (s *FlagViper) setConfigName(configName *string) { 22 | s.configName = configName 23 | } 24 | 25 | func (s *FlagViper) GetConfigName() *string { 26 | return s.configName 27 | } 28 | 29 | // SetConfigType 30 | func (s *FlagViper) setConfigType(configType *string) { 31 | s.configType = configType 32 | } 33 | 34 | func (s *FlagViper) GetConfigType() *string { 35 | return s.configType 36 | } 37 | 38 | // SetConfigPath 39 | func (s *FlagViper) setConfigPath(configPath *string) { 40 | s.configPath = configPath 41 | } 42 | 43 | func (s *FlagViper) GetConfigPath() *string { 44 | return s.configPath 45 | } 46 | -------------------------------------------------------------------------------- /pkg/infer_samples/bloom.go: -------------------------------------------------------------------------------- 1 | package infer_samples 2 | 3 | import ( 4 | "infer-microservices/internal/flags" 5 | 6 | bloomv3 "github.com/bits-and-blooms/bloom/v3" 7 | ) 8 | 9 | var userBloomFilterInstance *bloomv3.BloomFilter 10 | var itemBloomFilterInstance *bloomv3.BloomFilter 11 | 12 | func init() { 13 | flagFactory := flags.FlagFactory{} 14 | flagBloom := flagFactory.CreateFlagBloom() 15 | userBloomFilterInstance = bloomv3.NewWithEstimates(*flagBloom.GetUserCountLevel(), 0.01) 16 | itemBloomFilterInstance = bloomv3.NewWithEstimates(*flagBloom.GetItemCountLevel(), 0.01) 17 | } 18 | 19 | // INFO: singleton instance 20 | func GetUserBloomFilterInstance() *bloomv3.BloomFilter { 21 | return userBloomFilterInstance 22 | } 23 | 24 | // INFO: singleton instance 25 | func GetItemBloomFilterInstance() *bloomv3.BloomFilter { 26 | return itemBloomFilterInstance 27 | } 28 | 29 | func BloomPush(filter *bloomv3.BloomFilter, id string) { 30 | filter.Add([]byte(id)) 31 | } 32 | 33 | func BloomClean(filter *bloomv3.BloomFilter) { 34 | filter.ClearAll() 35 | } 36 | -------------------------------------------------------------------------------- /api/dubbo_api/client/dubbo_api_client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | dubbo_api "infer-microservices/api/dubbo_api/server" 6 | "infer-microservices/internal/logs" 7 | "infer-microservices/pkg/infer_services/io" 8 | 9 | "dubbo.apache.org/dubbo-go/v3/config" 10 | _ "dubbo.apache.org/dubbo-go/v3/imports" 11 | ) 12 | 13 | type DubboApiClient struct { 14 | dubboConfigFile string 15 | } 16 | 17 | func (d *DubboApiClient) setDubboConfigFile(dubboConfigFile string) { 18 | d.dubboConfigFile = dubboConfigFile 19 | } 20 | 21 | func (c *DubboApiClient) dubboServiceApiInfer(req io.RecRequest) (*io.RecResponse, error) { 22 | // export DUBBO_GO_CONFIG_PATH=dubbogo.yml or load it in code. 23 | if err := config.Load(config.WithPath(c.dubboConfigFile)); err != nil { 24 | panic(err) 25 | } 26 | 27 | // request dubbo infer service. use to test serivce. 28 | rsp, err := dubbo_api.DubboServiceApiClient.RecommenderInfer(context.TODO(), &req) 29 | if err != nil { 30 | logs.Error(err) 31 | return nil, err 32 | } 33 | 34 | return rsp, nil 35 | } 36 | -------------------------------------------------------------------------------- /Dockerfile/infer/infer_docker_file: -------------------------------------------------------------------------------- 1 | ############################### 2 | # 3 | #build go project docker mir 4 | # 5 | ############################### 6 | 7 | FROM golang:alpine 8 | MAINTAINER loki 9 | ENV VERSION 1.0.0 10 | 11 | WORKDIR /data/infer 12 | 13 | # copy start file and config files 14 | #copy go file 15 | COPY /data/loki/the-infer/cmd/infer/main/recsys-go-infer /data/infer 16 | #copy config file 17 | COPY /data/loki/the-infer/configs/dubbo/dubbo_server-1_config.yml /data/infer/configs 18 | COPY /data/loki/the-infer/configs/dubbo/dubbo_server-2_config.yml /data/infer/configs 19 | COPY /data/loki/the-infer/configs/nacos/service_start_config.json /data/infer/configs 20 | #copy start scripts 21 | COPY /data/loki/the-infer/scripts/deployments/infer/services_start.sh /data/infer 22 | 23 | # Expose ports 24 | # REST 25 | EXPOSE 6020 26 | 27 | # gRPC 28 | EXPOSE 6021 29 | 30 | # Set where models should be stored in the container 31 | ENV MODEL_BASE_PATH=/data/infer/configs 32 | RUN mkdir -p ${MODEL_BASE_PATH} 33 | 34 | ENTRYPOINT ["/data/infer/services_start.sh"] 35 | -------------------------------------------------------------------------------- /internal/utils/request_id_util.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "fmt" 7 | io "infer-microservices/pkg/infer_services/io" 8 | "time" 9 | ) 10 | 11 | func CreateRequestId(in *io.RecRequest) string { 12 | //Multiple requests from a user to a model within 120 seconds are considered a single request 13 | timestamp := time.Now().Unix() / 120 14 | value := in.GetDataId() + in.GetGroupId() + in.GetUserId() + fmt.Sprintf("%d", timestamp) 15 | data := []byte(value) 16 | md5New := md5.New() 17 | md5New.Write(data) 18 | 19 | requestId := hex.EncodeToString(md5New.Sum(nil)) 20 | return requestId 21 | } 22 | 23 | func CreateRequestId2(dataId string, groupId string, userId string) string { 24 | //Multiple requests from a user to a model within 120 seconds are considered a single request 25 | timestamp := time.Now().Unix() / 120 26 | value := dataId + groupId + userId + fmt.Sprintf("%d", timestamp) 27 | data := []byte(value) 28 | md5New := md5.New() 29 | md5New.Write(data) 30 | 31 | requestId := hex.EncodeToString(md5New.Sum(nil)) 32 | return requestId 33 | } 34 | -------------------------------------------------------------------------------- /internal/jwt/jwt_echo.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | jwt "github.com/dgrijalva/jwt-go" 8 | "github.com/labstack/echo" 9 | ) 10 | 11 | // jwtCustomClaims are custom claims extending default ones. 12 | type JwtCustomClaims struct { 13 | Name string `json:"name"` 14 | Admin bool `json:"admin"` 15 | jwt.StandardClaims 16 | } 17 | 18 | func Login(c echo.Context) error { 19 | username := c.FormValue("username") 20 | password := c.FormValue("password") 21 | 22 | if username == "infer" && password == "!@#$1234" { 23 | 24 | // Set custom claims 25 | claims := &JwtCustomClaims{ 26 | username, 27 | true, 28 | jwt.StandardClaims{ 29 | ExpiresAt: time.Now().Add(time.Hour * 72).Unix(), 30 | }, 31 | } 32 | 33 | // Create token with claims 34 | token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) 35 | 36 | // Generate encoded token and send it as response. 37 | t, err := token.SignedString([]byte("secret")) 38 | if err != nil { 39 | return err 40 | } 41 | return c.JSON(http.StatusOK, echo.Map{ 42 | "token": t, 43 | }) 44 | } 45 | 46 | return echo.ErrUnauthorized 47 | } 48 | -------------------------------------------------------------------------------- /pkg/config_loader/redis_config/redis_config_loader.go: -------------------------------------------------------------------------------- 1 | package redis_config 2 | 3 | import ( 4 | redis_v8 "infer-microservices/internal/db/redis" 5 | "infer-microservices/internal/utils" 6 | ) 7 | 8 | type RedisConfig struct { 9 | redisPool *redis_v8.InferRedisClient `validate:"required"` 10 | } 11 | 12 | // // INFO: singleton instance 13 | // func init() { 14 | // RedisClientInstance = new(RedisClient) 15 | // } 16 | 17 | // func getRedisClientInstance() *RedisClient { 18 | // return RedisClientInstance 19 | // } 20 | 21 | // redis pool 22 | func (r *RedisConfig) setRedisPool(redisPool *redis_v8.InferRedisClient) { 23 | r.redisPool = redisPool 24 | } 25 | 26 | func (r *RedisConfig) GetRedisPool() *redis_v8.InferRedisClient { 27 | return r.redisPool 28 | } 29 | 30 | // @implement ConfigLoadInterface 31 | func (r *RedisConfig) ConfigLoad(dataId string, redisConfStr string) error { 32 | confMap := utils.ConvertJsonToStruct(redisConfStr) 33 | redisClusterInfo := confMap["redisCluster"].(map[string]interface{}) 34 | redisConnPool := redis_v8.NewRedisClusterClient(redisClusterInfo) 35 | 36 | r.setRedisPool(redisConnPool) 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /scripts/deployments/elkb/es.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $ docker run -d --name elasticsearch docker.elastic.co/elasticsearch/elasticsearch:5.4.0 4 | 5 | 6 | # 7 | $ docker exec -it elasticsearch bash 8 | 9 | # edit config file 10 | $ vim config/elasticsearch.yml 11 | cluster.name: "docker-cluster" 12 | node.name: node-101 13 | path.data: /data/esdata #data dir 14 | path.logs: /data/eslogs #logs dir 15 | network.host: 0.0.0.0 16 | http.cors.enabled: true 17 | http.cors.allow-origin: "*" 18 | xpack.security.enabled: false 19 | # minimum_master_nodes need to be explicitly set when bound on a public IP 20 | # set to 1 to allow single node clusters 21 | # Details: https://github.com/elastic/elasticsearch/pull/17288 22 | discovery.zen.ping.unicast.hosts: ["192.168.9.101", "192.168.9.102","192.168.9.103"] 23 | discovery.zen.minimum_master_nodes: 1 24 | http.port: 9200 #es port 25 | 26 | 27 | docker commit -a "add config" -m "dev" a404c6c174a2 es:latest 28 | docker run -d --name es -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" es:latest 29 | $ curl 'http://localhost:9200/_nodes/http?pretty' 30 | 31 | 32 | docker run -p 9100:9100 mobz/elasticsearch-head:5 33 | 34 | -------------------------------------------------------------------------------- /Protofile/faiss/faissIndex.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "./;tfserving_go"; 3 | 4 | 5 | message ItemInfo { 6 | string ItemId = 1; 7 | float Score = 2; 8 | float Embedding = 3; 9 | } 10 | 11 | 12 | message UserVectorInfo { 13 | string UserId = 1; 14 | repeated float UserVector = 2; 15 | } 16 | 17 | 18 | message RecallRequest { 19 | string IndexName = 1; 20 | UserVectorInfo UserVectorInfo_ = 2; 21 | int32 RecallNum = 3; 22 | } 23 | 24 | 25 | message RecallResponse { 26 | string UserId = 1; 27 | repeated ItemInfo ItemInfo_ = 2; 28 | } 29 | 30 | 31 | message IndexInfo { 32 | string IndexName = 1; 33 | string IndexMd5 = 2; 34 | string IndexType = 3; 35 | string IndexLoadTime = 4; 36 | int32 IndexDim = 5; 37 | int32 IndexVectorsSize = 6; 38 | } 39 | 40 | message GetIndexInfoRequest { 41 | } 42 | 43 | message GetIndexInfoResponse { 44 | repeated IndexInfo IndexInfo_ = 1; 45 | } 46 | 47 | service GrpcRecallServerService { 48 | rpc GrpcRecall(RecallRequest) returns(RecallResponse); 49 | rpc GetIndexInfo(GetIndexInfoRequest) returns(GetIndexInfoResponse); 50 | } 51 | -------------------------------------------------------------------------------- /pkg/infer_models/model_strategy_context.go: -------------------------------------------------------------------------------- 1 | package infer_model 2 | 3 | import ( 4 | "infer-microservices/pkg/config_loader/model_config" 5 | feature "infer-microservices/pkg/infer_features" 6 | 7 | "net/http" 8 | ) 9 | 10 | type ModelStrategyContext struct { 11 | modelStrategy ModelStrategyInterface 12 | } 13 | 14 | func (m *ModelStrategyContext) SetModelStrategy(strategy ModelStrategyInterface) { 15 | m.modelStrategy = strategy 16 | } 17 | 18 | func (m *ModelStrategyContext) ModelInferSkywalking(model model_config.ModelConfig, requestId string, exposureList []string, r *http.Request, inferSample feature.ExampleFeatures, retNum int) (map[string][]map[string]interface{}, error) { 19 | response, err := m.modelStrategy.ModelInferSkywalking(model, requestId, exposureList, r, inferSample, retNum) 20 | return response, err 21 | } 22 | 23 | func (m *ModelStrategyContext) ModelInferNoSkywalking(model model_config.ModelConfig, requestId string, exposureList []string, inferSample feature.ExampleFeatures, retNum int) (map[string][]map[string]interface{}, error) { 24 | response, err := m.modelStrategy.ModelInferNoSkywalking(model, requestId, exposureList, inferSample, retNum) 25 | return response, err 26 | } 27 | -------------------------------------------------------------------------------- /internal/tfserving_gogofaster/proto/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "google/protobuf/wrappers.proto"; 7 | 8 | // Metadata for an inference request such as the model name and version. 9 | message ModelSpec { 10 | // Required servable name. 11 | string name = 1; 12 | 13 | // Optional choice of which version of the model to use. 14 | // 15 | // Recommended to be left unset in the common case. Should be specified only 16 | // when there is a strong version consistency requirement. 17 | // 18 | // When left unspecified, the system will serve the best available version. 19 | // This is typically the latest version, though during version transitions, 20 | // notably when serving on a fleet of instances, may be either the previous or 21 | // new version. 22 | oneof version_choice { 23 | // Use this specific version number. 24 | google.protobuf.Int64Value version = 2; 25 | 26 | // Use the version associated with the given label. 27 | string version_label = 4; 28 | } 29 | 30 | // A named signature to evaluate. If unspecified, the default signature will 31 | // be used. 32 | string signature_name = 3; 33 | } -------------------------------------------------------------------------------- /internal/flags/flag_logs.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagsLogInstance *flagsLog 4 | 5 | type flagsLog struct { 6 | // logs 7 | logMaxSize *int 8 | logSaveDays *int 9 | logFileName *string 10 | logLevel *string 11 | } 12 | 13 | // singleton instance 14 | func init() { 15 | flagsLogInstance = new(flagsLog) 16 | } 17 | 18 | func getFlagLogInstance() *flagsLog { 19 | return flagsLogInstance 20 | } 21 | 22 | // log_max_size 23 | func (s *flagsLog) setLogMaxSize(logMaxSize *int) { 24 | s.logMaxSize = logMaxSize 25 | } 26 | 27 | func (s *flagsLog) GetLogMaxSize() *int { 28 | return s.logMaxSize 29 | } 30 | 31 | // log_save_days 32 | func (s *flagsLog) setLogSaveDays(logSaveDays *int) { 33 | s.logSaveDays = logSaveDays 34 | } 35 | 36 | func (s *flagsLog) GetLogSaveDays() *int { 37 | return s.logSaveDays 38 | } 39 | 40 | // log_file_name 41 | func (s *flagsLog) setLogFileName(logFileName *string) { 42 | s.logFileName = logFileName 43 | } 44 | 45 | func (s *flagsLog) GetLogFileName() *string { 46 | return s.logFileName 47 | } 48 | 49 | // log_level 50 | func (s *flagsLog) setLogLevel(logLevel *string) { 51 | s.logLevel = logLevel 52 | } 53 | 54 | func (s *flagsLog) GetLogLevel() *string { 55 | return s.logLevel 56 | } 57 | -------------------------------------------------------------------------------- /internal/tfserving_gogofaster/proto/prediction_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | option cc_enable_arenas = true; 5 | 6 | import "tensorflow_serving/apis/classification.proto"; 7 | import "tensorflow_serving/apis/get_model_metadata.proto"; 8 | import "tensorflow_serving/apis/inference.proto"; 9 | import "tensorflow_serving/apis/predict.proto"; 10 | import "tensorflow_serving/apis/regression.proto"; 11 | 12 | // open source marker; do not remove 13 | // PredictionService provides access to machine-learned models loaded by 14 | // model_servers. 15 | service PredictionService { 16 | // Classify. 17 | rpc Classify(ClassificationRequest) returns (ClassificationResponse); 18 | 19 | // Regress. 20 | rpc Regress(RegressionRequest) returns (RegressionResponse); 21 | 22 | // Predict -- provides access to loaded TensorFlow model. 23 | rpc Predict(PredictRequest) returns (PredictResponse); 24 | 25 | // MultiInference API for multi-headed models. 26 | rpc MultiInference(MultiInferenceRequest) returns (MultiInferenceResponse); 27 | 28 | // GetModelMetadata - provides access to metadata for loaded models. 29 | rpc GetModelMetadata(GetModelMetadataRequest) 30 | returns (GetModelMetadataResponse); 31 | } -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/versions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "VersionsProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/versions_go_proto"; 10 | 11 | // Version information for a piece of serialized data 12 | // 13 | // There are different types of versions for each type of data 14 | // (GraphDef, etc.), but they all have the same common shape 15 | // described here. 16 | // 17 | // Each consumer has "consumer" and "min_producer" versions (specified 18 | // elsewhere). A consumer is allowed to consume this data if 19 | // 20 | // producer >= min_producer 21 | // consumer >= min_consumer 22 | // consumer not in bad_consumers 23 | // 24 | message VersionDef { 25 | // The version of the code that produced this data. 26 | int32 producer = 1; 27 | 28 | // Any consumer below this version is not allowed to consume this data. 29 | int32 min_consumer = 2; 30 | 31 | // Specific consumer versions which are disallowed (e.g. due to bugs). 32 | repeated int32 bad_consumers = 3; 33 | } 34 | -------------------------------------------------------------------------------- /pkg/infer_samples/infer_sample_subject.go: -------------------------------------------------------------------------------- 1 | package infer_samples 2 | 3 | //Subject 4 | type Subject interface { 5 | //notice 6 | NotifyObservers() 7 | 8 | //add observer 9 | AddObserver(observer Observer) 10 | 11 | //add observers 12 | AddObservers(observers ...Observer) 13 | 14 | //remove observer 15 | RemoveObserver(observer Observer) 16 | 17 | //clean observer 18 | RemoveAllObservers() 19 | } 20 | 21 | // SampleSubject implement Subject interface 22 | type SampleSubject struct { 23 | // observers 24 | observers []Observer 25 | } 26 | 27 | // NotifyObservers 28 | func (ss *SampleSubject) NotifyObservers() { 29 | for _, o := range ss.observers { 30 | o.notify(ss) 31 | } 32 | } 33 | 34 | // AddObserver 35 | func (ss *SampleSubject) AddObserver(observer Observer) { 36 | ss.observers = append(ss.observers, observer) 37 | } 38 | 39 | // AddObservers 40 | func (ss *SampleSubject) AddObservers(observers ...Observer) { 41 | ss.observers = append(ss.observers, observers...) 42 | } 43 | 44 | // RemoveObserver 45 | func (ss *SampleSubject) RemoveObserver(observer Observer) { 46 | for i := 0; i < len(ss.observers); i++ { 47 | if ss.observers[i] == observer { 48 | ss.observers = append(ss.observers[:i], ss.observers[i+1:]...) 49 | } 50 | } 51 | } 52 | 53 | // RemoveAllObservers 54 | func (ss *SampleSubject) RemoveAllObservers() { 55 | ss.observers = ss.observers[:0] 56 | } 57 | -------------------------------------------------------------------------------- /api/grpc_api/server/grpc_server_api.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "infer-microservices/internal/logs" 6 | grpc_service "infer-microservices/pkg/infer_services/grpc_service" 7 | "net" 8 | "runtime" 9 | 10 | "google.golang.org/grpc" 11 | ) 12 | 13 | type GrpcServiceApi struct { 14 | maxCpuNum int 15 | serverPort uint 16 | grpcService *grpc_service.GrpcService 17 | } 18 | 19 | func (s *GrpcServiceApi) SetMaxCpuNum(maxCpuNum int) { 20 | s.maxCpuNum = maxCpuNum 21 | } 22 | 23 | func (s *GrpcServiceApi) SetServicePort(serverPort uint) { 24 | s.serverPort = serverPort 25 | } 26 | 27 | func (s *GrpcServiceApi) SetGrpcService(grpcService *grpc_service.GrpcService) { 28 | s.grpcService = grpcService 29 | } 30 | 31 | // @implement start infertace 32 | func (s *GrpcServiceApi) ServiceStart() { 33 | cpuNum := runtime.NumCPU() 34 | if s.maxCpuNum <= cpuNum { 35 | cpuNum = s.maxCpuNum 36 | } 37 | runtime.GOMAXPROCS(cpuNum) 38 | logs.Info("cup num:", cpuNum) 39 | 40 | addr := fmt.Sprintf(":%d", s.serverPort) 41 | lis, err := net.Listen("tcp", addr) 42 | if err != nil { 43 | logs.Fatal("failed to listen: %v", err) 44 | panic(err) 45 | } else { 46 | logs.Info("listen to port:", addr) 47 | } 48 | 49 | gserver := grpc.NewServer() 50 | grpc_service.RegisterRecommenderInferServiceServer(gserver, s.grpcService) 51 | gserver.Serve(lis) 52 | if err != nil { 53 | logs.Error(err) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/nacos/nacos_config_parser.go: -------------------------------------------------------------------------------- 1 | package nacos 2 | 3 | import ( 4 | "encoding/json" 5 | "infer-microservices/internal/logs" 6 | "infer-microservices/internal/utils" 7 | 8 | validator "github.com/go-playground/validator/v10" 9 | ) 10 | 11 | type NacosContent struct { 12 | // author string `validate:"required"` 13 | // update string `validate:"required"` 14 | // version string `validate:"required"` 15 | Config Config_ `validate:"required"` 16 | } 17 | 18 | type Config_ struct { 19 | redisConfNacos map[string]interface{} `validate:"required"` //features redis conf. 20 | modelConfNacos map[string]interface{} `validate:"required"` //model trainning and model infer conf. 21 | indexConfNacos map[string]interface{} //faiss index conf. 22 | } 23 | 24 | // parse service config file, which contains index info、redis info and model info etc. 25 | func (s *NacosContent) InputServiceConfigParse(content string) (string, string, string) { 26 | tmpNacos := &NacosContent{} 27 | json.Unmarshal([]byte(string(content)), tmpNacos) 28 | validate := validator.New() 29 | err := validate.Struct(s) 30 | if err != nil { 31 | logs.Error(err) 32 | return "", "", "" 33 | } 34 | 35 | redisConfStr := utils.ConvertStructToJson(s.Config.redisConfNacos) 36 | modelConfStr := utils.ConvertStructToJson(s.Config.modelConfNacos) 37 | indexConfStr := utils.ConvertStructToJson(s.Config.indexConfNacos) 38 | 39 | return redisConfStr, modelConfStr, indexConfStr 40 | } 41 | -------------------------------------------------------------------------------- /internal/flags/flag_start_info.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagServiceStartInfoInstance *FlagServiceStartInfo 4 | 5 | type FlagServiceStartInfo struct { 6 | serviceConfig *string 7 | restPort *uint 8 | grpcPort *uint 9 | maxCpuNum *int 10 | } 11 | 12 | // singleton instance 13 | func init() { 14 | flagServiceStartInfoInstance = new(FlagServiceStartInfo) 15 | } 16 | 17 | func getFlagServiceStartInfoInstance() *FlagServiceStartInfo { 18 | return flagServiceStartInfoInstance 19 | } 20 | 21 | // service_start_file 22 | func (s *FlagServiceStartInfo) setServiceConfigFile(serviceConfig *string) { 23 | s.serviceConfig = serviceConfig 24 | } 25 | 26 | func (s *FlagServiceStartInfo) GetServiceConfigFile() *string { 27 | return s.serviceConfig 28 | } 29 | 30 | // rest_server_port 31 | func (s *FlagServiceStartInfo) setServiceRestPort(restPort *uint) { 32 | s.restPort = restPort 33 | } 34 | 35 | func (s *FlagServiceStartInfo) GetServiceRestPort() *uint { 36 | return s.restPort 37 | } 38 | 39 | // grpc_server_port 40 | func (s *FlagServiceStartInfo) setServiceGrpcPort(grpcPort *uint) { 41 | s.grpcPort = grpcPort 42 | } 43 | 44 | func (s *FlagServiceStartInfo) GetServiceGrpcPort() *uint { 45 | return s.grpcPort 46 | } 47 | 48 | // max_cpu_num 49 | func (s *FlagServiceStartInfo) setServiceMaxCpuNum(maxCpuNum *int) { 50 | s.maxCpuNum = maxCpuNum 51 | } 52 | 53 | func (s *FlagServiceStartInfo) GetServiceMaxCpuNum() *int { 54 | return s.maxCpuNum 55 | } 56 | -------------------------------------------------------------------------------- /internal/logs/README.md: -------------------------------------------------------------------------------- 1 | ## logs 2 | logs is a Go logs manager. It can use many logs adapters. The repo is inspired by `database/sql` . 3 | 4 | 5 | ## How to install? 6 | 7 | go get github.com/astaxie/beego/logs 8 | 9 | 10 | ## What adapters are supported? 11 | 12 | As of now this logs support console, file,smtp and conn. 13 | 14 | 15 | ## How to use it? 16 | 17 | First you must import it 18 | 19 | import ( 20 | "github.com/astaxie/beego/logs" 21 | ) 22 | 23 | Then init a Log (example with console adapter) 24 | 25 | log := NewLogger(10000) 26 | log.SetLogger("console", "") 27 | 28 | > the first params stand for how many channel 29 | 30 | Use it like this: 31 | 32 | log.Trace("trace") 33 | log.Info("info") 34 | log.Warn("warning") 35 | log.Debug("debug") 36 | log.Critical("critical") 37 | 38 | 39 | ## File adapter 40 | 41 | Configure file adapter like this: 42 | 43 | log := NewLogger(10000) 44 | log.SetLogger("file", `{"filename":"test.log"}`) 45 | 46 | 47 | ## Conn adapter 48 | 49 | Configure like this: 50 | 51 | log := NewLogger(1000) 52 | log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`) 53 | log.Info("info") 54 | 55 | 56 | ## Smtp adapter 57 | 58 | Configure like this: 59 | 60 | log := NewLogger(10000) 61 | log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`) 62 | log.Critical("sendmail critical") 63 | time.Sleep(time.Second * 30) 64 | -------------------------------------------------------------------------------- /api/dubbo_api/server/dubbo_server_api.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "infer-microservices/internal/logs" 6 | dubbo_service "infer-microservices/pkg/infer_services/dubbo_service" 7 | "infer-microservices/pkg/infer_services/io" 8 | "time" 9 | 10 | "dubbo.apache.org/dubbo-go/v3/config" 11 | hessian "github.com/apache/dubbo-go-hessian2" 12 | ) 13 | 14 | var ( 15 | DubboServiceApiClient = &DubboServiceApi{} 16 | ) 17 | 18 | type DubboServiceApi struct { 19 | dubboConfFile string 20 | dubboService *dubbo_service.DubboService 21 | // define service func name. 22 | RecommenderInfer func(ctx context.Context, req *io.RecRequest) (*io.RecResponse, error) 23 | } 24 | 25 | func init() { 26 | //regisger dubbo service. 27 | //INFO: both input and output need to register. 28 | hessian.RegisterPOJO(&io.RecRequest{}) 29 | hessian.RegisterPOJO(&io.RecResponse{}) 30 | config.SetConsumerService(DubboServiceApiClient) 31 | } 32 | 33 | func (s *DubboServiceApi) SetDubboConfFile(dubboConfFile string) { 34 | s.dubboConfFile = dubboConfFile 35 | } 36 | 37 | func (s *DubboServiceApi) SetDubboService(dubboService *dubbo_service.DubboService) { 38 | s.dubboService = dubboService 39 | } 40 | 41 | // @implement start infertace 42 | func (s *DubboServiceApi) ServiceStart() { 43 | config.SetProviderService(s) 44 | logs.Debug(time.Now(), "DubboServiceApi", s) 45 | if err := config.Load(config.WithPath(s.dubboConfFile)); err != nil { 46 | panic(err) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /internal/flags/flag_skywalking.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagsSkywalkingInstance *flagsSkywalking 4 | 5 | type flagsSkywalking struct { 6 | skywalkingWhetheropen *bool 7 | skywalkingServername *string 8 | skywalkingIp *string 9 | skywalkingPort *int 10 | } 11 | 12 | // singleton instance 13 | func init() { 14 | flagsSkywalkingInstance = new(flagsSkywalking) 15 | } 16 | 17 | func getFlagsSkywalkingInstance() *flagsSkywalking { 18 | return flagsSkywalkingInstance 19 | } 20 | 21 | // skywalking_whetheropen 22 | func (s *flagsSkywalking) setSkywalkingWhetheropen(skywalkingWhetheropen *bool) { 23 | s.skywalkingWhetheropen = skywalkingWhetheropen 24 | } 25 | 26 | func (s *flagsSkywalking) GetSkywalkingWhetheropen() *bool { 27 | return s.skywalkingWhetheropen 28 | } 29 | 30 | // skywalking_servername 31 | func (s *flagsSkywalking) setSkywalkingServername(skywalkingServername *string) { 32 | s.skywalkingServername = skywalkingServername 33 | } 34 | 35 | func (s *flagsSkywalking) GetSkywalkingServername() *string { 36 | return s.skywalkingServername 37 | } 38 | 39 | // skywalking_ip 40 | func (s *flagsSkywalking) setSkywalkingIp(skywalkingIp *string) { 41 | s.skywalkingIp = skywalkingIp 42 | } 43 | 44 | func (s *flagsSkywalking) GetSkywalkingIp() *string { 45 | return s.skywalkingIp 46 | } 47 | 48 | // skywalking_port 49 | func (s *flagsSkywalking) setSkywalkingPort(skywalkingPort *int) { 50 | s.skywalkingPort = skywalkingPort 51 | } 52 | 53 | func (s *flagsSkywalking) GetSkywalkingPort() *int { 54 | return s.skywalkingPort 55 | } 56 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/tensor_slice.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffer representing slices of a tensor 2 | 3 | syntax = "proto3"; 4 | 5 | package tensorflow; 6 | 7 | option cc_enable_arenas = true; 8 | option java_outer_classname = "TensorSliceProtos"; 9 | option java_multiple_files = true; 10 | option java_package = "org.tensorflow.framework"; 11 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_slice_go_proto"; 12 | 13 | // Can only be interpreted if you know the corresponding TensorShape. 14 | message TensorSliceProto { 15 | // Extent of the slice in one dimension. 16 | message Extent { 17 | // Either both or no attributes must be set. When no attribute is set 18 | // means: All data in that dimension. 19 | 20 | // Start index of the slice, starting at 0. 21 | int64 start = 1; 22 | 23 | // Length of the slice: if the length is missing or -1 we will 24 | // interpret this as "everything in this dimension". We use 25 | // "oneof" to preserve information about whether the length is 26 | // present without changing the serialization format from the 27 | // prior proto2 version of this proto. 28 | oneof has_length { 29 | int64 length = 2; 30 | } 31 | } 32 | 33 | // Extent of the slice in all tensor dimensions. 34 | // 35 | // Must have one entry for each of the dimension of the tensor that this 36 | // slice belongs to. The order of sizes is the same as the order of 37 | // dimensions in the TensorShape. 38 | repeated Extent extent = 1; 39 | } 40 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/resource_handle.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/tensor_shape.proto"; 6 | import "tensorflow/core/framework/types.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | option java_outer_classname = "ResourceHandle"; 10 | option java_multiple_files = true; 11 | option java_package = "org.tensorflow.framework"; 12 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/resource_handle_go_proto"; 13 | 14 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 15 | // not valid across executions, but can be serialized back and forth from within 16 | // a single run. 17 | message ResourceHandleProto { 18 | // Unique name for the device containing the resource. 19 | string device = 1; 20 | 21 | // Container in which this resource is placed. 22 | string container = 2; 23 | 24 | // Unique name of this resource. 25 | string name = 3; 26 | 27 | // Hash code for the type of the resource. Is only valid in the same device 28 | // and in the same execution. 29 | uint64 hash_code = 4; 30 | 31 | // For debug-only, the name of the type pointed to by this handle, if 32 | // available. 33 | string maybe_type_name = 5; 34 | 35 | // Protocol buffer representing a pair of (data type, tensor shape). 36 | message DtypeAndShape { 37 | DataType dtype = 1; 38 | TensorShapeProto shape = 2; 39 | } 40 | 41 | // Data types and shapes for the underlying resource. 42 | repeated DtypeAndShape dtypes_and_shapes = 6; 43 | 44 | reserved 7; 45 | } 46 | -------------------------------------------------------------------------------- /internal/tfserving_gogofaster/proto/predict.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.serving; 4 | 5 | import "tensorflow/core/framework/tensor.proto"; 6 | import "tensorflow_serving/apis/model.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | 10 | // PredictRequest specifies which TensorFlow model to run, as well as 11 | // how inputs are mapped to tensors and how outputs are filtered before 12 | // returning to user. 13 | message PredictRequest { 14 | // Model Specification. If version is not specified, will use the latest 15 | // (numerical) version. 16 | ModelSpec model_spec = 1; 17 | 18 | // Input tensors. 19 | // Names of input tensor are alias names. The mapping from aliases to real 20 | // input tensor names is stored in the SavedModel export as a prediction 21 | // SignatureDef under the 'inputs' field. 22 | map inputs = 2; 23 | 24 | // Output filter. 25 | // Names specified are alias names. The mapping from aliases to real output 26 | // tensor names is stored in the SavedModel export as a prediction 27 | // SignatureDef under the 'outputs' field. 28 | // Only tensors specified here will be run/fetched and returned, with the 29 | // exception that when none is specified, all tensors specified in the 30 | // named signature will be run/fetched and returned. 31 | repeated string output_filter = 3; 32 | 33 | // Reserved field 4. 34 | reserved 4; 35 | } 36 | 37 | // Response for PredictRequest on successful run. 38 | message PredictResponse { 39 | // Effective Model Specification used to process PredictRequest. 40 | ModelSpec model_spec = 2; 41 | 42 | // Output tensors. 43 | map outputs = 1; 44 | } -------------------------------------------------------------------------------- /pkg/ann/faiss/faiss_index_search.go: -------------------------------------------------------------------------------- 1 | package faiss 2 | 3 | import ( 4 | "context" 5 | feature "infer-microservices/pkg/infer_features" 6 | 7 | faiss_index "infer-microservices/internal/faiss_gogofaster" 8 | "infer-microservices/internal/flags" 9 | "infer-microservices/pkg/config_loader/faiss_config" 10 | "time" 11 | ) 12 | 13 | var grpcTimeout int64 14 | 15 | func init() { 16 | flagFactory := flags.FlagFactory{} 17 | flagTensorflow := flagFactory.CreateFlagTensorflow() 18 | grpcTimeout = *flagTensorflow.GetTfservingTimeoutMs() 19 | } 20 | func FaissVectorSearch(f *faiss_config.FaissIndexConfig, example feature.ExampleFeatures, vector []float32) ([]*faiss_index.ItemInfo, error) { 21 | 22 | faissIndexs := f.GetFaissIndexs() 23 | faissGrpcConn, err := f.GetFaissGrpcPool().Get() 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | defer f.GetFaissGrpcPool().Put(faissGrpcConn) 29 | 30 | faissClient := faiss_index.NewGrpcRecallServerServiceClient(faissGrpcConn) 31 | vector_info := faiss_index.UserVectorInfo{ 32 | UserVector: vector, 33 | } 34 | 35 | index_conf_tmp := &faiss_index.RecallRequest{ 36 | IndexName: faissIndexs.IndexName, 37 | UserVectorInfo_: &vector_info, 38 | RecallNum: faissIndexs.RecallNum, 39 | } 40 | 41 | if len(*example.UserExampleFeatures.Buff) == 0 || len(*example.UserContextExampleFeatures.Buff) == 0 { 42 | return make([]*faiss_index.ItemInfo, 0), nil 43 | } 44 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(grpcTimeout)*time.Millisecond) 45 | defer cancel() 46 | 47 | rst, err := faissClient.GrpcRecall(ctx, index_conf_tmp) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | return rst.ItemInfo_, nil 53 | } 54 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/kernel_def.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/attr_value.proto"; 6 | 7 | option cc_enable_arenas = true; 8 | option java_outer_classname = "KernelDefProtos"; 9 | option java_multiple_files = true; 10 | option java_package = "org.tensorflow.framework"; 11 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/kernel_def_go_proto"; 12 | 13 | message KernelDef { 14 | // Must match the name of an Op. 15 | string op = 1; 16 | 17 | // Type of device this kernel runs on. 18 | string device_type = 2; 19 | 20 | message AttrConstraint { 21 | // Name of an attr from the Op. 22 | string name = 1; 23 | 24 | // A list of values that this kernel supports for this attr. 25 | // Like OpDef.AttrDef.allowed_values, except for kernels instead of Ops. 26 | AttrValue allowed_values = 2; 27 | } 28 | repeated AttrConstraint constraint = 3; 29 | 30 | // Names of the Op's input_/output_args that reside in host memory 31 | // instead of device memory. 32 | repeated string host_memory_arg = 4; 33 | 34 | // This allows experimental kernels to be registered for an op that 35 | // won't be used unless the user specifies a "_kernel" attr with 36 | // value matching this. 37 | string label = 5; 38 | 39 | // Prioritization of kernel amongst different devices. By default we assume 40 | // priority is 0. The higher the priority the better. By default (i.e. if 41 | // this is not set), we prefer GPU kernels over CPU. 42 | int32 priority = 6; 43 | } 44 | 45 | // A collection of KernelDefs 46 | message KernelList { 47 | repeated KernelDef kernel = 1; 48 | } 49 | -------------------------------------------------------------------------------- /pkg/config_loader/service_config_builder.go: -------------------------------------------------------------------------------- 1 | package service_config_loader 2 | 3 | type ServiceConfigBuilder struct { 4 | serviceConfig ServiceConfig 5 | } 6 | 7 | // serviceConfig 8 | func (b *ServiceConfigBuilder) SetServiceConfig(serviceConfig ServiceConfig) { 9 | b.serviceConfig = serviceConfig 10 | } 11 | 12 | func (b *ServiceConfigBuilder) GetServiceConfig() ServiceConfig { 13 | return b.serviceConfig 14 | } 15 | 16 | // redis builder 17 | func (b *ServiceConfigBuilder) RedisConfigBuilder(dataId string, redisConfStr string) *ServiceConfigBuilder { 18 | configFactory := &ConfigFactory{} 19 | redisConfig := configFactory.createRedisConfig(dataId, redisConfStr) 20 | b.serviceConfig.setRedisConfig(*redisConfig) 21 | 22 | return b 23 | } 24 | 25 | // faiss builder 26 | func (b *ServiceConfigBuilder) FaissConfigBuilder(dataId string, indexConfStr string) *ServiceConfigBuilder { 27 | configFactory := &ConfigFactory{} 28 | faissConfigs := configFactory.createFaissConfig(dataId, indexConfStr) 29 | b.serviceConfig.SetFaissIndexConfigs(*faissConfigs) 30 | 31 | return b 32 | } 33 | 34 | // model builder 35 | func (b *ServiceConfigBuilder) ModelConfigBuilder(dataId string, modelConfStr string) *ServiceConfigBuilder { 36 | configFactory := &ConfigFactory{} 37 | modelConfigs := configFactory.createModelConfig(dataId, modelConfStr) 38 | b.serviceConfig.setModelsConfig(*modelConfigs) 39 | 40 | return b 41 | } 42 | 43 | // pipeline builder 44 | func (b *ServiceConfigBuilder) PipelineConfigBuilder(dataId string, pipelineConfStr string) *ServiceConfigBuilder { 45 | configFactory := &ConfigFactory{} 46 | PipelineConfig := configFactory.createPipelineConfig(dataId, pipelineConfStr) 47 | b.serviceConfig.setPipelineConfig(*PipelineConfig) 48 | 49 | return b 50 | } 51 | -------------------------------------------------------------------------------- /pkg/config_loader/service_config_director.go: -------------------------------------------------------------------------------- 1 | package service_config_loader 2 | 3 | import ( 4 | "infer-microservices/internal/logs" 5 | "time" 6 | 7 | validator "github.com/go-playground/validator/v10" 8 | ) 9 | 10 | type ServiceConfigDirector struct { 11 | configBuilder ServiceConfigBuilder 12 | } 13 | 14 | // set 15 | func (s *ServiceConfigDirector) SetConfigBuilder(builder ServiceConfigBuilder) { 16 | s.configBuilder = builder 17 | } 18 | 19 | // build contain index 20 | func (s *ServiceConfigDirector) ServiceConfigUpdateContainIndexDirector(dataId string, 21 | redisConfStr string, modelConfStr string, indexConfStr string) ServiceConfig { 22 | builder := s.configBuilder.RedisConfigBuilder(dataId, redisConfStr).FaissConfigBuilder(dataId, indexConfStr).ModelConfigBuilder(dataId, modelConfStr) 23 | serviceConfig := builder.GetServiceConfig() 24 | serviceConfig.setServiceId(dataId) 25 | 26 | //validete serviceConfig 27 | validate := validator.New() 28 | err := validate.Struct(serviceConfig) 29 | if err != nil { 30 | logs.Error(dataId, time.Now(), err) 31 | return ServiceConfig{} 32 | } 33 | 34 | return serviceConfig 35 | } 36 | 37 | // build not contain index 38 | func (s *ServiceConfigDirector) ServiceConfigUpdaterNotContainIndexDirector(dataId string, 39 | redisConfStr string, modelConfStr string) ServiceConfig { 40 | builder := s.configBuilder.RedisConfigBuilder(dataId, redisConfStr).ModelConfigBuilder(dataId, modelConfStr) 41 | serviceConfig := builder.GetServiceConfig() 42 | serviceConfig.setServiceId(dataId) 43 | 44 | //validete serviceConfig 45 | validate := validator.New() 46 | err := validate.Struct(serviceConfig) 47 | if err != nil { 48 | logs.Error(dataId, time.Now(), err) 49 | return ServiceConfig{} 50 | } 51 | 52 | return serviceConfig 53 | } 54 | -------------------------------------------------------------------------------- /pkg/config_loader/model_config/model_config_loader_test.go: -------------------------------------------------------------------------------- 1 | package model_config 2 | 3 | import ( 4 | "infer-microservices/pkg/config_loader/redis_config" 5 | "testing" 6 | ) 7 | 8 | func TestModelConfigLoader(t *testing.T) { 9 | 10 | testUserId := "u1111" 11 | testItemId := "i7001" 12 | 13 | redisTestStr := ` 14 | { 15 | "redis_conf": { 16 | "redisCluster": { 17 | "addrs": [], 18 | "password": "", 19 | "idleTimeoutMs": 100, 20 | "readTimeoutMs": 100, 21 | "writeTimeoutMs":100, 22 | "dialTimeoutS":600, 23 | "maxRetries":2, 24 | "minIdleConns":50 25 | } 26 | } 27 | } 28 | ` 29 | 30 | modelTestStr := ` 31 | { 32 | "model_conf": { 33 | "model-001": { 34 | "fieldsSpec": [{},{},{}], 35 | "tfservingGrpcAddr": { 36 | "tfservingModelName": "models", 37 | "addrs": [], 38 | "pool_size": 50, 39 | "initCap":10, 40 | "idleTimeoutMs": 100, 41 | "readTimeoutMs": 100, 42 | "writeTimeoutMs":100, 43 | "dialTimeoutS":600 44 | }, 45 | "user_feature_rediskey_pre": "", 46 | "item_feature_rediskey_pre": "" 47 | } 48 | }, 49 | } 50 | ` 51 | redisConf := redis_config.RedisConfig{} 52 | redisConf.ConfigLoad("testId", redisTestStr) 53 | 54 | modelConf := ModelConfig{} 55 | modelConf.ConfigLoad("testId", modelTestStr) 56 | t.Log("modelConf:", modelConf) 57 | 58 | //test user features 59 | rst, err := redisConf.GetRedisPool().HGetAll(modelConf.userRedisKeyPreOffline + testUserId) 60 | if err != nil { 61 | t.Errorf("redis init failed") 62 | } 63 | t.Log("user features:", rst) 64 | 65 | //test item features 66 | rst, err = redisConf.GetRedisPool().HGetAll(modelConf.itemRedisKeyPre + testItemId) 67 | if err != nil { 68 | t.Errorf("redis init failed") 69 | } 70 | t.Log("item features:", rst) 71 | } 72 | -------------------------------------------------------------------------------- /cmd/infer/main/infer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "infer-microservices/api" 6 | "infer-microservices/internal/logs" 7 | "infer-microservices/pkg/infer_samples" 8 | "time" 9 | ) 10 | 11 | var apiFactory api.ApiFactory 12 | 13 | func init() { 14 | apiFactory = api.ApiFactory{} 15 | } 16 | 17 | // reset bloomfilter 18 | func resetBloom() { 19 | //TODO: Cuckoo Filter better 20 | var ticker *time.Ticker = time.NewTicker(7 * 24 * time.Hour) //every 7 days clean bloom filter 21 | defer ticker.Stop() 22 | 23 | for t := range ticker.C { 24 | logs.Info("Start to reset bloom filter.", t) 25 | infer_samples.BloomClean(infer_samples.GetUserBloomFilterInstance()) 26 | infer_samples.BloomClean(infer_samples.GetItemBloomFilterInstance()) 27 | } 28 | } 29 | 30 | // start service 31 | func serviceStart(serviceApi api.ServiceStartInterface) { 32 | logs.Info("starting dubbo servivce...") 33 | serviceApi.ServiceStart() 34 | logs.Info("successed start dubbo servivce.") 35 | } 36 | 37 | func main() { 38 | //init 39 | flag.Parse() 40 | logs.InitLog() 41 | 42 | //watch and reset bloom fliter 43 | go infer_samples.WatchBloomConfig() //0 o'clock start service and load all users and all items into bloom filter. 44 | go resetBloom() //0 o'clock clean bloom filter, every 7 days. 45 | 46 | //start services. 47 | dubboServiceApi := apiFactory.CreateDubboServiceApi() 48 | grpcServiceApi := apiFactory.CreateGrpcServiceApi() 49 | //restServiceHttpApi := apiFactory.CreateRestServiceHttpApi() // @deprecated 50 | restServiceEchoApi := apiFactory.CreateRestServiceEchoApi() 51 | 52 | go serviceStart(dubboServiceApi) 53 | go serviceStart(grpcServiceApi) 54 | //go serviceStart(restServiceApi) // @deprecated 55 | go serviceStart(restServiceEchoApi) 56 | 57 | select {} 58 | } 59 | -------------------------------------------------------------------------------- /scripts/deployments/elkb/logstash.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | wget https://artifacts.elastic.co/downloads/logstash/logstash-5.4.3.tar.gz 4 | tar -zxvf logstash-5.4.3.tar.gz 5 | 6 | 7 | vi logstash-5.4.3/client.conf 8 | input { # input config, use filebeat 9 | beats { 10 | port => 5044 #filebeat and logstash deploy at the same servers 11 | codec => "json" 12 | } 13 | } 14 | 15 | #multi log files and multi es index 16 | filter{ 17 | if "infer-info-1" in [tags] { 18 | grok { 19 | match => { 20 | "message"=>"" #filter depends on bussiness 21 | } 22 | } 23 | } 24 | 25 | if "infer-info-2" in [tags] { 26 | grok { 27 | match=>{ 28 | "message"=>"" #filter depends on bussiness 29 | } 30 | } 31 | } 32 | 33 | mutate { #remove 34 | remove_field =>["message"] 35 | remove_field =>["beat"] 36 | remove_field =>["host"] 37 | #remove_field =>["fields"] 38 | remove_field =>["input"] 39 | remove_field =>["prospector"] 40 | } 41 | } 42 | 43 | output{ 44 | if "infer-info-1" in [tags] { 45 | elasticsearch { 46 | hosts => ["http://x.x.x.130:9200"] 47 | index => "infer-info-1-%{+yyyy.MM.dd}" 48 | #user => "elastic" 49 | #password => "changeme" 50 | } 51 | stdout {codec => rubydebug} 52 | } 53 | if "infer-info-1" in [tags] { 54 | elasticsearch { 55 | hosts => ["http://x.x.x.130:9200"] 56 | index => "infer-info-2-%{+yyyy.MM.dd}" 57 | #user => "elastic" 58 | #password => "changeme" 59 | } 60 | stdout {codec => rubydebug} 61 | } 62 | } 63 | 64 | 65 | 66 | bin/logstash -f client.conf -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/dataset.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.data; 4 | 5 | import "tensorflow/core/framework/tensor.proto"; 6 | import "tensorflow/core/framework/tensor_shape.proto"; 7 | import "tensorflow/core/framework/types.proto"; 8 | 9 | option cc_enable_arenas = true; 10 | 11 | // This file contains protocol buffers for working with tf.data Datasets. 12 | 13 | // Metadata describing a compressed component of a dataset element. 14 | message CompressedComponentMetadata { 15 | // The dtype of the component tensor. 16 | .tensorflow.DataType dtype = 1; 17 | 18 | // The shape of the component tensor. 19 | .tensorflow.TensorShapeProto tensor_shape = 2; 20 | 21 | // The amount of uncompressed tensor data. 22 | // - For string tensors, there is an element for each string indicating the 23 | // size of the string. 24 | // - For all other tensors, there is a single element indicating the size of 25 | // the tensor. 26 | repeated uint64 uncompressed_bytes = 4; 27 | 28 | reserved 3; 29 | } 30 | 31 | message CompressedElement { 32 | // Compressed tensor bytes for all components of the element. 33 | bytes data = 1; 34 | // Metadata for the components of the element. 35 | repeated CompressedComponentMetadata component_metadata = 2; 36 | // Version of the CompressedElement. CompressedElements may be stored on disk 37 | // and read back by later versions of code, so we store a version number to 38 | // help readers understand which version they are reading. When you add a new 39 | // field to this proto, you need to increment kCompressedElementVersion in 40 | // tensorflow/core/data/compression_utils.cc. 41 | int32 version = 3; 42 | } 43 | 44 | // An uncompressed dataset element. 45 | message UncompressedElement { 46 | repeated TensorProto components = 1; 47 | } 48 | -------------------------------------------------------------------------------- /pkg/infer_services/io/request.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | import ( 4 | "errors" 5 | "infer-microservices/internal/logs" 6 | ) 7 | 8 | type RecRequest struct { 9 | userId string 10 | dataId string //nacos dataid 11 | groupId string //nacos groupId 12 | namespaceId string //nacos namespaceId 13 | exposureList []string // filter the exposed items recently 14 | } 15 | 16 | // userId 17 | func (r *RecRequest) SetUserId(userId string) { 18 | r.userId = userId 19 | } 20 | 21 | func (r *RecRequest) GetUserId() string { 22 | return r.userId 23 | } 24 | 25 | // dataId 26 | func (r *RecRequest) SetDataId(dataId string) { 27 | r.dataId = dataId 28 | } 29 | 30 | func (r *RecRequest) GetDataId() string { 31 | return r.dataId 32 | } 33 | 34 | // groupId 35 | func (r *RecRequest) SetGroupId(groupId string) { 36 | r.groupId = groupId 37 | } 38 | 39 | func (r *RecRequest) GetGroupId() string { 40 | return r.groupId 41 | } 42 | 43 | // namespaceId 44 | func (r *RecRequest) SetNamespaceId(namespaceId string) { 45 | r.namespaceId = namespaceId 46 | } 47 | 48 | func (r *RecRequest) GetNamespaceId() string { 49 | return r.namespaceId 50 | } 51 | 52 | // exposureList 53 | func (r *RecRequest) SetExposureList(exposureList []string) { 54 | r.exposureList = exposureList 55 | } 56 | 57 | func (r *RecRequest) GetExposureList() []string { 58 | return r.exposureList 59 | } 60 | 61 | // JavaClassName 62 | func (r *RecRequest) JavaClassName() string { 63 | return "com.loki.www.infer.RecRequest" 64 | } 65 | 66 | func (r *RecRequest) Check() bool { 67 | //check dataid 68 | if r.dataId == "" { 69 | err := errors.New("dataid can not be empty") 70 | logs.Error(err) 71 | return false 72 | } 73 | 74 | //check userid 75 | if r.userId == "" { 76 | err := errors.New("userid can not be empty") 77 | logs.Error(err) 78 | return false 79 | } 80 | 81 | return true 82 | } 83 | -------------------------------------------------------------------------------- /Dockerfile/faiss/faiss_docker_file: -------------------------------------------------------------------------------- 1 | ############################### 2 | # 3 | #build faiss docker mir 4 | # 5 | ############################### 6 | 7 | FROM python:3.8-bullseye 8 | RUN sed -i -E "s/\w+.debian.org/mirrors.tuna.tsinghua.edu.cn/g" /etc/apt/sources.list 9 | 10 | # install miniconda 11 | ARG CONDA_MIRROR="https://mirrors.tuna.tsinghua.edu.cn/anaconda" 12 | ENV CONDA_MIRROR=${CONDA_MIRROR} 13 | ENV CONDA_SRC="https://repo.anaconda.com/miniconda" 14 | RUN CONDA_SRC="${CONDA_MIRROR}/miniconda"; \ 15 | curl -fsSL -v -o ~/miniconda.sh -O "$CONDA_SRC/Miniconda3-latest-Linux-x86_64.sh" && \ 16 | chmod +x ~/miniconda.sh && \ 17 | ~/miniconda.sh -b -p ~/miniconda && \ 18 | rm ~/miniconda.sh && \ 19 | echo "channels:" > $HOME/.condarc && \ 20 | echo " - ${CONDA_MIRROR}/pkgs/free/" >> $HOME/.condarc && \ 21 | echo " - ${CONDA_MIRROR}/pkgs/main/" >> $HOME/.condarc && \ 22 | echo " - ${CONDA_MIRROR}/cloud/pytorch/" >> $HOME/.condarc && \ 23 | echo " - defaults" >> $HOME/.condarc && \ 24 | echo "show_channel_urls: true" >> $HOME/.condarc; 25 | SHELL ["/bin/bash", "-c"] 26 | ENV PATH="~/miniconda/bin:${PATH}" 27 | ARG PATH="~/miniconda/bin:${PATH}" 28 | 29 | # install faiss 30 | ARG PYTHON_VERSION=3.8 31 | RUN conda install -y pytorch python=${PYTHON_VERSION} faiss-cpu pandas && \ 32 | conda clean -ya && \ 33 | conda init bash 34 | 35 | # install other 36 | RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ 37 | pip install sentence_transformers 38 | 39 | 40 | WORKDIR /data/faiss 41 | 42 | #copy start scripts 43 | COPY /data/loki/the-infer/scripts/deployments/faiss/faiss_start.sh /data/faiss 44 | 45 | # gRPC 46 | EXPOSE 5000 47 | 48 | # Set where models should be stored in the container 49 | ENV MODEL_BASE_PATH=/data/faiss 50 | RUN mkdir -p ${MODEL_BASE_PATH} 51 | 52 | ENTRYPOINT ["/data/faiss/faiss_start.sh"] 53 | 54 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/tensor_shape.proto: -------------------------------------------------------------------------------- 1 | // Protocol buffer representing the shape of tensors. 2 | 3 | syntax = "proto3"; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "TensorShapeProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_shape_go_proto"; 9 | 10 | package tensorflow; 11 | 12 | // Dimensions of a tensor. 13 | message TensorShapeProto { 14 | // One dimension of the tensor. 15 | message Dim { 16 | // Size of the tensor in that dimension. 17 | // This value must be >= -1, but values of -1 are reserved for "unknown" 18 | // shapes (values of -1 mean "unknown" dimension). Certain wrappers 19 | // that work with TensorShapeProto may fail at runtime when deserializing 20 | // a TensorShapeProto containing a dim value of -1. 21 | int64 size = 1; 22 | 23 | // Optional name of the tensor dimension. 24 | string name = 2; 25 | }; 26 | 27 | // Dimensions of the tensor, such as {"input", 30}, {"output", 40} 28 | // for a 30 x 40 2D tensor. If an entry has size -1, this 29 | // corresponds to a dimension of unknown size. The names are 30 | // optional. 31 | // 32 | // The order of entries in "dim" matters: It indicates the layout of the 33 | // values in the tensor in-memory representation. 34 | // 35 | // The first entry in "dim" is the outermost dimension used to layout the 36 | // values, the last entry is the innermost dimension. This matches the 37 | // in-memory layout of RowMajor Eigen tensors. 38 | // 39 | // If "dim.size()" > 0, "unknown_rank" must be false. 40 | repeated Dim dim = 2; 41 | 42 | // If true, the number of dimensions in the shape is unknown. 43 | // 44 | // If true, "dim.size()" must be 0. 45 | bool unknown_rank = 3; 46 | }; 47 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/device_attributes.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "DeviceAttributesProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/device_attributes_go_proto"; 10 | 11 | message InterconnectLink { 12 | int32 device_id = 1; 13 | string type = 2; 14 | int32 strength = 3; 15 | } 16 | 17 | message LocalLinks { 18 | repeated InterconnectLink link = 1; 19 | } 20 | 21 | message DeviceLocality { 22 | // Optional bus locality of device. Default value of 0 means 23 | // no specific locality. Specific localities are indexed from 1. 24 | int32 bus_id = 1; 25 | 26 | // Optional NUMA locality of device. 27 | int32 numa_node = 2; 28 | 29 | // Optional local interconnect links to other devices. 30 | LocalLinks links = 3; 31 | } 32 | 33 | message DeviceAttributes { 34 | // Fully specified name of the device within a cluster. 35 | string name = 1; 36 | 37 | // String representation of device_type. 38 | string device_type = 2; 39 | 40 | // Memory capacity of device in bytes. 41 | int64 memory_limit = 4; 42 | 43 | // Platform-specific data about device that may be useful 44 | // for supporting efficient data transfers. 45 | DeviceLocality locality = 5; 46 | 47 | // A device is assigned a global unique number each time it is 48 | // initialized. "incarnation" should never be 0. 49 | fixed64 incarnation = 6; 50 | 51 | // String representation of the physical device that this device maps to. 52 | string physical_device_desc = 7; 53 | 54 | // A physical device ID for use in XLA DeviceAssignments, unique across 55 | // clients in a multi-client setup. Set to -1 if unavailable, non-negative 56 | // otherwise. 57 | int64 xla_global_id = 8; 58 | } 59 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/optimized_function_graph.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/graph.proto"; 6 | import "tensorflow/core/framework/types.proto"; 7 | 8 | // Optimized function graph after instantiation-related graph optimization 9 | // passes (up till before graph partitioning). The first half of the proto is 10 | // representing a GraphDef and the rest of the fields are extra information from 11 | // graph optimizations. 12 | message OptimizedFunctionGraph { 13 | // Function name. It can be a human-readable SignatureDef's method name, or a 14 | // FunctionDef name. 15 | string name = 1; 16 | // Optimized function graph. 17 | GraphDef function_graph = 2; 18 | // Maps from node name to control ret. This is an output from running TF/XLA 19 | // bridge. 20 | map node_name_to_control_ret = 3; 21 | // Return node types of the function. This is an output of graph 22 | // preprocessing. 23 | repeated DataType ret_types = 4; 24 | // Number of return nodes. This is an output of graph preprocessing. 25 | uint32 num_return_nodes = 5; 26 | 27 | reserved 6; 28 | 29 | // Enum for distinguishing the origin where the proto is created. 30 | // 31 | // AOT: proto is created in ahead-of-time environment, which can be different 32 | // from the environment where the graph is actually executed. 33 | // 34 | // JIT: proto is created in just-in-time execution, which has the same 35 | // environment as the one the graph is actually executed. 36 | enum OptimizationSource { 37 | SOURCE_UNSPECIFIED = 0; 38 | AOT = 1; 39 | JIT = 2; 40 | } 41 | 42 | // Indicates the source environment where this proto is generated. 43 | optional OptimizationSource source = 7; 44 | 45 | // Time (in microseconds) spent on running the graph optimization passes for 46 | // this function. 47 | optional uint64 optimization_time_usecs = 8; 48 | } 49 | -------------------------------------------------------------------------------- /pkg/infer_samples/bloom_filter_warmup.go: -------------------------------------------------------------------------------- 1 | package infer_samples 2 | 3 | import ( 4 | "infer-microservices/internal/flags" 5 | "infer-microservices/internal/logs" 6 | 7 | "strings" 8 | 9 | "github.com/fsnotify/fsnotify" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | var viperConfig *viper.Viper 14 | 15 | // var baseModelObserver Observer 16 | var inferSampleSubject *SampleSubject 17 | 18 | func init() { 19 | flagFactory := flags.FlagFactory{} 20 | flagViper := flagFactory.CreateFlagViper() 21 | 22 | viperConfig = viper.New() 23 | viperConfig.SetConfigName(*flagViper.GetConfigName()) 24 | viperConfig.SetConfigType(*flagViper.GetConfigType()) 25 | viperConfig.AddConfigPath(*flagViper.GetConfigPath()) 26 | 27 | // //callback func config 28 | // basemodel0 := BaseModel{} 29 | // SampleCallBackFuncMap["recall"] = basemodel0.GetInferExampleFeaturesNotContainItems 30 | // SampleCallBackFuncMap["rank"] = basemodel0.GetInferExampleFeaturesContainItems 31 | } 32 | 33 | // TODO:listen kafka,not file. 34 | func loadViperConfigFile() { 35 | err := viperConfig.ReadInConfig() 36 | if err != nil { 37 | logs.Error(err) 38 | } 39 | userIdStr := viperConfig.GetString("userIdList") // "user1,user2,user3" 40 | itemIdStr := viperConfig.GetString("itemIdList") //"item1,item2,item3" 41 | 42 | userIdList := strings.Split(userIdStr, ",") 43 | itemIdList := strings.Split(itemIdStr, ",") 44 | 45 | for _, userId := range userIdList { 46 | BloomPush(GetUserBloomFilterInstance(), userId) 47 | } 48 | 49 | for _, itemId := range itemIdList { 50 | BloomPush(GetUserBloomFilterInstance(), itemId) 51 | } 52 | 53 | // //update bloom filter 54 | inferSampleSubject.NotifyObservers() 55 | } 56 | 57 | func WatchBloomConfig() { 58 | viperConfig.WatchConfig() 59 | viperConfig.OnConfigChange(func(e fsnotify.Event) { 60 | logs.Info("Config file changed:", e.Name) //the file only contains new users and new items in past 1 hour. 61 | loadViperConfigFile() 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/config_loader/config_factory.go: -------------------------------------------------------------------------------- 1 | package service_config_loader 2 | 3 | import ( 4 | "infer-microservices/internal/utils" 5 | "infer-microservices/pkg/config_loader/faiss_config" 6 | "infer-microservices/pkg/config_loader/model_config" 7 | "infer-microservices/pkg/config_loader/pipeline_config" 8 | "infer-microservices/pkg/config_loader/redis_config" 9 | ) 10 | 11 | // config interface 12 | type ConfigLoadInterface interface { 13 | ConfigLoad(domain string, dataId string, confStr string) error 14 | } 15 | 16 | type ConfigFactory struct { 17 | } 18 | 19 | // faiss config factory 20 | func (f *ConfigFactory) createFaissConfig(dataId string, indexConfStr string) *faiss_config.FaissIndexConfigs { 21 | faissConfigs := new(faiss_config.FaissIndexConfigs) 22 | faissConfigs.ConfigLoad(dataId, indexConfStr) 23 | 24 | return faissConfigs 25 | } 26 | 27 | // model config factory 28 | func (m *ConfigFactory) createModelConfig(dataId string, modelsConfStr string) *map[string]model_config.ModelConfig { 29 | modelsConfig := make(map[string]model_config.ModelConfig, 0) 30 | dataConf := utils.ConvertJsonToStruct(modelsConfStr) 31 | for modleIndex, modelConfStr := range dataConf { // multi models 32 | modelConfig := new(model_config.ModelConfig) 33 | modelConfig.ConfigLoad(dataId, modelConfStr.(string)) 34 | modelsConfig[modleIndex] = *modelConfig 35 | } 36 | 37 | return &modelsConfig 38 | } 39 | 40 | // redis config factory 41 | func (r *ConfigFactory) createRedisConfig(dataId string, redisConfStr string) *redis_config.RedisConfig { 42 | redisConfig := new(redis_config.RedisConfig) 43 | redisConfig.ConfigLoad(dataId, redisConfStr) 44 | 45 | return redisConfig 46 | } 47 | 48 | // pipeline config factory 49 | func (r *ConfigFactory) createPipelineConfig(dataId string, pipelineConfStr string) *pipeline_config.PipelineConfig { 50 | pipelineConfig := new(pipeline_config.PipelineConfig) 51 | pipelineConfig.ConfigLoad(dataId, pipelineConfStr) 52 | 53 | return pipelineConfig 54 | } 55 | -------------------------------------------------------------------------------- /internal/hystrix.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "infer-microservices/internal/flags" 5 | 6 | "github.com/afex/hystrix-go/hystrix" 7 | ) 8 | 9 | var hystrixTimeout int 10 | var hystrixMaxConcurrentRequests int 11 | var hystrixRequestVolumeThreshold int 12 | var hystrixSleepWindow int 13 | var hystrixErrorPercentThreshold int 14 | 15 | func init() { 16 | flagFactory := flags.FlagFactory{} 17 | flagHystrix := flagFactory.CreateFlagHystrix() 18 | 19 | hystrixTimeout = *flagHystrix.GetHystrixTimeoutMs() 20 | hystrixMaxConcurrentRequests = *flagHystrix.GetHystrixMaxConcurrentRequests() 21 | hystrixRequestVolumeThreshold = *flagHystrix.GetHystrixErrorPercentThreshold() 22 | hystrixSleepWindow = *flagHystrix.GetHystrixSleepWindow() 23 | hystrixErrorPercentThreshold = *flagHystrix.GetHystrixErrorPercentThreshold() 24 | 25 | //all ConfigureCommands use the same conf, we can use different conf if we need. 26 | hystrix.ConfigureCommand("dubboServer", hystrix.CommandConfig{ 27 | Timeout: hystrixTimeout, 28 | MaxConcurrentRequests: hystrixMaxConcurrentRequests, 29 | RequestVolumeThreshold: hystrixRequestVolumeThreshold, 30 | SleepWindow: hystrixSleepWindow, 31 | ErrorPercentThreshold: hystrixErrorPercentThreshold, 32 | }) 33 | 34 | hystrix.ConfigureCommand("restServer", hystrix.CommandConfig{ 35 | Timeout: hystrixTimeout, 36 | MaxConcurrentRequests: hystrixMaxConcurrentRequests, 37 | RequestVolumeThreshold: hystrixRequestVolumeThreshold, 38 | SleepWindow: hystrixSleepWindow, 39 | ErrorPercentThreshold: hystrixErrorPercentThreshold, 40 | }) 41 | 42 | hystrix.ConfigureCommand("grpcServer", hystrix.CommandConfig{ 43 | Timeout: hystrixTimeout, 44 | MaxConcurrentRequests: hystrixMaxConcurrentRequests, 45 | RequestVolumeThreshold: hystrixRequestVolumeThreshold, 46 | SleepWindow: hystrixSleepWindow, 47 | ErrorPercentThreshold: hystrixErrorPercentThreshold, 48 | }) 49 | 50 | } 51 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/graph_debug_info.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "GraphDebugInfoProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf/for_core_protos_go_proto"; 10 | 11 | message GraphDebugInfo { 12 | // This represents a file/line location in the source code. 13 | message FileLineCol { 14 | // File name index, which can be used to retrieve the file name string from 15 | // `files`. The value should be between 0 and (len(files)-1) 16 | int32 file_index = 1; 17 | 18 | // Line number in the file. 19 | int32 line = 2; 20 | 21 | // Col number in the file line. 22 | int32 col = 3; 23 | 24 | // Name of function contains the file line. 25 | string func = 4; 26 | 27 | // Source code contained in this file line. 28 | string code = 5; 29 | } 30 | 31 | // This represents a stack trace which is a ordered list of `FileLineCol`. 32 | message StackTrace { 33 | // Each line in the stack trace. 34 | repeated FileLineCol file_line_cols = 1; 35 | } 36 | 37 | // This stores all the source code file names and can be indexed by the 38 | // `file_index`. 39 | repeated string files = 1; 40 | 41 | // This maps a node name to a stack trace in the source code. 42 | // The map key is a mangling of the containing function and op name with 43 | // syntax: 44 | // op.name '@' func_name 45 | // For ops in the top-level graph, the func_name is the empty string. 46 | // Note that op names are restricted to a small number of characters which 47 | // exclude '@', making it impossible to collide keys of this form. Function 48 | // names accept a much wider set of characters. 49 | // It would be preferable to avoid mangling and use a tuple key of (op.name, 50 | // func_name), but this is not supported with protocol buffers. 51 | map traces = 2; 52 | } 53 | -------------------------------------------------------------------------------- /internal/jwt/jwt_http.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "errors" 5 | "infer-microservices/internal/flags" 6 | "net/http" 7 | "time" 8 | 9 | jwt "github.com/golang-jwt/jwt/v4" 10 | ) 11 | 12 | var jwtKey []byte 13 | 14 | type Claims struct { 15 | Username string `json:"username"` 16 | jwt.StandardClaims 17 | } 18 | 19 | func init() { 20 | flagFactory := flags.FlagFactory{} 21 | flagJwt := flagFactory.CreateFlagJwt() 22 | jwtKey = []byte(*flagJwt.GetJwtKey()) 23 | } 24 | 25 | func JwtAuthMiddleware(hd http.Handler) http.Handler { 26 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 27 | tokenString := r.Header.Get("Authorization") 28 | if tokenString == "" { 29 | w.WriteHeader(http.StatusUnauthorized) 30 | return 31 | } 32 | 33 | claims := &Claims{} 34 | token, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { 35 | if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { 36 | err := errors.New("unexpected signing method") 37 | return nil, err 38 | } 39 | return jwtKey, nil 40 | }) 41 | 42 | if err != nil { 43 | if err == jwt.ErrSignatureInvalid { 44 | w.WriteHeader(http.StatusUnauthorized) 45 | return 46 | } 47 | w.WriteHeader(http.StatusBadRequest) 48 | return 49 | } 50 | 51 | if !token.Valid { 52 | w.WriteHeader(http.StatusUnauthorized) 53 | return 54 | } 55 | 56 | hd.ServeHTTP(w, r) 57 | }) 58 | } 59 | 60 | func AuthHandler(w http.ResponseWriter, r *http.Request) { 61 | username := r.FormValue("username") 62 | //password := r.FormValue("password") 63 | 64 | // TODO: check username and password against database 65 | 66 | expirationTime := time.Now().Add(30 * time.Minute) 67 | claims := &Claims{ 68 | Username: username, 69 | StandardClaims: jwt.StandardClaims{ 70 | ExpiresAt: expirationTime.Unix(), 71 | }, 72 | } 73 | 74 | token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) 75 | 76 | tokenString, err := token.SignedString(jwtKey) 77 | if err != nil { 78 | w.WriteHeader(http.StatusInternalServerError) 79 | return 80 | } 81 | 82 | w.Write([]byte(tokenString)) 83 | } 84 | -------------------------------------------------------------------------------- /internal/flags/flag_nacos.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagsNacosInstance *flagsNacos 4 | 5 | type flagsNacos struct { 6 | //nacos 7 | nacosIp *string 8 | nacosPort *int 9 | nacosUsername *string 10 | nacosPassword *string 11 | nacosLogdir *string 12 | nacosCachedir *string 13 | nacosLoglevel *string 14 | nacosTimeoutMs *int 15 | } 16 | 17 | // singleton instance 18 | func init() { 19 | flagsNacosInstance = new(flagsNacos) 20 | } 21 | 22 | func getFlagsNacosInstance() *flagsNacos { 23 | return flagsNacosInstance 24 | } 25 | 26 | // nacos_ip 27 | func (s *flagsNacos) setNacosIp(nacosIp *string) { 28 | s.nacosIp = nacosIp 29 | } 30 | 31 | func (s *flagsNacos) GetNacosIp() *string { 32 | return s.nacosIp 33 | } 34 | 35 | // nacos_port 36 | func (s *flagsNacos) setNacosPort(nacosPort *int) { 37 | s.nacosPort = nacosPort 38 | } 39 | 40 | func (s *flagsNacos) GetNacosPort() *int { 41 | return s.nacosPort 42 | } 43 | 44 | // nacos_username 45 | func (s *flagsNacos) setNacosUsername(nacosUsername *string) { 46 | s.nacosUsername = nacosUsername 47 | } 48 | 49 | func (s *flagsNacos) GetNacosUsername() *string { 50 | return s.nacosUsername 51 | } 52 | 53 | // nacos_password 54 | func (s *flagsNacos) setNacosPassword(nacosPassword *string) { 55 | s.nacosPassword = nacosPassword 56 | } 57 | 58 | func (s *flagsNacos) GetNacosPassword() *string { 59 | return s.nacosPassword 60 | } 61 | 62 | // nacos_logdir 63 | func (s *flagsNacos) setNacosLogdir(nacosLogdir *string) { 64 | s.nacosLogdir = nacosLogdir 65 | } 66 | 67 | func (s *flagsNacos) GetNacosLogdir() *string { 68 | return s.nacosLogdir 69 | } 70 | 71 | // nacos_cachedir 72 | func (s *flagsNacos) setNacosCachedir(nacosCachedir *string) { 73 | s.nacosCachedir = nacosCachedir 74 | } 75 | 76 | func (s *flagsNacos) GetacosCachedir() *string { 77 | return s.nacosCachedir 78 | } 79 | 80 | // nacos_loglevel 81 | func (s *flagsNacos) setNacosLoglevel(nacosLoglevel *string) { 82 | s.nacosLoglevel = nacosLoglevel 83 | } 84 | 85 | func (s *flagsNacos) GetNacosLoglevel() *string { 86 | return s.nacosLoglevel 87 | } 88 | 89 | // nacos_timeoutMS 90 | func (s *flagsNacos) setNacosTimeoutMs(nacosTimeoutMs *int) { 91 | s.nacosTimeoutMs = nacosTimeoutMs 92 | } 93 | 94 | func (s *flagsNacos) GetNacosTimeoutMs() *int { 95 | return s.nacosTimeoutMs 96 | } 97 | -------------------------------------------------------------------------------- /Dockerfile/tfserving/tfserving_docker_file: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARG TF_SERVING_VERSION=latest 16 | ARG TF_SERVING_BUILD_IMAGE=tensorflow/serving:${TF_SERVING_VERSION}-devel 17 | 18 | FROM ${TF_SERVING_BUILD_IMAGE} as build_image 19 | FROM ubuntu:20.04 20 | 21 | ARG TF_SERVING_VERSION_GIT_BRANCH=master 22 | ARG TF_SERVING_VERSION_GIT_COMMIT=head 23 | ARG DEBIAN_FRONTEND=noninteractive 24 | 25 | LABEL maintainer="gvasudevan@google.com" 26 | LABEL tensorflow_serving_github_branchtag=${TF_SERVING_VERSION_GIT_BRANCH} 27 | LABEL tensorflow_serving_github_commit=${TF_SERVING_VERSION_GIT_COMMIT} 28 | 29 | RUN apt-get update && apt-get install -y --no-install-recommends \ 30 | ca-certificates \ 31 | && \ 32 | apt-get clean && \ 33 | rm -rf /var/lib/apt/lists/* 34 | 35 | # Install TF Serving pkg 36 | COPY --from=build_image /usr/local/bin/tensorflow_model_server /usr/bin/tensorflow_model_server 37 | 38 | # Expose ports 39 | # gRPC 40 | EXPOSE 8500 41 | 42 | # REST 43 | EXPOSE 8501 44 | 45 | # Set where models should be stored in the container 46 | ENV MODEL_BASE_PATH=/models 47 | RUN mkdir -p ${MODEL_BASE_PATH} 48 | 49 | # The only required piece is the model name in order to differentiate endpoints 50 | ENV MODEL_NAME=model 51 | 52 | # Create a script that runs the model server so we can use environment variables 53 | # while also passing in arguments from the docker command line 54 | RUN echo '#!/bin/bash \n\n\ 55 | tensorflow_model_server --port=8500 --rest_api_port=8501 \ 56 | --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} \ 57 | "$@"' > /usr/bin/tf_serving_entrypoint.sh \ 58 | && chmod +x /usr/bin/tf_serving_entrypoint.sh 59 | 60 | ENTRYPOINT ["/usr/bin/tf_serving_entrypoint.sh"] 61 | -------------------------------------------------------------------------------- /internal/kafka.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "context" 5 | "infer-microservices/internal/flags" 6 | "infer-microservices/internal/logs" 7 | "strings" 8 | "time" 9 | 10 | kafka "github.com/segmentio/kafka-go" 11 | ) 12 | 13 | var ( 14 | kafkaWriter *kafka.Writer 15 | kafkaURL string 16 | kafkaTopic string 17 | kafkaGroup string 18 | ) 19 | 20 | type CallbackFunc func(string, string) 21 | 22 | func init() { 23 | flagFactory := flags.FlagFactory{} 24 | flagKafka := flagFactory.CreateFlagKafka() 25 | kafkaURL = flagKafka.GetKafkaUrl() 26 | kafkaTopic = flagKafka.GetKafkaTopic() 27 | kafkaGroup = flagKafka.GetKafkaGroup() 28 | 29 | } 30 | 31 | func getKafkaWriter(kafkaURL, topic string) *kafka.Writer { 32 | return &kafka.Writer{ 33 | Addr: kafka.TCP(kafkaURL), 34 | Topic: topic, 35 | Balancer: &kafka.LeastBytes{}, 36 | } 37 | } 38 | 39 | func getKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader { 40 | brokers := strings.Split(kafkaURL, ",") 41 | return kafka.NewReader(kafka.ReaderConfig{ 42 | Brokers: brokers, 43 | GroupID: groupID, 44 | Topic: topic, 45 | MinBytes: 10e3, // 10KB 46 | MaxBytes: 10e7, // 100MB 47 | CommitInterval: time.Second, 48 | StartOffset: kafka.LastOffset, // kafka.FirstOffset, 49 | }) 50 | } 51 | 52 | // 从消息队列里监听来自用户管理后台传来的信息 53 | func KafkaConsumer(callback CallbackFunc) { 54 | reader := getKafkaReader(kafkaURL, kafkaTopic, kafkaGroup) 55 | defer reader.Close() 56 | 57 | index0 := 0 58 | for { 59 | m, err := reader.ReadMessage(context.Background()) 60 | if err != nil { 61 | logs.Error(err) 62 | continue 63 | } 64 | 65 | //do something 66 | callback(string(m.Key), string(m.Value)) 67 | 68 | if index0%100000 == 0 { 69 | logs.Info(" topic, partition, offset, time, key ,value = ", m.Topic, m.Partition, m.Offset, m.Time.Unix(), string(m.Key), string(m.Value)) 70 | } 71 | 72 | index0 += 1 73 | } 74 | } 75 | 76 | func KafkaProducer(msgKey string, msgValue string) { 77 | kafkaWriter = getKafkaWriter(kafkaURL, kafkaTopic) 78 | defer kafkaWriter.Close() 79 | 80 | msg := kafka.Message{ 81 | Key: []byte(msgKey), 82 | Value: []byte(msgValue), 83 | } 84 | 85 | err := kafkaWriter.WriteMessages(context.Background(), msg) 86 | if err != nil { 87 | logs.Error(err) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/graph_transfer_info.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/types.proto"; 6 | 7 | option cc_enable_arenas = true; 8 | option java_outer_classname = "GraphTransferInfoProto"; 9 | option java_multiple_files = true; 10 | option java_package = "org.tensorflow.framework"; 11 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/graph_transfer_info_go_proto"; 12 | 13 | message GraphTransferNodeInput { 14 | int32 node_id = 1; 15 | int32 output_port = 2; 16 | } 17 | message GraphTransferNodeInfo { 18 | string name = 1; 19 | int32 node_id = 2; 20 | string type_name = 3; 21 | int32 soc_op_id = 4; 22 | int32 padding_id = 5; 23 | int32 input_count = 6; 24 | int32 output_count = 7; 25 | } 26 | message GraphTransferConstNodeInfo { 27 | string name = 1; 28 | int32 node_id = 2; 29 | repeated int64 shape = 3; 30 | bytes data = 4; 31 | DataType dtype = 5; 32 | } 33 | message GraphTransferNodeInputInfo { 34 | int32 node_id = 1; 35 | repeated GraphTransferNodeInput node_input = 2; 36 | } 37 | message GraphTransferNodeOutputInfo { 38 | int32 node_id = 1; 39 | repeated int32 max_byte_size = 2; 40 | } 41 | message GraphTransferGraphInputNodeInfo { 42 | string name = 1; 43 | repeated int64 shape = 2; 44 | DataType dtype = 3; 45 | } 46 | 47 | message GraphTransferGraphOutputNodeInfo { 48 | string name = 1; 49 | repeated int64 shape = 2; 50 | DataType dtype = 3; 51 | } 52 | 53 | // Protocol buffer representing a handle to a tensorflow resource. Handles are 54 | // not valid across executions, but can be serialized back and forth from within 55 | // a single run. 56 | message GraphTransferInfo { 57 | enum Destination { 58 | NOP = 0; 59 | HEXAGON = 1; 60 | } 61 | 62 | repeated GraphTransferNodeInfo node_info = 1; 63 | repeated GraphTransferConstNodeInfo const_node_info = 2; 64 | repeated GraphTransferNodeInputInfo node_input_info = 3; 65 | repeated GraphTransferNodeOutputInfo node_output_info = 4; 66 | // Input Node parameters of transferred graph 67 | repeated GraphTransferGraphInputNodeInfo graph_input_node_info = 5; 68 | repeated GraphTransferGraphOutputNodeInfo graph_output_node_info = 6; 69 | // Destination of graph transfer 70 | Destination destination = 7; 71 | } 72 | -------------------------------------------------------------------------------- /pkg/infer_models/model_strategy_factory.go: -------------------------------------------------------------------------------- 1 | package infer_model 2 | 3 | import ( 4 | config_loader "infer-microservices/pkg/config_loader" 5 | "infer-microservices/pkg/config_loader/model_config" 6 | feature "infer-microservices/pkg/infer_features" 7 | "infer-microservices/pkg/infer_models/base_model" 8 | "infer-microservices/pkg/infer_models/ranking/heavy_ranker" 9 | dssm "infer-microservices/pkg/infer_models/recall/u2i" 10 | "net/http" 11 | ) 12 | 13 | var baseModel *base_model.BaseModel 14 | var modelStrategyMap map[string]ModelStrategyInterface 15 | 16 | type ModelStrategyInterface interface { 17 | //model infer. 18 | GetModelType() string 19 | ModelInferSkywalking(model model_config.ModelConfig, requestId string, exposureList []string, r *http.Request, inferSample feature.ExampleFeatures, retNum int) (map[string][]map[string]interface{}, error) 20 | ModelInferNoSkywalking(model model_config.ModelConfig, requestId string, exposureList []string, inferSample feature.ExampleFeatures, retNum int) (map[string][]map[string]interface{}, error) 21 | } 22 | 23 | type ModelStrategyFactory struct { 24 | } 25 | 26 | func init() { 27 | modelStrategyMap = make(map[string]ModelStrategyInterface, 0) 28 | } 29 | 30 | func SetModelStrategyMap(modelStrategy map[string]ModelStrategyInterface) { 31 | modelStrategyMap = modelStrategy 32 | } 33 | 34 | func GetModelStrategyMap() map[string]ModelStrategyInterface { 35 | return modelStrategyMap 36 | } 37 | 38 | // TODO: 改成传参行为，便于pipline 39 | func (m *ModelStrategyFactory) CreateModelStrategy(modelName string, serverConn *config_loader.ServiceConfig) ModelStrategyInterface { 40 | baseModel = base_model.GetBaseModelInstance() 41 | // baseModel.SetUserBloomFilter(internal.GetUserBloomFilterInstance()) 42 | // baseModel.SetItemBloomFilter(internal.GetItemBloomFilterInstance()) 43 | baseModel.SetServiceConfig(serverConn) 44 | 45 | //dssm model 46 | dssmModel := &dssm.Dssm{} 47 | dssmModel.SetBaseModel(*baseModel) 48 | dssmModel.SetModelType("recall") 49 | modelStrategyMap["dssm"] = dssmModel 50 | // tensorName := "user_embedding" 51 | 52 | //DIN model 53 | DINModel := &heavy_ranker.DIN{} 54 | DINModel.SetBaseModel(*baseModel) 55 | DINModel.SetModelType("rank") 56 | modelStrategyMap["DIN"] = DINModel 57 | // tensorName := "scores" 58 | 59 | // modelStrategyMap["lr"] = lrModel 60 | // modelStrategyMap["fm"] = fmModel 61 | 62 | return modelStrategyMap[modelName] 63 | } 64 | -------------------------------------------------------------------------------- /internal/flags/flag_cache.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagCacheInstance *flagCache 4 | 5 | type flagCache struct { 6 | //cache 7 | bigcacheShards *int 8 | bigcacheLifeWindowS *int 9 | bigcacheCleanWindowS *int 10 | bigcacheHardMaxCacheSize *int 11 | bigcacheMaxEntrySize *int 12 | bigcacheMaxEntriesInWindow *int 13 | bigcacheVerbose *bool 14 | } 15 | 16 | // singleton instance 17 | func init() { 18 | flagCacheInstance = new(flagCache) 19 | } 20 | 21 | func getFlagCacheInstance() *flagCache { 22 | return flagCacheInstance 23 | } 24 | 25 | // bigcahe_shards 26 | func (s *flagCache) setBigcacheShards(bigcacheShards *int) { 27 | s.bigcacheShards = bigcacheShards 28 | } 29 | 30 | func (s *flagCache) GetBigcacheShards() *int { 31 | return s.bigcacheShards 32 | } 33 | 34 | // bigcahe_lifeWindowS 35 | func (s *flagCache) setBigcacheLifeWindowS(bigcacheLifeWindowS *int) { 36 | s.bigcacheLifeWindowS = bigcacheLifeWindowS 37 | } 38 | 39 | func (s *flagCache) GetBigcacheLifeWindowS() *int { 40 | return s.bigcacheLifeWindowS 41 | } 42 | 43 | // bigcache_cleanWindowS 44 | func (s *flagCache) setBigcacheCleanWindowS(bigcacheCleanWindowS *int) { 45 | s.bigcacheCleanWindowS = bigcacheCleanWindowS 46 | } 47 | 48 | func (s *flagCache) GetBigcacheCleanWindowS() *int { 49 | return s.bigcacheCleanWindowS 50 | } 51 | 52 | // bigcache_hardMaxCacheSize 53 | func (s *flagCache) setBigcacheHardMaxCacheSize(bigcacheHardMaxCacheSize *int) { 54 | s.bigcacheHardMaxCacheSize = bigcacheHardMaxCacheSize 55 | } 56 | 57 | func (s *flagCache) GetBigcacheHardMaxCacheSize() *int { 58 | return s.bigcacheHardMaxCacheSize 59 | } 60 | 61 | // bigcache_maxEntrySize 62 | func (s *flagCache) setBigcacheMaxEntrySize(bigcacheMaxEntrySize *int) { 63 | s.bigcacheMaxEntrySize = bigcacheMaxEntrySize 64 | } 65 | 66 | func (s *flagCache) GetBigcacheMaxEntrySize() *int { 67 | return s.bigcacheMaxEntrySize 68 | } 69 | 70 | // bigcache_maxEntriesInWindow 71 | func (s *flagCache) setBigcacheMaxEntriesInWindow(bigcacheMaxEntriesInWindow *int) { 72 | s.bigcacheMaxEntriesInWindow = bigcacheMaxEntriesInWindow 73 | } 74 | 75 | func (s *flagCache) GetBigcacheMaxEntriesInWindow() *int { 76 | return s.bigcacheMaxEntriesInWindow 77 | } 78 | 79 | // bigcache_verbose 80 | func (s *flagCache) setBigcacheVerbose(bigcacheVerbose *bool) { 81 | s.bigcacheVerbose = bigcacheVerbose 82 | } 83 | 84 | func (s *flagCache) GetBigcacheVerbose() *bool { 85 | return s.bigcacheVerbose 86 | } 87 | -------------------------------------------------------------------------------- /pkg/config_loader/service_config.go: -------------------------------------------------------------------------------- 1 | package service_config_loader 2 | 3 | import ( 4 | "infer-microservices/pkg/config_loader/faiss_config" 5 | "infer-microservices/pkg/config_loader/model_config" 6 | "infer-microservices/pkg/config_loader/pipeline_config" 7 | "infer-microservices/pkg/config_loader/redis_config" 8 | ) 9 | 10 | var serviceConfigs = make(map[string]*ServiceConfig, 0) //one server/dataid,one service conn 11 | 12 | func SetServiceConfigs(serviceConfigs_ map[string]*ServiceConfig) { 13 | serviceConfigs = serviceConfigs_ 14 | } 15 | 16 | func GetServiceConfigs() map[string]*ServiceConfig { 17 | return serviceConfigs 18 | } 19 | 20 | type ServiceConfig struct { 21 | serviceId string `validate:"required,unique,min=4,max=10"` //dataid 22 | redisConfig redis_config.RedisConfig `validate:"required"` //redis conn info 23 | faissIndexConfigs faiss_config.FaissIndexConfigs //index conn info 24 | modelsConfig map[string]model_config.ModelConfig `validate:"required"` //model conn info 25 | pipelineCnfig pipeline_config.PipelineConfig `validate:"required"` //infer pipeline info 26 | } 27 | 28 | func init() { 29 | } 30 | 31 | // serviceId 32 | func (s *ServiceConfig) setServiceId(serviceId string) { 33 | s.serviceId = serviceId 34 | } 35 | 36 | func (s *ServiceConfig) GetServiceId() string { 37 | return s.serviceId 38 | } 39 | 40 | // redisConfig 41 | func (s *ServiceConfig) setRedisConfig(redisConfig redis_config.RedisConfig) { 42 | s.redisConfig = redisConfig 43 | } 44 | 45 | func (s *ServiceConfig) GetRedisConfig() *redis_config.RedisConfig { 46 | return &s.redisConfig 47 | } 48 | 49 | // faissIndexConfig 50 | func (s *ServiceConfig) SetFaissIndexConfigs(faissIndexConfigs faiss_config.FaissIndexConfigs) { 51 | s.faissIndexConfigs = faissIndexConfigs 52 | } 53 | 54 | func (s *ServiceConfig) GetFaissIndexConfigs() *faiss_config.FaissIndexConfigs { 55 | return &s.faissIndexConfigs 56 | } 57 | 58 | // modelConfig 59 | func (s *ServiceConfig) setModelsConfig(modelsConfig map[string]model_config.ModelConfig) { 60 | s.modelsConfig = modelsConfig 61 | } 62 | 63 | func (s *ServiceConfig) GetModelsConfig() map[string]model_config.ModelConfig { 64 | return s.modelsConfig 65 | } 66 | 67 | // piplineCnfig 68 | func (s *ServiceConfig) setPipelineConfig(piplineCnfig pipeline_config.PipelineConfig) { 69 | s.pipelineCnfig = piplineCnfig 70 | } 71 | 72 | func (s *ServiceConfig) GetPipelineConfig() *pipeline_config.PipelineConfig { 73 | return &s.pipelineCnfig 74 | } 75 | -------------------------------------------------------------------------------- /api/api_factory.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | dubbo_api "infer-microservices/api/dubbo_api/server" 5 | grpc_api "infer-microservices/api/grpc_api/server" 6 | rest_api "infer-microservices/api/rest_api/server" 7 | "infer-microservices/internal/flags" 8 | infer_services "infer-microservices/pkg/infer_services" 9 | ) 10 | 11 | var restPort uint 12 | var grpcPort uint 13 | var maxCpuNum int 14 | var dubboConfFile string 15 | var serviceFactory infer_services.ServiceFactory 16 | 17 | type ApiFactory struct { 18 | } 19 | 20 | type ServiceStartInterface interface { 21 | ServiceStart() 22 | } 23 | 24 | func init() { 25 | flagFactory := flags.FlagFactory{} 26 | //flagServiceConfig 27 | flagServiceConfig := flagFactory.CreateFlagServiceConfig() 28 | grpcPort = *flagServiceConfig.GetServiceRestPort() 29 | restPort = *flagServiceConfig.GetServiceGrpcPort() 30 | maxCpuNum = *flagServiceConfig.GetServiceMaxCpuNum() 31 | 32 | //flagDubbo 33 | flagDubbo := flagFactory.CreateFlagDubbo() 34 | dubboConfFile = *flagDubbo.GetDubboServiceFile() 35 | } 36 | 37 | // create dubbo server 38 | func (f ApiFactory) CreateDubboServiceApi() *dubbo_api.DubboServiceApi { 39 | dubboServiceApi := new(dubbo_api.DubboServiceApi) 40 | dubboServiceApi.SetDubboConfFile(dubboConfFile) 41 | 42 | dubboService := serviceFactory.CreateDubboService() 43 | dubboServiceApi.SetDubboService(dubboService) 44 | 45 | return dubboServiceApi 46 | } 47 | 48 | // create grpc server 49 | func (f ApiFactory) CreateGrpcServiceApi() *grpc_api.GrpcServiceApi { 50 | grpcServiceApi := new(grpc_api.GrpcServiceApi) 51 | grpcServiceApi.SetServicePort(grpcPort) 52 | grpcServiceApi.SetMaxCpuNum(maxCpuNum) 53 | 54 | grpcService := serviceFactory.CreateGrpcService() 55 | grpcServiceApi.SetGrpcService(grpcService) 56 | 57 | return grpcServiceApi 58 | } 59 | 60 | // create rest server 61 | // @deprecated 62 | func (f ApiFactory) CreateRestServiceHttpApi() *rest_api.HttpServiceApi { 63 | httpServiceApi := new(rest_api.HttpServiceApi) 64 | httpServiceApi.SetServicePort(restPort) 65 | httpServiceApi.SetMaxCpuNum(maxCpuNum) 66 | 67 | httpService := serviceFactory.CreateHttpService() 68 | httpServiceApi.SetRestService(httpService) 69 | 70 | return httpServiceApi 71 | } 72 | 73 | // create rest server 74 | func (f ApiFactory) CreateRestServiceEchoApi() *rest_api.EchoServiceApi { 75 | echoServiceApi := new(rest_api.EchoServiceApi) 76 | echoServiceApi.SetServicePort(restPort) 77 | echoServiceApi.SetMaxCpuNum(maxCpuNum) 78 | 79 | echoService := serviceFactory.CreateEchoService() 80 | echoServiceApi.SetRestService(echoService) 81 | 82 | return echoServiceApi 83 | } 84 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/graph.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/function.proto"; 6 | import "tensorflow/core/framework/graph_debug_info.proto"; 7 | import "tensorflow/core/framework/node_def.proto"; 8 | import "tensorflow/core/framework/versions.proto"; 9 | 10 | option cc_enable_arenas = true; 11 | option java_outer_classname = "GraphProtos"; 12 | option java_multiple_files = true; 13 | option java_package = "org.tensorflow.framework"; 14 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/graph_go_proto"; 15 | 16 | // Represents the graph of operations 17 | message GraphDef { 18 | repeated NodeDef node = 1; 19 | 20 | // Compatibility versions of the graph. See core/public/version.h for version 21 | // history. The GraphDef version is distinct from the TensorFlow version, and 22 | // each release of TensorFlow will support a range of GraphDef versions. 23 | VersionDef versions = 4; 24 | 25 | // Deprecated single version field; use versions above instead. Since all 26 | // GraphDef changes before "versions" was introduced were forward 27 | // compatible, this field is entirely ignored. 28 | int32 version = 3 [deprecated = true]; 29 | 30 | // "library" provides user-defined functions. 31 | // 32 | // Naming: 33 | // * library.function.name are in a flat namespace. 34 | // NOTE: We may need to change it to be hierarchical to support 35 | // different orgs. E.g., 36 | // { "/google/nn", { ... }}, 37 | // { "/google/vision", { ... }} 38 | // { "/org_foo/module_bar", { ... }} 39 | // map named_lib; 40 | // * If node[i].op is the name of one function in "library", 41 | // node[i] is deemed as a function call. Otherwise, node[i].op 42 | // must be a primitive operation supported by the runtime. 43 | // 44 | // 45 | // Function call semantics: 46 | // 47 | // * The callee may start execution as soon as some of its inputs 48 | // are ready. The caller may want to use Tuple() mechanism to 49 | // ensure all inputs are ready in the same time. 50 | // 51 | // * The consumer of return values may start executing as soon as 52 | // the return values the consumer depends on are ready. The 53 | // consumer may want to use Tuple() mechanism to ensure the 54 | // consumer does not start until all return values of the callee 55 | // function are ready. 56 | FunctionDefLibrary library = 2; 57 | 58 | // Stack traces for the nodes in this graph. 59 | GraphDebugInfo debug_info = 5; 60 | } 61 | -------------------------------------------------------------------------------- /pkg/config_loader/pipeline_config/pipeline_config_loader.go: -------------------------------------------------------------------------------- 1 | package pipeline_config 2 | 3 | import ( 4 | "infer-microservices/internal/utils" 5 | ) 6 | 7 | type PipelineConfig struct { 8 | recallNum int32 //recall num 9 | preRankingNum int32 //filter recall items, use to ranking 10 | recallNumLight int32 //recall num ,visit it when service degradation 11 | preRankingNumLight int32 //filter recall items, use to ranking.visit it when service degradation 12 | pipeline []string 13 | lightPipeline []string 14 | } 15 | 16 | // recallNum 17 | func (r *PipelineConfig) SetRecallNum(recallNum int32) { 18 | r.recallNum = recallNum 19 | } 20 | 21 | func (r *PipelineConfig) GetRecallNum() int32 { 22 | return r.recallNum 23 | } 24 | 25 | // preRankingNum 26 | func (r *PipelineConfig) SetPreRankingNum(preRankingNum int32) { 27 | r.preRankingNum = preRankingNum 28 | } 29 | 30 | // recallNumLight 31 | func (r *PipelineConfig) SetRecallNumLight(recallNumLight int32) { 32 | r.recallNumLight = recallNumLight 33 | } 34 | 35 | func (r *PipelineConfig) GetRecallNumLight() int32 { 36 | return r.recallNumLight 37 | } 38 | 39 | // preRankingNumLight 40 | func (r *PipelineConfig) SetPreRankingNumLight(preRankingNumLight int32) { 41 | r.preRankingNumLight = preRankingNumLight 42 | } 43 | 44 | func (r *PipelineConfig) GetPreRankingNumLight() int32 { 45 | return r.preRankingNumLight 46 | } 47 | 48 | // pipeline 49 | func (r *PipelineConfig) SetPipeline(pipeline []string) { 50 | r.pipeline = pipeline 51 | } 52 | 53 | func (r *PipelineConfig) GetPipeline() []string { 54 | return r.pipeline 55 | } 56 | 57 | // lightPipeline 58 | func (r *PipelineConfig) SetLightPipeline(lightPipeline []string) { 59 | r.lightPipeline = lightPipeline 60 | } 61 | 62 | func (r *PipelineConfig) GetLightPipeline() []string { 63 | return r.lightPipeline 64 | } 65 | 66 | // @implement ConfigLoadInterface 67 | func (f *PipelineConfig) ConfigLoad(dataId string, pipelineConfStr string) error { 68 | dataConf := utils.ConvertJsonToStruct(pipelineConfStr) 69 | recallNum := dataConf["recallNum"].(int32) 70 | preRankingNum := dataConf["preRankingNum"].(int32) 71 | recallNumLight := dataConf["recallNumLight"].(int32) 72 | preRankingNumLight := dataConf["preRankingNumLight"].(int32) 73 | pipline := dataConf["pipeline"].([]string) 74 | lightPipline := dataConf["light_pipeline"].([]string) 75 | 76 | f.SetRecallNum(recallNum) 77 | f.SetPreRankingNum(preRankingNum) 78 | f.SetRecallNumLight(recallNumLight) 79 | f.SetRecallNumLight(recallNumLight) 80 | f.SetPreRankingNumLight(preRankingNumLight) 81 | f.SetLightPipeline(lightPipline) 82 | f.SetPipeline(pipline) 83 | 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /internal/flags/flag_hystrix.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | var flagsHystrixInstance *flagsHystrix 4 | 5 | type flagsHystrix struct { 6 | // hystrix 7 | hystrixTimeoutMs *int 8 | hystrixMaxConcurrentRequests *int 9 | hystrixRequestVolumeThreshold *int 10 | hystrixSleepWindow *int 11 | hystrixErrorPercentThreshold *int 12 | hystrixLowerRecallNum *int 13 | hystrixLowerRankNum *int 14 | } 15 | 16 | // singleton instance 17 | func init() { 18 | flagsHystrixInstance = new(flagsHystrix) 19 | } 20 | 21 | func getFlagsHystrixInstance() *flagsHystrix { 22 | return flagsHystrixInstance 23 | } 24 | 25 | // hystrix_timeoutMS 26 | func (s *flagsHystrix) setHystrixTimeoutMs(hystrixTimeoutMs *int) { 27 | s.hystrixTimeoutMs = hystrixTimeoutMs 28 | } 29 | 30 | func (s *flagsHystrix) GetHystrixTimeoutMs() *int { 31 | return s.hystrixTimeoutMs 32 | } 33 | 34 | // hystrix_MaxConcurrentRequests 35 | func (s *flagsHystrix) setHystrixMaxConcurrentRequests(hystrixMaxConcurrentRequests *int) { 36 | s.hystrixMaxConcurrentRequests = hystrixMaxConcurrentRequests 37 | } 38 | 39 | func (s *flagsHystrix) GetHystrixMaxConcurrentRequests() *int { 40 | return s.hystrixMaxConcurrentRequests 41 | } 42 | 43 | // hystrix_RequestVolumeThreshold 44 | func (s *flagsHystrix) setHystrixRequestVolumeThreshold(hystrixRequestVolumeThreshold *int) { 45 | s.hystrixRequestVolumeThreshold = hystrixRequestVolumeThreshold 46 | } 47 | 48 | func (s *flagsHystrix) GetHystrixRequestVolumeThreshold() *int { 49 | return s.hystrixRequestVolumeThreshold 50 | } 51 | 52 | // hystrix_timeoutMS 53 | func (s *flagsHystrix) setHystrixSleepWindow(hystrixSleepWindow *int) { 54 | s.hystrixSleepWindow = hystrixSleepWindow 55 | } 56 | 57 | func (s *flagsHystrix) GetHystrixSleepWindow() *int { 58 | return s.hystrixSleepWindow 59 | } 60 | 61 | // hystrix_ErrorPercentThreshold 62 | func (s *flagsHystrix) setHystrixErrorPercentThreshold(hystrixErrorPercentThreshold *int) { 63 | s.hystrixErrorPercentThreshold = hystrixErrorPercentThreshold 64 | } 65 | 66 | func (s *flagsHystrix) GetHystrixErrorPercentThreshold() *int { 67 | return s.hystrixErrorPercentThreshold 68 | } 69 | 70 | // hystrix_lowerRecallNum 71 | func (s *flagsHystrix) setHystrixLowerRecallNum(hystrixLowerRecallNum *int) { 72 | s.hystrixLowerRecallNum = hystrixLowerRecallNum 73 | } 74 | 75 | func (s *flagsHystrix) GetHystrixLowerRecallNum() *int { 76 | return s.hystrixLowerRecallNum 77 | } 78 | 79 | // hystrix_lowerRankNum 80 | func (s *flagsHystrix) setHystrixLowerRankNum(hystrixLowerRankNum *int) { 81 | s.hystrixLowerRankNum = hystrixLowerRankNum 82 | } 83 | 84 | func (s *flagsHystrix) GetHystrixLowerRankNum() *int { 85 | return s.hystrixLowerRankNum 86 | } 87 | -------------------------------------------------------------------------------- /api/rest_api/server/rest_server_echo_api.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "infer-microservices/internal" 6 | "infer-microservices/internal/jwt" 7 | "infer-microservices/internal/logs" 8 | "infer-microservices/pkg/infer_services/rest_service" 9 | "runtime" 10 | 11 | "github.com/labstack/echo" 12 | "github.com/labstack/echo/middleware" 13 | ) 14 | 15 | //TODO: test gin rest api, test dubbo restful api 16 | 17 | var echoApi *echo.Echo 18 | var skywalkingOpen bool 19 | var skywalkingAddr string 20 | var skywalkingServerName string 21 | 22 | type EchoServiceApi struct { 23 | serverPort uint 24 | maxCpuNum int 25 | echoService *rest_service.EchoService 26 | } 27 | 28 | func init() { 29 | echoApi = echo.New() 30 | echoApi.Debug = true 31 | echoApi.Use(middleware.Recover()) 32 | echoApi.Use(middleware.Logger()) 33 | echoApi.Use(middleware.RequestID()) 34 | } 35 | 36 | func GetEcho() *echo.Echo { 37 | return echoApi 38 | } 39 | 40 | // set func 41 | func (s *EchoServiceApi) SetServicePort(serverPort uint) { 42 | s.serverPort = serverPort 43 | } 44 | 45 | func (s *EchoServiceApi) SetMaxCpuNum(maxCpuNum int) { 46 | s.maxCpuNum = maxCpuNum 47 | } 48 | 49 | func (s *EchoServiceApi) SetRestService(echoService *rest_service.EchoService) { 50 | s.echoService = echoService 51 | } 52 | 53 | // skywalkingMiddleware 54 | func skywalkingMiddleware(next echo.HandlerFunc) echo.HandlerFunc { 55 | return func(c echo.Context) error { 56 | if skywalkingOpen { 57 | internal.SkywalkingTracer(skywalkingAddr, skywalkingServerName) 58 | } 59 | 60 | return next(c) 61 | } 62 | } 63 | 64 | // @implement start infertace 65 | func (s *EchoServiceApi) ServiceStart() { 66 | echoApi.POST("/login2", jwt.Login) 67 | 68 | // Restricted group 69 | r := echoApi.Group("/infer2") 70 | 71 | // Configure middleware with the custom claims type 72 | config := middleware.JWTConfig{ 73 | Claims: &jwt.JwtCustomClaims{}, 74 | SigningKey: []byte("secret"), 75 | } 76 | r.Use(middleware.JWTWithConfig(config)) 77 | echoApi.POST("/infer2", s.echoService.SyncRecommenderInfer) 78 | 79 | skywalkingOpen = s.echoService.GetBaseService().GetSkywalkingWeatherOpen() 80 | skywalkingAddr = s.echoService.GetBaseService().GetSkywalkingIp() + ":" + fmt.Sprintf(":%d", s.echoService.GetBaseService().GetSkywalkingPort()) 81 | skywalkingServerName = s.echoService.GetBaseService().GetSkywalkingServerName() 82 | echoApi.Use(skywalkingMiddleware) 83 | 84 | cpuNum := runtime.NumCPU() 85 | if s.maxCpuNum <= cpuNum { 86 | cpuNum = s.maxCpuNum 87 | } 88 | runtime.GOMAXPROCS(cpuNum) 89 | logs.Info("cup num:", cpuNum) 90 | 91 | addr := fmt.Sprintf(":%d", s.serverPort) 92 | echoApi.Logger.Fatal(echoApi.Start(addr)) 93 | logs.Info("server addr:", addr) 94 | } 95 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/log_memory.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/tensor_description.proto"; 6 | 7 | option cc_enable_arenas = true; 8 | option java_outer_classname = "LogMemoryProtos"; 9 | option java_multiple_files = true; 10 | option java_package = "org.tensorflow.framework"; 11 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/log_memory_go_proto"; 12 | 13 | message MemoryLogStep { 14 | // Process-unique step id. 15 | int64 step_id = 1; 16 | 17 | // Handle describing the feeds and fetches of the step. 18 | string handle = 2; 19 | } 20 | 21 | message MemoryLogTensorAllocation { 22 | // Process-unique step id. 23 | int64 step_id = 1; 24 | 25 | // Name of the kernel making the allocation as set in GraphDef, 26 | // e.g., "affine2/weights/Assign". 27 | string kernel_name = 2; 28 | 29 | // Allocated tensor details. 30 | TensorDescription tensor = 3; 31 | } 32 | 33 | message MemoryLogTensorDeallocation { 34 | // Id of the tensor buffer being deallocated, used to match to a 35 | // corresponding allocation. 36 | int64 allocation_id = 1; 37 | 38 | // Name of the allocator used. 39 | string allocator_name = 2; 40 | } 41 | 42 | message MemoryLogTensorOutput { 43 | // Process-unique step id. 44 | int64 step_id = 1; 45 | 46 | // Name of the kernel producing an output as set in GraphDef, e.g., 47 | // "affine2/weights/Assign". 48 | string kernel_name = 2; 49 | 50 | // Index of the output being set. 51 | int32 index = 3; 52 | 53 | // Output tensor details. 54 | TensorDescription tensor = 4; 55 | } 56 | 57 | message MemoryLogRawAllocation { 58 | // Process-unique step id. 59 | int64 step_id = 1; 60 | 61 | // Name of the operation making the allocation. 62 | string operation = 2; 63 | 64 | // Number of bytes in the allocation. 65 | int64 num_bytes = 3; 66 | 67 | // Address of the allocation. 68 | uint64 ptr = 4; 69 | 70 | // Id of the tensor buffer being allocated, used to match to a 71 | // corresponding deallocation. 72 | int64 allocation_id = 5; 73 | 74 | // Name of the allocator used. 75 | string allocator_name = 6; 76 | } 77 | 78 | message MemoryLogRawDeallocation { 79 | // Process-unique step id. 80 | int64 step_id = 1; 81 | 82 | // Name of the operation making the deallocation. 83 | string operation = 2; 84 | 85 | // Id of the tensor buffer being deallocated, used to match to a 86 | // corresponding allocation. 87 | int64 allocation_id = 3; 88 | 89 | // Name of the allocator used. 90 | string allocator_name = 4; 91 | 92 | // True if the deallocation is queued and will be performed later, 93 | // e.g. for GPU lazy freeing of buffers. 94 | bool deferred = 5; 95 | } 96 | -------------------------------------------------------------------------------- /internal/logs/accesslog.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "bytes" 19 | "encoding/json" 20 | "fmt" 21 | "time" 22 | ) 23 | 24 | const ( 25 | apacheFormatPattern = "%s - - [%s] \"%s %d %d\" %f %s %s\n" 26 | apacheFormat = "APACHE_FORMAT" 27 | jsonFormat = "JSON_FORMAT" 28 | ) 29 | 30 | // AccessLogRecord struct for holding access log data. 31 | type AccessLogRecord struct { 32 | RemoteAddr string `json:"remote_addr"` 33 | RequestTime time.Time `json:"request_time"` 34 | RequestMethod string `json:"request_method"` 35 | Request string `json:"request"` 36 | ServerProtocol string `json:"server_protocol"` 37 | Host string `json:"host"` 38 | Status int `json:"status"` 39 | BodyBytesSent int64 `json:"body_bytes_sent"` 40 | ElapsedTime time.Duration `json:"elapsed_time"` 41 | HTTPReferrer string `json:"http_referrer"` 42 | HTTPUserAgent string `json:"http_user_agent"` 43 | RemoteUser string `json:"remote_user"` 44 | } 45 | 46 | func (r *AccessLogRecord) json() ([]byte, error) { 47 | buffer := &bytes.Buffer{} 48 | encoder := json.NewEncoder(buffer) 49 | disableEscapeHTML(encoder) 50 | 51 | err := encoder.Encode(r) 52 | return buffer.Bytes(), err 53 | } 54 | 55 | func disableEscapeHTML(i interface{}) { 56 | e, ok := i.(interface { 57 | SetEscapeHTML(bool) 58 | }) 59 | if ok { 60 | e.SetEscapeHTML(false) 61 | } 62 | } 63 | 64 | // AccessLog - Format and print access log. 65 | func AccessLog(r *AccessLogRecord, format string) { 66 | var msg string 67 | 68 | switch format { 69 | 70 | case apacheFormat: 71 | timeFormatted := r.RequestTime.Format("02/Jan/2014 00:00:00") 72 | msg = fmt.Sprintf(apacheFormatPattern, r.RemoteAddr, timeFormatted, r.Request, r.Status, r.BodyBytesSent, 73 | r.ElapsedTime.Seconds(), r.HTTPReferrer, r.HTTPUserAgent) 74 | case jsonFormat: 75 | fallthrough 76 | default: 77 | jsonData, err := r.json() 78 | if err != nil { 79 | msg = fmt.Sprintf(`{"Error": "%s"}`, err) 80 | } else { 81 | msg = string(jsonData) 82 | } 83 | } 84 | 85 | beeLogger.Debug(msg) 86 | } 87 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/attr_value.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/tensor.proto"; 6 | import "tensorflow/core/framework/tensor_shape.proto"; 7 | import "tensorflow/core/framework/types.proto"; 8 | 9 | option cc_enable_arenas = true; 10 | option java_outer_classname = "AttrValueProtos"; 11 | option java_multiple_files = true; 12 | option java_package = "org.tensorflow.framework"; 13 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/attr_value_go_proto"; 14 | 15 | // Protocol buffer representing the value for an attr used to configure an Op. 16 | // Comment indicates the corresponding attr type. Only the field matching the 17 | // attr type may be filled. 18 | message AttrValue { 19 | // LINT.IfChange 20 | message ListValue { 21 | repeated bytes s = 2; // "list(string)" 22 | repeated int64 i = 3 [packed = true]; // "list(int)" 23 | repeated float f = 4 [packed = true]; // "list(float)" 24 | repeated bool b = 5 [packed = true]; // "list(bool)" 25 | repeated DataType type = 6 [packed = true]; // "list(type)" 26 | repeated TensorShapeProto shape = 7; // "list(shape)" 27 | repeated TensorProto tensor = 8; // "list(tensor)" 28 | repeated NameAttrList func = 9; // "list(attr)" 29 | } 30 | // LINT.ThenChange(//tensorflow/c/c_api.cc) 31 | 32 | oneof value { 33 | bytes s = 2; // "string" 34 | int64 i = 3; // "int" 35 | float f = 4; // "float" 36 | bool b = 5; // "bool" 37 | DataType type = 6; // "type" 38 | TensorShapeProto shape = 7; // "shape" 39 | TensorProto tensor = 8; // "tensor" 40 | ListValue list = 1; // any "list(...)" 41 | 42 | // "func" represents a function. func.name is a function's name or 43 | // a primitive op's name. func.attr.first is the name of an attr 44 | // defined for that function. func.attr.second is the value for 45 | // that attr in the instantiation. 46 | NameAttrList func = 10; 47 | 48 | // This is a placeholder only used in nodes defined inside a 49 | // function. It indicates the attr value will be supplied when 50 | // the function is instantiated. For example, let us suppose a 51 | // node "N" in function "FN". "N" has an attr "A" with value 52 | // placeholder = "foo". When FN is instantiated with attr "foo" 53 | // set to "bar", the instantiated node N's attr A will have been 54 | // given the value "bar". 55 | string placeholder = 9; 56 | } 57 | } 58 | 59 | // A list of attr names and their values. The whole list is attached 60 | // with a string name. E.g., MatMul[T=float]. 61 | message NameAttrList { 62 | string name = 1; 63 | map attr = 2; 64 | } 65 | -------------------------------------------------------------------------------- /configs/nacos/service_start_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataId": "inferid-001", 3 | "group": "infer", 4 | "namespace": "", 5 | "content": { "_comment":" // from local only include content. whole info from nacos", 6 | "author": "loki", 7 | "update": "", 8 | "version": "v1.0", 9 | "desc": "", 10 | "config": { 11 | "model_conf": { 12 | "model-001": { 13 | "modelName":"dssm", 14 | "modelType":"recall", 15 | "featureList": [], 16 | "tfservingGrpcAddr": { 17 | "tfservingModelName": "models", 18 | "tfservingAddrs": [], 19 | "pool_size": 50, 20 | "initCap":10, 21 | "idleTimeoutMs": 100, 22 | "readTimeoutMs": 100, 23 | "writeTimeoutMs":100, 24 | "dialTimeoutS":600 25 | }, 26 | "user_feature_rediskey_pre_offline": "", 27 | "user_feature_rediskey_pre_realtime": "", 28 | "item_feature_rediskey_pre": "" 29 | }, 30 | "model-002": { 31 | "modelName":"deepfm", 32 | "modelType":"rank", 33 | "featureList": [], 34 | "tfservingGrpcAddr": { 35 | "tfservingModelName": "models", 36 | "tfservingAddrs": [], 37 | "pool_size": 50, 38 | "initCap":10, 39 | "idleTimeoutMs": 100, 40 | "readTimeoutMs": 100, 41 | "writeTimeoutMs":100, 42 | "dialTimeoutS":600 43 | }, 44 | "user_feature_rediskey_pre_offline": "", 45 | "user_feature_rediskey_pre_realtime": "", 46 | "item_feature_rediskey_pre": "" 47 | } 48 | }, 49 | "index_conf": { 50 | "faissGrpcAddr": { 51 | "addrs": [], 52 | "pool_size": 50, 53 | "initCap":10, 54 | "idleTimeoutMs": 100, 55 | "readTimeoutMs": 100, 56 | "writeTimeoutMs":100, 57 | "dialTimeoutS":600 58 | }, 59 | "indexInfo":[ 60 | { 61 | "recallNum" :1000 , 62 | "indexName": "index-001" 63 | }, 64 | { 65 | "recallNum" :1000 , 66 | "indexName": "index-001" 67 | } 68 | ] 69 | }, 70 | "redis_conf": { 71 | "redisCluster": { 72 | "addrs": [], 73 | "password": "", 74 | "idleTimeoutMs": 100, 75 | "readTimeoutMs": 100, 76 | "writeTimeoutMs":100, 77 | "dialTimeoutS":600, 78 | "maxRetries":2, 79 | "minIdleConns":50 80 | } 81 | }, 82 | "pipiline_conf":{ 83 | "pipeline":[ 84 | "('recall_sample','recallSample')","('dssm_recall','dssmRecall')", 85 | "('pre_ranking_sample','preRankingSample')","('pre_ranking','dssmRank')", 86 | "('ranking_sample','rankingSample')","('ranking','deepfm')", 87 | "('re_rank','reRank')" 88 | ], 89 | "recallNum" :1000 , 90 | "preRankingNum" : 200, 91 | 92 | "pipeline_light":[ 93 | "('recall_sample','recallSample')","('fm_recall','fmRecall')","('light_ranking','lrRank')" 94 | ], 95 | "recallNumLight" : 100 , 96 | "preRankingNumLight" :10 97 | } 98 | } 99 | } 100 | } -------------------------------------------------------------------------------- /internal/logs/console.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "encoding/json" 19 | "os" 20 | "runtime" 21 | "time" 22 | ) 23 | 24 | // brush is a color join function 25 | type brush func(string) string 26 | 27 | // newBrush return a fix color Brush 28 | func newBrush(color string) brush { 29 | pre := "\033[" 30 | reset := "\033[0m" 31 | return func(text string) string { 32 | return pre + color + "m" + text + reset 33 | } 34 | } 35 | 36 | var colors = []brush{ 37 | newBrush("1;37"), // Emergency white 38 | newBrush("1;36"), // Alert cyan 39 | newBrush("1;35"), // Critical magenta 40 | newBrush("1;31"), // Error red 41 | newBrush("1;33"), // Warning yellow 42 | newBrush("1;32"), // Notice green 43 | newBrush("1;34"), // Informational blue 44 | newBrush("1;44"), // Debug Background blue 45 | } 46 | 47 | // consoleWriter implements LoggerInterface and writes messages to terminal. 48 | type consoleWriter struct { 49 | lg *logWriter 50 | Level int `json:"level"` 51 | Colorful bool `json:"color"` //this filed is useful only when system's terminal supports color 52 | } 53 | 54 | // NewConsole create ConsoleWriter returning as LoggerInterface. 55 | func NewConsole() Logger { 56 | cw := &consoleWriter{ 57 | lg: newLogWriter(os.Stdout), 58 | Level: LevelDebug, 59 | Colorful: runtime.GOOS != "windows", 60 | } 61 | return cw 62 | } 63 | 64 | // Init init console logger. 65 | // jsonConfig like '{"level":LevelTrace}'. 66 | func (c *consoleWriter) Init(jsonConfig string) error { 67 | if len(jsonConfig) == 0 { 68 | return nil 69 | } 70 | err := json.Unmarshal([]byte(jsonConfig), c) 71 | if runtime.GOOS == "windows" { 72 | c.Colorful = false 73 | } 74 | return err 75 | } 76 | 77 | // WriteMsg write message in console. 78 | func (c *consoleWriter) WriteMsg(when time.Time, msg string, level int) error { 79 | if level > c.Level { 80 | return nil 81 | } 82 | if c.Colorful { 83 | msg = colors[level](msg) 84 | } 85 | c.lg.println(when, msg) 86 | return nil 87 | } 88 | 89 | // Destroy implementing method. empty. 90 | func (c *consoleWriter) Destroy() { 91 | 92 | } 93 | 94 | // Flush implementing method. empty. 95 | func (c *consoleWriter) Flush() { 96 | 97 | } 98 | 99 | func init() { 100 | Register(AdapterConsole, NewConsole) 101 | } 102 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "TypesProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/types_go_proto"; 10 | 11 | // (== suppress_warning documentation-presence ==) 12 | // LINT.IfChange 13 | enum DataType { 14 | // Not a legal value for DataType. Used to indicate a DataType field 15 | // has not been set. 16 | DT_INVALID = 0; 17 | 18 | // Data types that all computation devices are expected to be 19 | // capable to support. 20 | DT_FLOAT = 1; 21 | DT_DOUBLE = 2; 22 | DT_INT32 = 3; 23 | DT_UINT8 = 4; 24 | DT_INT16 = 5; 25 | DT_INT8 = 6; 26 | DT_STRING = 7; 27 | DT_COMPLEX64 = 8; // Single-precision complex 28 | DT_INT64 = 9; 29 | DT_BOOL = 10; 30 | DT_QINT8 = 11; // Quantized int8 31 | DT_QUINT8 = 12; // Quantized uint8 32 | DT_QINT32 = 13; // Quantized int32 33 | DT_BFLOAT16 = 14; // Float32 truncated to 16 bits. 34 | DT_QINT16 = 15; // Quantized int16 35 | DT_QUINT16 = 16; // Quantized uint16 36 | DT_UINT16 = 17; 37 | DT_COMPLEX128 = 18; // Double-precision complex 38 | DT_HALF = 19; 39 | DT_RESOURCE = 20; 40 | DT_VARIANT = 21; // Arbitrary C++ data types 41 | DT_UINT32 = 22; 42 | DT_UINT64 = 23; 43 | DT_FLOAT8_E5M2 = 24; // 5 exponent bits, 2 mantissa bits. 44 | DT_FLOAT8_E4M3FN = 25; // 4 exponent bits, 3 mantissa bits, finite-only, with 45 | // 2 NaNs (0bS1111111). 46 | 47 | // Do not use! These are only for parameters. Every enum above 48 | // should have a corresponding value below (verified by types_test). 49 | DT_FLOAT_REF = 101; 50 | DT_DOUBLE_REF = 102; 51 | DT_INT32_REF = 103; 52 | DT_UINT8_REF = 104; 53 | DT_INT16_REF = 105; 54 | DT_INT8_REF = 106; 55 | DT_STRING_REF = 107; 56 | DT_COMPLEX64_REF = 108; 57 | DT_INT64_REF = 109; 58 | DT_BOOL_REF = 110; 59 | DT_QINT8_REF = 111; 60 | DT_QUINT8_REF = 112; 61 | DT_QINT32_REF = 113; 62 | DT_BFLOAT16_REF = 114; 63 | DT_QINT16_REF = 115; 64 | DT_QUINT16_REF = 116; 65 | DT_UINT16_REF = 117; 66 | DT_COMPLEX128_REF = 118; 67 | DT_HALF_REF = 119; 68 | DT_RESOURCE_REF = 120; 69 | DT_VARIANT_REF = 121; 70 | DT_UINT32_REF = 122; 71 | DT_UINT64_REF = 123; 72 | DT_FLOAT8_E5M2_REF = 124; 73 | DT_FLOAT8_E4M3FN_REF = 125; 74 | } 75 | // LINT.ThenChange( 76 | // https://www.tensorflow.org/code/tensorflow/c/tf_datatype.h, 77 | // https://www.tensorflow.org/code/tensorflow/go/tensor.go, 78 | // https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc, 79 | // https://www.tensorflow.org/code/tensorflow/core/framework/types.h, 80 | // https://www.tensorflow.org/code/tensorflow/core/framework/types.cc, 81 | // https://www.tensorflow.org/code/tensorflow/python/framework/dtypes.py, 82 | // https://www.tensorflow.org/code/tensorflow/python/framework/function.py) 83 | 84 | // Represents a serialized tf.dtypes.Dtype 85 | message SerializedDType { 86 | DataType datatype = 1; 87 | } 88 | -------------------------------------------------------------------------------- /internal/logs/conn.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "encoding/json" 19 | "io" 20 | "net" 21 | "time" 22 | ) 23 | 24 | // connWriter implements LoggerInterface. 25 | // it writes messages in keep-live tcp connection. 26 | type connWriter struct { 27 | lg *logWriter 28 | innerWriter io.WriteCloser 29 | ReconnectOnMsg bool `json:"reconnectOnMsg"` 30 | Reconnect bool `json:"reconnect"` 31 | Net string `json:"net"` 32 | Addr string `json:"addr"` 33 | Level int `json:"level"` 34 | } 35 | 36 | // NewConn create new ConnWrite returning as LoggerInterface. 37 | func NewConn() Logger { 38 | conn := new(connWriter) 39 | conn.Level = LevelTrace 40 | return conn 41 | } 42 | 43 | // Init init connection writer with json config. 44 | // json config only need key "level". 45 | func (c *connWriter) Init(jsonConfig string) error { 46 | return json.Unmarshal([]byte(jsonConfig), c) 47 | } 48 | 49 | // WriteMsg write message in connection. 50 | // if connection is down, try to re-connect. 51 | func (c *connWriter) WriteMsg(when time.Time, msg string, level int) error { 52 | if level > c.Level { 53 | return nil 54 | } 55 | if c.needToConnectOnMsg() { 56 | err := c.connect() 57 | if err != nil { 58 | return err 59 | } 60 | } 61 | 62 | if c.ReconnectOnMsg { 63 | defer c.innerWriter.Close() 64 | } 65 | 66 | c.lg.println(when, msg) 67 | return nil 68 | } 69 | 70 | // Flush implementing method. empty. 71 | func (c *connWriter) Flush() { 72 | 73 | } 74 | 75 | // Destroy destroy connection writer and close tcp listener. 76 | func (c *connWriter) Destroy() { 77 | if c.innerWriter != nil { 78 | c.innerWriter.Close() 79 | } 80 | } 81 | 82 | func (c *connWriter) connect() error { 83 | if c.innerWriter != nil { 84 | c.innerWriter.Close() 85 | c.innerWriter = nil 86 | } 87 | 88 | conn, err := net.Dial(c.Net, c.Addr) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | if tcpConn, ok := conn.(*net.TCPConn); ok { 94 | tcpConn.SetKeepAlive(true) 95 | } 96 | 97 | c.innerWriter = conn 98 | c.lg = newLogWriter(conn) 99 | return nil 100 | } 101 | 102 | func (c *connWriter) needToConnectOnMsg() bool { 103 | if c.Reconnect { 104 | c.Reconnect = false 105 | return true 106 | } 107 | 108 | if c.innerWriter == nil { 109 | return true 110 | } 111 | 112 | return c.ReconnectOnMsg 113 | } 114 | 115 | func init() { 116 | Register(AdapterConn, NewConn) 117 | } 118 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/step_stats.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/allocation_description.proto"; 6 | import "tensorflow/core/framework/tensor_description.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | option java_outer_classname = "StepStatsProtos"; 10 | option java_multiple_files = true; 11 | option java_package = "org.tensorflow.framework"; 12 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/step_stats_go_proto"; 13 | 14 | // An allocation/de-allocation operation performed by the allocator. 15 | message AllocationRecord { 16 | // The timestamp of the operation. 17 | int64 alloc_micros = 1; 18 | // Number of bytes allocated, or de-allocated if negative. 19 | int64 alloc_bytes = 2; 20 | } 21 | 22 | message AllocatorMemoryUsed { 23 | string allocator_name = 1; 24 | // These are per-node allocator memory stats. 25 | int64 total_bytes = 2; 26 | int64 peak_bytes = 3; 27 | // The bytes that are not deallocated. 28 | int64 live_bytes = 4; 29 | // The allocation and deallocation timeline. 30 | repeated AllocationRecord allocation_records = 6; 31 | 32 | // These are snapshots of the overall allocator memory stats. 33 | // The number of live bytes currently allocated by the allocator. 34 | int64 allocator_bytes_in_use = 5; 35 | } 36 | 37 | // Output sizes recorded for a single execution of a graph node. 38 | message NodeOutput { 39 | int32 slot = 1; 40 | TensorDescription tensor_description = 3; 41 | } 42 | 43 | // For memory tracking. 44 | message MemoryStats { 45 | int64 temp_memory_size = 1; 46 | int64 persistent_memory_size = 3; 47 | repeated int64 persistent_tensor_alloc_ids = 5; 48 | 49 | int64 device_temp_memory_size = 2 [deprecated = true]; 50 | int64 device_persistent_memory_size = 4 [deprecated = true]; 51 | repeated int64 device_persistent_tensor_alloc_ids = 6 [deprecated = true]; 52 | } 53 | 54 | // Time/size stats recorded for a single execution of a graph node. 55 | message NodeExecStats { 56 | // TODO(tucker): Use some more compact form of node identity than 57 | // the full string name. Either all processes should agree on a 58 | // global id (cost_id?) for each node, or we should use a hash of 59 | // the name. 60 | string node_name = 1; 61 | int64 all_start_micros = 2; 62 | int64 op_start_rel_micros = 3; 63 | int64 op_end_rel_micros = 4; 64 | int64 all_end_rel_micros = 5; 65 | repeated AllocatorMemoryUsed memory = 6; 66 | repeated NodeOutput output = 7; 67 | string timeline_label = 8; 68 | int64 scheduled_micros = 9; 69 | uint32 thread_id = 10; 70 | repeated AllocationDescription referenced_tensor = 11; 71 | MemoryStats memory_stats = 12; 72 | int64 all_start_nanos = 13; 73 | int64 op_start_rel_nanos = 14; 74 | int64 op_end_rel_nanos = 15; 75 | int64 all_end_rel_nanos = 16; 76 | int64 scheduled_nanos = 17; 77 | } 78 | 79 | message DeviceStepStats { 80 | string device = 1; 81 | repeated NodeExecStats node_stats = 2; 82 | // Its key is thread id. 83 | map thread_names = 3; 84 | } 85 | 86 | message StepStats { 87 | repeated DeviceStepStats dev_stats = 1; 88 | } 89 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/cost_graph.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/tensor_shape.proto"; 6 | import "tensorflow/core/framework/types.proto"; 7 | 8 | option cc_enable_arenas = true; 9 | option java_outer_classname = "CostGraphProtos"; 10 | option java_multiple_files = true; 11 | option java_package = "org.tensorflow.framework"; 12 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/cost_graph_go_proto"; 13 | 14 | message CostGraphDef { 15 | message Node { 16 | // The name of the node. Names are globally unique. 17 | string name = 1; 18 | 19 | // The device of the node. Can be empty if the node is mapped to the 20 | // default partition or partitioning hasn't been run yet. 21 | string device = 2; 22 | 23 | // The id of the node. Node ids are only unique inside a partition. 24 | int32 id = 3; 25 | 26 | // Inputs of this node. They must be executed before this node can be 27 | // executed. An input is a particular output of another node, specified 28 | // by the node id and the output index. 29 | message InputInfo { 30 | int32 preceding_node = 1; 31 | int32 preceding_port = 2; 32 | } 33 | repeated InputInfo input_info = 4; 34 | 35 | // Outputs of this node. 36 | message OutputInfo { 37 | int64 size = 1; 38 | // If >= 0, the output is an alias of an input. Note that an alias input 39 | // may itself be an alias. The algorithm will therefore need to follow 40 | // those pointers. 41 | int64 alias_input_port = 2; 42 | TensorShapeProto shape = 3; 43 | DataType dtype = 4; 44 | } 45 | repeated OutputInfo output_info = 5; 46 | 47 | // Temporary memory used by this node. 48 | int64 temporary_memory_size = 6; 49 | 50 | // Persistent memory used by this node. 51 | int64 persistent_memory_size = 12; 52 | 53 | int64 host_temp_memory_size = 10 [deprecated = true]; 54 | int64 device_temp_memory_size = 11 [deprecated = true]; 55 | int64 device_persistent_memory_size = 16 [deprecated = true]; 56 | 57 | // Estimate of the computational cost of this node, in microseconds. 58 | int64 compute_cost = 9; 59 | 60 | // Analytical estimate of the computational cost of this node, in 61 | // microseconds. 62 | int64 compute_time = 14; 63 | 64 | // Analytical estimate of the memory access cost of this node, in 65 | // microseconds. 66 | int64 memory_time = 15; 67 | 68 | // If true, the output is permanent: it can't be discarded, because this 69 | // node is part of the "final output". Nodes may depend on final nodes. 70 | bool is_final = 7; 71 | 72 | // Ids of the control inputs for this node. 73 | repeated int32 control_input = 8; 74 | 75 | // Are the costs inaccurate? 76 | bool inaccurate = 17; 77 | } 78 | repeated Node node = 1; 79 | 80 | // Total cost of this graph, typically used for balancing decisions. 81 | message AggregatedCost { 82 | // Aggregated cost value. 83 | float cost = 1; 84 | 85 | // Aggregated cost dimension (e.g. 'memory', 'compute', 'network'). 86 | string dimension = 2; 87 | } 88 | repeated AggregatedCost cost = 2; 89 | } 90 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/variable.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | option cc_enable_arenas = true; 6 | option java_outer_classname = "VariableProtos"; 7 | option java_multiple_files = true; 8 | option java_package = "org.tensorflow.framework"; 9 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/variable_go_proto"; 10 | 11 | // Indicates when a distributed variable will be synced. 12 | enum VariableSynchronization { 13 | // `AUTO`: Indicates that the synchronization will be determined by the 14 | // current `DistributionStrategy` (eg. With `MirroredStrategy` this would be 15 | // `ON_WRITE`). 16 | VARIABLE_SYNCHRONIZATION_AUTO = 0; 17 | // `NONE`: Indicates that there will only be one copy of the variable, so 18 | // there is no need to sync. 19 | VARIABLE_SYNCHRONIZATION_NONE = 1; 20 | // `ON_WRITE`: Indicates that the variable will be updated across devices 21 | // every time it is written. 22 | VARIABLE_SYNCHRONIZATION_ON_WRITE = 2; 23 | // `ON_READ`: Indicates that the variable will be aggregated across devices 24 | // when it is read (eg. when checkpointing or when evaluating an op that uses 25 | // the variable). 26 | VARIABLE_SYNCHRONIZATION_ON_READ = 3; 27 | } 28 | 29 | // Indicates how a distributed variable will be aggregated. 30 | enum VariableAggregation { 31 | // `NONE`: This is the default, giving an error if you use a 32 | // variable-update operation with multiple replicas. 33 | VARIABLE_AGGREGATION_NONE = 0; 34 | // `SUM`: Add the updates across replicas. 35 | VARIABLE_AGGREGATION_SUM = 1; 36 | // `MEAN`: Take the arithmetic mean ("average") of the updates across 37 | // replicas. 38 | VARIABLE_AGGREGATION_MEAN = 2; 39 | // `ONLY_FIRST_REPLICA`: This is for when every replica is performing the same 40 | // update, but we only want to perform the update once. Used, e.g., for the 41 | // global step counter. 42 | VARIABLE_AGGREGATION_ONLY_FIRST_REPLICA = 3; 43 | } 44 | 45 | // Protocol buffer representing a Variable. 46 | message VariableDef { 47 | // Name of the variable tensor. 48 | string variable_name = 1; 49 | 50 | // Name of the tensor holding the variable's initial value. 51 | string initial_value_name = 6; 52 | 53 | // Name of the initializer op. 54 | string initializer_name = 2; 55 | 56 | // Name of the snapshot tensor. 57 | string snapshot_name = 3; 58 | 59 | // Support for saving variables as slices of a larger variable. 60 | SaveSliceInfoDef save_slice_info_def = 4; 61 | 62 | // Whether to represent this as a ResourceVariable. 63 | bool is_resource = 5; 64 | 65 | // Whether this variable should be trained. 66 | bool trainable = 7; 67 | 68 | // Indicates when a distributed variable will be synced. 69 | VariableSynchronization synchronization = 8; 70 | 71 | // Indicates how a distributed variable will be aggregated. 72 | VariableAggregation aggregation = 9; 73 | } 74 | 75 | message SaveSliceInfoDef { 76 | // Name of the full variable of which this is a slice. 77 | string full_name = 1; 78 | // Shape of the full variable. 79 | repeated int64 full_shape = 2; 80 | // Offset of this variable into the full variable. 81 | repeated int64 var_offset = 3; 82 | // Shape of this variable. 83 | repeated int64 var_shape = 4; 84 | } 85 | -------------------------------------------------------------------------------- /pkg/infer_services/service_factory.go: -------------------------------------------------------------------------------- 1 | package infer_services 2 | 3 | import ( 4 | "infer-microservices/internal/flags" 5 | "infer-microservices/pkg/infer_services/base_service" 6 | dubbo_service "infer-microservices/pkg/infer_services/dubbo_service" 7 | grpc_service "infer-microservices/pkg/infer_services/grpc_service" 8 | rest_service "infer-microservices/pkg/infer_services/rest_service" 9 | ) 10 | 11 | var skywalkingWeatherOpen bool 12 | var skywalkingIp string 13 | var skywalkingPort int 14 | var skywalkingServerName string 15 | var nacosIp string 16 | var nacosPort uint64 17 | var lowerRankNum int 18 | var lowerRecallNum int 19 | 20 | type RecommenderInferInterface interface { 21 | RecommenderInfer() 22 | } 23 | 24 | type ServiceFactory struct { 25 | } 26 | 27 | func init() { 28 | flagFactory := flags.FlagFactory{} 29 | //flagSkywalking 30 | flagSkywalking := flagFactory.CreateFlagSkywalking() 31 | skywalkingWeatherOpen = *flagSkywalking.GetSkywalkingWhetheropen() 32 | skywalkingIp = *flagSkywalking.GetSkywalkingIp() 33 | skywalkingPort = *flagSkywalking.GetSkywalkingPort() 34 | skywalkingServerName = *flagSkywalking.GetSkywalkingServername() 35 | 36 | //flagHystrix 37 | flagHystrix := flagFactory.CreateFlagHystrix() 38 | lowerRecallNum = *flagHystrix.GetHystrixLowerRecallNum() 39 | lowerRankNum = *flagHystrix.GetHystrixLowerRankNum() 40 | } 41 | 42 | // create base server 43 | func (f ServiceFactory) createBaseServiceSkywalking() *base_service.BaseService { 44 | baseService := new(base_service.BaseService) 45 | baseService.SetNacosIp(nacosIp) 46 | baseService.SetNacosPort(uint(nacosPort)) 47 | baseService.SetSkywalkingWeatherOpen(skywalkingWeatherOpen) 48 | baseService.SetSkywalkingIp(skywalkingIp) 49 | baseService.SetSkywalkingPort(uint(skywalkingPort)) 50 | baseService.SetSkywalkingServerName(skywalkingServerName) 51 | baseService.SetLowerRankNum(lowerRankNum) 52 | baseService.SetLowerRecallNum(lowerRecallNum) 53 | 54 | return baseService 55 | } 56 | 57 | // create base server 58 | func (f ServiceFactory) createBaseServiceNoSkywalking() *base_service.BaseService { 59 | baseService := new(base_service.BaseService) 60 | baseService.SetNacosIp(nacosIp) 61 | baseService.SetNacosPort(uint(nacosPort)) 62 | baseService.SetSkywalkingWeatherOpen(false) 63 | baseService.SetLowerRankNum(lowerRankNum) 64 | baseService.SetLowerRecallNum(lowerRecallNum) 65 | 66 | return baseService 67 | } 68 | 69 | // create dubbo server 70 | func (f ServiceFactory) CreateDubboService() *dubbo_service.DubboService { 71 | dubboService := new(dubbo_service.DubboService) 72 | dubboService.SetBaseService(f.createBaseServiceNoSkywalking()) 73 | 74 | return dubboService 75 | } 76 | 77 | // create grpc server 78 | func (f ServiceFactory) CreateGrpcService() *grpc_service.GrpcService { 79 | grpcService := new(grpc_service.GrpcService) 80 | grpcService.SetBaseService(f.createBaseServiceNoSkywalking()) 81 | 82 | return grpcService 83 | } 84 | 85 | // create http server 86 | // @deprecated 87 | func (f ServiceFactory) CreateHttpService() *rest_service.HttpService { 88 | httpService := new(rest_service.HttpService) 89 | httpService.SetBaseService(f.createBaseServiceSkywalking()) 90 | 91 | return httpService 92 | } 93 | 94 | // create echo server 95 | func (f ServiceFactory) CreateEchoService() *rest_service.EchoService { 96 | echoService := new(rest_service.EchoService) 97 | echoService.SetBaseService(f.createBaseServiceSkywalking()) 98 | 99 | return echoService 100 | } 101 | -------------------------------------------------------------------------------- /api/rest_api/server/rest_server_http_api_deprecated.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "infer-microservices/internal" 6 | "infer-microservices/internal/jwt" 7 | "infer-microservices/internal/logs" 8 | "infer-microservices/pkg/infer_services/rest_service" 9 | "net/http" 10 | "runtime" 11 | 12 | httpPlugin "github.com/SkyAPM/go2sky/plugins/http" 13 | ) 14 | 15 | //TODO: test gin rest api, test dubbo restful api 16 | 17 | type CallbackFunc func(w http.ResponseWriter, r *http.Request) 18 | 19 | type HttpServiceApi struct { 20 | serverPort uint 21 | maxCpuNum int 22 | httpService *rest_service.HttpService 23 | } 24 | 25 | // set func 26 | func (s *HttpServiceApi) SetServicePort(serverPort uint) { 27 | s.serverPort = serverPort 28 | } 29 | 30 | func (s *HttpServiceApi) SetMaxCpuNum(maxCpuNum int) { 31 | s.maxCpuNum = maxCpuNum 32 | } 33 | 34 | func (s *HttpServiceApi) SetRestService(httpService *rest_service.HttpService) { 35 | s.httpService = httpService 36 | } 37 | 38 | func (s *HttpServiceApi) restNoskywalkingServerRunner(path []string, CallbackFunc []CallbackFunc) error { 39 | for idx, p := range path { 40 | //http.HandleFunc(p, CallbackFunc [idx]) 41 | http.Handle(p, jwt.JwtAuthMiddleware(http.HandlerFunc(CallbackFunc[idx]))) 42 | } 43 | 44 | cpuNum := runtime.NumCPU() 45 | if s.maxCpuNum <= cpuNum { 46 | cpuNum = s.maxCpuNum 47 | } 48 | 49 | runtime.GOMAXPROCS(cpuNum) 50 | logs.Info("cup num:", cpuNum) 51 | 52 | addr := fmt.Sprintf(":%d", s.serverPort) 53 | err := http.ListenAndServe(addr, nil) 54 | if err == nil { 55 | logs.Error("server start succ ip:port ", addr) 56 | return err 57 | } 58 | 59 | return nil 60 | } 61 | 62 | func (s *HttpServiceApi) restSkywalkingServerRunner(go2skyAddr string, serverName string, path []string, CallbackFunc []CallbackFunc) error { 63 | internal.SkywalkingTracer(go2skyAddr, serverName) 64 | 65 | sm, err := httpPlugin.NewServerMiddleware(internal.GetTracer()) 66 | if err != nil { 67 | logs.Error("create server middleware error %v \n", err) 68 | } 69 | logs.Info("path:", path) 70 | logs.Info("CallbackFunc :", CallbackFunc) 71 | 72 | route := http.NewServeMux() 73 | for idx, p := range path { 74 | logs.Info("p CallbackFunc []:", p, CallbackFunc[idx]) 75 | //route.HandleFunc(p, CallbackFunc [idx]) 76 | route.Handle(p, jwt.JwtAuthMiddleware(http.HandlerFunc(CallbackFunc[idx]))) 77 | 78 | } 79 | 80 | cpuNum := runtime.NumCPU() 81 | if s.maxCpuNum <= cpuNum { 82 | cpuNum = s.maxCpuNum 83 | } 84 | 85 | runtime.GOMAXPROCS(cpuNum) 86 | logs.Info("cup num:", cpuNum) 87 | 88 | addr := fmt.Sprintf(":%d", s.serverPort) 89 | err = http.ListenAndServe(addr, sm(route)) 90 | if err == nil { 91 | logs.Error("server start succ ip:port ", err) 92 | return nil 93 | } 94 | 95 | return nil 96 | } 97 | 98 | // @implement start infertace 99 | func (s *HttpServiceApi) ServiceStart() { 100 | paths := []string{ 101 | "/login", "/infer", 102 | } 103 | 104 | CallbackFuncs := []CallbackFunc{ 105 | jwt.AuthHandler, s.httpService.SyncRecommenderInfer, 106 | } 107 | 108 | if s.httpService.GetBaseService().GetSkywalkingWeatherOpen() { 109 | go2skyAddr := s.httpService.GetBaseService().GetSkywalkingIp() + ":" + fmt.Sprintf(":%d", s.httpService.GetBaseService().GetSkywalkingPort()) 110 | go s.restSkywalkingServerRunner(go2skyAddr, s.httpService.GetBaseService().GetSkywalkingServerName(), paths, CallbackFuncs) 111 | } else { 112 | go s.restNoskywalkingServerRunner(paths, CallbackFuncs) 113 | } 114 | 115 | select {} 116 | } 117 | -------------------------------------------------------------------------------- /pkg/infer_services/dubbo_service/dubbo_service.go: -------------------------------------------------------------------------------- 1 | package dubbo_service 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | config_loader "infer-microservices/pkg/config_loader" 8 | "infer-microservices/pkg/infer_services/base_service" 9 | 10 | "infer-microservices/internal/logs" 11 | "infer-microservices/internal/utils" 12 | "infer-microservices/pkg/infer_services/io" 13 | nacos "infer-microservices/pkg/nacos" 14 | "time" 15 | 16 | _ "dubbo.apache.org/dubbo-go/v3/imports" 17 | ) 18 | 19 | // extend from baseservice 20 | type DubboService struct { 21 | baseService *base_service.BaseService 22 | } 23 | 24 | func (s *DubboService) SetBaseService(baseService *base_service.BaseService) { 25 | s.baseService = baseService 26 | } 27 | 28 | func (s *DubboService) GetBaseService() *base_service.BaseService { 29 | return s.baseService 30 | } 31 | 32 | //INFO:DONT REMOVE. JAVA request service need it. 33 | // // MethodMapper mapper upper func name to lower func name ,for java request. 34 | // func (s *InferDubbogoService) MethodMapper() map[string]string { 35 | // return map[string]string{ 36 | // "DubboRecommendServer": "dubboRecommendServer", 37 | // } 38 | // } 39 | 40 | // Implement interface methods. 41 | func (s *DubboService) RecommenderInfer(ctx context.Context, in *io.RecRequest) (*io.RecResponse, error) { 42 | response := &io.RecResponse{} 43 | response.SetCode(404) 44 | requestId := utils.CreateRequestId(in) 45 | logs.Debug(requestId, time.Now(), "RecRequest:", in) 46 | 47 | //check input 48 | checkStatus := in.Check() 49 | if !checkStatus { 50 | err := errors.New("input check failed") 51 | logs.Error(requestId, time.Now(), err) 52 | return response, err 53 | } 54 | 55 | //INFO: set timeout by context, degraded service by hystix. 56 | ctx, cancelFunc := context.WithTimeout(ctx, time.Millisecond*100) 57 | defer cancelFunc() 58 | 59 | respCh := make(chan *io.RecResponse, 100) 60 | go s.recommenderInferContext(ctx, in, respCh) 61 | 62 | select { 63 | case <-ctx.Done(): 64 | switch ctx.Err() { 65 | case context.DeadlineExceeded: 66 | logs.Error(requestId, time.Now(), ctx.Err()) 67 | case context.Canceled: 68 | logs.Error(requestId, time.Now(), ctx.Err()) 69 | } 70 | return response, ctx.Err() 71 | case responseCh := <-respCh: 72 | response = responseCh 73 | logs.Info(requestId, time.Now(), "response:", response) 74 | return response, nil 75 | } 76 | } 77 | 78 | func (s *DubboService) recommenderInferContext(ctx context.Context, in *io.RecRequest, respCh chan *io.RecResponse) { 79 | defer func() { 80 | if info := recover(); info != nil { 81 | logs.Fatal("panic", info) 82 | } //else { 83 | // logs.Info("finish.") 84 | //} 85 | }() 86 | 87 | response := &io.RecResponse{} 88 | response.SetCode(404) 89 | requestId := utils.CreateRequestId(in) 90 | 91 | //nacos listen 92 | nacosFactory := nacos.NacosFactory{} 93 | nacosConfig := nacosFactory.CreateNacosConfig(s.baseService.GetNacosIp(), uint64(s.baseService.GetNacosPort()), in) 94 | logs.Debug(requestId, time.Now(), "nacosConfig:", nacosConfig) 95 | 96 | nacosConfig.StartListenNacos() 97 | 98 | //infer 99 | ServiceConfig := config_loader.GetServiceConfigs()[in.GetDataId()] 100 | response_, err := s.baseService.RecommenderInferHystrix(nil, "dubboServer", in, ServiceConfig) 101 | if err != nil || len(response_) == 0 { 102 | response.SetMessage(fmt.Sprintf("%s", err)) 103 | panic(err) 104 | } else { 105 | response.SetCode(response_["code"].(int)) 106 | response.SetMessage(response_["message"].(string)) 107 | response.SetData(response_["data"].([]string)) 108 | } 109 | 110 | respCh <- response 111 | } 112 | -------------------------------------------------------------------------------- /scripts/deployments/infer/services_start.sh: -------------------------------------------------------------------------------- 1 | # start script in docker. 2 | 3 | #!/bin/bash 4 | 5 | today=`date -d "0 days ago" "+%Y%m%d"` 6 | go_project=recsys-go-infer 7 | WORK_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 8 | LOGPATH=${WORK_PATH}/logs 9 | 10 | echo $WORK_PATH 11 | echo $LOGPATH 12 | 13 | #conf file 14 | service_start_file="./configs/server_start_config.json" 15 | 16 | #dubbo 17 | dubbo_server_conf1="./configs/dubbogo_server-1_config.yml" 18 | dubbo_server_conf2="./configs/dubbogo_server-2_config.yml" 19 | 20 | #nacos 21 | nacos_ip="10.10.10.11" 22 | nacos_port=1022 23 | nacos_username="nacos" 24 | nacos_password="nacos" 25 | nacos_loglevel="error" 26 | 27 | #hystrix 28 | hystrix_timeoutMS=100 29 | hystrix_lowerRecallNum=100 30 | hystrix_lowerRankNum=20 31 | 32 | #skywalking 33 | skywalking_ip="10.10.10.10" 34 | skywalking_port=1020 35 | skywalking_whetheropen=true 36 | skywalking_servername="infer" 37 | 38 | #redis 39 | redis_password="#67fag!@#" 40 | 41 | #port 42 | rest_server_port=6020 43 | grpc_server_port=6021 44 | max_cpu_num=20 45 | 46 | #bigcache 47 | bigcahe_lifeWindowS=300 48 | bigcache_cleanWindowS=120 49 | bigcache_hardMaxCacheSize=409600 50 | 51 | #tensorflow 52 | tfserving_timeoutms=100 53 | 54 | #logs 55 | log_level="error" 56 | 57 | function status(){ 58 | ps -ef | grep ${go_project} 59 | } 60 | 61 | function start(){ 62 | 63 | echo "starting servers ..." 64 | cd ${WORK_PATH}/ 65 | dubbo_conf=$1 66 | 67 | nohup ${WORK_PATH}/${go_project} \ 68 | 69 | #start file 70 | --service_start_file 71 | 72 | #dubbo 73 | --dubbo_server_conf=${dubbo_conf} \ 74 | 75 | #nacos 76 | --nacos_ip=${nacos_ip} \ 77 | --nacos_port=${nacos_port} 78 | --nacos_username=${nacos_username} \ 79 | --nacos_password=${nacos_password} \ 80 | --nacos_loglevel=${nacos_loglevel} \ 81 | 82 | #hystrix 83 | --hystrix_timeoutMS=${hystrix_timeoutMS} \ 84 | --hystrix_lowerRecallNum=${hystrix_lowerRecallNum} \ 85 | --hystrix_lowerRankNum=${hystrix_lowerRankNum} \ 86 | 87 | #skywalking 88 | --skywalking_ip=${skywalking_ip} \ 89 | --skywalking_port=${skywalking_port} \ 90 | --skywalking_whetheropen=${skywalking_whetheropen} \ 91 | --skywalking_servername=${skywalking_servername} \ 92 | 93 | #redis 94 | --redis_password=${redis_password} \ 95 | 96 | #server port 97 | --rest_server_port=${rest_server_port} \ 98 | --grpc_server_port=${grpc_server_port} \ 99 | --max_cpu_num=${max_cpu_num} \ 100 | 101 | #bigcache 102 | --bigcahe_lifeWindowS=${bigcahe_lifeWindowS} \ 103 | --bigcache_cleanWindowS=${bigcache_cleanWindowS} \ 104 | --bigcache_hardMaxCacheSize=${bigcache_hardMaxCacheSize} \ 105 | 106 | #tensorflow 107 | --tfserving_timeoutms=${tfserving_timeoutms} \ 108 | 109 | #logs 110 | --log_level=${log_level} > ${LOGPATH}/${go_project}_${today}.log 2>&1 & 111 | 112 | } 113 | 114 | function stop(){ 115 | ps -ef | grep ${go_project} | grep -v grep | awk '{print $2}' | xargs kill -9 116 | } 117 | 118 | case $1 in 119 | "status") 120 | status 121 | ;; 122 | 123 | "start") 124 | #start 2 services. 125 | start ${dubbo_server_conf1} 126 | start ${dubbo_server_conf2} 127 | status 128 | ;; 129 | 130 | "stop") 131 | status 132 | stop 133 | ;; 134 | 135 | *) 136 | echo "run args err, please input (status | start | stop) " 137 | esac 138 | 139 | -------------------------------------------------------------------------------- /pkg/infer_services/grpc_service/grpc_service.go: -------------------------------------------------------------------------------- 1 | package grpc_service 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "infer-microservices/internal/utils" 7 | config_loader "infer-microservices/pkg/config_loader" 8 | "time" 9 | 10 | "infer-microservices/internal/logs" 11 | "infer-microservices/pkg/infer_services/base_service" 12 | "infer-microservices/pkg/infer_services/io" 13 | nacos "infer-microservices/pkg/nacos" 14 | 15 | "golang.org/x/net/context" 16 | ) 17 | 18 | // extend from baseservice 19 | type GrpcService struct { 20 | baseservice *base_service.BaseService 21 | } 22 | 23 | func (s *GrpcService) SetBaseService(baseservice *base_service.BaseService) { 24 | s.baseservice = baseservice 25 | } 26 | 27 | func (s *GrpcService) GetBaseService() *base_service.BaseService { 28 | return s.baseservice 29 | } 30 | 31 | // INFO: implement grpc func which defined by proto. 32 | func (s *GrpcService) RecommenderInfer(ctx context.Context, in *RecommendRequest) (*RecommendResponse, error) { 33 | //INFO: set timeout by context, degraded service by hystix. 34 | response := &RecommendResponse{ 35 | Code: 404, 36 | } 37 | request := convertGrpcRequestToRecRequest(in) 38 | requestId := utils.CreateRequestId(&request) 39 | logs.Debug(requestId, time.Now(), "RecRequest:", requestId) 40 | 41 | ctx, cancelFunc := context.WithTimeout(ctx, time.Millisecond*100) 42 | defer cancelFunc() 43 | 44 | respCh := make(chan *RecommendResponse, 100) 45 | go s.recommenderInferContext(ctx, in, respCh) 46 | 47 | select { 48 | case <-ctx.Done(): 49 | switch ctx.Err() { 50 | case context.DeadlineExceeded: 51 | logs.Error(requestId, time.Now(), ctx.Err()) 52 | case context.Canceled: 53 | logs.Error(requestId, time.Now(), ctx.Err()) 54 | } 55 | return response, ctx.Err() 56 | case responseCh := <-respCh: 57 | response = responseCh 58 | logs.Info(requestId, time.Now(), "response:", response) 59 | return response, nil 60 | } 61 | } 62 | 63 | func (s *GrpcService) recommenderInferContext(ctx context.Context, in *RecommendRequest, respCh chan *RecommendResponse) { 64 | defer func() { 65 | if info := recover(); info != nil { 66 | logs.Fatal("panic", info) 67 | } //else { 68 | //fmt.Println("") 69 | //} 70 | }() 71 | 72 | response := &RecommendResponse{ 73 | Code: 404, 74 | } 75 | request := convertGrpcRequestToRecRequest(in) 76 | requestId := utils.CreateRequestId(&request) 77 | 78 | //check input 79 | checkStatus := request.Check() 80 | if !checkStatus { 81 | err := errors.New("input check failed") 82 | logs.Error(requestId, time.Now(), err) 83 | panic(err) 84 | } 85 | 86 | //nacos listen 87 | nacosFactory := nacos.NacosFactory{} 88 | nacosConfig := nacosFactory.CreateNacosConfig(s.baseservice.GetNacosIp(), uint64(s.baseservice.GetNacosPort()), &request) 89 | logs.Debug(requestId, time.Now(), "nacosConfig:", nacosConfig) 90 | 91 | nacosConfig.StartListenNacos() 92 | 93 | //infer 94 | ServiceConfig := config_loader.GetServiceConfigs()[in.GetDataId()] 95 | response_, err := s.baseservice.RecommenderInferHystrix(nil, "GrpcService", &request, ServiceConfig) 96 | if err != nil { 97 | response.Message = fmt.Sprintf("%s", err) 98 | panic(err) 99 | } else { 100 | response = &RecommendResponse{ 101 | Code: response_["code"].(int32), 102 | Message: response_["message"].(string), 103 | Data: &ItemInfoList{ 104 | Iteminfo_: response_["data"].([]*ItemInfo), 105 | }, 106 | } 107 | 108 | } 109 | respCh <- response 110 | } 111 | 112 | func convertGrpcRequestToRecRequest(in *RecommendRequest) io.RecRequest { 113 | request := io.RecRequest{} 114 | request.SetDataId(in.GetDataId()) 115 | request.SetGroupId(in.GetGroupId()) 116 | request.SetNamespaceId(in.GetNamespace()) 117 | request.SetUserId(in.UserId) 118 | 119 | return request 120 | } 121 | -------------------------------------------------------------------------------- /internal/logs/multifile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "encoding/json" 19 | "time" 20 | ) 21 | 22 | // A filesLogWriter manages several fileLogWriter 23 | // filesLogWriter will write logs to the file in json configuration and write the same level log to correspond file 24 | // means if the file name in configuration is project.log filesLogWriter will create project.error.log/project.debug.log 25 | // and write the error-level logs to project.error.log and write the debug-level logs to project.debug.log 26 | // the rotate attribute also acts like fileLogWriter 27 | type multiFileLogWriter struct { 28 | writers [LevelDebug + 1 + 1]*fileLogWriter // the last one for fullLogWriter 29 | fullLogWriter *fileLogWriter 30 | Separate []string `json:"separate"` 31 | } 32 | 33 | var levelNames = [...]string{"emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"} 34 | 35 | // Init file logger with json config. 36 | // jsonConfig like: 37 | // { 38 | // "filename":"logs/beego.log", 39 | // "maxLines":0, 40 | // "maxsize":0, 41 | // "daily":true, 42 | // "maxDays":15, 43 | // "rotate":true, 44 | // "perm":0600, 45 | // "separate":["emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"], 46 | // } 47 | 48 | func (f *multiFileLogWriter) Init(config string) error { 49 | writer := newFileWriter().(*fileLogWriter) 50 | err := writer.Init(config) 51 | if err != nil { 52 | return err 53 | } 54 | f.fullLogWriter = writer 55 | f.writers[LevelDebug+1] = writer 56 | 57 | //unmarshal "separate" field to f.Separate 58 | json.Unmarshal([]byte(config), f) 59 | 60 | jsonMap := map[string]interface{}{} 61 | json.Unmarshal([]byte(config), &jsonMap) 62 | 63 | for i := LevelEmergency; i < LevelDebug+1; i++ { 64 | for _, v := range f.Separate { 65 | if v == levelNames[i] { 66 | jsonMap["filename"] = f.fullLogWriter.fileNameOnly + "." + levelNames[i] + f.fullLogWriter.suffix 67 | jsonMap["level"] = i 68 | bs, _ := json.Marshal(jsonMap) 69 | writer = newFileWriter().(*fileLogWriter) 70 | writer.Init(string(bs)) 71 | f.writers[i] = writer 72 | } 73 | } 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func (f *multiFileLogWriter) Destroy() { 80 | for i := 0; i < len(f.writers); i++ { 81 | if f.writers[i] != nil { 82 | f.writers[i].Destroy() 83 | } 84 | } 85 | } 86 | 87 | func (f *multiFileLogWriter) WriteMsg(when time.Time, msg string, level int) error { 88 | if f.fullLogWriter != nil { 89 | f.fullLogWriter.WriteMsg(when, msg, level) 90 | } 91 | for i := 0; i < len(f.writers)-1; i++ { 92 | if f.writers[i] != nil { 93 | if level == f.writers[i].Level { 94 | f.writers[i].WriteMsg(when, msg, level) 95 | } 96 | } 97 | } 98 | return nil 99 | } 100 | 101 | func (f *multiFileLogWriter) Flush() { 102 | for i := 0; i < len(f.writers); i++ { 103 | if f.writers[i] != nil { 104 | f.writers[i].Flush() 105 | } 106 | } 107 | } 108 | 109 | // newFilesWriter create a FileLogWriter returning as LoggerInterface. 110 | func newFilesWriter() Logger { 111 | return &multiFileLogWriter{} 112 | } 113 | 114 | func init() { 115 | Register(AdapterMultiFile, newFilesWriter) 116 | } 117 | -------------------------------------------------------------------------------- /internal/db/redis/redis.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "context" 5 | "infer-microservices/internal/flags" 6 | "time" 7 | 8 | redis "github.com/go-redis/redis/v8" 9 | ) 10 | 11 | var ctx = context.Background() 12 | var password string 13 | var flagRedis flags.FlagRedis 14 | 15 | type InferRedisClient struct { 16 | cli *redis.ClusterClient 17 | } 18 | 19 | func init() { 20 | flagFactory := flags.FlagFactory{} 21 | flagRedis := flagFactory.CreateFlagRedis() 22 | password = *flagRedis.GetRedisPassword() 23 | } 24 | 25 | func NewRedisClusterClient(data map[string]interface{}) *InferRedisClient { 26 | addrs_raw := data["addrs"].([]interface{}) 27 | readTimeout := time.Duration(int64(data["readTimeoutMs"].(float64))) * time.Millisecond 28 | writeTimeout := time.Duration(int64(data["writeTimeoutMs"].(float64))) * time.Millisecond 29 | dialTimeout := time.Duration(int64(data["dialTimeoutMs"].(float64))) * time.Millisecond 30 | idleTimeout := time.Duration(int64(data["idleTimeoutS"].(float64))) * time.Second 31 | maxRetries := int(data["maxRetries"].(float64)) 32 | minIdleConns := int(data["minIdleConns"].(float64)) 33 | 34 | addrs := make([]string, 0) 35 | for _, addr := range addrs_raw { 36 | addrs = append(addrs, addr.(string)) 37 | } 38 | cli := redis.NewClusterClient(&redis.ClusterOptions{ 39 | Addrs: addrs, 40 | Password: password, 41 | ReadTimeout: readTimeout, 42 | WriteTimeout: writeTimeout, 43 | DialTimeout: dialTimeout, 44 | IdleTimeout: idleTimeout, 45 | PoolTimeout: 4 * time.Second, 46 | MaxRetries: maxRetries, 47 | MinIdleConns: minIdleConns, 48 | ReadOnly: true, 49 | RouteRandomly: true, 50 | }) 51 | 52 | return &InferRedisClient{cli: cli} 53 | } 54 | 55 | func (m *InferRedisClient) Get(key string) (string, error) { 56 | cmd := m.cli.Get(ctx, key) 57 | value, err := cmd.Result() 58 | if err != nil { 59 | return "", err 60 | } 61 | 62 | return value, nil 63 | } 64 | 65 | func (m *InferRedisClient) Set(key string, value string, expire time.Duration) error { 66 | return m.cli.Set(ctx, key, value, expire).Err() 67 | } 68 | 69 | func (m *InferRedisClient) HGet(key string, field string) (string, error) { 70 | value, err := m.cli.HGet(ctx, key, field).Result() 71 | if err != nil { 72 | return "", err 73 | } 74 | return value, nil 75 | } 76 | 77 | func (m *InferRedisClient) HSet(key string, field string, value string) error { 78 | return m.cli.HSet(ctx, key, field, value).Err() 79 | } 80 | 81 | func (m *InferRedisClient) HGetAll(key string) (map[string]string, error) { 82 | value, err := m.cli.HGetAll(ctx, key).Result() 83 | if err != nil { 84 | return map[string]string{}, err 85 | } 86 | return value, nil 87 | } 88 | 89 | // list 90 | func (m *InferRedisClient) Lrange(key string) ([]string, error) { 91 | cmd := m.cli.LRange(ctx, key, 0, -1) 92 | value, err := cmd.Result() 93 | if err != nil { 94 | return make([]string, 0), err 95 | } 96 | 97 | return value, nil 98 | } 99 | 100 | func (m *InferRedisClient) Lrem(key string, value string) error { 101 | return m.cli.LRem(ctx, key, 0, value).Err() 102 | } 103 | 104 | func (m *InferRedisClient) Exists(key string) (bool, error) { 105 | ok, err := m.cli.Exists(ctx, key).Result() 106 | if err != nil { 107 | return false, err 108 | } 109 | 110 | if ok == 1 { 111 | return true, nil 112 | } else { 113 | return false, nil 114 | } 115 | } 116 | 117 | // INFO: Expire shoud after Push 118 | func (m *InferRedisClient) LPush(key string, value string, expire time.Duration) error { 119 | m.cli.LPush(ctx, key, value) 120 | return m.cli.Expire(ctx, key, expire).Err() 121 | } 122 | 123 | func (m *InferRedisClient) RPush(key string, value string, expire time.Duration) error { 124 | m.cli.RPush(ctx, key, value) 125 | return m.cli.Expire(ctx, key, expire).Err() 126 | } 127 | 128 | // del 129 | func (m *InferRedisClient) Delete(key string) error { 130 | return m.cli.Del(ctx, key).Err() 131 | } 132 | -------------------------------------------------------------------------------- /pkg/infer_services/rest_service/echo_service.go: -------------------------------------------------------------------------------- 1 | package rest_service 2 | 3 | import ( 4 | "errors" 5 | "infer-microservices/internal/logs" 6 | "infer-microservices/internal/utils" 7 | config_loader "infer-microservices/pkg/config_loader" 8 | "infer-microservices/pkg/infer_services/base_service" 9 | "infer-microservices/pkg/infer_services/io" 10 | nacos "infer-microservices/pkg/nacos" 11 | "net/http" 12 | "time" 13 | 14 | jsoniter "github.com/json-iterator/go" 15 | "github.com/labstack/echo" 16 | ) 17 | 18 | // extend from baseservice 19 | type EchoService struct { 20 | baseservice *base_service.BaseService 21 | } 22 | 23 | func (s *EchoService) SetBaseService(baseservice *base_service.BaseService) { 24 | s.baseservice = baseservice 25 | } 26 | 27 | func (s *EchoService) GetBaseService() *base_service.BaseService { 28 | return s.baseservice 29 | } 30 | 31 | // sync server 32 | func (s *EchoService) SyncRecommenderInfer(c echo.Context) error { 33 | respCh := make(chan map[string]interface{}, 100) 34 | go s.RecommenderInfer(c, respCh) 35 | 36 | select { 37 | case <-time.After(time.Millisecond * 100): 38 | rsp := make(map[string]interface{}, 0) 39 | return c.JSON(http.StatusRequestTimeout, rsp) 40 | case responseCh := <-respCh: 41 | return c.JSON(http.StatusOK, responseCh) 42 | } 43 | 44 | } 45 | 46 | // infer 47 | func (s *EchoService) RecommenderInfer(c echo.Context, ch chan<- map[string]interface{}) { 48 | defer func() { 49 | if info := recover(); info != nil { 50 | logs.Fatal("panic", info) 51 | rsp := make(map[string]interface{}, 0) 52 | ch <- rsp 53 | } //else { 54 | //fmt.Println("") 55 | //} 56 | }() 57 | 58 | rsp := make(map[string]interface{}, 0) 59 | //INFO: convert http string data to struct data. 60 | request, err := s.convertHttpRequstToRecRequest(c) 61 | requestId := utils.CreateRequestId(&request) 62 | 63 | if err != nil { 64 | logs.Error(requestId, time.Now(), err) 65 | panic(err) 66 | } 67 | 68 | //check http input 69 | checkStatus := s.check(c, requestId) 70 | if !checkStatus { 71 | err := errors.New("http input check failed") 72 | logs.Error(err) 73 | panic(err) 74 | } 75 | //check input 76 | checkStatus = request.Check() 77 | if !checkStatus { 78 | err := errors.New("input check failed") 79 | logs.Error(requestId, time.Now(), err) 80 | panic(err) 81 | } 82 | 83 | //nacos listen 84 | nacosFactory := nacos.NacosFactory{} 85 | nacosConfig := nacosFactory.CreateNacosConfig(s.baseservice.GetNacosIp(), uint64(s.baseservice.GetNacosPort()), &request) 86 | logs.Debug(requestId, time.Now(), "nacosConfig:", nacosConfig) 87 | 88 | nacosConfig.StartListenNacos() 89 | 90 | //infer 91 | ServiceConfig := config_loader.GetServiceConfigs()[request.GetDataId()] 92 | response, err := s.baseservice.RecommenderInferHystrix(c.Request(), "restServer", &request, ServiceConfig) 93 | if err != nil { 94 | logs.Error(requestId, time.Now(), err) 95 | panic(err) 96 | } else { 97 | rsp["data"] = response 98 | } 99 | 100 | ch <- rsp 101 | } 102 | 103 | func (s *EchoService) check(c echo.Context, requestId string) bool { 104 | err := c.Request().ParseForm() 105 | if err != nil { 106 | logs.Error(requestId, time.Now(), err) 107 | return false 108 | } 109 | 110 | method := c.Request().Method 111 | if method != "POST" { 112 | logs.Error(requestId, time.Now(), err) 113 | return false 114 | } 115 | 116 | data := c.Request().Form["data"] 117 | if len(data) == 0 { 118 | logs.Error(requestId, time.Now(), err) 119 | return false 120 | } 121 | 122 | return true 123 | } 124 | 125 | func (s *EchoService) convertHttpRequstToRecRequest(c echo.Context) (io.RecRequest, error) { 126 | request := io.RecRequest{} 127 | data := c.Request().Form["data"] 128 | requestMap := make(map[string]interface{}, 0) 129 | err := jsoniter.Unmarshal([]byte(data[0]), &requestMap) 130 | if err != nil { 131 | return request, err 132 | } 133 | 134 | return request, nil 135 | } 136 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/tensor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | 5 | import "tensorflow/core/framework/resource_handle.proto"; 6 | import "tensorflow/core/framework/tensor_shape.proto"; 7 | import "tensorflow/core/framework/types.proto"; 8 | 9 | option cc_enable_arenas = true; 10 | option java_outer_classname = "TensorProtos"; 11 | option java_multiple_files = true; 12 | option java_package = "org.tensorflow.framework"; 13 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_go_proto"; 14 | 15 | // Protocol buffer representing a tensor. 16 | message TensorProto { 17 | DataType dtype = 1; 18 | 19 | // Shape of the tensor. TODO(touts): sort out the 0-rank issues. 20 | TensorShapeProto tensor_shape = 2; 21 | 22 | // Only one of the representations below is set, one of "tensor_contents" and 23 | // the "xxx_val" attributes. We are not using oneof because as oneofs cannot 24 | // contain repeated fields it would require another extra set of messages. 25 | 26 | // Version number. 27 | // 28 | // In version 0, if the "repeated xxx" representations contain only one 29 | // element, that element is repeated to fill the shape. This makes it easy 30 | // to represent a constant Tensor with a single value. 31 | int32 version_number = 3; 32 | 33 | // Serialized raw tensor content from either Tensor::AsProtoTensorContent or 34 | // memcpy in tensorflow::grpc::EncodeTensorToByteBuffer. This representation 35 | // can be used for all tensor types. The purpose of this representation is to 36 | // reduce serialization overhead during RPC call by avoiding serialization of 37 | // many repeated small items. 38 | bytes tensor_content = 4; 39 | 40 | // Type specific representations that make it easy to create tensor protos in 41 | // all languages. Only the representation corresponding to "dtype" can 42 | // be set. The values hold the flattened representation of the tensor in 43 | // row major order. 44 | 45 | // DT_HALF, DT_BFLOAT16. Note that since protobuf has no int16 type, we'll 46 | // have some pointless zero padding for each value here. 47 | repeated int32 half_val = 13 [packed = true]; 48 | 49 | // DT_FLOAT. 50 | repeated float float_val = 5 [packed = true]; 51 | 52 | // DT_DOUBLE. 53 | repeated double double_val = 6 [packed = true]; 54 | 55 | // DT_INT32, DT_INT16, DT_UINT16, DT_INT8, DT_UINT8. 56 | repeated int32 int_val = 7 [packed = true]; 57 | 58 | // DT_STRING 59 | repeated bytes string_val = 8; 60 | 61 | // DT_COMPLEX64. scomplex_val(2*i) and scomplex_val(2*i+1) are real 62 | // and imaginary parts of i-th single precision complex. 63 | repeated float scomplex_val = 9 [packed = true]; 64 | 65 | // DT_INT64 66 | repeated int64 int64_val = 10 [packed = true]; 67 | 68 | // DT_BOOL 69 | repeated bool bool_val = 11 [packed = true]; 70 | 71 | // DT_COMPLEX128. dcomplex_val(2*i) and dcomplex_val(2*i+1) are real 72 | // and imaginary parts of i-th double precision complex. 73 | repeated double dcomplex_val = 12 [packed = true]; 74 | 75 | // DT_RESOURCE 76 | repeated ResourceHandleProto resource_handle_val = 14; 77 | 78 | // DT_VARIANT 79 | repeated VariantTensorDataProto variant_val = 15; 80 | 81 | // DT_UINT32 82 | repeated uint32 uint32_val = 16 [packed = true]; 83 | 84 | // DT_UINT64 85 | repeated uint64 uint64_val = 17 [packed = true]; 86 | 87 | // DT_FLOAT8_*, use variable-sized set of bytes 88 | // (i.e. the equivalent of repeated uint8, if such a thing existed). 89 | bytes float8_val = 18; 90 | } 91 | 92 | // Protocol buffer representing the serialization format of DT_VARIANT tensors. 93 | message VariantTensorDataProto { 94 | // Name of the type of objects being serialized. 95 | string type_name = 1; 96 | // Portions of the object that are not Tensors. 97 | bytes metadata = 2; 98 | // Tensors contained within objects being serialized. 99 | repeated TensorProto tensors = 3; 100 | } 101 | -------------------------------------------------------------------------------- /pkg/infer_samples/ready_samples.go: -------------------------------------------------------------------------------- 1 | package infer_samples 2 | 3 | import ( 4 | "infer-microservices/internal/flags" 5 | "infer-microservices/internal/logs" 6 | config_loader "infer-microservices/pkg/config_loader" 7 | feature "infer-microservices/pkg/infer_features" 8 | 9 | "time" 10 | 11 | "github.com/allegro/bigcache" 12 | bloomv3 "github.com/bits-and-blooms/bloom/v3" 13 | ) 14 | 15 | 16 | //INFO:solution-A : All tfrecored format samples have been preprocessed and stored in Redis 17 | 18 | 19 | var bigCacheConfSample bigcache.Config 20 | var lifeWindowS time.Duration 21 | var cleanWindowS time.Duration 22 | var hardMaxCacheSize int 23 | var maxEntrySize int 24 | var maxEntriesInWindow int 25 | var verbose bool 26 | var shards int 27 | var bigCacheSample *bigcache.BigCache 28 | var err error 29 | 30 | type InferSample struct { 31 | modelName string 32 | serviceConfig *config_loader.ServiceConfig 33 | bigCacheSample *bigcache.BigCache 34 | userBloomFilter *bloomv3.BloomFilter 35 | itemBloomFilter *bloomv3.BloomFilter 36 | userOfflineSample *feature.SeqExampleBuff 37 | userRealtimeSample *feature.SeqExampleBuff 38 | itemsSample *[]feature.SeqExampleBuff 39 | } 40 | 41 | func init() { 42 | flagFactory := flags.FlagFactory{} 43 | flagCache := flagFactory.CreateFlagCache() 44 | lifeWindowS = time.Duration(*flagCache.GetBigcacheLifeWindowS()) 45 | cleanWindowS = time.Duration(*flagCache.GetBigcacheCleanWindowS()) 46 | hardMaxCacheSize = *flagCache.GetBigcacheHardMaxCacheSize() 47 | maxEntrySize = *flagCache.GetBigcacheMaxEntrySize() 48 | bigCacheConfSample = bigcache.Config{ 49 | Shards: shards, 50 | LifeWindow: lifeWindowS * time.Minute, 51 | CleanWindow: cleanWindowS * time.Minute, 52 | MaxEntriesInWindow: maxEntriesInWindow, 53 | MaxEntrySize: maxEntrySize, 54 | Verbose: verbose, 55 | HardMaxCacheSize: hardMaxCacheSize, 56 | OnRemove: nil, 57 | OnRemoveWithReason: nil, 58 | } 59 | 60 | //set cache 61 | bigCacheSample, err = bigcache.NewBigCache(bigCacheConfSample) 62 | if err != nil { 63 | logs.Error(err) 64 | } 65 | } 66 | 67 | // userid 68 | func (b InferSample) SetModelName(modelName string) { 69 | b.modelName = modelName 70 | } 71 | 72 | func (b InferSample) GetModelName() string { 73 | return b.modelName 74 | } 75 | 76 | // serviceConfig *service_config.ServiceConfig 77 | func (b InferSample) SetServiceConfig(serviceConfig *config_loader.ServiceConfig) { 78 | b.serviceConfig = serviceConfig 79 | } 80 | 81 | func (b InferSample) GetServiceConfig() *config_loader.ServiceConfig { 82 | return b.serviceConfig 83 | } 84 | 85 | func (s InferSample) SetUserBloomFilter(filter *bloomv3.BloomFilter) { 86 | s.userBloomFilter = filter 87 | } 88 | 89 | func (s InferSample) GetUserBloomFilter() *bloomv3.BloomFilter { 90 | return s.userBloomFilter 91 | } 92 | 93 | func (s InferSample) SetItemBloomFilter(filter *bloomv3.BloomFilter) { 94 | s.itemBloomFilter = filter 95 | } 96 | 97 | func (s InferSample) GetItemBloomFilter() *bloomv3.BloomFilter { 98 | return s.itemBloomFilter 99 | } 100 | 101 | func (s InferSample) GetUserOfflineSample() *feature.SeqExampleBuff { 102 | return s.userOfflineSample 103 | } 104 | 105 | func (s InferSample) SetUserOfflineSample(userOfflineSample *feature.SeqExampleBuff) { 106 | s.userOfflineSample = userOfflineSample 107 | } 108 | 109 | func (s InferSample) GetUserRealtimeSample() *feature.SeqExampleBuff { 110 | return s.userRealtimeSample 111 | } 112 | 113 | func (s InferSample) SetUserRealtimeSample(userRealtimeSample *feature.SeqExampleBuff) { 114 | s.userRealtimeSample = userRealtimeSample 115 | } 116 | 117 | func (s InferSample) GetItemsSample() *[]feature.SeqExampleBuff { 118 | return s.itemsSample 119 | } 120 | 121 | func (s InferSample) SetItemsSample(itemsSample *[]feature.SeqExampleBuff) { 122 | s.itemsSample = itemsSample 123 | } 124 | 125 | // observer nontify 126 | func (s InferSample) notify(sub Subject) { 127 | //reload baseModel 128 | s.SetUserBloomFilter(GetUserBloomFilterInstance()) 129 | s.SetItemBloomFilter(GetItemBloomFilterInstance()) 130 | } 131 | -------------------------------------------------------------------------------- /internal/tensorflow_gogofaster/core/framework/proto/model.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow.data.model; 4 | 5 | option cc_enable_arenas = true; 6 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/model_go_proto"; 7 | 8 | // Class of a node in the performance model. 9 | enum NodeClass { 10 | UNKNOWN = 0; 11 | INTERLEAVE_MANY = 1; 12 | ASYNC_INTERLEAVE_MANY = 2; 13 | KNOWN_RATIO = 3; 14 | ASYNC_KNOWN_RATIO = 4; 15 | UNKNOWN_RATIO = 5; 16 | ASYNC_UNKNOWN_RATIO = 6; 17 | } 18 | 19 | // Algorithm used for model autotuning optimization. 20 | enum AutotuneAlgorithm { 21 | DEFAULT = 0; 22 | HILL_CLIMB = 1; 23 | GRADIENT_DESCENT = 2; 24 | MAX_PARALLELISM = 3; 25 | STAGE_BASED = 4; 26 | } 27 | 28 | // Protocol buffer representing the data used by the autotuning modeling 29 | // framework. 30 | message ModelProto { 31 | // General representation of a node in the model. 32 | message Node { 33 | // Unique node ID. 34 | int64 id = 1; 35 | 36 | // Human-readable name of the node. 37 | string name = 2; 38 | 39 | // An indication whether autotuning is enabled for this node. 40 | bool autotune = 3; 41 | 42 | // The number of bytes stored in this node's buffer. 43 | int64 buffered_bytes = 4; 44 | 45 | // The number of elements stored in this node's buffer. 46 | int64 buffered_elements = 5; 47 | 48 | // The number of bytes consumed by the node. 49 | int64 bytes_consumed = 6; 50 | 51 | // The number of bytes produced by the node. 52 | int64 bytes_produced = 7; 53 | 54 | // The number of elements produced by the node. 55 | int64 num_elements = 8; 56 | 57 | // The aggregate processing time spent in this node in nanoseconds. 58 | int64 processing_time = 9; 59 | 60 | // An indication whether this node records metrics about produced and 61 | // consumed elements. 62 | bool record_metrics = 10; 63 | 64 | // Represents a node parameter. 65 | message Parameter { 66 | // Human-readable name of the parameter. 67 | string name = 1; 68 | 69 | // Identifies the model value of the parameter. This can be different from 70 | // the actual value (e.g. during optimization search). 71 | double value = 2; 72 | 73 | // The actual value of the parameter. 74 | double state_value = 3; 75 | 76 | // Minimum value of the parameter. 77 | double min = 4; 78 | 79 | // Maximum value of the parameter. 80 | double max = 5; 81 | 82 | // Identifies whether the parameter should participate in autotuning. 83 | bool tunable = 6; 84 | } 85 | 86 | // Parameters of this node. 87 | repeated Parameter parameters = 11; 88 | 89 | // Statistic of inputs processing time history. 90 | double input_processing_time_sum = 12; 91 | int64 input_processing_time_count = 13; 92 | 93 | // IDs of inputs of this node. 94 | repeated int64 inputs = 14; 95 | 96 | // Class of this node. 97 | NodeClass node_class = 15; 98 | 99 | // Ratio of input to output elements. This is only used by KNOWN_RATIO and 100 | // ASYNC_KNOWN_RATIO nodes. 101 | double ratio = 16; 102 | 103 | // Ratio identifies how many parallelism calls are introduced by one 104 | // buffered element. This is only used by ASYNC_KNOWN_RATIO nodes. 105 | double memory_ratio = 17; 106 | } 107 | 108 | // Map of node IDs to nodes of this model. 109 | map nodes = 1; 110 | 111 | // ID of the output node of this model. 112 | int64 output = 2; 113 | 114 | // Counter for node IDs of this model. 115 | int64 id_counter = 3; 116 | 117 | reserved 4; 118 | 119 | // Contains parameters of the model autotuning optimization. 120 | message OptimizationParams { 121 | // Algorithm used for autotuning optimization. 122 | AutotuneAlgorithm algorithm = 1; 123 | 124 | // Number of available logical threads. 125 | int64 cpu_budget = 2; 126 | 127 | // Amount of available memory in bytes. 128 | int64 ram_budget = 3; 129 | 130 | // Time between two consecutive `GetNext` calls to the iterator represented 131 | // by the output node. 132 | double model_input_time = 4; 133 | } 134 | 135 | OptimizationParams optimization_params = 5; 136 | 137 | repeated uint64 gap_times = 6; 138 | } 139 | -------------------------------------------------------------------------------- /pkg/config_loader/faiss_config/faiss_config_loader.go: -------------------------------------------------------------------------------- 1 | package faiss_config 2 | 3 | import ( 4 | "infer-microservices/internal" 5 | faiss_index "infer-microservices/internal/faiss_gogofaster" 6 | "infer-microservices/internal/logs" 7 | "infer-microservices/internal/utils" 8 | "reflect" 9 | "strconv" 10 | ) 11 | 12 | type FaissIndexConfigs struct { 13 | faissIndexConfigs []FaissIndexConfig 14 | } 15 | 16 | func (f *FaissIndexConfigs) SetFaissIndexConfig(faissIndexConfigs []FaissIndexConfig) { 17 | f.faissIndexConfigs = faissIndexConfigs 18 | } 19 | 20 | func (f *FaissIndexConfigs) GetFaissIndexConfig() []FaissIndexConfig { 21 | return f.faissIndexConfigs 22 | } 23 | 24 | type FaissIndexConfig struct { 25 | indexName string `validate:"required,unique,min=4,max=10"` //index name. 26 | faissGrpcPool *internal.GRPCPool `validate:"required"` //faiss grpc pool. 27 | faissIndexs *faiss_index.RecallRequest `validate:"required"` // faiss index. 28 | recallNum int `validate:"required"` // faiss recall num. 29 | } 30 | 31 | // index name 32 | func (f *FaissIndexConfig) setIndexName(indexName string) { 33 | f.indexName = indexName 34 | } 35 | 36 | func (f *FaissIndexConfig) GetIndexName() string { 37 | return f.indexName 38 | } 39 | 40 | // grpc pool 41 | func (f *FaissIndexConfig) setFaissGrpcPool(faissGrpcPool *internal.GRPCPool) { 42 | f.faissGrpcPool = faissGrpcPool 43 | } 44 | 45 | func (f *FaissIndexConfig) GetFaissGrpcPool() *internal.GRPCPool { 46 | return f.faissGrpcPool 47 | } 48 | 49 | // FaissIndexs 50 | func (f *FaissIndexConfig) setFaissIndexs(faissIndexs *faiss_index.RecallRequest) { 51 | f.faissIndexs = faissIndexs 52 | } 53 | 54 | func (f *FaissIndexConfig) GetFaissIndexs() *faiss_index.RecallRequest { 55 | return f.faissIndexs 56 | } 57 | 58 | // recall num 59 | func (f *FaissIndexConfig) SetRecallNum(recallNum int) { 60 | f.recallNum = recallNum 61 | } 62 | 63 | func (f *FaissIndexConfig) GetRecallNum() int { 64 | return f.recallNum 65 | } 66 | 67 | // @implement ConfigLoadInterface 68 | func (f *FaissIndexConfigs) ConfigLoad(dataId string, indexConfStr string) error { 69 | dataConf := utils.ConvertJsonToStruct(indexConfStr) 70 | 71 | faissIndexConfigs := make([]FaissIndexConfig, 0) 72 | // create faiss grpc pool 73 | faissGrpcConf := dataConf["faissGrpcAddr"].(map[string]interface{}) 74 | faissGrpcPool, err := internal.CreateGrpcConn(faissGrpcConf) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | //INFO:the recallNum param from http request,maybe int/ float /string。 user reflect to convert to int32. 80 | //INFO:Processing multiple recalls simultaneously to save network overhead 81 | recallNum := int32(100) 82 | indexInfo := dataConf["indexInfo"].([]interface{}) 83 | for _, tmpIndexConf := range indexInfo { 84 | tmpIndexConfMap := tmpIndexConf.(map[string]interface{}) 85 | faissIndexConfig := FaissIndexConfig{} 86 | 87 | //recallNum := int32(tmpIndexConfMap["recall_num"].(float64)) 88 | recallNumType := reflect.TypeOf(tmpIndexConfMap["recallNum"]) 89 | recallNumTypeKind := recallNumType.Kind() 90 | switch recallNumTypeKind { 91 | case reflect.String: 92 | recallNumStr, ok := tmpIndexConfMap["recallNum"].(string) 93 | if ok { 94 | recallNum64, err := strconv.ParseInt(recallNumStr, 10, 64) 95 | if err != nil { 96 | logs.Warn(err) 97 | } else { 98 | recallNum = int32(recallNum64) 99 | } 100 | } 101 | case reflect.Float32, reflect.Float64, reflect.Int16, reflect.Int, reflect.Int64, reflect.Int8, 102 | reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: 103 | recallNum, _ = tmpIndexConfMap["recallNum"].(int32) 104 | default: 105 | logs.Warn("unkown type, set recallnum to 100") 106 | } 107 | 108 | indexName_ := tmpIndexConfMap["indexName"].(string) 109 | indexInfoStruct := &faiss_index.RecallRequest{ 110 | IndexName: indexName_, 111 | RecallNum: recallNum, 112 | } 113 | 114 | faissIndexConfig.setIndexName(indexName_) 115 | faissIndexConfig.setFaissGrpcPool(faissGrpcPool) 116 | faissIndexConfig.setFaissIndexs(indexInfoStruct) 117 | faissIndexConfigs = append(faissIndexConfigs, faissIndexConfig) 118 | 119 | f.SetFaissIndexConfig(faissIndexConfigs) 120 | } 121 | 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /pkg/config_loader/model_config/model_config_loader.go: -------------------------------------------------------------------------------- 1 | package model_config 2 | 3 | import ( 4 | "infer-microservices/internal" 5 | "infer-microservices/internal/utils" 6 | ) 7 | 8 | type ModelConfig struct { 9 | modelName string `validate:"required,unique,min=4,max=10"` //model name, dssm 、 deepfm 10 | modelType string `validate:"required,unique,min=4,max=10"` //recall 、rank 11 | tfservingModelName string `validate:"required,min=4,max=10"` //model name of tfserving config list. 12 | tfservingGrpcPool *internal.GRPCPool `validate:"required"` //tfserving grpc pool. 13 | //fieldsSpec map[string]interface{} //feaure engine conf. 14 | userRedisKeyPreOffline string `validate:"required,min=4,max=10"` //user offline feature redis key pre. 15 | userRedisKeyPreRealtime string `validate:"required,min=4,max=10"` //user Realtime feature redis key pre. 16 | itemRedisKeyPre string `validate:"required,min=4,max=10"` //item feature redis key pre. 17 | featureList []string 18 | } 19 | 20 | func init() { 21 | } 22 | 23 | // modelName 24 | func (f *ModelConfig) setModelName(modelName string) { 25 | f.modelName = modelName 26 | } 27 | 28 | func (f *ModelConfig) GetModelName() string { 29 | return f.modelName 30 | } 31 | 32 | // modelType 33 | func (f *ModelConfig) setModelType(modelType string) { 34 | f.modelType = modelType 35 | } 36 | 37 | func (f *ModelConfig) GetModelType() string { 38 | return f.modelType 39 | } 40 | 41 | // tfservingModelName 42 | func (f *ModelConfig) setTfservingModelName(tfservingModelName string) { 43 | f.tfservingModelName = tfservingModelName 44 | } 45 | 46 | func (f *ModelConfig) GetTfservingModelName() string { 47 | return f.tfservingModelName 48 | } 49 | 50 | // tfservingGrpcPool 51 | func (f *ModelConfig) setTfservingGrpcPool(tfservingGrpcPool *internal.GRPCPool) { 52 | f.tfservingGrpcPool = tfservingGrpcPool 53 | } 54 | 55 | func (f *ModelConfig) GetTfservingGrpcPool() *internal.GRPCPool { 56 | return f.tfservingGrpcPool 57 | } 58 | 59 | // userRedisKeyPre 60 | func (f *ModelConfig) setUserRedisKeyPreOffline(userRedisKeyPreOffline string) { 61 | f.userRedisKeyPreOffline = userRedisKeyPreOffline 62 | } 63 | 64 | func (f *ModelConfig) GetUserRedisKeyPreOffline() string { 65 | return f.userRedisKeyPreOffline 66 | } 67 | 68 | // userRedisKeyPre 69 | func (f *ModelConfig) setUserRedisKeyPreRealtime(userRedisKeyPreRealtime string) { 70 | f.userRedisKeyPreRealtime = userRedisKeyPreRealtime 71 | } 72 | 73 | func (f *ModelConfig) GetUserRedisKeyPreRealtime() string { 74 | return f.userRedisKeyPreRealtime 75 | } 76 | 77 | // itemRedisKeyPre 78 | func (f *ModelConfig) setItemRedisKeyPre(itemRedisKeyPre string) { 79 | f.itemRedisKeyPre = itemRedisKeyPre 80 | } 81 | 82 | func (f *ModelConfig) GetItemRedisKeyPre() string { 83 | return f.itemRedisKeyPre 84 | } 85 | 86 | // featureList 87 | func (f *ModelConfig) setFeatureList(featureList []string) { 88 | f.featureList = featureList 89 | } 90 | 91 | func (f *ModelConfig) GetFeatureList() []string { 92 | return f.featureList 93 | } 94 | 95 | // @implement ConfigLoadInterface 96 | func (m *ModelConfig) ConfigLoad(dataId string, modelConfStr string) error { 97 | 98 | modelConfTmp := utils.ConvertJsonToStruct(modelConfStr) 99 | tfservingGrpcConf := modelConfTmp["tfservingGrpcAddr"].(map[string]interface{}) 100 | modelName := tfservingGrpcConf["tfservingModelName"].(string) //tfserving config list modelname 101 | 102 | // create tfserving grpc pool 103 | tfservingGrpcPool, err := internal.CreateGrpcConn(tfservingGrpcConf) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | //fieldsSpec := modelConfTmp["fieldsSpec"].(map[string]interface{}) 109 | userRedisKeyPreOffline := modelConfTmp["userRedisKeyPreOffline"].(string) 110 | userRedisKeyPreRealtime := modelConfTmp["userRedisKeyPreRealtime"].(string) 111 | itemRedisKeyPre := modelConfTmp["itemRedisKeyPre"].(string) 112 | featureList := modelConfTmp["featureList"].([]string) 113 | 114 | //set 115 | m.setModelName(dataId) 116 | m.setTfservingModelName(modelName) 117 | m.setTfservingGrpcPool(tfservingGrpcPool) 118 | m.setUserRedisKeyPreOffline(userRedisKeyPreOffline) 119 | m.setUserRedisKeyPreRealtime(userRedisKeyPreRealtime) 120 | m.setItemRedisKeyPre(itemRedisKeyPre) 121 | m.setFeatureList(featureList) 122 | 123 | return nil 124 | } 125 | --------------------------------------------------------------------------------