├── LICENSE
├── README.md
├── data
└── data_B10W.csv
├── docs
├── imgs
│ ├── architecture.png
│ ├── deploy_arch.png
│ ├── mysql_service_svc.png
│ ├── p1.png
│ ├── p2.png
│ ├── redis_service_svc.PNG
│ ├── trainer.png
│ └── 联邦学习.png
├── k8s_deploy.md
├── nfs_deploy.md
├── prop
│ ├── application.properties
│ └── psi.properties
├── sql
│ ├── coordinator_init.sql
│ ├── mpc_init.sql
│ └── nacos_init.sql
└── yamls
│ ├── coordinator_deployment.yaml
│ ├── coordinator_k8s_cofigmap.yaml
│ ├── coordinator_nacos_configmap.yaml
│ ├── coordinator_service.yaml
│ ├── fileservice_deployment.yaml
│ ├── mysql_deployment.yaml
│ ├── mysql_secret.yaml
│ ├── mysql_service.yaml
│ ├── nacos_configmap.yaml
│ ├── nacos_deployment.yaml
│ ├── nacos_service.yaml
│ ├── proxy_cert_configmap.yaml
│ ├── proxy_deployment.yaml
│ ├── proxy_lua_configmap.yaml
│ ├── proxy_mpc_conf_configmap.yaml
│ ├── proxy_ngnix_conf_configmap.yaml
│ ├── psi.yaml
│ ├── redis_configmap.yaml
│ ├── redis_deployment.yaml
│ └── redis_service.yaml
├── nacos_init.sh
├── src
├── Coordinator
│ ├── Makefile
│ ├── Makefile.win
│ ├── docker
│ │ ├── Dockerfile
│ │ ├── Dockerfile_base
│ │ ├── README.md
│ │ └── start.sh
│ ├── pom.xml
│ └── src
│ │ └── main
│ │ ├── java
│ │ └── com
│ │ │ └── jd
│ │ │ └── mpc
│ │ │ ├── MpcApplication.java
│ │ │ ├── aces
│ │ │ └── TdeService.java
│ │ │ ├── cert
│ │ │ ├── CSRUtil.java
│ │ │ ├── CertGenerator.java
│ │ │ ├── KeyGenerator.java
│ │ │ ├── KeyPairPojo.java
│ │ │ └── SignUtil.java
│ │ │ ├── common
│ │ │ ├── advice
│ │ │ │ ├── ByteArrayServletInputStream.java
│ │ │ │ ├── CoordinatorRequestFilter.java
│ │ │ │ ├── GlobalExceptionHandler.java
│ │ │ │ ├── TeeServletOutputStream.java
│ │ │ │ └── ThreadPoolConfig.java
│ │ │ ├── config
│ │ │ │ └── NacosListenerConfig.java
│ │ │ ├── constant
│ │ │ │ ├── CommonConstant.java
│ │ │ │ ├── DeploymentPathConstant.java
│ │ │ │ ├── OfflineTaskMap.java
│ │ │ │ ├── ServicePathConstant.java
│ │ │ │ └── TargetMap.java
│ │ │ ├── enums
│ │ │ │ ├── IsDeletedEnum.java
│ │ │ │ ├── IsLocalEnum.java
│ │ │ │ ├── IsRootEnum.java
│ │ │ │ ├── K8sResourceTypeEnum.java
│ │ │ │ ├── LogLevelEnum.java
│ │ │ │ ├── OperatorStatusEnum.java
│ │ │ │ ├── StoreTypeEnum.java
│ │ │ │ ├── TaskStatusEnum.java
│ │ │ │ └── TaskTypeEnum.java
│ │ │ ├── response
│ │ │ │ ├── CommonException.java
│ │ │ │ ├── CommonResponse.java
│ │ │ │ ├── ErrorStatus.java
│ │ │ │ └── ProcessCommonResponse.java
│ │ │ └── util
│ │ │ │ ├── CommonUtils.java
│ │ │ │ ├── EncryptUtils.java
│ │ │ │ ├── GsonUtil.java
│ │ │ │ ├── HttpUtil.java
│ │ │ │ ├── JNISigner.java
│ │ │ │ ├── MapTypeAdapter.java
│ │ │ │ ├── MyX509TrustManager.java
│ │ │ │ ├── ParameterParseUtil.java
│ │ │ │ └── RestTemplateConfig.java
│ │ │ ├── domain
│ │ │ ├── cert
│ │ │ │ ├── CertInfo.java
│ │ │ │ ├── JobTaskStub.java
│ │ │ │ └── SignCertVo.java
│ │ │ ├── config
│ │ │ │ └── ResourceLimitPolicy.java
│ │ │ ├── form
│ │ │ │ └── EsQueryForm.java
│ │ │ ├── offline
│ │ │ │ ├── commons
│ │ │ │ │ ├── Job.java
│ │ │ │ │ ├── OfflineTask.java
│ │ │ │ │ ├── PreJob.java
│ │ │ │ │ └── SubTask.java
│ │ │ │ ├── jxz
│ │ │ │ │ └── JxzTaskTypeEnum.java
│ │ │ │ └── vo
│ │ │ │ │ └── WorkerStatusVo.java
│ │ │ ├── online
│ │ │ │ ├── OnlineJob.java
│ │ │ │ ├── OnlineResponse.java
│ │ │ │ ├── OnlineSubTask.java
│ │ │ │ ├── OnlineTask.java
│ │ │ │ └── Result.java
│ │ │ ├── param
│ │ │ │ ├── ExistParam.java
│ │ │ │ ├── GetConfigParam.java
│ │ │ │ ├── GetCoordinatorLogParam.java
│ │ │ │ ├── GetFileServiceLogParam.java
│ │ │ │ ├── GetNodeLogParam.java
│ │ │ │ ├── TaskInfoParam.java
│ │ │ │ └── WorkerInfoParam.java
│ │ │ ├── task
│ │ │ │ ├── AuthInfo.java
│ │ │ │ ├── AuthStatusEnum.java
│ │ │ │ ├── CertTypeEnum.java
│ │ │ │ ├── ChildrenTask.java
│ │ │ │ ├── ChildrenTaskExample.java
│ │ │ │ ├── ParentTask.java
│ │ │ │ └── ParentTaskExample.java
│ │ │ └── vo
│ │ │ │ ├── AuthInfoDto.java
│ │ │ │ ├── BlockVo.java
│ │ │ │ ├── CallbackBody.java
│ │ │ │ ├── Data.java
│ │ │ │ ├── EtlHeaderParam.java
│ │ │ │ ├── FeatureDetail.java
│ │ │ │ ├── FileSchemaInfo.java
│ │ │ │ ├── FileSizeInfo.java
│ │ │ │ ├── GrpcResourceLimitResult.java
│ │ │ │ ├── JobInfos.java
│ │ │ │ ├── JoinRawInfo.java
│ │ │ │ ├── MailInfo.java
│ │ │ │ ├── PodInfos.java
│ │ │ │ ├── PredictQuery.java
│ │ │ │ ├── PredictResult.java
│ │ │ │ ├── ProxyInfo.java
│ │ │ │ ├── ResourcesInfo.java
│ │ │ │ ├── SqlParserParam.java
│ │ │ │ ├── SyncInfo.java
│ │ │ │ ├── SyncRequest.java
│ │ │ │ ├── SyncResInfo.java
│ │ │ │ ├── SyncResponse.java
│ │ │ │ ├── TableInfo.java
│ │ │ │ ├── TargetInfo.java
│ │ │ │ ├── TaskIdList.java
│ │ │ │ ├── TaskInfo.java
│ │ │ │ ├── TaskStatusInfo.java
│ │ │ │ ├── VerifyVo.java
│ │ │ │ └── WorkerInfo.java
│ │ │ ├── grpc
│ │ │ ├── GrpcClient.java
│ │ │ ├── GrpcOfflineClient.java
│ │ │ ├── GrpcOfflineService.java
│ │ │ ├── GrpcOuterClient.java
│ │ │ ├── GrpcOuterService.java
│ │ │ ├── GrpcPredictClient.java
│ │ │ ├── GrpcRetryExceptionHandler.java
│ │ │ └── GrpcSignClient.java
│ │ │ ├── mapper
│ │ │ ├── AuthInfoMapper.java
│ │ │ ├── CertMapper.java
│ │ │ ├── ChildrenTaskMapper.java
│ │ │ ├── JobTaskStubMapper.java
│ │ │ └── ParentTaskMapper.java
│ │ │ ├── quartz
│ │ │ ├── QuartzConfig.java
│ │ │ └── job
│ │ │ │ ├── FinishTaskJob.java
│ │ │ │ └── StartTaskJob.java
│ │ │ ├── redis
│ │ │ ├── RedisConfig.java
│ │ │ ├── RedisLock.java
│ │ │ ├── RedisServer.java
│ │ │ └── RedisService.java
│ │ │ ├── service
│ │ │ ├── AuthInfoService.java
│ │ │ ├── FileService.java
│ │ │ ├── K8sService.java
│ │ │ ├── OfflineService.java
│ │ │ ├── OuterService.java
│ │ │ ├── OuterSupport.java
│ │ │ ├── ParaCompiler.java
│ │ │ ├── PodFactory.java
│ │ │ ├── TaskFactory.java
│ │ │ ├── TaskPersistenceService.java
│ │ │ ├── TaskSupport.java
│ │ │ ├── cert
│ │ │ │ ├── CertPersistenceService.java
│ │ │ │ ├── JobCertValidateService.java
│ │ │ │ └── JobTaskStubService.java
│ │ │ ├── task
│ │ │ │ ├── AbstractTaskService.java
│ │ │ │ ├── FileSvcTaskService.java
│ │ │ │ └── ITaskService.java
│ │ │ └── zeebe
│ │ │ │ ├── AbstractZeebeService.java
│ │ │ │ ├── BuffaloZeebeService.java
│ │ │ │ ├── FileTransferZeebeService.java
│ │ │ │ ├── IZeebeService.java
│ │ │ │ ├── PsiZeebeService.java
│ │ │ │ ├── ZeebeDispatcher.java
│ │ │ │ ├── ZeebeGateWay.java
│ │ │ │ ├── Zeebes.java
│ │ │ │ └── domain
│ │ │ │ └── param
│ │ │ │ └── ProcessResultParam.java
│ │ │ ├── storage
│ │ │ ├── OfflineTaskMapHolder.java
│ │ │ └── TargetMapHolder.java
│ │ │ └── web
│ │ │ ├── AuthController.java
│ │ │ ├── OfflineController.java
│ │ │ └── OuterController.java
│ │ ├── proto
│ │ ├── api.proto
│ │ ├── ast.proto
│ │ ├── authprotocol.proto
│ │ ├── external_service.proto
│ │ ├── feature.proto
│ │ ├── internal_service.proto
│ │ ├── offline.proto
│ │ ├── online.proto
│ │ ├── outer.proto
│ │ ├── parameter.proto
│ │ ├── service.proto
│ │ └── table.proto
│ │ └── resources
│ │ ├── application.properties
│ │ ├── bootstrap.properties
│ │ ├── logback-spring.xml
│ │ ├── mapper
│ │ ├── AuthInfoMapper.xml
│ │ ├── CertMapper.xml
│ │ ├── ChildrenTaskMapper.xml
│ │ ├── JobTaskStubMapper.xml
│ │ └── ParentTaskMapper.xml
│ │ ├── mybatis
│ │ └── generatorConfig.xml
│ │ └── nacos-application.properties
├── FileService
│ ├── Dockerfile
│ ├── README.md
│ ├── app
│ │ ├── file_service.py
│ │ └── redis_client.py
│ ├── fileservice.yaml
│ └── start.sh
└── PSI
│ ├── Dockerfile
│ ├── README.md
│ ├── crypto
│ ├── .gitignore
│ ├── Cargo.lock
│ ├── Cargo.toml
│ ├── pyproject.toml
│ └── src
│ │ ├── curve.rs
│ │ ├── hash.rs
│ │ ├── hash_set.rs
│ │ └── lib.rs
│ ├── image
│ └── psi.png
│ ├── link_py
│ ├── .bazelrc
│ ├── .gitignore
│ ├── BUILD.bazel
│ ├── WORKSPACE
│ └── link_py.cc
│ ├── psi
│ ├── .gitignore
│ ├── interconnection_psi
│ │ ├── __init__.py
│ │ ├── api.py
│ │ ├── base.py
│ │ ├── cipher_store.py
│ │ ├── config.py
│ │ ├── flow.py
│ │ ├── interconnection
│ │ │ ├── __init__.py
│ │ │ ├── common
│ │ │ │ ├── __init__.py
│ │ │ │ └── header.proto
│ │ │ ├── handshake
│ │ │ │ ├── __init__.py
│ │ │ │ ├── algos
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── psi.proto
│ │ │ │ ├── entry.proto
│ │ │ │ └── protocol_family
│ │ │ │ │ ├── __init__.py
│ │ │ │ │ └── ecc.proto
│ │ │ └── runtime
│ │ │ │ ├── __init__.py
│ │ │ │ └── ecdh_psi.proto
│ │ ├── log.py
│ │ └── register_uuid.py
│ └── setup.py
│ ├── psi_actors.py
│ └── start.sh
└── 隐私计算PSI标品部署操作文档.md
/README.md:
--------------------------------------------------------------------------------
1 | # 九数联邦学习整体解决方案
2 |
3 |
4 |
5 | ## 概述
6 |
7 | 数据是人工智能的基石,打破数据孤岛,实现数据共享是加速人工智能高速发展的必要条件。联邦学习,作为数据安全计算的业界前沿技术方案,在保护数据隐私的同时深度连接各个合作方,达到技术赋能、共创共赢的目标。
8 |
9 |
10 |
11 |
12 |
13 | ## 1 整体架构
14 |
15 | - 整个系统分为四个大模块
16 | - 整体调度与转发模块
17 | - 资源管理与调度模块
18 | - 数据求交模块
19 | - 训练器模块
20 |
21 | 
22 |
23 | ## 2 整体调度与转发模块
24 |
25 | - 整体控制数据求交与训练的调度
26 | - 训练器的配对工作
27 | - 高效的流量转发
28 |
29 |
30 |
31 | ## 3 资源调度与管理模块
32 |
33 | - 使用k8s屏蔽底层资源差异
34 | - 使用k8s进行资源的动态调度
35 |
36 |
37 |
38 | ## 4 数据求交
39 |
40 | - 异步分布式框架提升拼接效率
41 |
42 |
43 |
44 | ## LICNESE
45 |
46 | 9nFL使用apache2.0许可
47 |
48 |
49 |
--------------------------------------------------------------------------------
/docs/imgs/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/architecture.png
--------------------------------------------------------------------------------
/docs/imgs/deploy_arch.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/deploy_arch.png
--------------------------------------------------------------------------------
/docs/imgs/mysql_service_svc.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/mysql_service_svc.png
--------------------------------------------------------------------------------
/docs/imgs/p1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/p1.png
--------------------------------------------------------------------------------
/docs/imgs/p2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/p2.png
--------------------------------------------------------------------------------
/docs/imgs/redis_service_svc.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/redis_service_svc.PNG
--------------------------------------------------------------------------------
/docs/imgs/trainer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/trainer.png
--------------------------------------------------------------------------------
/docs/imgs/联邦学习.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/联邦学习.png
--------------------------------------------------------------------------------
/docs/k8s_deploy.md:
--------------------------------------------------------------------------------
1 | # K8S cluster deployment
2 |
--------------------------------------------------------------------------------
/docs/nfs_deploy.md:
--------------------------------------------------------------------------------
1 | # NFS服务部署
2 | ## 服务端
3 | ### NFS服务端软件包
4 | ```
5 | yum install -y nfs-utils
6 | ```
7 | ### 设置可访问NFS的主机并增加配置
8 | ```
9 | yum install -y nfs-utils
10 | ```
11 | 向其中增加配置{shared_path} {IP_Cluster}/IP(rw,sync,fsid=0)
12 | 比如 /home/nfs/ 192.168.241.0/24(rw,sync,fsid=0)
13 | ### 启动NFS服务
14 | ```
15 | systemctl enable rpcbind.service
16 | systemctl enable nfs-server.service
17 |
18 | systemctl start rpcbind.service
19 | systemctl start nfs-server.service
20 | ```
21 | ### 确认NFS服务生效
22 | ```
23 | rpcinfo -p
24 | exportfs -r
25 | exportfs
26 | ```
27 |
28 | ## 客服端
29 | ### 启动基础服务
30 | ```
31 | systemctl enable rpcbind.service
32 | systemctl start rpcbind.service
33 | ```
34 | ### 检查NFS服务目录
35 | ```
36 | showmount -e {nfs_server_ip}
37 | ```
38 | ### 挂载到客服端
39 | ```
40 | cd /mnt && mkdir /nfs
41 | mount -t nfs {nfs_server_ip}:{nfs_path} /mnt/nfs
42 | ```
43 | 以上步骤操作完毕后,可以进一步设置客服端到k8的Pod的挂载目录
--------------------------------------------------------------------------------
/docs/prop/psi.properties:
--------------------------------------------------------------------------------
1 | tmp-dir=/mnt/tmp
2 | send-back=true
3 | log-level=DEBUG
4 | cpu-cores=16
5 | csv-header=true
6 |
--------------------------------------------------------------------------------
/docs/yamls/coordinator_deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | labels:
5 | app: coordinator
6 | name: coordinator
7 | spec:
8 | replicas: 2
9 | selector:
10 | matchLabels:
11 | app: coordinator
12 | template:
13 | metadata:
14 | labels:
15 | app: coordinator
16 | spec:
17 | containers:
18 | - image: $COORDINATOR_IMAGE
19 | imagePullPolicy: Always
20 | name: coordinator
21 | volumeMounts:
22 | - mountPath: /home/config/application.properties
23 | name: coordinator-conf
24 | readOnly: true
25 | subPath: application.properties
26 | - mountPath: /home/config/k8sconfig.yaml
27 | name: k8s-conf
28 | readOnly: true
29 | subPath: k8sconfig.yaml
30 | - mountPath: /k8s
31 | name: k8s
32 | - mountPath: /mnt/logs
33 | name: logs
34 | resources:
35 | limits:
36 | cpu: "4"
37 | memory: 8Gi
38 | requests:
39 | cpu: "4"
40 | memory: 8Gi
41 | restartPolicy: Always
42 | volumes:
43 | - hostPath:
44 | path: $VOLUME_LOGS
45 | type: DirectoryOrCreate
46 | name: logs
47 | - hostPath:
48 | path: $VOLUME_DATA
49 | type: DirectoryOrCreate
50 | name: k8s
51 | - configMap:
52 | defaultMode: 420
53 | name: coordinator-conf
54 | name: coordinator-conf
55 | - configMap:
56 | defaultMode: 420
57 | name: k8s-conf
58 | name: k8s-conf
59 |
--------------------------------------------------------------------------------
/docs/yamls/coordinator_k8s_cofigmap.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: k8s-conf
5 | data:
6 | k8sconfig.yaml: |+
7 | apiVersion: v1
8 | clusters:
9 | - cluster:
10 | server: $K8S_SERVER_URL
11 | insecure-skip-tls-verify: true
12 | name: kubernetes
13 | contexts:
14 | - context:
15 | cluster: kubernetes
16 | user: basic-authentication
17 | name: basic-authentication@kubernetes
18 | - context:
19 | cluster: kubernetes
20 | user: cert-authentication
21 | name: cert-authentication@kubernetes
22 | current-context: cert-authentication@kubernetes
23 | kind: Config
24 | preferences: {}
25 | users:
26 | - name: basic-authentication
27 | user:
28 | username: $K8S_USERNAME
29 | password: $K8S_PASSWORD
30 | - name: cert-authentication
31 | user:
32 | client-certificate-data: $K8S_CLIENT_CERTIFICATE_DATA
33 | client-key-data: $K8S_CLIENT_KEY_DATA
34 |
--------------------------------------------------------------------------------
/docs/yamls/coordinator_nacos_configmap.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: $COORDINATOR_CONF
5 | data:
6 | application.properties: |
7 | nacos.config.server-addr=$NACOS_DOMAIN
8 | nacos.config.remote-first=true
9 | nacos.config.data-id=application.properties
10 | nacos.config.namespace=$NAMESPACE
11 | nacos.config.group=APPLICATION_GROUP
12 | nacos.config.type=properties
13 | nacos.config.auto-refresh=true
14 | nacos.config.local-disk-cache-dir=/k8s/nacos
15 |
--------------------------------------------------------------------------------
/docs/yamls/coordinator_service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | labels:
5 | name: coordinator
6 | name: coordinator
7 | spec:
8 | ports:
9 | - name: coo-http
10 | nodePort: $COORDINATOR_NODE_PORT_OUT
11 | port: $COORDINATOR_SERVICE_PORT_OUT
12 | protocol: TCP
13 | targetPort: $COORDINATOR_POD_PORT_OUT
14 | - name: coo-http2
15 | nodePort: $COORDINATOR_NODE_PORT_IN
16 | port: $COORDINATOR_SERVICE_PORT_IN
17 | protocol: TCP
18 | targetPort: $COORDINATOR_POD_PORT_IN
19 | selector:
20 | app: coordinator
21 | type: NodePort
22 |
--------------------------------------------------------------------------------
/docs/yamls/fileservice_deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: file-service
5 | labels:
6 | app: file-service
7 | spec:
8 | replicas: 1
9 | selector:
10 | matchLabels:
11 | app: file-service
12 | template:
13 | metadata:
14 | labels:
15 | app: file-service
16 | spec:
17 | containers:
18 | - name: file-service
19 | image: $FILESERVICE_IMAGE
20 | ports:
21 | - containerPort: $FILESERVICE_PORT
22 | env:
23 | - name: REDIS_HOST
24 | value: $REDIS_SERVICE_IP
25 | - name: REDIS_PORT
26 | value: REDIS_SERVICE_PORT
27 | - name: REDIS_PASSWORD
28 | value: $REDIS_PASSWORD
29 | volumeMounts:
30 | - name: file-data
31 | mountPath: /mnt/data
32 | - name: logs
33 | mountPath: /mnt/logs
34 | volumes:
35 | - name: logs
36 | hostPath:
37 | path: $VOLUME_LOGS
38 | type: DirectoryOrCreate
39 | - name: file-data
40 | hostPath:
41 | path: $VOLUME_DATA
42 | type: DirectoryOrCreate
43 |
--------------------------------------------------------------------------------
/docs/yamls/mysql_deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: mysql
5 | spec:
6 | replicas: 1
7 | selector:
8 | matchLabels:
9 | app: mysql
10 | template:
11 | metadata:
12 | labels:
13 | app: mysql
14 | spec:
15 | containers:
16 | - name: mysql
17 | image: $MYSQL_IMAGE
18 | ports:
19 | - containerPort: $MYSQL_POD_PORT
20 | resources:
21 | limits:
22 | cpu: "1"
23 | memory: "1Gi"
24 | requests:
25 | cpu: "250m"
26 | memory: "256Mi"
27 | env:
28 | - name: MYSQL_ROOT_PASSWORD
29 | valueFrom:
30 | secretKeyRef:
31 | name: $MYSQL_PASSWORD_SECRET
32 | key: password
33 | volumeMounts:
34 | - name: mysql-volume
35 | mountPath: /var/lib/mysql
36 | volumes:
37 | - hostPath:
38 | path: $MYSQL_VOLUME_PATH
39 | type: DirectoryOrCreate
40 | name: mysql-volume
41 |
--------------------------------------------------------------------------------
/docs/yamls/mysql_secret.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Secret
3 | metadata:
4 | name: $MYSQL_PASSWORD_SECRET
5 | type: Opaque
6 | data:
7 | username: $MYSQL_USERNAME
8 | password: $MYSQL_PASSWORD
9 |
--------------------------------------------------------------------------------
/docs/yamls/mysql_service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: mysql
5 | spec:
6 | selector:
7 | app: mysql
8 | ports:
9 | - protocol: TCP
10 | port: $MYSQL_SERVICE_PORT
11 | targetPort: $MYSQL_POD_PORT
12 | type: ClusterIP
13 |
--------------------------------------------------------------------------------
/docs/yamls/nacos_deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | labels:
5 | name: nacos
6 | name: nacos
7 | spec:
8 | progressDeadlineSeconds: 600
9 | replicas: 1
10 | revisionHistoryLimit: 10
11 | selector:
12 | matchLabels:
13 | name: nacos
14 | strategy:
15 | rollingUpdate:
16 | maxSurge: 25%
17 | maxUnavailable: 25%
18 | type: RollingUpdate
19 | template:
20 | metadata:
21 | creationTimestamp: null
22 | labels:
23 | name: nacos
24 | spec:
25 | containers:
26 | - env:
27 | - name: MODE
28 | value: standalone
29 | - name: SPRING_DATASOURCE_PLATFORM
30 | value: mysql
31 | - name: PREFER_HOST_MODE
32 | value: hostname
33 | image: $NACOS_IMAGE
34 | imagePullPolicy: Always
35 | name: nacos
36 | resources:
37 | limits:
38 | cpu: "1"
39 | memory: 2Gi
40 | requests:
41 | cpu: "1"
42 | memory: 2Gi
43 | terminationMessagePath: /dev/termination-log
44 | terminationMessagePolicy: File
45 | volumeMounts:
46 | - mountPath: /home/nacos/conf/application.properties
47 | name: nacos-cm
48 | subPath: application.properties
49 | dnsPolicy: ClusterFirst
50 | restartPolicy: Always
51 | schedulerName: default-scheduler
52 | securityContext: {}
53 | terminationGracePeriodSeconds: 30
54 | volumes:
55 | - configMap:
56 | defaultMode: 420
57 | items:
58 | - key: application.properties
59 | path: application.properties
60 | name: nacos-cm
61 | name: nacos-cm
62 |
--------------------------------------------------------------------------------
/docs/yamls/nacos_service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | labels:
5 | name: nacos-svc
6 | name: nacos-svc
7 | spec:
8 | ports:
9 | - name: http
10 | nodePort: 30009
11 | port: 8848
12 | protocol: TCP
13 | targetPort: 8848
14 | - name: http1
15 | nodePort: 31397
16 | port: 9848
17 | protocol: TCP
18 | targetPort: 9848
19 | - name: http2
20 | nodePort: 30140
21 | port: 9555
22 | protocol: TCP
23 | targetPort: 9555
24 | selector:
25 | name: nacos
26 | sessionAffinity: None
27 | type: NodePort
28 | status:
29 | loadBalancer: {}
30 |
--------------------------------------------------------------------------------
/docs/yamls/proxy_cert_configmap.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: cert
5 | namespace: mpc-chk-test
6 | data:
7 | ca.crt: |
8 | -----BEGIN CERTIFICATE-----
9 | -----END CERTIFICATE-----
10 | server_cert.pem: |
11 | -----BEGIN CERTIFICATE-----
12 | -----END CERTIFICATE-----
13 | server_private.pem: |
14 | -----BEGIN PRIVATE KEY-----
15 | -----END PRIVATE KEY-----
16 |
--------------------------------------------------------------------------------
/docs/yamls/proxy_deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | labels:
5 | app: proxy
6 | name: proxy
7 | spec:
8 | replicas: 1
9 | revisionHistoryLimit: 10
10 | selector:
11 | matchLabels:
12 | app: proxy
13 | template:
14 | metadata:
15 | labels:
16 | app: proxy
17 | spec:
18 | containers:
19 | - image: $PROXY_IMAGE
20 | imagePullPolicy: Always
21 | name: proxy
22 | resources:
23 | limits:
24 | cpu: "4"
25 | memory: 8Gi
26 | requests:
27 | cpu: "0.5"
28 | memory: 1Gi
29 | volumeMounts:
30 | - mountPath: /usr/local/openresty/nginx/conf/nginx.conf
31 | name: nginx-conf
32 | subPath: nginx.conf
33 | - mountPath: /usr/local/openresty/nginx/conf/conf.d/mpc-proxy.conf
34 | name: mpc-conf
35 | subPath: mpc-proxy.conf
36 | - mountPath: /usr/local/openresty/nginx/lua_src
37 | name: lua-src
38 | - mountPath: /cert
39 | name: cert
40 | hostNetwork: true
41 | nodeName: $NODENAME
42 | restartPolicy: Always
43 | volumes:
44 | - configMap:
45 | defaultMode: 420
46 | name: nginx-conf
47 | name: nginx-conf
48 | - configMap:
49 | defaultMode: 420
50 | name: mpc-conf
51 | name: mpc-conf
52 | - configMap:
53 | defaultMode: 420
54 | name: lua-src
55 | name: lua-src
56 | - configMap:
57 | defaultMode: 420
58 | name: cert
59 | name: cert
60 |
--------------------------------------------------------------------------------
/docs/yamls/proxy_ngnix_conf_configmap.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: $NGINX_CONF
5 | data:
6 | nginx.conf: |
7 | # 设置为nginx的CPU核数
8 | worker_processes 8;
9 | worker_rlimit_nofile 65535;
10 | error_log /usr/local/openresty/nginx/logs/error.log info;
11 | worker_shutdown_timeout 1h;
12 |
13 | events {
14 | use epoll;
15 | }
16 |
17 | http {
18 | sendfile on;
19 | aio threads;
20 | aio_write on;
21 | directio 8m;
22 | tcp_nopush on;
23 | tcp_nodelay on;
24 |
25 | keepalive_timeout 65;
26 | keepalive_requests 10000;
27 |
28 | include /usr/local/openresty/nginx/conf/mime.types;
29 | default_type application/octet-stream;
30 | lua_package_path "$prefix/lua_src/?.lua;/usr/local/openresty/lualib/resty/?.lua;;";
31 | log_format xlog '$http_host $remote_addr $remote_port $remote_user [$time_local] $request_time '
32 | '"$request" $status $body_bytes_sent '
33 | '"$http_referer" "$http_user_agent" "$http_host" "$http_cookie" '
34 | '"$upstream_response_time" $upstream_addr "$http_x_forwarded_for" $scheme '
35 | '"$upstream_http_set_cookie"';
36 |
37 | access_log ./logs/access.log xlog;
38 |
39 | client_max_body_size 0;
40 | client_body_buffer_size 32m;
41 |
42 | ssl_session_cache shared:SSL:10m;
43 | ssl_session_timeout 10m;
44 |
45 | resolver local=on ipv6=off;
46 |
47 | include conf.d/mpc-proxy.conf;
48 | }
49 |
--------------------------------------------------------------------------------
/docs/yamls/psi.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | labels:
5 | name: psi-worker
6 | name: psi-worker
7 | namespace: $NAMESPACE
8 | spec:
9 | replicas: 1
10 | selector:
11 | matchLabels:
12 | name: psi-worker
13 | template:
14 | metadata:
15 | labels:
16 | name: psi-worker
17 | spec:
18 | containers:
19 | - image: $PSI_IMAGE
20 | imagePullPolicy: IfNotPresent
21 | name: psi-worker
22 | env:
23 | - name: Local_IP
24 | valueFrom:
25 | fieldRef:
26 | apiVersion: v1
27 | fieldPath: status.podIP
28 | ports:
29 | - containerPort: 22020
30 | name: http
31 | protocol: TCP
32 | resources:
33 | limits:
34 | cpu: "16"
35 | memory: 16Gi
36 | requests:
37 | cpu: "16"
38 | memory: 16Gi
39 | volumeMounts:
40 | - mountPath: /mnt/data
41 | name: data
42 | - mountPath: /mnt/logs
43 | name: logs
44 | - mountPath: /dev/shm
45 | name: dshm
46 | dnsPolicy: ClusterFirst
47 | restartPolicy: Always
48 | volumes:
49 | - name: data
50 | hostPath:
51 | path: $VOLUME_DATA
52 | type: DirectoryOrCreate
53 | - name: logs
54 | hostPath:
55 | path: $VOLUME_LOGS
56 | type: DirectoryOrCreate
57 | - name: dshm
58 | emptyDir:
59 | medium: Memory
60 |
--------------------------------------------------------------------------------
/docs/yamls/redis_configmap.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ConfigMap
3 | metadata:
4 | name: redis
5 | data:
6 | redis.conf: |
7 | bind 0.0.0.0
8 | protected-mode no
9 | port $REDIS_POD_PORT
10 | tcp-backlog 511
11 | timeout 0
12 | tcp-keepalive 300
13 | daemonize no
14 | supervised no
15 | pidfile /var/run/redis_6379.pid
16 | loglevel notice
17 | logfile /data/redis.log
18 | databases 16
19 | save 900 1
20 | save 300 10
21 | save 60 10000
22 | stop-writes-on-bgsave-error yes
23 | rdbcompression yes
24 | rdbchecksum yes
25 | dbfilename dump.rdb
26 | dir /data
27 | slave-serve-stale-data yes
28 | slave-read-only yes
29 | repl-diskless-sync no
30 | repl-diskless-sync-delay 5
31 | repl-disable-tcp-nodelay no
32 | slave-priority 100
33 | requirepass $REDIS_PASSWORD
34 | rename-command FLUSHALL ""
35 | rename-command FLUSHDB ""
36 | rename-command KEYS ""
37 | appendonly no
38 | appendfilename "appendonly.aof"
39 | appendfsync everysec
40 | no-appendfsync-on-rewrite no
41 | auto-aof-rewrite-percentage 100
42 | auto-aof-rewrite-min-size 64mb
43 | aof-load-truncated yes
44 | lua-time-limit 5000
45 | slowlog-log-slower-than 10000
46 | slowlog-max-len 128
47 | latency-monitor-threshold 0
48 | notify-keyspace-events ""
49 | hash-max-ziplist-entries 512
50 | hash-max-ziplist-value 64
51 | list-max-ziplist-size -2
52 | list-compress-depth 0
53 | set-max-intset-entries 512
54 | zset-max-ziplist-entries 128
55 | zset-max-ziplist-value 64
56 | hll-sparse-max-bytes 3000
57 | activerehashing yes
58 | client-output-buffer-limit normal 0 0 0
59 | client-output-buffer-limit slave 256mb 64mb 60
60 | client-output-buffer-limit pubsub 32mb 8mb 60
61 | hz 10
62 | aof-rewrite-incremental-fsync yes
63 |
--------------------------------------------------------------------------------
/docs/yamls/redis_deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: redis
5 | spec:
6 | replicas: 1
7 | selector:
8 | matchLabels:
9 | app: redis
10 | template:
11 | metadata:
12 | labels:
13 | app: redis
14 | spec:
15 | containers:
16 | - name: redis
17 | image: $REDIS_IMAGE
18 | ports:
19 | - containerPort: $REDIS_POD_PORT
20 | resources:
21 | limits:
22 | cpu: "1"
23 | memory: "1Gi"
24 | requests:
25 | cpu: "250m"
26 | memory: "256Mi"
27 | volumeMounts:
28 | - name: redis-volume
29 | mountPath: /data
30 | - name: redis-conf
31 | mountPath: /etc/redis.conf
32 | subPath: redis.conf
33 | readOnly: true
34 | volumes:
35 | - hostPath:
36 | path: $REDIS_VOLUME_PATH
37 | type: DirectoryOrCreate
38 | name: redis-volume
39 | - name: redis-conf
40 | configMap:
41 | name: $REDIS_CONF
--------------------------------------------------------------------------------
/docs/yamls/redis_service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: redis
5 | spec:
6 | selector:
7 | app: redis
8 | ports:
9 | - protocol: TCP
10 | port: $REDIS_SERVICE_PORT
11 | targetPort: $REDIS_POD_PORT
12 |
--------------------------------------------------------------------------------
/src/Coordinator/Makefile:
--------------------------------------------------------------------------------
1 | export JAVA_HOME=/usr
2 |
3 | jarName=mpc-coordinator-1.0-SNAPSHOT.jar
4 | ifeq ($(FLAG),jcjd)
5 | # pre1
6 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_pk_release
7 | restartScriptPath=/Users/feiguodong1/Documents/project/k8/k8shell/pre1_restart_pod.sh
8 | endif
9 | ifeq ($(FLAG),fljd)
10 | # 9nmpc-fl
11 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coodinator_jttest
12 | restartScriptPath=/Users/feiguodong1/Documents/project/k8/k8shell/fl_restart_pod.sh
13 | endif
14 | ifeq ($(FLAG),pejd)
15 | # pre
16 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coodinator_pre
17 | restartScriptPath=1.sh
18 | endif
19 | #ifeq ($(FLAG),oljd)
20 | # # mpc-online
21 | # imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_master
22 | #endif
23 | ifeq ($(FLAG),prodjd)
24 | # mpc-online
25 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_prod_v1.0.0
26 | restartScriptPath=111.sh
27 | endif
28 | ifeq ($(FLAG),tejd)
29 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_test_v1.0.0
30 | # imageName=yd-model-learning-cn-north-1.jcr.service.jdcloud.com/yd-model-store:coordinator_highly_available
31 | restartScriptPath=111.sh
32 | endif
33 | ifeq ($(FLAG),tjd)
34 | imageName=test.registry.net/9ntrain:coodinator_pre
35 | restartScriptPath=/Users/feiguodong1/Documents/project/k8/k8shell/tjd_restart_pod.sh
36 | endif
37 | ifeq ($(FLAG),btjd)
38 | imageName=test.registry.net/9ntrain:coodinator_busi_test
39 | restartScriptPath=tmp.sh
40 | endif
41 | ifndef imageName
42 | return -1
43 | endif
44 | ifndef restartScriptPath
45 | return -1
46 | endif
47 |
48 | restart: push
49 | sh $(restartScriptPath) coordinator
50 |
51 | push: image
52 | docker push $(imageName)
53 |
54 | image: compile
55 | - rm -rf docker/$(jarName)
56 | - docker rmi $(imageName)
57 | cp target/$(jarName) docker/$(jarName)
58 | cd docker && docker build -f Dockerfile.jre8 -t $(imageName) .
59 |
60 | compile:
61 | /Users/chenghekai1/Desktop/software/apache-maven-3.8.6/bin/mvn clean package -Dmaven.test.skip=true
--------------------------------------------------------------------------------
/src/Coordinator/Makefile.win:
--------------------------------------------------------------------------------
1 | # This script is used to build an image on the WINDOWS.
2 | # You need to use 'git bash' to run it.
3 | # start cmd: sh Makefile.win
4 |
5 | # variables
6 | jarName=mpc-coordinator-1.0-SNAPSHOT.jar
7 | #imageName=test.registry.net/9ntrain:coordinator_highly_available
8 | #imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_highly_available
9 | imageName=yd-model-learning-cn-north-1.jcr.service.jdcloud.com/yd-model-store:coordinator_highly_available
10 | #imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coodinator_jttest
11 | #imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_pk_release
12 |
13 | # 1.compile
14 | # settins.xml's location need to be customized
15 | mvn -s /c/Users/Administrator/.m2/settings.xml clean package -Dmaven.test.skip=true
16 |
17 | # 2.build image
18 | rm ./docker/$jarName
19 | cp ./target/$jarName ./docker/$jarName
20 | cd docker
21 | docker rmi $imageName
22 | docker build -f Dockerfile.jre8 -t $imageName .
23 |
24 | # 3.push
25 | docker push $imageName
26 |
--------------------------------------------------------------------------------
/src/Coordinator/docker/Dockerfile:
--------------------------------------------------------------------------------
1 | # reference https://docs.spring.io/spring-boot/docs/2.5.5/reference/htmlsingle/#features.container-images
2 | FROM jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base1 AS builder
3 | ARG HOME=/home
4 | WORKDIR $HOME
5 | COPY mpc-coordinator-1.0-SNAPSHOT.jar mpc-coordinator-1.0-SNAPSHOT.jar
6 | RUN cat $(java -version)
7 | RUN java -Djarmode=layertools -jar mpc-coordinator-1.0-SNAPSHOT.jar extract
8 |
9 | FROM jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base1
10 | ARG HOME=/home
11 | WORKDIR $HOME
12 | COPY --from=builder $HOME/dependencies/ ./
13 | COPY --from=builder $HOME/spring-boot-loader/ ./
14 | COPY --from=builder $HOME/snapshot-dependencies/ ./
15 | COPY --from=builder $HOME/application/ ./
16 | #COPY --from=jni_sign /usr/local/lib/libjni_sign.so /usr/lib/libjni_sign.so
17 | ADD start.sh $HOME/start.sh
18 | CMD ["sh", "start.sh"]
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/src/Coordinator/docker/Dockerfile_base:
--------------------------------------------------------------------------------
1 | FROM debian:bullseye-slim
2 | # build jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base
3 |
4 | ## OpenJDK 8
5 |
6 | RUN set -eux; \
7 | apt-get update; \
8 | apt-get install -y --no-install-recommends \
9 | # utilities for keeping Debian and OpenJDK CA certificates in sync
10 | ca-certificates p11-kit \
11 | ; \
12 | rm -rf /var/lib/apt/lists/*
13 |
14 | ENV JAVA_HOME /usr/local/openjdk-17
15 | RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] # backwards compatibility
16 | ENV PATH $JAVA_HOME/bin:$PATH
17 |
18 | ENV LANG C.UTF-8
19 | ENV JAVA_VERSION 8u345
20 |
21 | RUN set -eux; \
22 | \
23 | arch="$(dpkg --print-architecture)"; \
24 | case "$arch" in \
25 | 'amd64') \
26 | downloadUrl='https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'; \
27 | ;; \
28 | *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \
29 | esac; \
30 | \
31 | savedAptMark="$(apt-mark showmanual)"; \
32 | apt-get update; \
33 | apt-get install -y --no-install-recommends \
34 | dirmngr \
35 | gnupg \
36 | wget \
37 | ; \
38 | rm -rf /var/lib/apt/lists/*; \
39 | \
40 | wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \
41 | \
42 | mkdir -p "$JAVA_HOME"; \
43 | tar --extract \
44 | --file openjdk.tgz \
45 | --directory "$JAVA_HOME" \
46 | --strip-components 1 \
47 | --no-same-owner \
48 | ; \
49 | rm openjdk.tgz*; \
50 | \
51 | # basic smoke test
52 | javac -version; \
53 | java -version
54 |
55 |
56 |
--------------------------------------------------------------------------------
/src/Coordinator/docker/README.md:
--------------------------------------------------------------------------------
1 | # coordinator构建说明
2 | 构建需求:
3 | jdk版本: 17
4 | 1. 在docker目录下 执行 docker build . -f Dockerfile_base -t xxxxx
5 | 2. 替换Dockerfile中jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base1为xxxxx
6 | 3. 构建项目 执行mvn clean package
7 | 4. 将target目录下的mpc-coordinator-1.0-SNAPSHOT.jar复制到docker目录下
8 | 5. 在docker目录下执行 docker build . -f Dockerfile_base -t coordinator镜像名
9 |
--------------------------------------------------------------------------------
/src/Coordinator/docker/start.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | java -Xmx6g -Xms6g org.springframework.boot.loader.JarLauncher
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/MpcApplication.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc;
2 |
3 | import com.alibaba.nacos.api.NacosFactory;
4 | import com.alibaba.nacos.api.PropertyKeyConst;
5 | import com.alibaba.nacos.api.annotation.NacosProperties;
6 | import com.alibaba.nacos.api.config.ConfigService;
7 | import com.alibaba.nacos.api.exception.NacosException;
8 | import com.alibaba.nacos.spring.context.annotation.EnableNacos;
9 | import com.alibaba.nacos.spring.context.annotation.config.EnableNacosConfig;
10 | import com.alibaba.nacos.spring.context.annotation.config.NacosPropertySource;
11 | import io.camunda.zeebe.spring.client.EnableZeebeClient;
12 | import org.mybatis.spring.annotation.MapperScan;
13 | import org.springframework.boot.SpringApplication;
14 | import org.springframework.boot.autoconfigure.SpringBootApplication;
15 | import org.springframework.retry.annotation.EnableRetry;
16 | import org.springframework.scheduling.annotation.EnableScheduling;
17 |
18 | import java.util.HashMap;
19 | import java.util.Properties;
20 |
21 | import static org.springframework.core.env.StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME;
22 | import static org.springframework.core.env.StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME;
23 |
24 | /**
25 | * 启动类
26 | *
27 | */
28 |
29 | @SpringBootApplication
30 | @MapperScan("com.jd.mpc.mapper")
31 | @EnableRetry
32 | @EnableNacosConfig(globalProperties = @NacosProperties(serverAddr = "${nacos.config.server-addr}",namespace = "${nacos.config.namespace}"))
33 | @NacosPropertySource(dataId = "application.properties", groupId = "APPLICATION_GROUP", autoRefreshed = true, first = true)
34 | public class MpcApplication
35 |
36 | public static void main(String[] args) {
37 | System.setProperty("nacos.logging.default.config.enabled", "false");
38 | SpringApplication sa = new SpringApplication(MpcApplication.class);
39 | sa.setAllowCircularReferences(Boolean.TRUE);// 加入的参数
40 | sa.run(args);
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/aces/TdeService.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.aces;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.springframework.stereotype.Component;
5 |
6 |
7 | /**
8 | */
9 | @Component
10 | @Slf4j
11 | public class TdeService {
12 |
13 | /**
14 | * 加密
15 | * @param input
16 | * @return
17 | */
18 | public String encryptString(String input){
19 | return input;
20 | }
21 |
22 | /**
23 | * 解密
24 | * @param input
25 | * @return
26 | */
27 | public String decryptString(String input){
28 | return input;
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/cert/CSRUtil.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.cert;
2 |
3 | import org.bouncycastle.asn1.x500.X500Name;
4 | import org.bouncycastle.operator.ContentSigner;
5 | import org.bouncycastle.operator.OperatorCreationException;
6 | import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
7 | import org.bouncycastle.pkcs.PKCS10CertificationRequest;
8 | import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
9 | import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest;
10 | import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
11 |
12 | import cn.hutool.core.codec.Base64;
13 | import java.io.IOException;
14 | import java.security.InvalidKeyException;
15 | import java.security.NoSuchAlgorithmException;
16 | import java.security.PrivateKey;
17 | import java.security.PublicKey;
18 | import java.security.cert.CertificateException;
19 |
20 | /**
21 | *
22 | * @date 2022-03-31 18:54
23 | */
24 | public class CSRUtil {
25 |
26 | public static final String SHA_SIGN_ALGORITHM = "SHA256withECDSA";
27 |
28 | /**
29 | * 生成CSR请求文件
30 | * @param reqName 请求者主体信息
31 | * @param userPublicKey 用户公钥
32 | * @param userPrivateKey 用户私钥
33 | * @return
34 | * @throws OperatorCreationException
35 | * @throws CertificateException
36 | * @throws IOException
37 | */
38 | public static String csrBuilder(X500Name reqName, PublicKey userPublicKey, PrivateKey userPrivateKey) throws OperatorCreationException, IOException {
39 |
40 | PKCS10CertificationRequestBuilder p10Builder = new JcaPKCS10CertificationRequestBuilder(reqName,userPublicKey );
41 | JcaContentSignerBuilder csBuilder = new JcaContentSignerBuilder(SHA_SIGN_ALGORITHM);
42 | ContentSigner csrSigner = csBuilder.build(userPrivateKey);
43 | PKCS10CertificationRequest csr = p10Builder.build(csrSigner);
44 |
45 | //处理证书 ANS.I DER 编码 =》 String Base64编码
46 | String encode = Base64.encode(csr.getEncoded());;
47 | StringBuilder sb = new StringBuilder();
48 | sb.append("-----BEGIN CERTIFICATE REQUEST-----"+"\n");
49 | sb.append(encode);
50 | sb.append("-----END CERTIFICATE REQUEST-----");
51 | return sb.toString();
52 | }
53 | /**
54 | * 根据CSR字符串转换成JcaPKCS10CertificationRequest对象
55 | * @param csrStr CSR字符串
56 | * @return JcaPKCS10CertificationRequest 包含证书申请方主体信息(jcaPKCS10CertificationRequest.getSubject())以及公钥信息(jcaPKCS10CertificationRequest.getPublicKey())
57 | * @throws NoSuchAlgorithmException
58 | * @throws InvalidKeyException
59 | */
60 | public static JcaPKCS10CertificationRequest parseCSRStr(String csrStr) throws NoSuchAlgorithmException, InvalidKeyException, IOException {
61 | if( !csrStr.startsWith("-----BEGIN CERTIFICATE REQUEST-----") || !csrStr.endsWith("-----END CERTIFICATE REQUEST-----")){
62 | throw new IOException("csr 信息不合法");
63 | }
64 | csrStr = csrStr.replace("-----BEGIN CERTIFICATE REQUEST-----"+"\n","");
65 | csrStr = csrStr.replace("-----END CERTIFICATE REQUEST-----","");
66 | byte[] bArray = Base64.decode(csrStr);
67 | PKCS10CertificationRequest csrRequest = new PKCS10CertificationRequest(bArray);
68 | return new JcaPKCS10CertificationRequest(csrRequest);
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/cert/KeyPairPojo.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.cert;
2 |
3 | import lombok.AllArgsConstructor;
4 | import lombok.Data;
5 | import lombok.NoArgsConstructor;
6 |
7 | import java.io.Serializable;
8 |
9 | /**
10 | *
11 | * @date 2022-03-31 17:47
12 | * 生成公私钥的模数,用于生成根证书,需要通过ACES加密保存,
13 | */
14 | @Data
15 | @AllArgsConstructor
16 | @NoArgsConstructor
17 | public class KeyPairPojo implements Serializable {
18 | private String publicExponent;
19 | private String privateExponent;
20 | private String modulus;
21 | }
22 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/cert/SignUtil.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.cert;
2 |
3 | import cn.hutool.core.codec.Base64;
4 | import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPrivateKey;
5 | import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPublicKey;
6 | import org.bouncycastle.jce.provider.BouncyCastleProvider;
7 |
8 | import java.io.IOException;
9 | import java.nio.charset.StandardCharsets;
10 | import java.security.*;
11 |
12 | public class SignUtil {
13 |
14 | private static final String SIGN_ALGORITHM = "SHA256withECDSA";
15 |
16 | /**
17 | * 签名
18 | * @param priKeyStr 私钥
19 | * @param oriData 原始数据
20 | * @return base64编码后的结果
21 | * @throws IOException
22 | * @throws NoSuchAlgorithmException
23 | * @throws InvalidKeyException
24 | * @throws SignatureException
25 | */
26 | public static String sign(String priKeyStr,String oriData) throws IOException, NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException {
27 | BCECPrivateKey privateKey = KeyGenerator.getPrivateKey(priKeyStr);
28 | return sign(privateKey,oriData);
29 | }
30 |
31 | public static String sign(PrivateKey privateKey,String oriData) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException {
32 | Signature signature = Signature.getInstance(SIGN_ALGORITHM,BouncyCastleProvider.PROVIDER_NAME);
33 | signature.initSign(privateKey);
34 | signature.update(oriData.getBytes(StandardCharsets.UTF_8));
35 | return Base64.encode(signature.sign());
36 | }
37 |
38 | /**
39 | * 对base64编码的数据进行验证
40 | * @param pubKeyStr 公钥
41 | * @param oriData 原始数据
42 | * @param signStr base64编码的签名字符串
43 | * @return
44 | */
45 | public static boolean verify(String pubKeyStr,String oriData,String signStr) throws IOException, NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException {
46 | BCECPublicKey publicKey = KeyGenerator.getPublicKey(pubKeyStr);
47 | return verify(publicKey,oriData,signStr);
48 | }
49 |
50 | public static boolean verify(PublicKey publicKey,String oriData,String signStr) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException {
51 | Signature signature = Signature.getInstance(SIGN_ALGORITHM,BouncyCastleProvider.PROVIDER_NAME);
52 | signature.initVerify(publicKey);
53 | signature.update(oriData.getBytes(StandardCharsets.UTF_8));
54 | return signature.verify(Base64.decode(signStr));
55 | }
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/advice/ByteArrayServletInputStream.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.advice;
2 |
3 | import jakarta.servlet.ReadListener;
4 | import jakarta.servlet.ServletInputStream;
5 |
6 | import java.io.ByteArrayInputStream;
7 | import java.io.IOException;
8 |
9 | /**
10 | * @Description: 输入流
11 | */
12 |
13 | public class ByteArrayServletInputStream extends ServletInputStream {
14 |
15 | private ByteArrayInputStream byteArrayInputStream;
16 |
17 | public ByteArrayServletInputStream(ByteArrayInputStream byteArrayInputStream) {
18 | this.byteArrayInputStream = byteArrayInputStream;
19 | }
20 |
21 | @Override
22 | public int read() throws IOException {
23 | return byteArrayInputStream.read();
24 | }
25 |
26 | @Override
27 | public boolean isFinished() {
28 | return false;
29 | }
30 |
31 | @Override
32 | public boolean isReady() {
33 | return false;
34 | }
35 |
36 | @Override
37 | public void setReadListener(ReadListener readListener) {
38 |
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/advice/GlobalExceptionHandler.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.advice;
2 |
3 | import java.util.List;
4 |
5 | import org.springframework.validation.BindException;
6 | import org.springframework.validation.ObjectError;
7 | import org.springframework.web.bind.annotation.ExceptionHandler;
8 | import org.springframework.web.bind.annotation.RestControllerAdvice;
9 | import org.springframework.web.method.HandlerMethod;
10 |
11 | import com.jd.mpc.common.response.CommonException;
12 | import com.jd.mpc.common.response.CommonResponse;
13 | import com.jd.mpc.common.response.ErrorStatus;
14 |
15 | import lombok.extern.slf4j.Slf4j;
16 |
17 | /**
18 | * 全局异常处理,该类处理时候注意考虑某些场景下新老版本兼容。
19 | */
20 | @Slf4j
21 | @RestControllerAdvice
22 | public class GlobalExceptionHandler {
23 | /**
24 | * 系统异常文案。
25 | */
26 | private static final String SYSTEM_ERROR_TEXT = "请求错误,请稍后重试";
27 |
28 | /**
29 | * 参数校验失败默认文案。
30 | */
31 | private static final String PARAMETER_VALIDATE_ERROR = "参数校验失败,请检查之后重试";
32 |
33 | /**
34 | * 处理业务中触发的异常。
35 | *
36 | * @param e
37 | * @return
38 | */
39 | @ExceptionHandler(value = CommonException.class)
40 | public Object handleException(CommonException e, HandlerMethod method) {
41 | log.error("处理业务异常", e);
42 |
43 | return buildResponse(method, e.getStatus(), e.getMessage());
44 | }
45 |
46 | /**
47 | * 处理通用异常。
48 | *
49 | * @return
50 | */
51 | @ExceptionHandler(value = BindException.class)
52 | public Object handleException(BindException bindException, HandlerMethod method) {
53 | log.error("处理BindException异常", bindException);
54 | List allErrors = bindException.getAllErrors();
55 | for (ObjectError error : allErrors) {
56 | return buildResponse(method, ErrorStatus.PARAMETER_ERROR, error.getDefaultMessage());
57 | }
58 | return buildResponse(method, ErrorStatus.BUSINESS_ERROR, SYSTEM_ERROR_TEXT);
59 | }
60 |
61 | /**
62 | * 处理通用异常。
63 | *
64 | * @return
65 | */
66 | @ExceptionHandler(value = Exception.class)
67 | public Object handleException(Exception e, HandlerMethod method) {
68 | log.error("处理通用异常", e);
69 |
70 | return buildResponse(method, ErrorStatus.BUSINESS_ERROR, e.getMessage());
71 | }
72 |
73 | /**
74 | * 构建异常返回的数据格式。
75 | *
76 | * @param method
77 | * @param status
78 | * @param message
79 | * @return
80 | */
81 | private CommonResponse buildResponse(HandlerMethod method, Integer status, String message) {
82 | CommonResponse response = new CommonResponse<>();
83 | response.setError(status, message);
84 | return response;
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/advice/TeeServletOutputStream.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.advice;
2 |
3 |
4 | import jakarta.servlet.ServletOutputStream;
5 | import jakarta.servlet.WriteListener;
6 | import org.apache.commons.io.output.TeeOutputStream;
7 |
8 | import java.io.IOException;
9 | import java.io.OutputStream;
10 |
11 | /**
12 | * @Description: 输出流
13 | *
14 | * @Date: 2021/11/29
15 | */
16 | public class TeeServletOutputStream extends ServletOutputStream {
17 |
18 | private final TeeOutputStream teeOutputStream;
19 |
20 | public TeeServletOutputStream(OutputStream one, OutputStream two) {
21 | this.teeOutputStream = new TeeOutputStream(one, two);
22 | }
23 |
24 | @Override
25 | public void write(byte[] b) throws IOException {
26 | this.teeOutputStream.write(b);
27 | }
28 |
29 | @Override
30 | public void write(byte[] b, int off, int len) throws IOException {
31 | this.teeOutputStream.write(b, off, len);
32 | }
33 |
34 | @Override
35 | public void write(int b) throws IOException {
36 | this.teeOutputStream.write(b);
37 | }
38 |
39 | @Override
40 | public void flush() throws IOException {
41 | super.flush();
42 | this.teeOutputStream.flush();
43 | }
44 |
45 | @Override
46 | public void close() throws IOException {
47 | super.close();
48 | this.teeOutputStream.close();
49 | }
50 |
51 | @Override
52 | public boolean isReady() {
53 | return false;
54 | }
55 |
56 | @Override
57 | public void setWriteListener(WriteListener writeListener) {
58 |
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/advice/ThreadPoolConfig.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.advice;
2 |
3 | import org.springframework.context.annotation.Bean;
4 | import org.springframework.context.annotation.Configuration;
5 | import org.springframework.scheduling.annotation.EnableAsync;
6 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
7 |
8 | import java.util.concurrent.ThreadPoolExecutor;
9 |
10 | /**
11 | * @Description: 线程池配置
12 | *
13 | * @Date: 2022/2/21
14 | */
15 | @Configuration
16 | @EnableAsync
17 | public class ThreadPoolConfig {
18 |
19 | private static final int corePoolSize = 20; // 核心线程数(默认线程数)线程池创建时候初始化的线程数
20 | private static final int maxPoolSize = 20; // 最大线程数 线程池最大的线程数,只有在缓冲队列满了之后才会申请超过核心线程数的线程
21 | private static final int keepAliveTime = 20; // 允许线程空闲时间(单位:默认为秒)当超过了核心线程之外的线程在空闲时间到达之后会被销毁
22 | private static final int queueCapacity = 200; // 缓冲队列数 用来缓冲执行任务的队列
23 | private static final String threadNamePrefix = "async-"; // 线程池名前缀 方便我们定位处理任务所在的线程池
24 |
25 | @Bean("threadPoolTaskExecutor") // bean的名称,默认为首字母小写的方法名
26 | public ThreadPoolTaskExecutor taskExecutor(){
27 | ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
28 | executor.setCorePoolSize(corePoolSize);
29 | executor.setMaxPoolSize(maxPoolSize);
30 | executor.setQueueCapacity(queueCapacity);
31 | executor.setKeepAliveSeconds(keepAliveTime);
32 | executor.setThreadNamePrefix(threadNamePrefix);
33 |
34 | // 线程池对拒绝任务的处理策略 采用了CallerRunsPolicy策略,当线程池没有处理能力的时候,该策略会直接在 execute 方法的调用线程中运行被拒绝的任务;如果执行程序已关闭,则会丢弃该任务
35 | executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
36 | // 初始化
37 | executor.initialize();
38 | return executor;
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/config/NacosListenerConfig.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.config;
2 |
3 | import com.alibaba.fastjson.JSONObject;
4 | import com.alibaba.nacos.api.annotation.NacosInjected;
5 | import com.alibaba.nacos.api.config.ConfigService;
6 | import com.alibaba.nacos.api.config.annotation.NacosConfigListener;
7 | import com.alibaba.nacos.api.config.convert.NacosConfigConverter;
8 | import com.alibaba.nacos.api.exception.NacosException;
9 | import com.alibaba.nacos.spring.context.event.config.TimeoutNacosConfigListener;
10 | import com.google.common.collect.Maps;
11 | import com.jd.mpc.common.constant.CommonConstant;
12 | import com.jd.mpc.common.enums.TaskTypeEnum;
13 | import com.jd.mpc.common.response.CommonException;
14 | import lombok.extern.slf4j.Slf4j;
15 | import org.springframework.context.annotation.Bean;
16 | import org.springframework.context.annotation.Configuration;
17 | import org.springframework.util.ReflectionUtils;
18 |
19 | import java.io.IOException;
20 | import java.io.StringReader;
21 | import java.util.Map;
22 | import java.util.Properties;
23 |
24 | /**
25 | * @Description: TODO
26 | *
27 | * @Date: 2022/12/23
28 | */
29 | @Slf4j
30 | @Configuration
31 | public class NacosListenerConfig {
32 |
33 | @NacosInjected
34 | private ConfigService configService;
35 |
36 | @Bean
37 | public Map functorGroup() throws NacosException, IOException {
38 | Map funtorMap = Maps.newConcurrentMap();
39 | try {
40 | for (TaskTypeEnum taskType : TaskTypeEnum.values()) {
41 | String dataId = taskType.getName() + ".properties";
42 | long timeout = 5000;
43 | String config = configService.getConfig(dataId, CommonConstant.FUNCTOR_GROUP, timeout);
44 | if (config == null) {
45 | log.warn("nacos default config of " + taskType.getName() + " is null!");
46 | continue;
47 | }
48 | Properties properties = new Properties();
49 | properties.load(new StringReader(config));
50 | funtorMap.put(taskType, properties);
51 | configService.addListener(dataId, CommonConstant.FUNCTOR_GROUP, new TimeoutNacosConfigListener(dataId, CommonConstant.FUNCTOR_GROUP, timeout) {
52 | @Override
53 | protected void onReceived(String config) {
54 | try {
55 | Properties properties = new Properties();
56 | properties.load(new StringReader(config));
57 | funtorMap.put(taskType, properties);
58 | } catch (Exception e) {
59 | }
60 | }
61 | });
62 | }
63 | }catch (Exception e){
64 |
65 | }
66 | return funtorMap;
67 | }
68 |
69 | }
70 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/constant/CommonConstant.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.constant;
2 |
3 | /**
4 | * @Description: 常量
5 | *
6 | * @Date: 2022/3/22
7 | */
8 | public interface CommonConstant {
9 |
10 | /** coordinator服务名称 */
11 | String COORDINATOR_NAME = "coordinator";
12 |
13 | String BIN_BASH = "/bin/bash";
14 |
15 | String NN_MPC_POD_NAME_STR = "-mpc-nn-worker-";
16 |
17 | String K8S_DATA_VOLUME = "data";
18 |
19 | String K8S_LOG_VOLUME = "logs";
20 |
21 | /**
22 | * k8s's yaml in nacos
23 | */
24 | String K8S_GROUP = "K8S_GROUP";
25 |
26 | /**
27 | * functor's default properties in nacos
28 | */
29 | String FUNCTOR_GROUP = "FUNCTOR_GROUP";
30 | /**
31 | * DEFAULT_GROUP
32 | */
33 | String DEFAULT_GROUP = "DEFAULT_GROUP";
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/constant/DeploymentPathConstant.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.constant;
2 |
3 | /**
4 | * @Description: 部署文件路径常量
5 | *
6 | * @Date: 2022/2/10
7 | */
8 | public interface DeploymentPathConstant {
9 | String INTERSECTION = "intersection.yaml";
10 |
11 | String PSI = "psi.yaml";
12 |
13 | String FEATURE = "feature.yaml";
14 |
15 | String FEATURE_FL = "feature-fl.yaml";
16 |
17 | String TRAIN = "train.yaml";
18 |
19 | String JXZ_MPC = "jxz-mpc.yaml";
20 |
21 | String JXZ_LOCAL = "jxz-local.yaml";
22 |
23 | String UNICOM = "unicom.yaml";
24 |
25 | String HRZ_FL_BASE = "hrz-fl-base.yaml";
26 |
27 | String TRAIN_BASE = "train-base.yaml";
28 |
29 | String TRAIN_BASE_VIF = "train-base-vif.yaml";
30 |
31 | String PREDICT_VERTICAL = "predict-vertical.yaml";
32 |
33 | String PREDICT_HORIZONTAL = "predict-horizontal.yaml";
34 |
35 | String PREDICT_NN = "predict-nn.yaml";
36 |
37 | String PREDICT_EVAL = "predict-estimate.yaml";
38 |
39 | /**
40 | * XGB树模型
41 | */
42 | String TREE_TRAIN_XGB = "tree-train-base.yaml";
43 |
44 | /**
45 | * 增加随机森林树模型
46 | */
47 | String TREE_TRAIN_RF = "tree-train-rf.yaml";
48 |
49 | String HRZ_FL_PREDICT_BASE = "hrz-fl-predict-base.yaml";
50 |
51 | String CUT_DATAFRAME_BASE = "feature-cut-dataframe-base.yaml";
52 |
53 | String XGBOOST_TRAIN_BASE = "xgboost-train-base.yaml";
54 |
55 | String XGBBOOST_TRAIN_BASE_WITH_RAYGBO = "ray_cluster_xgb.yaml";
56 |
57 | String RAY_BASE = "ray_cluster.yaml";
58 |
59 | String NN_DC = "nn-dc.yaml";
60 |
61 | String NN_MPC = "mpc-nn-worker.yaml";
62 |
63 | String CODE_MPC = "mpc-code-worker.yaml";
64 |
65 | String NN_TRAINER = "nn-trainer.yaml";
66 |
67 | String STABILITY_INDEX = "stability-index.yaml";
68 |
69 | String PLUMBER_BASE = "plumber.yaml";
70 |
71 | String NEW_PSI = "new-psi.yaml";
72 |
73 | String LINEAR_EVALUATE = "linear-evaluate.yaml";
74 |
75 | String NN_EVALUATE = "nn-evaluate.yaml";
76 |
77 | String SHAPLEY_VALUE_EVALUATE = "shapley-value-evaluate.yaml";
78 |
79 | String SCORE_CARD = "score-card.yaml";
80 |
81 | String SPEARMANMPC = "spearman-mpc.yaml";
82 |
83 | String JTPSI_MASTER = "jingteng-master-psi.yaml";
84 |
85 | String JTPSI_WORKER = "jingteng-worker-psi.yaml";
86 |
87 | String ETL = "etl.yaml";
88 |
89 | String FILE_SERVICE = "file-service.yaml";
90 |
91 | String LOCAL_WORKER = "local-worker.yaml";
92 |
93 | String FILE_TRANSFER = "file-transfer.yaml";
94 |
95 | String BUFFALO_WORKER = "buffalo-worker.yaml";
96 |
97 | String BDP_DECRYPT = "bdp-decrypt.yaml";
98 |
99 | String BUSI_DOWNLOAD = "busi-download.yaml";
100 |
101 | }
102 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/constant/OfflineTaskMap.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.constant;
2 |
3 | import com.jd.mpc.domain.offline.commons.SubTask;
4 |
5 | import java.util.List;
6 | import java.util.concurrent.ConcurrentHashMap;
7 |
8 | public class OfflineTaskMap extends ConcurrentHashMap> {
9 |
10 | private static volatile OfflineTaskMap taskMap = null;
11 |
12 |
13 | private OfflineTaskMap() {
14 | }
15 |
16 | public static OfflineTaskMap getInstance() {
17 | //第一次校验GlobalLock是否为空
18 | if (taskMap == null) {
19 | synchronized (OfflineTaskMap.class) {
20 | //第二次校验GlobalLock是否为空
21 | if (taskMap == null) {
22 | taskMap = new OfflineTaskMap();
23 | }
24 | }
25 | }
26 | return taskMap;
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/constant/ServicePathConstant.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.constant;
2 |
3 | /**
4 | * @Description: service配置文件
5 | *
6 | */
7 | public interface ServicePathConstant {
8 |
9 | String MPC_NN_SERVICE = "/k8s/mpc-nn-worker-service.yaml";
10 | }
11 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/constant/TargetMap.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.constant;
2 |
3 | import java.util.Set;
4 | import java.util.concurrent.ConcurrentHashMap;
5 |
6 | /**s
7 | *
8 | * @date 2021/11/24 6:32 下午
9 | * 该类已由TargetMapHolder代替update by yezhenyue on 20220411
10 | * @see com.jd.mpc.storage.TargetMapHolder
11 | *
12 | */
13 | public class TargetMap extends ConcurrentHashMap> {
14 |
15 | private static volatile TargetMap targetMap = null;
16 |
17 |
18 | private TargetMap() {
19 | }
20 |
21 | public static TargetMap getInstance() {
22 | //第一次校验GlobalLock是否为空
23 | if (targetMap == null) {
24 | synchronized (TargetMap.class) {
25 | //第二次校验GlobalLock是否为空
26 | if (targetMap == null) {
27 | targetMap = new TargetMap();
28 | }
29 | }
30 | }
31 | return targetMap;
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/IsDeletedEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | import lombok.Getter;
4 |
5 | /**
6 | * 任务类型枚举
7 | *
8 | *
9 | * @date 2021/9/26 8:48 下午
10 | */
11 | @Getter
12 | public enum IsDeletedEnum {
13 |
14 | /**
15 | * 未删除
16 | */
17 | FALSE((byte) 0),
18 |
19 | /**
20 | * 已删除
21 | */
22 | TRUE((byte) 1);
23 |
24 |
25 | private final Byte status;
26 |
27 | IsDeletedEnum(byte status) {
28 | this.status = status;
29 | }
30 |
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/IsLocalEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | import lombok.Getter;
4 |
5 | /**
6 | *
7 | * @date 2022-04-02 11:53
8 | */
9 | @Getter
10 | public enum IsLocalEnum {
11 | /**
12 | * 非本地任务
13 | */
14 | FALSE((byte) 0),
15 |
16 | /**
17 | * 本地任务
18 | */
19 | TRUE((byte) 1);
20 |
21 |
22 | private final Byte status;
23 |
24 | IsLocalEnum(byte status) {
25 | this.status = status;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/IsRootEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | import lombok.Getter;
4 |
5 | /**
6 | *
7 | * @date 2022-04-02 11:51
8 | */
9 | @Getter
10 | public enum IsRootEnum {
11 | /**
12 | * 非根证书
13 | */
14 | FALSE((byte) 0),
15 |
16 | /**
17 | * 根证书
18 | */
19 | TRUE((byte) 1);
20 |
21 |
22 | private final Byte status;
23 |
24 | IsRootEnum(byte status) {
25 | this.status = status;
26 | }
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/K8sResourceTypeEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | import lombok.Getter;
4 |
5 | import java.util.Objects;
6 |
7 | /**
8 | * k8s资源类型枚举
9 | *
10 | *
11 | * @date 2021/9/26 8:48 下午
12 | */
13 | @Getter
14 | public enum K8sResourceTypeEnum {
15 |
16 | /**
17 | * deployment
18 | */
19 | DEPLOYMENT("deployment"),
20 |
21 | /**
22 | * crd
23 | */
24 | CRD("crd");
25 |
26 |
27 |
28 | private final String name;
29 |
30 | K8sResourceTypeEnum(String name) {
31 | this.name = name;
32 | }
33 |
34 | public static K8sResourceTypeEnum getByValue(String name) {
35 | for (K8sResourceTypeEnum taskTypeEnum : values()) {
36 | if (Objects.equals(taskTypeEnum.name, name)) {
37 | return taskTypeEnum;
38 | }
39 | }
40 | return K8sResourceTypeEnum.DEPLOYMENT;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/LogLevelEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | public enum LogLevelEnum {
4 | DEBUG("debug"),
5 | INFO("info"),
6 | WARNING("warning"),
7 | ERROR("error"),
8 | // ALL("all"),
9 | ;
10 |
11 | private String desc;
12 |
13 | LogLevelEnum(String desc) {
14 | this.desc = desc;
15 | }
16 |
17 | public String getDesc() {
18 | return desc;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/OperatorStatusEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | import com.fasterxml.jackson.annotation.JsonFormat;
4 |
5 | /**
6 | * @Description: 算子状态
7 | *
8 | * @Date: 2022/4/1
9 | */
10 | @JsonFormat(shape = JsonFormat.Shape.OBJECT)
11 | public enum OperatorStatusEnum {
12 | INTERNAL_ERROR(500, "内部错误"),
13 | LOSS_LARGE(501, "学习率过大,模型不再收敛"),
14 | ORI_DATA_ERROR(501, "原始数据错误"),
15 | PENDING_ERROR(999, "资源不足"),;
16 |
17 | private final Integer code;
18 |
19 | private final String desc;
20 |
21 | OperatorStatusEnum(Integer code, String desc) {
22 | this.code = code;
23 | this.desc = desc;
24 | }
25 |
26 | public Integer getCode() {
27 | return code;
28 | }
29 |
30 | public String getDesc() {
31 | return desc;
32 | }
33 |
34 | public static OperatorStatusEnum getByCode(Integer code) {
35 | for (OperatorStatusEnum value : OperatorStatusEnum.values()) {
36 | if (value.code.equals(code)) {
37 | return value;
38 | }
39 | }
40 | return null;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/StoreTypeEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | /**
4 | * @Description: 存储类型
5 | *
6 | * @Date: 2022/8/10
7 | */
8 | public enum StoreTypeEnum {
9 | HDFS("HDFS"),
10 | CFS("CFS"),
11 | ;
12 | private final String desc;
13 |
14 | StoreTypeEnum(String desc) {
15 | this.desc = desc;
16 | }
17 |
18 | public String getDesc() {
19 | return desc;
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/enums/TaskStatusEnum.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.enums;
2 |
3 | import lombok.Getter;
4 |
5 | /**
6 | * 任务状态枚举
7 | *
8 | *
9 | * @date 2021/9/26 8:48 下午
10 | */
11 | @Getter
12 | public enum TaskStatusEnum {
13 |
14 | /**
15 | * 新建
16 | */
17 | NEW(0),
18 |
19 | /**
20 | * 运行中
21 | */
22 | RUNNING(1),
23 |
24 | /**
25 | * 运行结束
26 | */
27 | COMPLETED(2),
28 |
29 | /**
30 | * 运行异常
31 | */
32 | ERROR(3),
33 |
34 | /**
35 | * 运行停止
36 | */
37 | STOPPED(4);
38 |
39 | private final Integer status;
40 |
41 | TaskStatusEnum(int status) {
42 | this.status = status;
43 | }
44 |
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/response/CommonException.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.response;
2 |
3 |
4 | public class CommonException extends RuntimeException {
5 | private final Integer status;
6 | private Object[] params;
7 |
8 | public CommonException(String msg) {
9 | super(msg);
10 | this.status = ErrorStatus.BUSINESS_ERROR;
11 | this.params = null;
12 | }
13 |
14 | public CommonException(Integer code, String msg) {
15 | super(msg);
16 | this.status = code;
17 | this.params = null;
18 | }
19 |
20 | public CommonException(Integer code, String msg, Exception e) {
21 | super(msg, e);
22 | this.status = code;
23 | this.params = null;
24 | }
25 |
26 | public CommonException(Integer code, String msg, Throwable t) {
27 | super(msg, t);
28 | this.status = code;
29 | this.params = null;
30 | }
31 |
32 | public Integer getStatus() {
33 | return this.status;
34 | }
35 |
36 | public Object[] getParams() {
37 | return this.params;
38 | }
39 |
40 | public void setParams(Object[] params) {
41 | this.params = params;
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/response/CommonResponse.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.response;
2 |
3 |
4 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
5 | import com.fasterxml.jackson.annotation.JsonInclude;
6 | import com.fasterxml.jackson.annotation.JsonInclude.Include;
7 |
8 | import java.io.Serializable;
9 |
10 | @JsonInclude(Include.NON_NULL)
11 | @JsonIgnoreProperties(ignoreUnknown = true)
12 | public class CommonResponse implements Serializable {
13 | private Integer status;
14 | private String errMsg;
15 | private T result;
16 | /**
17 | * 重定向url
18 | */
19 | private String url;
20 |
21 | public CommonResponse() {
22 | this.status = ErrorStatus.SUCCESS;
23 | }
24 |
25 | public Integer getStatus() {
26 | return this.status;
27 | }
28 |
29 | public String getErrMsg() {
30 | return this.errMsg;
31 | }
32 |
33 | public T getResult() {
34 | return this.result;
35 | }
36 |
37 | public void setResult(T result) {
38 | this.result = result;
39 | }
40 |
41 | public String getUrl() {
42 | return url;
43 | }
44 |
45 | public void setUrl(String url) {
46 | this.url = url;
47 | }
48 |
49 | public void setError(Integer status, String errMsg) {
50 | this.status = status;
51 | this.errMsg = errMsg;
52 | }
53 |
54 | /**
55 | * @param t
56 | * @param
57 | * @return
58 | */
59 | public static CommonResponse ok(T t) {
60 | CommonResponse r = ok();
61 | r.setResult(t);
62 | return r;
63 | }
64 |
65 | public static CommonResponse ok() {
66 | CommonResponse r = new CommonResponse<>();
67 | r.status = ErrorStatus.SUCCESS;
68 | return r;
69 | }
70 |
71 | public static CommonResponse fail(String msg) {
72 | CommonResponse r = new CommonResponse<>();
73 | r.status = ErrorStatus.BUSINESS_ERROR;
74 | r.errMsg = msg;
75 | return r;
76 | }
77 |
78 | public static CommonResponse fail(Integer code, String msg) {
79 | CommonResponse r = new CommonResponse<>();
80 | r.status = code;
81 | r.errMsg = msg;
82 | return r;
83 | }
84 |
85 | public static void convertThrowException(CommonResponse commonResponse){
86 | if (!ErrorStatus.SUCCESS.equals(commonResponse.getStatus())){
87 | throw new CommonException(commonResponse.getStatus(), commonResponse.getErrMsg());
88 | }
89 | }
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/response/ErrorStatus.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.response;
2 |
3 | public class ErrorStatus {
4 | public static final Integer SUCCESS = 0;
5 | public static final Integer SERVER_INTERNAL_ERROR = 100001;
6 | public static final Integer BUSINESS_ERROR = 100002;
7 | public static final Integer AUTH_ERROR = 100003;
8 | public static final Integer PARAMETER_EMPTY = 100004;
9 | public static final Integer STATUS_ERROR = 100005;
10 | public static final Integer DATA_NOT_EXIST = 100006;
11 | public static final Integer DATA_EXIST = 100007;
12 | public static final Integer HTTP_REQUEST_ERROR = 100008;
13 | public static final Integer PARAMETER_ERROR = 100009;
14 | }
15 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/response/ProcessCommonResponse.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.response;
2 |
3 |
4 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
5 | import com.fasterxml.jackson.annotation.JsonInclude;
6 | import com.fasterxml.jackson.annotation.JsonInclude.Include;
7 |
8 | import java.io.Serializable;
9 |
10 | @JsonInclude(Include.NON_NULL)
11 | @JsonIgnoreProperties(ignoreUnknown = true)
12 | public class ProcessCommonResponse implements Serializable {
13 | private Integer code;
14 | private String errMsg;
15 | private T result;
16 | /**
17 | * 重定向url
18 | */
19 | private String url;
20 |
21 | public ProcessCommonResponse() {
22 | this.code = ErrorStatus.SUCCESS;
23 | }
24 |
25 | public String getErrMsg() {
26 | return this.errMsg;
27 | }
28 |
29 | public T getResult() {
30 | return this.result;
31 | }
32 |
33 | public void setResult(T result) {
34 | this.result = result;
35 | }
36 |
37 | public String getUrl() {
38 | return url;
39 | }
40 |
41 | public void setUrl(String url) {
42 | this.url = url;
43 | }
44 |
45 | public void setError(Integer status, String errMsg) {
46 | this.code = status;
47 | this.errMsg = errMsg;
48 | }
49 |
50 | /**
51 | * @param t
52 | * @param
53 | * @return
54 | */
55 | public static ProcessCommonResponse ok(T t) {
56 | ProcessCommonResponse r = ok();
57 | r.setResult(t);
58 | return r;
59 | }
60 |
61 | public Integer getCode() {
62 | return code;
63 | }
64 |
65 | public static ProcessCommonResponse ok() {
66 | ProcessCommonResponse r = new ProcessCommonResponse<>();
67 | r.code = ErrorStatus.SUCCESS;
68 | return r;
69 | }
70 |
71 | public static ProcessCommonResponse fail(String msg) {
72 | ProcessCommonResponse r = new ProcessCommonResponse<>();
73 | r.code = ErrorStatus.BUSINESS_ERROR;
74 | r.errMsg = msg;
75 | return r;
76 | }
77 |
78 | public static ProcessCommonResponse fail(Integer code, String msg) {
79 | ProcessCommonResponse r = new ProcessCommonResponse<>();
80 | r.code = code;
81 | r.errMsg = msg;
82 | return r;
83 | }
84 |
85 | public static void convertThrowException(ProcessCommonResponse commonResponse){
86 | if (!ErrorStatus.SUCCESS.equals(commonResponse.getCode())){
87 | throw new CommonException(commonResponse.getCode(), commonResponse.getErrMsg());
88 | }
89 | }
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/util/JNISigner.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.util;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 |
5 | import java.util.Base64;
6 |
7 | @Slf4j
8 | public class JNISigner {
9 | // This declares that the static `hello` method will be provided
10 | // a native library.
11 | public static native byte[] sign(byte[] privateKey, byte[] message);
12 |
13 | public static native byte[] newPrivateKey();
14 |
15 | public static native byte[] publicKey(byte[] privateKey);
16 |
17 | static {
18 | // This actually loads the shared object that we'll be creating.
19 | // The actual location of the .so or .dll may differ based on your
20 | // platform.
21 | System.loadLibrary("jni_sign");
22 | }
23 |
24 | // The rest is just regular ol' Java!
25 | public static void gen() {
26 | byte[] msg = "dfasfadsad".getBytes();
27 | for (int i = 0; i < 1; i++) {
28 | // System.out.println(i);
29 | byte[] priv = newPrivateKey();
30 | log.info("java-private: " +
31 | Base64.getEncoder().encodeToString(priv));
32 | byte[] pub = publicKey(priv);
33 | log.info("java-public: " +
34 | Base64.getEncoder().encodeToString(pub));
35 | log.info("java-message: " +
36 | Base64.getEncoder().encodeToString(msg));
37 | byte[] sig = sign(priv, msg);
38 | log.info(Base64.getEncoder().encodeToString(sig));
39 | }
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/src/Coordinator/src/main/java/com/jd/mpc/common/util/MapTypeAdapter.java:
--------------------------------------------------------------------------------
1 | package com.jd.mpc.common.util;
2 |
3 | import com.google.gson.TypeAdapter;
4 | import com.google.gson.internal.LinkedTreeMap;
5 | import com.google.gson.stream.JsonReader;
6 | import com.google.gson.stream.JsonToken;
7 | import com.google.gson.stream.JsonWriter;
8 |
9 | import java.io.IOException;
10 | import java.util.ArrayList;
11 | import java.util.List;
12 | import java.util.Map;
13 |
14 | /**
15 | * @Date 2022/3/16
16 | * @Description
17 | */
18 | public class MapTypeAdapter extends TypeAdapter