├── LICENSE ├── README.md ├── data └── data_B10W.csv ├── docs ├── imgs │ ├── architecture.png │ ├── deploy_arch.png │ ├── mysql_service_svc.png │ ├── p1.png │ ├── p2.png │ ├── redis_service_svc.PNG │ ├── trainer.png │ └── 联邦学习.png ├── k8s_deploy.md ├── nfs_deploy.md ├── prop │ ├── application.properties │ └── psi.properties ├── sql │ ├── coordinator_init.sql │ ├── mpc_init.sql │ └── nacos_init.sql └── yamls │ ├── coordinator_deployment.yaml │ ├── coordinator_k8s_cofigmap.yaml │ ├── coordinator_nacos_configmap.yaml │ ├── coordinator_service.yaml │ ├── fileservice_deployment.yaml │ ├── mysql_deployment.yaml │ ├── mysql_secret.yaml │ ├── mysql_service.yaml │ ├── nacos_configmap.yaml │ ├── nacos_deployment.yaml │ ├── nacos_service.yaml │ ├── proxy_cert_configmap.yaml │ ├── proxy_deployment.yaml │ ├── proxy_lua_configmap.yaml │ ├── proxy_mpc_conf_configmap.yaml │ ├── proxy_ngnix_conf_configmap.yaml │ ├── psi.yaml │ ├── redis_configmap.yaml │ ├── redis_deployment.yaml │ └── redis_service.yaml ├── nacos_init.sh ├── src ├── Coordinator │ ├── Makefile │ ├── Makefile.win │ ├── docker │ │ ├── Dockerfile │ │ ├── Dockerfile_base │ │ ├── README.md │ │ └── start.sh │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── jd │ │ │ └── mpc │ │ │ ├── MpcApplication.java │ │ │ ├── aces │ │ │ └── TdeService.java │ │ │ ├── cert │ │ │ ├── CSRUtil.java │ │ │ ├── CertGenerator.java │ │ │ ├── KeyGenerator.java │ │ │ ├── KeyPairPojo.java │ │ │ └── SignUtil.java │ │ │ ├── common │ │ │ ├── advice │ │ │ │ ├── ByteArrayServletInputStream.java │ │ │ │ ├── CoordinatorRequestFilter.java │ │ │ │ ├── GlobalExceptionHandler.java │ │ │ │ ├── TeeServletOutputStream.java │ │ │ │ └── ThreadPoolConfig.java │ │ │ ├── config │ │ │ │ └── NacosListenerConfig.java │ │ │ ├── constant │ │ │ │ ├── CommonConstant.java │ │ │ │ ├── DeploymentPathConstant.java │ │ │ │ ├── OfflineTaskMap.java │ │ │ │ ├── ServicePathConstant.java │ │ │ │ └── TargetMap.java │ │ │ ├── enums │ │ │ │ ├── IsDeletedEnum.java │ │ │ │ ├── IsLocalEnum.java │ │ │ │ ├── IsRootEnum.java │ │ │ │ ├── K8sResourceTypeEnum.java │ │ │ │ ├── LogLevelEnum.java │ │ │ │ ├── OperatorStatusEnum.java │ │ │ │ ├── StoreTypeEnum.java │ │ │ │ ├── TaskStatusEnum.java │ │ │ │ └── TaskTypeEnum.java │ │ │ ├── response │ │ │ │ ├── CommonException.java │ │ │ │ ├── CommonResponse.java │ │ │ │ ├── ErrorStatus.java │ │ │ │ └── ProcessCommonResponse.java │ │ │ └── util │ │ │ │ ├── CommonUtils.java │ │ │ │ ├── EncryptUtils.java │ │ │ │ ├── GsonUtil.java │ │ │ │ ├── HttpUtil.java │ │ │ │ ├── JNISigner.java │ │ │ │ ├── MapTypeAdapter.java │ │ │ │ ├── MyX509TrustManager.java │ │ │ │ ├── ParameterParseUtil.java │ │ │ │ └── RestTemplateConfig.java │ │ │ ├── domain │ │ │ ├── cert │ │ │ │ ├── CertInfo.java │ │ │ │ ├── JobTaskStub.java │ │ │ │ └── SignCertVo.java │ │ │ ├── config │ │ │ │ └── ResourceLimitPolicy.java │ │ │ ├── form │ │ │ │ └── EsQueryForm.java │ │ │ ├── offline │ │ │ │ ├── commons │ │ │ │ │ ├── Job.java │ │ │ │ │ ├── OfflineTask.java │ │ │ │ │ ├── PreJob.java │ │ │ │ │ └── SubTask.java │ │ │ │ ├── jxz │ │ │ │ │ └── JxzTaskTypeEnum.java │ │ │ │ └── vo │ │ │ │ │ └── WorkerStatusVo.java │ │ │ ├── online │ │ │ │ ├── OnlineJob.java │ │ │ │ ├── OnlineResponse.java │ │ │ │ ├── OnlineSubTask.java │ │ │ │ ├── OnlineTask.java │ │ │ │ └── Result.java │ │ │ ├── param │ │ │ │ ├── ExistParam.java │ │ │ │ ├── GetConfigParam.java │ │ │ │ ├── GetCoordinatorLogParam.java │ │ │ │ ├── GetFileServiceLogParam.java │ │ │ │ ├── GetNodeLogParam.java │ │ │ │ ├── TaskInfoParam.java │ │ │ │ └── WorkerInfoParam.java │ │ │ ├── task │ │ │ │ ├── AuthInfo.java │ │ │ │ ├── AuthStatusEnum.java │ │ │ │ ├── CertTypeEnum.java │ │ │ │ ├── ChildrenTask.java │ │ │ │ ├── ChildrenTaskExample.java │ │ │ │ ├── ParentTask.java │ │ │ │ └── ParentTaskExample.java │ │ │ └── vo │ │ │ │ ├── AuthInfoDto.java │ │ │ │ ├── BlockVo.java │ │ │ │ ├── CallbackBody.java │ │ │ │ ├── Data.java │ │ │ │ ├── EtlHeaderParam.java │ │ │ │ ├── FeatureDetail.java │ │ │ │ ├── FileSchemaInfo.java │ │ │ │ ├── FileSizeInfo.java │ │ │ │ ├── GrpcResourceLimitResult.java │ │ │ │ ├── JobInfos.java │ │ │ │ ├── JoinRawInfo.java │ │ │ │ ├── MailInfo.java │ │ │ │ ├── PodInfos.java │ │ │ │ ├── PredictQuery.java │ │ │ │ ├── PredictResult.java │ │ │ │ ├── ProxyInfo.java │ │ │ │ ├── ResourcesInfo.java │ │ │ │ ├── SqlParserParam.java │ │ │ │ ├── SyncInfo.java │ │ │ │ ├── SyncRequest.java │ │ │ │ ├── SyncResInfo.java │ │ │ │ ├── SyncResponse.java │ │ │ │ ├── TableInfo.java │ │ │ │ ├── TargetInfo.java │ │ │ │ ├── TaskIdList.java │ │ │ │ ├── TaskInfo.java │ │ │ │ ├── TaskStatusInfo.java │ │ │ │ ├── VerifyVo.java │ │ │ │ └── WorkerInfo.java │ │ │ ├── grpc │ │ │ ├── GrpcClient.java │ │ │ ├── GrpcOfflineClient.java │ │ │ ├── GrpcOfflineService.java │ │ │ ├── GrpcOuterClient.java │ │ │ ├── GrpcOuterService.java │ │ │ ├── GrpcPredictClient.java │ │ │ ├── GrpcRetryExceptionHandler.java │ │ │ └── GrpcSignClient.java │ │ │ ├── mapper │ │ │ ├── AuthInfoMapper.java │ │ │ ├── CertMapper.java │ │ │ ├── ChildrenTaskMapper.java │ │ │ ├── JobTaskStubMapper.java │ │ │ └── ParentTaskMapper.java │ │ │ ├── quartz │ │ │ ├── QuartzConfig.java │ │ │ └── job │ │ │ │ ├── FinishTaskJob.java │ │ │ │ └── StartTaskJob.java │ │ │ ├── redis │ │ │ ├── RedisConfig.java │ │ │ ├── RedisLock.java │ │ │ ├── RedisServer.java │ │ │ └── RedisService.java │ │ │ ├── service │ │ │ ├── AuthInfoService.java │ │ │ ├── FileService.java │ │ │ ├── K8sService.java │ │ │ ├── OfflineService.java │ │ │ ├── OuterService.java │ │ │ ├── OuterSupport.java │ │ │ ├── ParaCompiler.java │ │ │ ├── PodFactory.java │ │ │ ├── TaskFactory.java │ │ │ ├── TaskPersistenceService.java │ │ │ ├── TaskSupport.java │ │ │ ├── cert │ │ │ │ ├── CertPersistenceService.java │ │ │ │ ├── JobCertValidateService.java │ │ │ │ └── JobTaskStubService.java │ │ │ ├── task │ │ │ │ ├── AbstractTaskService.java │ │ │ │ ├── FileSvcTaskService.java │ │ │ │ └── ITaskService.java │ │ │ └── zeebe │ │ │ │ ├── AbstractZeebeService.java │ │ │ │ ├── BuffaloZeebeService.java │ │ │ │ ├── FileTransferZeebeService.java │ │ │ │ ├── IZeebeService.java │ │ │ │ ├── PsiZeebeService.java │ │ │ │ ├── ZeebeDispatcher.java │ │ │ │ ├── ZeebeGateWay.java │ │ │ │ ├── Zeebes.java │ │ │ │ └── domain │ │ │ │ └── param │ │ │ │ └── ProcessResultParam.java │ │ │ ├── storage │ │ │ ├── OfflineTaskMapHolder.java │ │ │ └── TargetMapHolder.java │ │ │ └── web │ │ │ ├── AuthController.java │ │ │ ├── OfflineController.java │ │ │ └── OuterController.java │ │ ├── proto │ │ ├── api.proto │ │ ├── ast.proto │ │ ├── authprotocol.proto │ │ ├── external_service.proto │ │ ├── feature.proto │ │ ├── internal_service.proto │ │ ├── offline.proto │ │ ├── online.proto │ │ ├── outer.proto │ │ ├── parameter.proto │ │ ├── service.proto │ │ └── table.proto │ │ └── resources │ │ ├── application.properties │ │ ├── bootstrap.properties │ │ ├── logback-spring.xml │ │ ├── mapper │ │ ├── AuthInfoMapper.xml │ │ ├── CertMapper.xml │ │ ├── ChildrenTaskMapper.xml │ │ ├── JobTaskStubMapper.xml │ │ └── ParentTaskMapper.xml │ │ ├── mybatis │ │ └── generatorConfig.xml │ │ └── nacos-application.properties ├── FileService │ ├── Dockerfile │ ├── README.md │ ├── app │ │ ├── file_service.py │ │ └── redis_client.py │ ├── fileservice.yaml │ └── start.sh └── PSI │ ├── Dockerfile │ ├── README.md │ ├── crypto │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ ├── pyproject.toml │ └── src │ │ ├── curve.rs │ │ ├── hash.rs │ │ ├── hash_set.rs │ │ └── lib.rs │ ├── image │ └── psi.png │ ├── link_py │ ├── .bazelrc │ ├── .gitignore │ ├── BUILD.bazel │ ├── WORKSPACE │ └── link_py.cc │ ├── psi │ ├── .gitignore │ ├── interconnection_psi │ │ ├── __init__.py │ │ ├── api.py │ │ ├── base.py │ │ ├── cipher_store.py │ │ ├── config.py │ │ ├── flow.py │ │ ├── interconnection │ │ │ ├── __init__.py │ │ │ ├── common │ │ │ │ ├── __init__.py │ │ │ │ └── header.proto │ │ │ ├── handshake │ │ │ │ ├── __init__.py │ │ │ │ ├── algos │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── psi.proto │ │ │ │ ├── entry.proto │ │ │ │ └── protocol_family │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── ecc.proto │ │ │ └── runtime │ │ │ │ ├── __init__.py │ │ │ │ └── ecdh_psi.proto │ │ ├── log.py │ │ └── register_uuid.py │ └── setup.py │ ├── psi_actors.py │ └── start.sh └── 隐私计算PSI标品部署操作文档.md /README.md: -------------------------------------------------------------------------------- 1 | # 九数联邦学习整体解决方案 2 | 3 | 4 | 5 | ## 概述 6 | 7 | 数据是人工智能的基石，打破数据孤岛，实现数据共享是加速人工智能高速发展的必要条件。联邦学习，作为数据安全计算的业界前沿技术方案，在保护数据隐私的同时深度连接各个合作方，达到技术赋能、共创共赢的目标。 8 | 9 | p1 10 | 11 | p2 12 | 13 | ## 1 整体架构 14 | 15 | - 整个系统分为四个大模块 16 | - 整体调度与转发模块 17 | - 资源管理与调度模块 18 | - 数据求交模块 19 | - 训练器模块 20 | 21 | ![architecture.png](docs/imgs/architecture.png) 22 | 23 | ## 2 整体调度与转发模块 24 | 25 | - 整体控制数据求交与训练的调度 26 | - 训练器的配对工作 27 | - 高效的流量转发 28 | 29 | 30 | 31 | ## 3 资源调度与管理模块 32 | 33 | - 使用k8s屏蔽底层资源差异 34 | - 使用k8s进行资源的动态调度 35 | 36 | 37 | 38 | ## 4 数据求交 39 | 40 | - 异步分布式框架提升拼接效率 41 | 42 | 43 | 44 | ## LICNESE 45 | 46 | 9nFL使用apache2.0许可 47 | 48 | 49 | -------------------------------------------------------------------------------- /docs/imgs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/architecture.png -------------------------------------------------------------------------------- /docs/imgs/deploy_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/deploy_arch.png -------------------------------------------------------------------------------- /docs/imgs/mysql_service_svc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/mysql_service_svc.png -------------------------------------------------------------------------------- /docs/imgs/p1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/p1.png -------------------------------------------------------------------------------- /docs/imgs/p2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/p2.png -------------------------------------------------------------------------------- /docs/imgs/redis_service_svc.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/redis_service_svc.PNG -------------------------------------------------------------------------------- /docs/imgs/trainer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/trainer.png -------------------------------------------------------------------------------- /docs/imgs/联邦学习.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/docs/imgs/联邦学习.png -------------------------------------------------------------------------------- /docs/k8s_deploy.md: -------------------------------------------------------------------------------- 1 | # K8S cluster deployment 2 | -------------------------------------------------------------------------------- /docs/nfs_deploy.md: -------------------------------------------------------------------------------- 1 | # NFS服务部署 2 | ## 服务端 3 | ### NFS服务端软件包 4 | ``` 5 | yum install -y nfs-utils 6 | ``` 7 | ### 设置可访问NFS的主机并增加配置 8 | ``` 9 | yum install -y nfs-utils 10 | ``` 11 | 向其中增加配置{shared_path} {IP_Cluster}/IP(rw,sync,fsid=0) 12 | 比如 /home/nfs/ 192.168.241.0/24(rw,sync,fsid=0) 13 | ### 启动NFS服务 14 | ``` 15 | systemctl enable rpcbind.service 16 | systemctl enable nfs-server.service 17 | 18 | systemctl start rpcbind.service 19 | systemctl start nfs-server.service 20 | ``` 21 | ### 确认NFS服务生效 22 | ``` 23 | rpcinfo -p 24 | exportfs -r 25 | exportfs 26 | ``` 27 | 28 | ## 客服端 29 | ### 启动基础服务 30 | ``` 31 | systemctl enable rpcbind.service 32 | systemctl start rpcbind.service 33 | ``` 34 | ### 检查NFS服务目录 35 | ``` 36 | showmount -e {nfs_server_ip} 37 | ``` 38 | ### 挂载到客服端 39 | ``` 40 | cd /mnt && mkdir /nfs 41 | mount -t nfs {nfs_server_ip}:{nfs_path} /mnt/nfs 42 | ``` 43 | 以上步骤操作完毕后，可以进一步设置客服端到k8的Pod的挂载目录 -------------------------------------------------------------------------------- /docs/prop/psi.properties: -------------------------------------------------------------------------------- 1 | tmp-dir=/mnt/tmp 2 | send-back=true 3 | log-level=DEBUG 4 | cpu-cores=16 5 | csv-header=true 6 | -------------------------------------------------------------------------------- /docs/yamls/coordinator_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: coordinator 6 | name: coordinator 7 | spec: 8 | replicas: 2 9 | selector: 10 | matchLabels: 11 | app: coordinator 12 | template: 13 | metadata: 14 | labels: 15 | app: coordinator 16 | spec: 17 | containers: 18 | - image: $COORDINATOR_IMAGE 19 | imagePullPolicy: Always 20 | name: coordinator 21 | volumeMounts: 22 | - mountPath: /home/config/application.properties 23 | name: coordinator-conf 24 | readOnly: true 25 | subPath: application.properties 26 | - mountPath: /home/config/k8sconfig.yaml 27 | name: k8s-conf 28 | readOnly: true 29 | subPath: k8sconfig.yaml 30 | - mountPath: /k8s 31 | name: k8s 32 | - mountPath: /mnt/logs 33 | name: logs 34 | resources: 35 | limits: 36 | cpu: "4" 37 | memory: 8Gi 38 | requests: 39 | cpu: "4" 40 | memory: 8Gi 41 | restartPolicy: Always 42 | volumes: 43 | - hostPath: 44 | path: $VOLUME_LOGS 45 | type: DirectoryOrCreate 46 | name: logs 47 | - hostPath: 48 | path: $VOLUME_DATA 49 | type: DirectoryOrCreate 50 | name: k8s 51 | - configMap: 52 | defaultMode: 420 53 | name: coordinator-conf 54 | name: coordinator-conf 55 | - configMap: 56 | defaultMode: 420 57 | name: k8s-conf 58 | name: k8s-conf 59 | -------------------------------------------------------------------------------- /docs/yamls/coordinator_k8s_cofigmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: k8s-conf 5 | data: 6 | k8sconfig.yaml: |+ 7 | apiVersion: v1 8 | clusters: 9 | - cluster: 10 | server: $K8S_SERVER_URL 11 | insecure-skip-tls-verify: true 12 | name: kubernetes 13 | contexts: 14 | - context: 15 | cluster: kubernetes 16 | user: basic-authentication 17 | name: basic-authentication@kubernetes 18 | - context: 19 | cluster: kubernetes 20 | user: cert-authentication 21 | name: cert-authentication@kubernetes 22 | current-context: cert-authentication@kubernetes 23 | kind: Config 24 | preferences: {} 25 | users: 26 | - name: basic-authentication 27 | user: 28 | username: $K8S_USERNAME 29 | password: $K8S_PASSWORD 30 | - name: cert-authentication 31 | user: 32 | client-certificate-data: $K8S_CLIENT_CERTIFICATE_DATA 33 | client-key-data: $K8S_CLIENT_KEY_DATA 34 | -------------------------------------------------------------------------------- /docs/yamls/coordinator_nacos_configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: $COORDINATOR_CONF 5 | data: 6 | application.properties: | 7 | nacos.config.server-addr=$NACOS_DOMAIN 8 | nacos.config.remote-first=true 9 | nacos.config.data-id=application.properties 10 | nacos.config.namespace=$NAMESPACE 11 | nacos.config.group=APPLICATION_GROUP 12 | nacos.config.type=properties 13 | nacos.config.auto-refresh=true 14 | nacos.config.local-disk-cache-dir=/k8s/nacos 15 | -------------------------------------------------------------------------------- /docs/yamls/coordinator_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | name: coordinator 6 | name: coordinator 7 | spec: 8 | ports: 9 | - name: coo-http 10 | nodePort: $COORDINATOR_NODE_PORT_OUT 11 | port: $COORDINATOR_SERVICE_PORT_OUT 12 | protocol: TCP 13 | targetPort: $COORDINATOR_POD_PORT_OUT 14 | - name: coo-http2 15 | nodePort: $COORDINATOR_NODE_PORT_IN 16 | port: $COORDINATOR_SERVICE_PORT_IN 17 | protocol: TCP 18 | targetPort: $COORDINATOR_POD_PORT_IN 19 | selector: 20 | app: coordinator 21 | type: NodePort 22 | -------------------------------------------------------------------------------- /docs/yamls/fileservice_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: file-service 5 | labels: 6 | app: file-service 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: file-service 12 | template: 13 | metadata: 14 | labels: 15 | app: file-service 16 | spec: 17 | containers: 18 | - name: file-service 19 | image: $FILESERVICE_IMAGE 20 | ports: 21 | - containerPort: $FILESERVICE_PORT 22 | env: 23 | - name: REDIS_HOST 24 | value: $REDIS_SERVICE_IP 25 | - name: REDIS_PORT 26 | value: REDIS_SERVICE_PORT 27 | - name: REDIS_PASSWORD 28 | value: $REDIS_PASSWORD 29 | volumeMounts: 30 | - name: file-data 31 | mountPath: /mnt/data 32 | - name: logs 33 | mountPath: /mnt/logs 34 | volumes: 35 | - name: logs 36 | hostPath: 37 | path: $VOLUME_LOGS 38 | type: DirectoryOrCreate 39 | - name: file-data 40 | hostPath: 41 | path: $VOLUME_DATA 42 | type: DirectoryOrCreate 43 | -------------------------------------------------------------------------------- /docs/yamls/mysql_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: mysql 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: mysql 10 | template: 11 | metadata: 12 | labels: 13 | app: mysql 14 | spec: 15 | containers: 16 | - name: mysql 17 | image: $MYSQL_IMAGE 18 | ports: 19 | - containerPort: $MYSQL_POD_PORT 20 | resources: 21 | limits: 22 | cpu: "1" 23 | memory: "1Gi" 24 | requests: 25 | cpu: "250m" 26 | memory: "256Mi" 27 | env: 28 | - name: MYSQL_ROOT_PASSWORD 29 | valueFrom: 30 | secretKeyRef: 31 | name: $MYSQL_PASSWORD_SECRET 32 | key: password 33 | volumeMounts: 34 | - name: mysql-volume 35 | mountPath: /var/lib/mysql 36 | volumes: 37 | - hostPath: 38 | path: $MYSQL_VOLUME_PATH 39 | type: DirectoryOrCreate 40 | name: mysql-volume 41 | -------------------------------------------------------------------------------- /docs/yamls/mysql_secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: $MYSQL_PASSWORD_SECRET 5 | type: Opaque 6 | data: 7 | username: $MYSQL_USERNAME 8 | password: $MYSQL_PASSWORD 9 | -------------------------------------------------------------------------------- /docs/yamls/mysql_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: mysql 5 | spec: 6 | selector: 7 | app: mysql 8 | ports: 9 | - protocol: TCP 10 | port: $MYSQL_SERVICE_PORT 11 | targetPort: $MYSQL_POD_PORT 12 | type: ClusterIP 13 | -------------------------------------------------------------------------------- /docs/yamls/nacos_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | name: nacos 6 | name: nacos 7 | spec: 8 | progressDeadlineSeconds: 600 9 | replicas: 1 10 | revisionHistoryLimit: 10 11 | selector: 12 | matchLabels: 13 | name: nacos 14 | strategy: 15 | rollingUpdate: 16 | maxSurge: 25% 17 | maxUnavailable: 25% 18 | type: RollingUpdate 19 | template: 20 | metadata: 21 | creationTimestamp: null 22 | labels: 23 | name: nacos 24 | spec: 25 | containers: 26 | - env: 27 | - name: MODE 28 | value: standalone 29 | - name: SPRING_DATASOURCE_PLATFORM 30 | value: mysql 31 | - name: PREFER_HOST_MODE 32 | value: hostname 33 | image: $NACOS_IMAGE 34 | imagePullPolicy: Always 35 | name: nacos 36 | resources: 37 | limits: 38 | cpu: "1" 39 | memory: 2Gi 40 | requests: 41 | cpu: "1" 42 | memory: 2Gi 43 | terminationMessagePath: /dev/termination-log 44 | terminationMessagePolicy: File 45 | volumeMounts: 46 | - mountPath: /home/nacos/conf/application.properties 47 | name: nacos-cm 48 | subPath: application.properties 49 | dnsPolicy: ClusterFirst 50 | restartPolicy: Always 51 | schedulerName: default-scheduler 52 | securityContext: {} 53 | terminationGracePeriodSeconds: 30 54 | volumes: 55 | - configMap: 56 | defaultMode: 420 57 | items: 58 | - key: application.properties 59 | path: application.properties 60 | name: nacos-cm 61 | name: nacos-cm 62 | -------------------------------------------------------------------------------- /docs/yamls/nacos_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | name: nacos-svc 6 | name: nacos-svc 7 | spec: 8 | ports: 9 | - name: http 10 | nodePort: 30009 11 | port: 8848 12 | protocol: TCP 13 | targetPort: 8848 14 | - name: http1 15 | nodePort: 31397 16 | port: 9848 17 | protocol: TCP 18 | targetPort: 9848 19 | - name: http2 20 | nodePort: 30140 21 | port: 9555 22 | protocol: TCP 23 | targetPort: 9555 24 | selector: 25 | name: nacos 26 | sessionAffinity: None 27 | type: NodePort 28 | status: 29 | loadBalancer: {} 30 | -------------------------------------------------------------------------------- /docs/yamls/proxy_cert_configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: cert 5 | namespace: mpc-chk-test 6 | data: 7 | ca.crt: | 8 | -----BEGIN CERTIFICATE----- 9 | -----END CERTIFICATE----- 10 | server_cert.pem: | 11 | -----BEGIN CERTIFICATE----- 12 | -----END CERTIFICATE----- 13 | server_private.pem: | 14 | -----BEGIN PRIVATE KEY----- 15 | -----END PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /docs/yamls/proxy_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: proxy 6 | name: proxy 7 | spec: 8 | replicas: 1 9 | revisionHistoryLimit: 10 10 | selector: 11 | matchLabels: 12 | app: proxy 13 | template: 14 | metadata: 15 | labels: 16 | app: proxy 17 | spec: 18 | containers: 19 | - image: $PROXY_IMAGE 20 | imagePullPolicy: Always 21 | name: proxy 22 | resources: 23 | limits: 24 | cpu: "4" 25 | memory: 8Gi 26 | requests: 27 | cpu: "0.5" 28 | memory: 1Gi 29 | volumeMounts: 30 | - mountPath: /usr/local/openresty/nginx/conf/nginx.conf 31 | name: nginx-conf 32 | subPath: nginx.conf 33 | - mountPath: /usr/local/openresty/nginx/conf/conf.d/mpc-proxy.conf 34 | name: mpc-conf 35 | subPath: mpc-proxy.conf 36 | - mountPath: /usr/local/openresty/nginx/lua_src 37 | name: lua-src 38 | - mountPath: /cert 39 | name: cert 40 | hostNetwork: true 41 | nodeName: $NODENAME 42 | restartPolicy: Always 43 | volumes: 44 | - configMap: 45 | defaultMode: 420 46 | name: nginx-conf 47 | name: nginx-conf 48 | - configMap: 49 | defaultMode: 420 50 | name: mpc-conf 51 | name: mpc-conf 52 | - configMap: 53 | defaultMode: 420 54 | name: lua-src 55 | name: lua-src 56 | - configMap: 57 | defaultMode: 420 58 | name: cert 59 | name: cert 60 | -------------------------------------------------------------------------------- /docs/yamls/proxy_ngnix_conf_configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: $NGINX_CONF 5 | data: 6 | nginx.conf: | 7 | # 设置为nginx的CPU核数 8 | worker_processes 8; 9 | worker_rlimit_nofile 65535; 10 | error_log /usr/local/openresty/nginx/logs/error.log info; 11 | worker_shutdown_timeout 1h; 12 | 13 | events { 14 | use epoll; 15 | } 16 | 17 | http { 18 | sendfile on; 19 | aio threads; 20 | aio_write on; 21 | directio 8m; 22 | tcp_nopush on; 23 | tcp_nodelay on; 24 | 25 | keepalive_timeout 65; 26 | keepalive_requests 10000; 27 | 28 | include /usr/local/openresty/nginx/conf/mime.types; 29 | default_type application/octet-stream; 30 | lua_package_path "$prefix/lua_src/?.lua;/usr/local/openresty/lualib/resty/?.lua;;"; 31 | log_format xlog '$http_host $remote_addr $remote_port $remote_user [$time_local] $request_time ' 32 | '"$request" $status $body_bytes_sent ' 33 | '"$http_referer" "$http_user_agent" "$http_host" "$http_cookie" ' 34 | '"$upstream_response_time" $upstream_addr "$http_x_forwarded_for" $scheme ' 35 | '"$upstream_http_set_cookie"'; 36 | 37 | access_log ./logs/access.log xlog; 38 | 39 | client_max_body_size 0; 40 | client_body_buffer_size 32m; 41 | 42 | ssl_session_cache shared:SSL:10m; 43 | ssl_session_timeout 10m; 44 | 45 | resolver local=on ipv6=off; 46 | 47 | include conf.d/mpc-proxy.conf; 48 | } 49 | -------------------------------------------------------------------------------- /docs/yamls/psi.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | name: psi-worker 6 | name: psi-worker 7 | namespace: $NAMESPACE 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | name: psi-worker 13 | template: 14 | metadata: 15 | labels: 16 | name: psi-worker 17 | spec: 18 | containers: 19 | - image: $PSI_IMAGE 20 | imagePullPolicy: IfNotPresent 21 | name: psi-worker 22 | env: 23 | - name: Local_IP 24 | valueFrom: 25 | fieldRef: 26 | apiVersion: v1 27 | fieldPath: status.podIP 28 | ports: 29 | - containerPort: 22020 30 | name: http 31 | protocol: TCP 32 | resources: 33 | limits: 34 | cpu: "16" 35 | memory: 16Gi 36 | requests: 37 | cpu: "16" 38 | memory: 16Gi 39 | volumeMounts: 40 | - mountPath: /mnt/data 41 | name: data 42 | - mountPath: /mnt/logs 43 | name: logs 44 | - mountPath: /dev/shm 45 | name: dshm 46 | dnsPolicy: ClusterFirst 47 | restartPolicy: Always 48 | volumes: 49 | - name: data 50 | hostPath: 51 | path: $VOLUME_DATA 52 | type: DirectoryOrCreate 53 | - name: logs 54 | hostPath: 55 | path: $VOLUME_LOGS 56 | type: DirectoryOrCreate 57 | - name: dshm 58 | emptyDir: 59 | medium: Memory 60 | -------------------------------------------------------------------------------- /docs/yamls/redis_configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: redis 5 | data: 6 | redis.conf: | 7 | bind 0.0.0.0 8 | protected-mode no 9 | port $REDIS_POD_PORT 10 | tcp-backlog 511 11 | timeout 0 12 | tcp-keepalive 300 13 | daemonize no 14 | supervised no 15 | pidfile /var/run/redis_6379.pid 16 | loglevel notice 17 | logfile /data/redis.log 18 | databases 16 19 | save 900 1 20 | save 300 10 21 | save 60 10000 22 | stop-writes-on-bgsave-error yes 23 | rdbcompression yes 24 | rdbchecksum yes 25 | dbfilename dump.rdb 26 | dir /data 27 | slave-serve-stale-data yes 28 | slave-read-only yes 29 | repl-diskless-sync no 30 | repl-diskless-sync-delay 5 31 | repl-disable-tcp-nodelay no 32 | slave-priority 100 33 | requirepass $REDIS_PASSWORD 34 | rename-command FLUSHALL "" 35 | rename-command FLUSHDB "" 36 | rename-command KEYS "" 37 | appendonly no 38 | appendfilename "appendonly.aof" 39 | appendfsync everysec 40 | no-appendfsync-on-rewrite no 41 | auto-aof-rewrite-percentage 100 42 | auto-aof-rewrite-min-size 64mb 43 | aof-load-truncated yes 44 | lua-time-limit 5000 45 | slowlog-log-slower-than 10000 46 | slowlog-max-len 128 47 | latency-monitor-threshold 0 48 | notify-keyspace-events "" 49 | hash-max-ziplist-entries 512 50 | hash-max-ziplist-value 64 51 | list-max-ziplist-size -2 52 | list-compress-depth 0 53 | set-max-intset-entries 512 54 | zset-max-ziplist-entries 128 55 | zset-max-ziplist-value 64 56 | hll-sparse-max-bytes 3000 57 | activerehashing yes 58 | client-output-buffer-limit normal 0 0 0 59 | client-output-buffer-limit slave 256mb 64mb 60 60 | client-output-buffer-limit pubsub 32mb 8mb 60 61 | hz 10 62 | aof-rewrite-incremental-fsync yes 63 | -------------------------------------------------------------------------------- /docs/yamls/redis_deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: redis 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: redis 10 | template: 11 | metadata: 12 | labels: 13 | app: redis 14 | spec: 15 | containers: 16 | - name: redis 17 | image: $REDIS_IMAGE 18 | ports: 19 | - containerPort: $REDIS_POD_PORT 20 | resources: 21 | limits: 22 | cpu: "1" 23 | memory: "1Gi" 24 | requests: 25 | cpu: "250m" 26 | memory: "256Mi" 27 | volumeMounts: 28 | - name: redis-volume 29 | mountPath: /data 30 | - name: redis-conf 31 | mountPath: /etc/redis.conf 32 | subPath: redis.conf 33 | readOnly: true 34 | volumes: 35 | - hostPath: 36 | path: $REDIS_VOLUME_PATH 37 | type: DirectoryOrCreate 38 | name: redis-volume 39 | - name: redis-conf 40 | configMap: 41 | name: $REDIS_CONF -------------------------------------------------------------------------------- /docs/yamls/redis_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: redis 5 | spec: 6 | selector: 7 | app: redis 8 | ports: 9 | - protocol: TCP 10 | port: $REDIS_SERVICE_PORT 11 | targetPort: $REDIS_POD_PORT 12 | -------------------------------------------------------------------------------- /src/Coordinator/Makefile: -------------------------------------------------------------------------------- 1 | export JAVA_HOME=/usr 2 | 3 | jarName=mpc-coordinator-1.0-SNAPSHOT.jar 4 | ifeq ($(FLAG),jcjd) 5 | # pre1 6 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_pk_release 7 | restartScriptPath=/Users/feiguodong1/Documents/project/k8/k8shell/pre1_restart_pod.sh 8 | endif 9 | ifeq ($(FLAG),fljd) 10 | # 9nmpc-fl 11 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coodinator_jttest 12 | restartScriptPath=/Users/feiguodong1/Documents/project/k8/k8shell/fl_restart_pod.sh 13 | endif 14 | ifeq ($(FLAG),pejd) 15 | # pre 16 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coodinator_pre 17 | restartScriptPath=1.sh 18 | endif 19 | #ifeq ($(FLAG),oljd) 20 | # # mpc-online 21 | # imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_master 22 | #endif 23 | ifeq ($(FLAG),prodjd) 24 | # mpc-online 25 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_prod_v1.0.0 26 | restartScriptPath=111.sh 27 | endif 28 | ifeq ($(FLAG),tejd) 29 | imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_test_v1.0.0 30 | # imageName=yd-model-learning-cn-north-1.jcr.service.jdcloud.com/yd-model-store:coordinator_highly_available 31 | restartScriptPath=111.sh 32 | endif 33 | ifeq ($(FLAG),tjd) 34 | imageName=test.registry.net/9ntrain:coodinator_pre 35 | restartScriptPath=/Users/feiguodong1/Documents/project/k8/k8shell/tjd_restart_pod.sh 36 | endif 37 | ifeq ($(FLAG),btjd) 38 | imageName=test.registry.net/9ntrain:coodinator_busi_test 39 | restartScriptPath=tmp.sh 40 | endif 41 | ifndef imageName 42 | return -1 43 | endif 44 | ifndef restartScriptPath 45 | return -1 46 | endif 47 | 48 | restart: push 49 | sh $(restartScriptPath) coordinator 50 | 51 | push: image 52 | docker push $(imageName) 53 | 54 | image: compile 55 | - rm -rf docker/$(jarName) 56 | - docker rmi $(imageName) 57 | cp target/$(jarName) docker/$(jarName) 58 | cd docker && docker build -f Dockerfile.jre8 -t $(imageName) . 59 | 60 | compile: 61 | /Users/chenghekai1/Desktop/software/apache-maven-3.8.6/bin/mvn clean package -Dmaven.test.skip=true -------------------------------------------------------------------------------- /src/Coordinator/Makefile.win: -------------------------------------------------------------------------------- 1 | # This script is used to build an image on the WINDOWS. 2 | # You need to use 'git bash' to run it. 3 | # start cmd: sh Makefile.win 4 | 5 | # variables 6 | jarName=mpc-coordinator-1.0-SNAPSHOT.jar 7 | #imageName=test.registry.net/9ntrain:coordinator_highly_available 8 | #imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_highly_available 9 | imageName=yd-model-learning-cn-north-1.jcr.service.jdcloud.com/yd-model-store:coordinator_highly_available 10 | #imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coodinator_jttest 11 | #imageName=jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coordinator_pk_release 12 | 13 | # 1.compile 14 | # settins.xml's location need to be customized 15 | mvn -s /c/Users/Administrator/.m2/settings.xml clean package -Dmaven.test.skip=true 16 | 17 | # 2.build image 18 | rm ./docker/$jarName 19 | cp ./target/$jarName ./docker/$jarName 20 | cd docker 21 | docker rmi $imageName 22 | docker build -f Dockerfile.jre8 -t $imageName . 23 | 24 | # 3.push 25 | docker push $imageName 26 | -------------------------------------------------------------------------------- /src/Coordinator/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # reference https://docs.spring.io/spring-boot/docs/2.5.5/reference/htmlsingle/#features.container-images 2 | FROM jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base1 AS builder 3 | ARG HOME=/home 4 | WORKDIR $HOME 5 | COPY mpc-coordinator-1.0-SNAPSHOT.jar mpc-coordinator-1.0-SNAPSHOT.jar 6 | RUN cat $(java -version) 7 | RUN java -Djarmode=layertools -jar mpc-coordinator-1.0-SNAPSHOT.jar extract 8 | 9 | FROM jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base1 10 | ARG HOME=/home 11 | WORKDIR $HOME 12 | COPY --from=builder $HOME/dependencies/ ./ 13 | COPY --from=builder $HOME/spring-boot-loader/ ./ 14 | COPY --from=builder $HOME/snapshot-dependencies/ ./ 15 | COPY --from=builder $HOME/application/ ./ 16 | #COPY --from=jni_sign /usr/local/lib/libjni_sign.so /usr/lib/libjni_sign.so 17 | ADD start.sh $HOME/start.sh 18 | CMD ["sh", "start.sh"] 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/Coordinator/docker/Dockerfile_base: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye-slim 2 | # build jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base 3 | 4 | ## OpenJDK 8 5 | 6 | RUN set -eux; \ 7 | apt-get update; \ 8 | apt-get install -y --no-install-recommends \ 9 | # utilities for keeping Debian and OpenJDK CA certificates in sync 10 | ca-certificates p11-kit \ 11 | ; \ 12 | rm -rf /var/lib/apt/lists/* 13 | 14 | ENV JAVA_HOME /usr/local/openjdk-17 15 | RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] # backwards compatibility 16 | ENV PATH $JAVA_HOME/bin:$PATH 17 | 18 | ENV LANG C.UTF-8 19 | ENV JAVA_VERSION 8u345 20 | 21 | RUN set -eux; \ 22 | \ 23 | arch="$(dpkg --print-architecture)"; \ 24 | case "$arch" in \ 25 | 'amd64') \ 26 | downloadUrl='https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'; \ 27 | ;; \ 28 | *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ 29 | esac; \ 30 | \ 31 | savedAptMark="$(apt-mark showmanual)"; \ 32 | apt-get update; \ 33 | apt-get install -y --no-install-recommends \ 34 | dirmngr \ 35 | gnupg \ 36 | wget \ 37 | ; \ 38 | rm -rf /var/lib/apt/lists/*; \ 39 | \ 40 | wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \ 41 | \ 42 | mkdir -p "$JAVA_HOME"; \ 43 | tar --extract \ 44 | --file openjdk.tgz \ 45 | --directory "$JAVA_HOME" \ 46 | --strip-components 1 \ 47 | --no-same-owner \ 48 | ; \ 49 | rm openjdk.tgz*; \ 50 | \ 51 | # basic smoke test 52 | javac -version; \ 53 | java -version 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/Coordinator/docker/README.md: -------------------------------------------------------------------------------- 1 | # coordinator构建说明 2 | 构建需求： 3 | jdk版本： 17 4 | 1. 在docker目录下 执行 docker build . -f Dockerfile_base -t xxxxx 5 | 2. 替换Dockerfile中jd-mpc-cn-north-1-inner.jcr.service.jdcloud.com/mpcimage/9ntrain:coor_base1为xxxxx 6 | 3. 构建项目 执行mvn clean package 7 | 4. 将target目录下的mpc-coordinator-1.0-SNAPSHOT.jar复制到docker目录下 8 | 5. 在docker目录下执行 docker build . -f Dockerfile_base -t coordinator镜像名 9 | -------------------------------------------------------------------------------- /src/Coordinator/docker/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | java -Xmx6g -Xms6g org.springframework.boot.loader.JarLauncher -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/MpcApplication.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc; 2 | 3 | import com.alibaba.nacos.api.NacosFactory; 4 | import com.alibaba.nacos.api.PropertyKeyConst; 5 | import com.alibaba.nacos.api.annotation.NacosProperties; 6 | import com.alibaba.nacos.api.config.ConfigService; 7 | import com.alibaba.nacos.api.exception.NacosException; 8 | import com.alibaba.nacos.spring.context.annotation.EnableNacos; 9 | import com.alibaba.nacos.spring.context.annotation.config.EnableNacosConfig; 10 | import com.alibaba.nacos.spring.context.annotation.config.NacosPropertySource; 11 | import io.camunda.zeebe.spring.client.EnableZeebeClient; 12 | import org.mybatis.spring.annotation.MapperScan; 13 | import org.springframework.boot.SpringApplication; 14 | import org.springframework.boot.autoconfigure.SpringBootApplication; 15 | import org.springframework.retry.annotation.EnableRetry; 16 | import org.springframework.scheduling.annotation.EnableScheduling; 17 | 18 | import java.util.HashMap; 19 | import java.util.Properties; 20 | 21 | import static org.springframework.core.env.StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME; 22 | import static org.springframework.core.env.StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME; 23 | 24 | /** 25 | * 启动类 26 | * 27 | */ 28 | 29 | @SpringBootApplication 30 | @MapperScan("com.jd.mpc.mapper") 31 | @EnableRetry 32 | @EnableNacosConfig(globalProperties = @NacosProperties(serverAddr = "${nacos.config.server-addr}",namespace = "${nacos.config.namespace}")) 33 | @NacosPropertySource(dataId = "application.properties", groupId = "APPLICATION_GROUP", autoRefreshed = true, first = true) 34 | public class MpcApplication 35 | 36 | public static void main(String[] args) { 37 | System.setProperty("nacos.logging.default.config.enabled", "false"); 38 | SpringApplication sa = new SpringApplication(MpcApplication.class); 39 | sa.setAllowCircularReferences(Boolean.TRUE);// 加入的参数 40 | sa.run(args); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/aces/TdeService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.aces; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.stereotype.Component; 5 | 6 | 7 | /** 8 | */ 9 | @Component 10 | @Slf4j 11 | public class TdeService { 12 | 13 | /** 14 | * 加密 15 | * @param input 16 | * @return 17 | */ 18 | public String encryptString(String input){ 19 | return input; 20 | } 21 | 22 | /** 23 | * 解密 24 | * @param input 25 | * @return 26 | */ 27 | public String decryptString(String input){ 28 | return input; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/cert/CSRUtil.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.cert; 2 | 3 | import org.bouncycastle.asn1.x500.X500Name; 4 | import org.bouncycastle.operator.ContentSigner; 5 | import org.bouncycastle.operator.OperatorCreationException; 6 | import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; 7 | import org.bouncycastle.pkcs.PKCS10CertificationRequest; 8 | import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder; 9 | import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest; 10 | import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder; 11 | 12 | import cn.hutool.core.codec.Base64; 13 | import java.io.IOException; 14 | import java.security.InvalidKeyException; 15 | import java.security.NoSuchAlgorithmException; 16 | import java.security.PrivateKey; 17 | import java.security.PublicKey; 18 | import java.security.cert.CertificateException; 19 | 20 | /** 21 | * 22 | * @date 2022-03-31 18:54 23 | */ 24 | public class CSRUtil { 25 | 26 | public static final String SHA_SIGN_ALGORITHM = "SHA256withECDSA"; 27 | 28 | /** 29 | * 生成CSR请求文件 30 | * @param reqName 请求者主体信息 31 | * @param userPublicKey 用户公钥 32 | * @param userPrivateKey 用户私钥 33 | * @return 34 | * @throws OperatorCreationException 35 | * @throws CertificateException 36 | * @throws IOException 37 | */ 38 | public static String csrBuilder(X500Name reqName, PublicKey userPublicKey, PrivateKey userPrivateKey) throws OperatorCreationException, IOException { 39 | 40 | PKCS10CertificationRequestBuilder p10Builder = new JcaPKCS10CertificationRequestBuilder(reqName,userPublicKey ); 41 | JcaContentSignerBuilder csBuilder = new JcaContentSignerBuilder(SHA_SIGN_ALGORITHM); 42 | ContentSigner csrSigner = csBuilder.build(userPrivateKey); 43 | PKCS10CertificationRequest csr = p10Builder.build(csrSigner); 44 | 45 | //处理证书 ANS.I DER 编码 =》 String Base64编码 46 | String encode = Base64.encode(csr.getEncoded());; 47 | StringBuilder sb = new StringBuilder(); 48 | sb.append("-----BEGIN CERTIFICATE REQUEST-----"+"\n"); 49 | sb.append(encode); 50 | sb.append("-----END CERTIFICATE REQUEST-----"); 51 | return sb.toString(); 52 | } 53 | /** 54 | * 根据CSR字符串转换成JcaPKCS10CertificationRequest对象 55 | * @param csrStr CSR字符串 56 | * @return JcaPKCS10CertificationRequest 包含证书申请方主体信息（jcaPKCS10CertificationRequest.getSubject()）以及公钥信息（jcaPKCS10CertificationRequest.getPublicKey()） 57 | * @throws NoSuchAlgorithmException 58 | * @throws InvalidKeyException 59 | */ 60 | public static JcaPKCS10CertificationRequest parseCSRStr(String csrStr) throws NoSuchAlgorithmException, InvalidKeyException, IOException { 61 | if( !csrStr.startsWith("-----BEGIN CERTIFICATE REQUEST-----") || !csrStr.endsWith("-----END CERTIFICATE REQUEST-----")){ 62 | throw new IOException("csr 信息不合法"); 63 | } 64 | csrStr = csrStr.replace("-----BEGIN CERTIFICATE REQUEST-----"+"\n",""); 65 | csrStr = csrStr.replace("-----END CERTIFICATE REQUEST-----",""); 66 | byte[] bArray = Base64.decode(csrStr); 67 | PKCS10CertificationRequest csrRequest = new PKCS10CertificationRequest(bArray); 68 | return new JcaPKCS10CertificationRequest(csrRequest); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/cert/KeyPairPojo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.cert; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * 11 | * @date 2022-03-31 17:47 12 | * 生成公私钥的模数，用于生成根证书，需要通过ACES加密保存， 13 | */ 14 | @Data 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | public class KeyPairPojo implements Serializable { 18 | private String publicExponent; 19 | private String privateExponent; 20 | private String modulus; 21 | } 22 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/cert/SignUtil.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.cert; 2 | 3 | import cn.hutool.core.codec.Base64; 4 | import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPrivateKey; 5 | import org.bouncycastle.jcajce.provider.asymmetric.ec.BCECPublicKey; 6 | import org.bouncycastle.jce.provider.BouncyCastleProvider; 7 | 8 | import java.io.IOException; 9 | import java.nio.charset.StandardCharsets; 10 | import java.security.*; 11 | 12 | public class SignUtil { 13 | 14 | private static final String SIGN_ALGORITHM = "SHA256withECDSA"; 15 | 16 | /** 17 | * 签名 18 | * @param priKeyStr 私钥 19 | * @param oriData 原始数据 20 | * @return base64编码后的结果 21 | * @throws IOException 22 | * @throws NoSuchAlgorithmException 23 | * @throws InvalidKeyException 24 | * @throws SignatureException 25 | */ 26 | public static String sign(String priKeyStr,String oriData) throws IOException, NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException { 27 | BCECPrivateKey privateKey = KeyGenerator.getPrivateKey(priKeyStr); 28 | return sign(privateKey,oriData); 29 | } 30 | 31 | public static String sign(PrivateKey privateKey,String oriData) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException { 32 | Signature signature = Signature.getInstance(SIGN_ALGORITHM,BouncyCastleProvider.PROVIDER_NAME); 33 | signature.initSign(privateKey); 34 | signature.update(oriData.getBytes(StandardCharsets.UTF_8)); 35 | return Base64.encode(signature.sign()); 36 | } 37 | 38 | /** 39 | * 对base64编码的数据进行验证 40 | * @param pubKeyStr 公钥 41 | * @param oriData 原始数据 42 | * @param signStr base64编码的签名字符串 43 | * @return 44 | */ 45 | public static boolean verify(String pubKeyStr,String oriData,String signStr) throws IOException, NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException { 46 | BCECPublicKey publicKey = KeyGenerator.getPublicKey(pubKeyStr); 47 | return verify(publicKey,oriData,signStr); 48 | } 49 | 50 | public static boolean verify(PublicKey publicKey,String oriData,String signStr) throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, NoSuchProviderException { 51 | Signature signature = Signature.getInstance(SIGN_ALGORITHM,BouncyCastleProvider.PROVIDER_NAME); 52 | signature.initVerify(publicKey); 53 | signature.update(oriData.getBytes(StandardCharsets.UTF_8)); 54 | return signature.verify(Base64.decode(signStr)); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/advice/ByteArrayServletInputStream.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.advice; 2 | 3 | import jakarta.servlet.ReadListener; 4 | import jakarta.servlet.ServletInputStream; 5 | 6 | import java.io.ByteArrayInputStream; 7 | import java.io.IOException; 8 | 9 | /** 10 | * @Description: 输入流 11 | */ 12 | 13 | public class ByteArrayServletInputStream extends ServletInputStream { 14 | 15 | private ByteArrayInputStream byteArrayInputStream; 16 | 17 | public ByteArrayServletInputStream(ByteArrayInputStream byteArrayInputStream) { 18 | this.byteArrayInputStream = byteArrayInputStream; 19 | } 20 | 21 | @Override 22 | public int read() throws IOException { 23 | return byteArrayInputStream.read(); 24 | } 25 | 26 | @Override 27 | public boolean isFinished() { 28 | return false; 29 | } 30 | 31 | @Override 32 | public boolean isReady() { 33 | return false; 34 | } 35 | 36 | @Override 37 | public void setReadListener(ReadListener readListener) { 38 | 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/advice/GlobalExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.advice; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.validation.BindException; 6 | import org.springframework.validation.ObjectError; 7 | import org.springframework.web.bind.annotation.ExceptionHandler; 8 | import org.springframework.web.bind.annotation.RestControllerAdvice; 9 | import org.springframework.web.method.HandlerMethod; 10 | 11 | import com.jd.mpc.common.response.CommonException; 12 | import com.jd.mpc.common.response.CommonResponse; 13 | import com.jd.mpc.common.response.ErrorStatus; 14 | 15 | import lombok.extern.slf4j.Slf4j; 16 | 17 | /** 18 | * 全局异常处理，该类处理时候注意考虑某些场景下新老版本兼容。 19 | */ 20 | @Slf4j 21 | @RestControllerAdvice 22 | public class GlobalExceptionHandler { 23 | /** 24 | * 系统异常文案。 25 | */ 26 | private static final String SYSTEM_ERROR_TEXT = "请求错误，请稍后重试"; 27 | 28 | /** 29 | * 参数校验失败默认文案。 30 | */ 31 | private static final String PARAMETER_VALIDATE_ERROR = "参数校验失败，请检查之后重试"; 32 | 33 | /** 34 | * 处理业务中触发的异常。 35 | * 36 | * @param e 37 | * @return 38 | */ 39 | @ExceptionHandler(value = CommonException.class) 40 | public Object handleException(CommonException e, HandlerMethod method) { 41 | log.error("处理业务异常", e); 42 | 43 | return buildResponse(method, e.getStatus(), e.getMessage()); 44 | } 45 | 46 | /** 47 | * 处理通用异常。 48 | * 49 | * @return 50 | */ 51 | @ExceptionHandler(value = BindException.class) 52 | public Object handleException(BindException bindException, HandlerMethod method) { 53 | log.error("处理BindException异常", bindException); 54 | List allErrors = bindException.getAllErrors(); 55 | for (ObjectError error : allErrors) { 56 | return buildResponse(method, ErrorStatus.PARAMETER_ERROR, error.getDefaultMessage()); 57 | } 58 | return buildResponse(method, ErrorStatus.BUSINESS_ERROR, SYSTEM_ERROR_TEXT); 59 | } 60 | 61 | /** 62 | * 处理通用异常。 63 | * 64 | * @return 65 | */ 66 | @ExceptionHandler(value = Exception.class) 67 | public Object handleException(Exception e, HandlerMethod method) { 68 | log.error("处理通用异常", e); 69 | 70 | return buildResponse(method, ErrorStatus.BUSINESS_ERROR, e.getMessage()); 71 | } 72 | 73 | /** 74 | * 构建异常返回的数据格式。 75 | * 76 | * @param method 77 | * @param status 78 | * @param message 79 | * @return 80 | */ 81 | private CommonResponse buildResponse(HandlerMethod method, Integer status, String message) { 82 | CommonResponse response = new CommonResponse<>(); 83 | response.setError(status, message); 84 | return response; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/advice/TeeServletOutputStream.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.advice; 2 | 3 | 4 | import jakarta.servlet.ServletOutputStream; 5 | import jakarta.servlet.WriteListener; 6 | import org.apache.commons.io.output.TeeOutputStream; 7 | 8 | import java.io.IOException; 9 | import java.io.OutputStream; 10 | 11 | /** 12 | * @Description: 输出流 13 | * 14 | * @Date: 2021/11/29 15 | */ 16 | public class TeeServletOutputStream extends ServletOutputStream { 17 | 18 | private final TeeOutputStream teeOutputStream; 19 | 20 | public TeeServletOutputStream(OutputStream one, OutputStream two) { 21 | this.teeOutputStream = new TeeOutputStream(one, two); 22 | } 23 | 24 | @Override 25 | public void write(byte[] b) throws IOException { 26 | this.teeOutputStream.write(b); 27 | } 28 | 29 | @Override 30 | public void write(byte[] b, int off, int len) throws IOException { 31 | this.teeOutputStream.write(b, off, len); 32 | } 33 | 34 | @Override 35 | public void write(int b) throws IOException { 36 | this.teeOutputStream.write(b); 37 | } 38 | 39 | @Override 40 | public void flush() throws IOException { 41 | super.flush(); 42 | this.teeOutputStream.flush(); 43 | } 44 | 45 | @Override 46 | public void close() throws IOException { 47 | super.close(); 48 | this.teeOutputStream.close(); 49 | } 50 | 51 | @Override 52 | public boolean isReady() { 53 | return false; 54 | } 55 | 56 | @Override 57 | public void setWriteListener(WriteListener writeListener) { 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/advice/ThreadPoolConfig.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.advice; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.scheduling.annotation.EnableAsync; 6 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; 7 | 8 | import java.util.concurrent.ThreadPoolExecutor; 9 | 10 | /** 11 | * @Description: 线程池配置 12 | * 13 | * @Date: 2022/2/21 14 | */ 15 | @Configuration 16 | @EnableAsync 17 | public class ThreadPoolConfig { 18 | 19 | private static final int corePoolSize = 20; // 核心线程数（默认线程数）线程池创建时候初始化的线程数 20 | private static final int maxPoolSize = 20; // 最大线程数 线程池最大的线程数，只有在缓冲队列满了之后才会申请超过核心线程数的线程 21 | private static final int keepAliveTime = 20; // 允许线程空闲时间（单位：默认为秒）当超过了核心线程之外的线程在空闲时间到达之后会被销毁 22 | private static final int queueCapacity = 200; // 缓冲队列数 用来缓冲执行任务的队列 23 | private static final String threadNamePrefix = "async-"; // 线程池名前缀 方便我们定位处理任务所在的线程池 24 | 25 | @Bean("threadPoolTaskExecutor") // bean的名称，默认为首字母小写的方法名 26 | public ThreadPoolTaskExecutor taskExecutor(){ 27 | ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); 28 | executor.setCorePoolSize(corePoolSize); 29 | executor.setMaxPoolSize(maxPoolSize); 30 | executor.setQueueCapacity(queueCapacity); 31 | executor.setKeepAliveSeconds(keepAliveTime); 32 | executor.setThreadNamePrefix(threadNamePrefix); 33 | 34 | // 线程池对拒绝任务的处理策略 采用了CallerRunsPolicy策略，当线程池没有处理能力的时候，该策略会直接在 execute 方法的调用线程中运行被拒绝的任务；如果执行程序已关闭，则会丢弃该任务 35 | executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); 36 | // 初始化 37 | executor.initialize(); 38 | return executor; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/config/NacosListenerConfig.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.config; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.alibaba.nacos.api.annotation.NacosInjected; 5 | import com.alibaba.nacos.api.config.ConfigService; 6 | import com.alibaba.nacos.api.config.annotation.NacosConfigListener; 7 | import com.alibaba.nacos.api.config.convert.NacosConfigConverter; 8 | import com.alibaba.nacos.api.exception.NacosException; 9 | import com.alibaba.nacos.spring.context.event.config.TimeoutNacosConfigListener; 10 | import com.google.common.collect.Maps; 11 | import com.jd.mpc.common.constant.CommonConstant; 12 | import com.jd.mpc.common.enums.TaskTypeEnum; 13 | import com.jd.mpc.common.response.CommonException; 14 | import lombok.extern.slf4j.Slf4j; 15 | import org.springframework.context.annotation.Bean; 16 | import org.springframework.context.annotation.Configuration; 17 | import org.springframework.util.ReflectionUtils; 18 | 19 | import java.io.IOException; 20 | import java.io.StringReader; 21 | import java.util.Map; 22 | import java.util.Properties; 23 | 24 | /** 25 | * @Description: TODO 26 | * 27 | * @Date: 2022/12/23 28 | */ 29 | @Slf4j 30 | @Configuration 31 | public class NacosListenerConfig { 32 | 33 | @NacosInjected 34 | private ConfigService configService; 35 | 36 | @Bean 37 | public Map functorGroup() throws NacosException, IOException { 38 | Map funtorMap = Maps.newConcurrentMap(); 39 | try { 40 | for (TaskTypeEnum taskType : TaskTypeEnum.values()) { 41 | String dataId = taskType.getName() + ".properties"; 42 | long timeout = 5000; 43 | String config = configService.getConfig(dataId, CommonConstant.FUNCTOR_GROUP, timeout); 44 | if (config == null) { 45 | log.warn("nacos default config of " + taskType.getName() + " is null!"); 46 | continue; 47 | } 48 | Properties properties = new Properties(); 49 | properties.load(new StringReader(config)); 50 | funtorMap.put(taskType, properties); 51 | configService.addListener(dataId, CommonConstant.FUNCTOR_GROUP, new TimeoutNacosConfigListener(dataId, CommonConstant.FUNCTOR_GROUP, timeout) { 52 | @Override 53 | protected void onReceived(String config) { 54 | try { 55 | Properties properties = new Properties(); 56 | properties.load(new StringReader(config)); 57 | funtorMap.put(taskType, properties); 58 | } catch (Exception e) { 59 | } 60 | } 61 | }); 62 | } 63 | }catch (Exception e){ 64 | 65 | } 66 | return funtorMap; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/constant/CommonConstant.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.constant; 2 | 3 | /** 4 | * @Description: 常量 5 | * 6 | * @Date: 2022/3/22 7 | */ 8 | public interface CommonConstant { 9 | 10 | /** coordinator服务名称 */ 11 | String COORDINATOR_NAME = "coordinator"; 12 | 13 | String BIN_BASH = "/bin/bash"; 14 | 15 | String NN_MPC_POD_NAME_STR = "-mpc-nn-worker-"; 16 | 17 | String K8S_DATA_VOLUME = "data"; 18 | 19 | String K8S_LOG_VOLUME = "logs"; 20 | 21 | /** 22 | * k8s's yaml in nacos 23 | */ 24 | String K8S_GROUP = "K8S_GROUP"; 25 | 26 | /** 27 | * functor's default properties in nacos 28 | */ 29 | String FUNCTOR_GROUP = "FUNCTOR_GROUP"; 30 | /** 31 | * DEFAULT_GROUP 32 | */ 33 | String DEFAULT_GROUP = "DEFAULT_GROUP"; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/constant/DeploymentPathConstant.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.constant; 2 | 3 | /** 4 | * @Description: 部署文件路径常量 5 | * 6 | * @Date: 2022/2/10 7 | */ 8 | public interface DeploymentPathConstant { 9 | String INTERSECTION = "intersection.yaml"; 10 | 11 | String PSI = "psi.yaml"; 12 | 13 | String FEATURE = "feature.yaml"; 14 | 15 | String FEATURE_FL = "feature-fl.yaml"; 16 | 17 | String TRAIN = "train.yaml"; 18 | 19 | String JXZ_MPC = "jxz-mpc.yaml"; 20 | 21 | String JXZ_LOCAL = "jxz-local.yaml"; 22 | 23 | String UNICOM = "unicom.yaml"; 24 | 25 | String HRZ_FL_BASE = "hrz-fl-base.yaml"; 26 | 27 | String TRAIN_BASE = "train-base.yaml"; 28 | 29 | String TRAIN_BASE_VIF = "train-base-vif.yaml"; 30 | 31 | String PREDICT_VERTICAL = "predict-vertical.yaml"; 32 | 33 | String PREDICT_HORIZONTAL = "predict-horizontal.yaml"; 34 | 35 | String PREDICT_NN = "predict-nn.yaml"; 36 | 37 | String PREDICT_EVAL = "predict-estimate.yaml"; 38 | 39 | /** 40 | * XGB树模型 41 | */ 42 | String TREE_TRAIN_XGB = "tree-train-base.yaml"; 43 | 44 | /** 45 | * 增加随机森林树模型 46 | */ 47 | String TREE_TRAIN_RF = "tree-train-rf.yaml"; 48 | 49 | String HRZ_FL_PREDICT_BASE = "hrz-fl-predict-base.yaml"; 50 | 51 | String CUT_DATAFRAME_BASE = "feature-cut-dataframe-base.yaml"; 52 | 53 | String XGBOOST_TRAIN_BASE = "xgboost-train-base.yaml"; 54 | 55 | String XGBBOOST_TRAIN_BASE_WITH_RAYGBO = "ray_cluster_xgb.yaml"; 56 | 57 | String RAY_BASE = "ray_cluster.yaml"; 58 | 59 | String NN_DC = "nn-dc.yaml"; 60 | 61 | String NN_MPC = "mpc-nn-worker.yaml"; 62 | 63 | String CODE_MPC = "mpc-code-worker.yaml"; 64 | 65 | String NN_TRAINER = "nn-trainer.yaml"; 66 | 67 | String STABILITY_INDEX = "stability-index.yaml"; 68 | 69 | String PLUMBER_BASE = "plumber.yaml"; 70 | 71 | String NEW_PSI = "new-psi.yaml"; 72 | 73 | String LINEAR_EVALUATE = "linear-evaluate.yaml"; 74 | 75 | String NN_EVALUATE = "nn-evaluate.yaml"; 76 | 77 | String SHAPLEY_VALUE_EVALUATE = "shapley-value-evaluate.yaml"; 78 | 79 | String SCORE_CARD = "score-card.yaml"; 80 | 81 | String SPEARMANMPC = "spearman-mpc.yaml"; 82 | 83 | String JTPSI_MASTER = "jingteng-master-psi.yaml"; 84 | 85 | String JTPSI_WORKER = "jingteng-worker-psi.yaml"; 86 | 87 | String ETL = "etl.yaml"; 88 | 89 | String FILE_SERVICE = "file-service.yaml"; 90 | 91 | String LOCAL_WORKER = "local-worker.yaml"; 92 | 93 | String FILE_TRANSFER = "file-transfer.yaml"; 94 | 95 | String BUFFALO_WORKER = "buffalo-worker.yaml"; 96 | 97 | String BDP_DECRYPT = "bdp-decrypt.yaml"; 98 | 99 | String BUSI_DOWNLOAD = "busi-download.yaml"; 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/constant/OfflineTaskMap.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.constant; 2 | 3 | import com.jd.mpc.domain.offline.commons.SubTask; 4 | 5 | import java.util.List; 6 | import java.util.concurrent.ConcurrentHashMap; 7 | 8 | public class OfflineTaskMap extends ConcurrentHashMap> { 9 | 10 | private static volatile OfflineTaskMap taskMap = null; 11 | 12 | 13 | private OfflineTaskMap() { 14 | } 15 | 16 | public static OfflineTaskMap getInstance() { 17 | //第一次校验GlobalLock是否为空 18 | if (taskMap == null) { 19 | synchronized (OfflineTaskMap.class) { 20 | //第二次校验GlobalLock是否为空 21 | if (taskMap == null) { 22 | taskMap = new OfflineTaskMap(); 23 | } 24 | } 25 | } 26 | return taskMap; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/constant/ServicePathConstant.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.constant; 2 | 3 | /** 4 | * @Description: service配置文件 5 | * 6 | */ 7 | public interface ServicePathConstant { 8 | 9 | String MPC_NN_SERVICE = "/k8s/mpc-nn-worker-service.yaml"; 10 | } 11 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/constant/TargetMap.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.constant; 2 | 3 | import java.util.Set; 4 | import java.util.concurrent.ConcurrentHashMap; 5 | 6 | /**s 7 | * 8 | * @date 2021/11/24 6:32 下午 9 | * 该类已由TargetMapHolder代替update by yezhenyue on 20220411 10 | * @see com.jd.mpc.storage.TargetMapHolder 11 | * 12 | */ 13 | public class TargetMap extends ConcurrentHashMap> { 14 | 15 | private static volatile TargetMap targetMap = null; 16 | 17 | 18 | private TargetMap() { 19 | } 20 | 21 | public static TargetMap getInstance() { 22 | //第一次校验GlobalLock是否为空 23 | if (targetMap == null) { 24 | synchronized (TargetMap.class) { 25 | //第二次校验GlobalLock是否为空 26 | if (targetMap == null) { 27 | targetMap = new TargetMap(); 28 | } 29 | } 30 | } 31 | return targetMap; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/IsDeletedEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | import lombok.Getter; 4 | 5 | /** 6 | * 任务类型枚举 7 | * 8 | * 9 | * @date 2021/9/26 8:48 下午 10 | */ 11 | @Getter 12 | public enum IsDeletedEnum { 13 | 14 | /** 15 | * 未删除 16 | */ 17 | FALSE((byte) 0), 18 | 19 | /** 20 | * 已删除 21 | */ 22 | TRUE((byte) 1); 23 | 24 | 25 | private final Byte status; 26 | 27 | IsDeletedEnum(byte status) { 28 | this.status = status; 29 | } 30 | 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/IsLocalEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | import lombok.Getter; 4 | 5 | /** 6 | * 7 | * @date 2022-04-02 11:53 8 | */ 9 | @Getter 10 | public enum IsLocalEnum { 11 | /** 12 | * 非本地任务 13 | */ 14 | FALSE((byte) 0), 15 | 16 | /** 17 | * 本地任务 18 | */ 19 | TRUE((byte) 1); 20 | 21 | 22 | private final Byte status; 23 | 24 | IsLocalEnum(byte status) { 25 | this.status = status; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/IsRootEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | import lombok.Getter; 4 | 5 | /** 6 | * 7 | * @date 2022-04-02 11:51 8 | */ 9 | @Getter 10 | public enum IsRootEnum { 11 | /** 12 | * 非根证书 13 | */ 14 | FALSE((byte) 0), 15 | 16 | /** 17 | * 根证书 18 | */ 19 | TRUE((byte) 1); 20 | 21 | 22 | private final Byte status; 23 | 24 | IsRootEnum(byte status) { 25 | this.status = status; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/K8sResourceTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | import lombok.Getter; 4 | 5 | import java.util.Objects; 6 | 7 | /** 8 | * k8s资源类型枚举 9 | * 10 | * 11 | * @date 2021/9/26 8:48 下午 12 | */ 13 | @Getter 14 | public enum K8sResourceTypeEnum { 15 | 16 | /** 17 | * deployment 18 | */ 19 | DEPLOYMENT("deployment"), 20 | 21 | /** 22 | * crd 23 | */ 24 | CRD("crd"); 25 | 26 | 27 | 28 | private final String name; 29 | 30 | K8sResourceTypeEnum(String name) { 31 | this.name = name; 32 | } 33 | 34 | public static K8sResourceTypeEnum getByValue(String name) { 35 | for (K8sResourceTypeEnum taskTypeEnum : values()) { 36 | if (Objects.equals(taskTypeEnum.name, name)) { 37 | return taskTypeEnum; 38 | } 39 | } 40 | return K8sResourceTypeEnum.DEPLOYMENT; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/LogLevelEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | public enum LogLevelEnum { 4 | DEBUG("debug"), 5 | INFO("info"), 6 | WARNING("warning"), 7 | ERROR("error"), 8 | // ALL("all"), 9 | ; 10 | 11 | private String desc; 12 | 13 | LogLevelEnum(String desc) { 14 | this.desc = desc; 15 | } 16 | 17 | public String getDesc() { 18 | return desc; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/OperatorStatusEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | import com.fasterxml.jackson.annotation.JsonFormat; 4 | 5 | /** 6 | * @Description: 算子状态 7 | * 8 | * @Date: 2022/4/1 9 | */ 10 | @JsonFormat(shape = JsonFormat.Shape.OBJECT) 11 | public enum OperatorStatusEnum { 12 | INTERNAL_ERROR(500, "内部错误"), 13 | LOSS_LARGE(501, "学习率过大,模型不再收敛"), 14 | ORI_DATA_ERROR(501, "原始数据错误"), 15 | PENDING_ERROR(999, "资源不足"),; 16 | 17 | private final Integer code; 18 | 19 | private final String desc; 20 | 21 | OperatorStatusEnum(Integer code, String desc) { 22 | this.code = code; 23 | this.desc = desc; 24 | } 25 | 26 | public Integer getCode() { 27 | return code; 28 | } 29 | 30 | public String getDesc() { 31 | return desc; 32 | } 33 | 34 | public static OperatorStatusEnum getByCode(Integer code) { 35 | for (OperatorStatusEnum value : OperatorStatusEnum.values()) { 36 | if (value.code.equals(code)) { 37 | return value; 38 | } 39 | } 40 | return null; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/StoreTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | /** 4 | * @Description: 存储类型 5 | * 6 | * @Date: 2022/8/10 7 | */ 8 | public enum StoreTypeEnum { 9 | HDFS("HDFS"), 10 | CFS("CFS"), 11 | ; 12 | private final String desc; 13 | 14 | StoreTypeEnum(String desc) { 15 | this.desc = desc; 16 | } 17 | 18 | public String getDesc() { 19 | return desc; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/enums/TaskStatusEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.enums; 2 | 3 | import lombok.Getter; 4 | 5 | /** 6 | * 任务状态枚举 7 | * 8 | * 9 | * @date 2021/9/26 8:48 下午 10 | */ 11 | @Getter 12 | public enum TaskStatusEnum { 13 | 14 | /** 15 | * 新建 16 | */ 17 | NEW(0), 18 | 19 | /** 20 | * 运行中 21 | */ 22 | RUNNING(1), 23 | 24 | /** 25 | * 运行结束 26 | */ 27 | COMPLETED(2), 28 | 29 | /** 30 | * 运行异常 31 | */ 32 | ERROR(3), 33 | 34 | /** 35 | * 运行停止 36 | */ 37 | STOPPED(4); 38 | 39 | private final Integer status; 40 | 41 | TaskStatusEnum(int status) { 42 | this.status = status; 43 | } 44 | 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/response/CommonException.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.response; 2 | 3 | 4 | public class CommonException extends RuntimeException { 5 | private final Integer status; 6 | private Object[] params; 7 | 8 | public CommonException(String msg) { 9 | super(msg); 10 | this.status = ErrorStatus.BUSINESS_ERROR; 11 | this.params = null; 12 | } 13 | 14 | public CommonException(Integer code, String msg) { 15 | super(msg); 16 | this.status = code; 17 | this.params = null; 18 | } 19 | 20 | public CommonException(Integer code, String msg, Exception e) { 21 | super(msg, e); 22 | this.status = code; 23 | this.params = null; 24 | } 25 | 26 | public CommonException(Integer code, String msg, Throwable t) { 27 | super(msg, t); 28 | this.status = code; 29 | this.params = null; 30 | } 31 | 32 | public Integer getStatus() { 33 | return this.status; 34 | } 35 | 36 | public Object[] getParams() { 37 | return this.params; 38 | } 39 | 40 | public void setParams(Object[] params) { 41 | this.params = params; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/response/CommonResponse.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.response; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 5 | import com.fasterxml.jackson.annotation.JsonInclude; 6 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 7 | 8 | import java.io.Serializable; 9 | 10 | @JsonInclude(Include.NON_NULL) 11 | @JsonIgnoreProperties(ignoreUnknown = true) 12 | public class CommonResponse implements Serializable { 13 | private Integer status; 14 | private String errMsg; 15 | private T result; 16 | /** 17 | * 重定向url 18 | */ 19 | private String url; 20 | 21 | public CommonResponse() { 22 | this.status = ErrorStatus.SUCCESS; 23 | } 24 | 25 | public Integer getStatus() { 26 | return this.status; 27 | } 28 | 29 | public String getErrMsg() { 30 | return this.errMsg; 31 | } 32 | 33 | public T getResult() { 34 | return this.result; 35 | } 36 | 37 | public void setResult(T result) { 38 | this.result = result; 39 | } 40 | 41 | public String getUrl() { 42 | return url; 43 | } 44 | 45 | public void setUrl(String url) { 46 | this.url = url; 47 | } 48 | 49 | public void setError(Integer status, String errMsg) { 50 | this.status = status; 51 | this.errMsg = errMsg; 52 | } 53 | 54 | /** 55 | * @param t 56 | * @param 57 | * @return 58 | */ 59 | public static CommonResponse ok(T t) { 60 | CommonResponse r = ok(); 61 | r.setResult(t); 62 | return r; 63 | } 64 | 65 | public static CommonResponse ok() { 66 | CommonResponse r = new CommonResponse<>(); 67 | r.status = ErrorStatus.SUCCESS; 68 | return r; 69 | } 70 | 71 | public static CommonResponse fail(String msg) { 72 | CommonResponse r = new CommonResponse<>(); 73 | r.status = ErrorStatus.BUSINESS_ERROR; 74 | r.errMsg = msg; 75 | return r; 76 | } 77 | 78 | public static CommonResponse fail(Integer code, String msg) { 79 | CommonResponse r = new CommonResponse<>(); 80 | r.status = code; 81 | r.errMsg = msg; 82 | return r; 83 | } 84 | 85 | public static void convertThrowException(CommonResponse commonResponse){ 86 | if (!ErrorStatus.SUCCESS.equals(commonResponse.getStatus())){ 87 | throw new CommonException(commonResponse.getStatus(), commonResponse.getErrMsg()); 88 | } 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/response/ErrorStatus.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.response; 2 | 3 | public class ErrorStatus { 4 | public static final Integer SUCCESS = 0; 5 | public static final Integer SERVER_INTERNAL_ERROR = 100001; 6 | public static final Integer BUSINESS_ERROR = 100002; 7 | public static final Integer AUTH_ERROR = 100003; 8 | public static final Integer PARAMETER_EMPTY = 100004; 9 | public static final Integer STATUS_ERROR = 100005; 10 | public static final Integer DATA_NOT_EXIST = 100006; 11 | public static final Integer DATA_EXIST = 100007; 12 | public static final Integer HTTP_REQUEST_ERROR = 100008; 13 | public static final Integer PARAMETER_ERROR = 100009; 14 | } 15 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/response/ProcessCommonResponse.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.response; 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 5 | import com.fasterxml.jackson.annotation.JsonInclude; 6 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 7 | 8 | import java.io.Serializable; 9 | 10 | @JsonInclude(Include.NON_NULL) 11 | @JsonIgnoreProperties(ignoreUnknown = true) 12 | public class ProcessCommonResponse implements Serializable { 13 | private Integer code; 14 | private String errMsg; 15 | private T result; 16 | /** 17 | * 重定向url 18 | */ 19 | private String url; 20 | 21 | public ProcessCommonResponse() { 22 | this.code = ErrorStatus.SUCCESS; 23 | } 24 | 25 | public String getErrMsg() { 26 | return this.errMsg; 27 | } 28 | 29 | public T getResult() { 30 | return this.result; 31 | } 32 | 33 | public void setResult(T result) { 34 | this.result = result; 35 | } 36 | 37 | public String getUrl() { 38 | return url; 39 | } 40 | 41 | public void setUrl(String url) { 42 | this.url = url; 43 | } 44 | 45 | public void setError(Integer status, String errMsg) { 46 | this.code = status; 47 | this.errMsg = errMsg; 48 | } 49 | 50 | /** 51 | * @param t 52 | * @param 53 | * @return 54 | */ 55 | public static ProcessCommonResponse ok(T t) { 56 | ProcessCommonResponse r = ok(); 57 | r.setResult(t); 58 | return r; 59 | } 60 | 61 | public Integer getCode() { 62 | return code; 63 | } 64 | 65 | public static ProcessCommonResponse ok() { 66 | ProcessCommonResponse r = new ProcessCommonResponse<>(); 67 | r.code = ErrorStatus.SUCCESS; 68 | return r; 69 | } 70 | 71 | public static ProcessCommonResponse fail(String msg) { 72 | ProcessCommonResponse r = new ProcessCommonResponse<>(); 73 | r.code = ErrorStatus.BUSINESS_ERROR; 74 | r.errMsg = msg; 75 | return r; 76 | } 77 | 78 | public static ProcessCommonResponse fail(Integer code, String msg) { 79 | ProcessCommonResponse r = new ProcessCommonResponse<>(); 80 | r.code = code; 81 | r.errMsg = msg; 82 | return r; 83 | } 84 | 85 | public static void convertThrowException(ProcessCommonResponse commonResponse){ 86 | if (!ErrorStatus.SUCCESS.equals(commonResponse.getCode())){ 87 | throw new CommonException(commonResponse.getCode(), commonResponse.getErrMsg()); 88 | } 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/util/JNISigner.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.util; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | 5 | import java.util.Base64; 6 | 7 | @Slf4j 8 | public class JNISigner { 9 | // This declares that the static `hello` method will be provided 10 | // a native library. 11 | public static native byte[] sign(byte[] privateKey, byte[] message); 12 | 13 | public static native byte[] newPrivateKey(); 14 | 15 | public static native byte[] publicKey(byte[] privateKey); 16 | 17 | static { 18 | // This actually loads the shared object that we'll be creating. 19 | // The actual location of the .so or .dll may differ based on your 20 | // platform. 21 | System.loadLibrary("jni_sign"); 22 | } 23 | 24 | // The rest is just regular ol' Java! 25 | public static void gen() { 26 | byte[] msg = "dfasfadsad".getBytes(); 27 | for (int i = 0; i < 1; i++) { 28 | // System.out.println(i); 29 | byte[] priv = newPrivateKey(); 30 | log.info("java-private: " + 31 | Base64.getEncoder().encodeToString(priv)); 32 | byte[] pub = publicKey(priv); 33 | log.info("java-public: " + 34 | Base64.getEncoder().encodeToString(pub)); 35 | log.info("java-message: " + 36 | Base64.getEncoder().encodeToString(msg)); 37 | byte[] sig = sign(priv, msg); 38 | log.info(Base64.getEncoder().encodeToString(sig)); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/util/MapTypeAdapter.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.util; 2 | 3 | import com.google.gson.TypeAdapter; 4 | import com.google.gson.internal.LinkedTreeMap; 5 | import com.google.gson.stream.JsonReader; 6 | import com.google.gson.stream.JsonToken; 7 | import com.google.gson.stream.JsonWriter; 8 | 9 | import java.io.IOException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | /** 15 | * @Date 2022/3/16 16 | * @Description 17 | */ 18 | public class MapTypeAdapter extends TypeAdapter { 19 | 20 | 21 | @Override 22 | public void write(JsonWriter out, Object value) throws IOException { 23 | // 序列化无需实现 24 | } 25 | 26 | @Override 27 | public Object read(JsonReader in) throws IOException { 28 | JsonToken token = in.peek(); 29 | switch (token) { 30 | case BEGIN_ARRAY: 31 | List list = new ArrayList(); 32 | in.beginArray(); 33 | while (in.hasNext()) { 34 | list.add(read(in)); 35 | } 36 | in.endArray(); 37 | return list; 38 | 39 | case BEGIN_OBJECT: 40 | Map map = new LinkedTreeMap(); 41 | in.beginObject(); 42 | while (in.hasNext()) { 43 | map.put(in.nextName(), read(in)); 44 | } 45 | in.endObject(); 46 | return map; 47 | 48 | case STRING: 49 | return in.nextString(); 50 | 51 | case NUMBER: 52 | /** 53 | * 改写数字的处理逻辑，将数字值分为整型与浮点型。 54 | */ 55 | String numberStr = in.nextString(); 56 | if (numberStr.contains(".") || numberStr.contains("e") 57 | || numberStr.contains("E")) { 58 | return Double.parseDouble(numberStr); 59 | } 60 | if (Long.parseLong(numberStr) <= Integer.MAX_VALUE) { 61 | return Integer.parseInt(numberStr); 62 | } 63 | return Long.parseLong(numberStr); 64 | 65 | case BOOLEAN: 66 | return in.nextBoolean(); 67 | 68 | case NULL: 69 | in.nextNull(); 70 | return null; 71 | 72 | default: 73 | throw new IllegalStateException(); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/util/MyX509TrustManager.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.util; 2 | 3 | import java.security.cert.CertificateException; 4 | import java.security.cert.X509Certificate; 5 | 6 | import javax.net.ssl.X509TrustManager; 7 | 8 | public class MyX509TrustManager implements X509TrustManager { 9 | @Override 10 | public void checkClientTrusted(X509Certificate[] chain, String authType) 11 | throws CertificateException { 12 | } 13 | 14 | @Override 15 | public void checkServerTrusted(X509Certificate[] chain, String authType) 16 | throws CertificateException { 17 | } 18 | 19 | @Override 20 | public X509Certificate[] getAcceptedIssuers() { 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/common/util/RestTemplateConfig.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.common.util; 2 | 3 | import org.apache.hc.client5.http.classic.HttpClient; 4 | import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; 5 | import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; 6 | import org.apache.hc.client5.http.io.HttpClientConnectionManager; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.http.client.ClientHttpRequestFactory; 10 | import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; 11 | import org.springframework.web.client.RestTemplate; 12 | 13 | 14 | @Configuration 15 | public class RestTemplateConfig { 16 | 17 | /** 18 | * http连接管理器 19 | * @return 20 | */ 21 | @Bean 22 | public HttpClientConnectionManager poolingHttpClientConnectionManager() { 23 | PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(); 24 | // 最大连接数 25 | poolingHttpClientConnectionManager.setMaxTotal(500); 26 | // 同路由并发数（每个主机的并发） 27 | poolingHttpClientConnectionManager.setDefaultMaxPerRoute(100); 28 | return poolingHttpClientConnectionManager; 29 | } 30 | 31 | /** 32 | * HttpClient 33 | * @param poolingHttpClientConnectionManager 34 | * @return 35 | */ 36 | @Bean 37 | public HttpClient httpClient(HttpClientConnectionManager poolingHttpClientConnectionManager) { 38 | HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); 39 | // 设置http连接管理器 40 | httpClientBuilder.setConnectionManager(poolingHttpClientConnectionManager); 41 | 42 | return httpClientBuilder.build(); 43 | } 44 | 45 | /** 46 | * 请求连接池配置 47 | * @param httpClient 48 | * @return 49 | */ 50 | @Bean 51 | public ClientHttpRequestFactory clientHttpRequestFactory(HttpClient httpClient) { 52 | HttpComponentsClientHttpRequestFactory clientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory(); 53 | // httpClient创建器 54 | clientHttpRequestFactory.setHttpClient(httpClient); 55 | // 连接超时时间/毫秒（连接上服务器(握手成功)的时间，超出抛出connect timeout） 56 | clientHttpRequestFactory.setConnectTimeout(5 * 1000); 57 | // 数据读取超时时间(socketTimeout)/毫秒（务器返回数据(response)的时间，超过抛出read timeout） 58 | clientHttpRequestFactory.setReadTimeout(10 * 1000); 59 | // 连接池获取请求连接的超时时间，不宜过长，必须设置/毫秒（超时间未拿到可用连接，会抛出org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection from pool） 60 | clientHttpRequestFactory.setConnectionRequestTimeout(10 * 1000); 61 | return clientHttpRequestFactory; 62 | } 63 | 64 | /** 65 | * rest模板 66 | * @return 67 | */ 68 | @Bean 69 | public RestTemplate restTemplate(ClientHttpRequestFactory clientHttpRequestFactory) { 70 | // boot中可使用RestTemplateBuilder.build创建 71 | RestTemplate restTemplate = new RestTemplate(); 72 | // 配置请求工厂 73 | restTemplate.setRequestFactory(clientHttpRequestFactory); 74 | return restTemplate; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/cert/CertInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.cert; 2 | 3 | import com.baomidou.mybatisplus.annotation.IdType; 4 | import com.baomidou.mybatisplus.annotation.TableId; 5 | import com.baomidou.mybatisplus.annotation.TableName; 6 | import lombok.Data; 7 | 8 | import java.time.LocalDateTime; 9 | 10 | /** 11 | * 12 | * @date 2022-04-01 15:24 13 | * 存储证书以及秘钥信息 14 | */ 15 | @SuppressWarnings("unused") 16 | @TableName("cert_info") 17 | @Data 18 | public class CertInfo { 19 | /** 20 | * 主键[自增] 21 | */ 22 | @TableId(type = IdType.AUTO) 23 | private Long id; 24 | /** 25 | * 证书文件的字符串内容 26 | */ 27 | private String certContent; 28 | /** 29 | * 公钥指数 ACES加密存储 30 | */ 31 | private String publicExponent; 32 | /** 33 | * 私钥指数 ACES加密存储 34 | */ 35 | private String privateExponent; 36 | /** 37 | * 模数 ACES加密存储 38 | */ 39 | private String modulus; 40 | /** 41 | * x509证书类型 42 | */ 43 | private Byte isRoot; 44 | /** 45 | * 创建时间 46 | */ 47 | private LocalDateTime createAt; 48 | 49 | /** 50 | * 更新时间 51 | */ 52 | private LocalDateTime updateAt; 53 | 54 | /** 55 | * 是否删除 56 | */ 57 | private Byte isDeleted; 58 | } 59 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/cert/JobTaskStub.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.cert; 2 | 3 | import com.baomidou.mybatisplus.annotation.IdType; 4 | import com.baomidou.mybatisplus.annotation.TableId; 5 | import com.baomidou.mybatisplus.annotation.TableName; 6 | import lombok.Data; 7 | 8 | import javax.validation.constraints.NotBlank; 9 | import javax.validation.constraints.Size; 10 | import java.time.LocalDateTime; 11 | 12 | /** 13 | * 14 | * @date 2022-04-01 18:20 15 | * 任务存根，包括任务信息以及发起方和执行方签名 16 | */ 17 | @SuppressWarnings("unused") 18 | @TableName("job_task_stub") 19 | @Data 20 | public class JobTaskStub { 21 | /** 22 | * 主键[自增] 23 | */ 24 | @TableId(type = IdType.AUTO) 25 | private Long id; 26 | /** 27 | * 任务id 28 | */ 29 | @NotBlank(message = "任务id不能为空") 30 | @Size(max = 100, message = "任务id不能超过100位") 31 | private String parentTaskId; 32 | /** 33 | * 任务详情，被签名的内容 34 | */ 35 | @NotBlank(message = "子任务详情") 36 | private String preJobJson; 37 | /** 38 | * 任务执行方 39 | */ 40 | private String jobTarget; 41 | /** 42 | * 任务发起方签名 43 | */ 44 | private String jobDistributorSign; 45 | /** 46 | * 任务执行方签名 47 | */ 48 | private String jobExecutorSign; 49 | /** 50 | * 任务发起方证书 51 | */ 52 | private String jobDistributorCert; 53 | /** 54 | * 任务执行方证书 55 | */ 56 | private String jobExecutorCert; 57 | /** 58 | * 任务发起方，1=发起方为自身，0=发起方为外部节点 59 | */ 60 | private Byte isLocal; 61 | /** 62 | * 创建时间 63 | */ 64 | private LocalDateTime createAt; 65 | 66 | /** 67 | * 更新时间 68 | */ 69 | private LocalDateTime updateAt; 70 | 71 | /** 72 | * 是否删除 73 | */ 74 | private Byte isDeleted; 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/cert/SignCertVo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.cert; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | /** 8 | * 9 | * @date 2022-04-02 14:29 10 | */ 11 | @Data 12 | @AllArgsConstructor 13 | @NoArgsConstructor 14 | public class SignCertVo { 15 | private String sign; 16 | private String certContent; 17 | } 18 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/config/ResourceLimitPolicy.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.config; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @Description: 资源限制策略 7 | * 8 | * @Date: 2022/4/6 9 | */ 10 | @Data 11 | public class ResourceLimitPolicy { 12 | /**最小cpu*/ 13 | private Integer minCpu = 32; 14 | /**最小memory*/ 15 | private Integer minMemory = 64; 16 | /**最大cpu*/ 17 | private Integer maxCpu = 50; 18 | /**最大内存*/ 19 | private Integer maxMemory = 100; 20 | /**内存记录重启次数*/ 21 | private Integer restartCount = 0; 22 | /**最大重启次数 */ 23 | private Integer maxRestartCount = 5; 24 | } 25 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/form/EsQueryForm.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.form; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.alibaba.fastjson.annotation.JSONField; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Builder; 7 | import lombok.Data; 8 | import lombok.NoArgsConstructor; 9 | 10 | @NoArgsConstructor 11 | @Data 12 | @Builder 13 | @AllArgsConstructor 14 | public class EsQueryForm { 15 | @JSONField(name = "query") 16 | private JSONObject query; 17 | @JSONField(name = "size") 18 | private Integer size; 19 | @JSONField(name = "from") 20 | private Integer from; 21 | } 22 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/offline/commons/Job.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.offline.commons; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * 12 | * @date 2022/1/11 2:38 下午 13 | */ 14 | @Data 15 | @Builder 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | public class Job { 19 | 20 | private String id; 21 | 22 | private String type; 23 | 24 | private List subTaskList; 25 | } 26 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/offline/commons/PreJob.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.offline.commons; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @date 2022/1/11 2:38 下午 10 | */ 11 | @Data 12 | public class PreJob { 13 | 14 | private String id; 15 | 16 | private String type; 17 | /**TODO 临时添加的前缀,用于下载oss代码 */ 18 | private String prefix; 19 | 20 | private String version = "new"; // 區分老ea算子和新平台算子 默認new 可選值old 21 | 22 | private Boolean isCustomer=false; 23 | /** 24 | * 区分测试环境、开发环境 25 | */ 26 | private String env = "test"; 27 | 28 | private List tasks; 29 | } 30 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/offline/commons/SubTask.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.offline.commons; 2 | 3 | import com.jd.mpc.domain.config.ResourceLimitPolicy; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.NoArgsConstructor; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * 13 | * @date 2022/1/14 5:40 下午 14 | */ 15 | @Data 16 | @Builder 17 | @AllArgsConstructor 18 | @NoArgsConstructor 19 | public class SubTask { 20 | 21 | private String id; 22 | 23 | private Integer subId; 24 | 25 | private Integer status; 26 | 27 | private String env; 28 | 29 | /** 资源限制策略 */ 30 | private ResourceLimitPolicy resourceLimitPolicy; 31 | 32 | private List tasks; 33 | } 34 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/offline/jxz/JxzTaskTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.offline.jxz; 2 | 3 | import lombok.Getter; 4 | 5 | /** 6 | * 7 | * @date 2022/1/17 2:31 下午 8 | */ 9 | @Getter 10 | public enum JxzTaskTypeEnum { 11 | /** 12 | * mpc 13 | */ 14 | MPC(0, "mpc"), 15 | 16 | /** 17 | * local 18 | */ 19 | LOCAL(1, "local"); 20 | 21 | 22 | private final Integer code; 23 | private final String name; 24 | 25 | JxzTaskTypeEnum(int code, String name) { 26 | this.code = code; 27 | this.name = name; 28 | } 29 | 30 | public static JxzTaskTypeEnum getByValue(int value) { 31 | for (JxzTaskTypeEnum code : values()) { 32 | if (code.getCode() == value) { 33 | return code; 34 | } 35 | } 36 | return JxzTaskTypeEnum.MPC; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/offline/vo/WorkerStatusVo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.offline.vo; 2 | 3 | import com.jd.mpc.domain.offline.commons.OfflineTask; 4 | import lombok.Data; 5 | 6 | import java.util.List; 7 | 8 | @Data 9 | public class WorkerStatusVo { 10 | 11 | private String uuid;//AppID_Service_index 12 | 13 | private int status; //-1 运行中 0 完成 大于0 错误 14 | 15 | private String msg; 16 | 17 | private String result ; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/online/OnlineJob.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.online; 2 | 3 | import lombok.Data; 4 | import lombok.EqualsAndHashCode; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * json任务对象【实时】 10 | * 11 | * 12 | * @date 2021/9/26 2:33 下午 13 | */ 14 | @EqualsAndHashCode 15 | @Data 16 | public class OnlineJob { 17 | /** 18 | * 全局唯一的任务ID 19 | */ 20 | private String id; 21 | 22 | /** 23 | * 项目ID 24 | */ 25 | private String project_id; 26 | 27 | /** 28 | * 子任务列表 29 | */ 30 | private List tasks; 31 | 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/online/OnlineResponse.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.online; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 返回对象【实时】 9 | * 10 | * 11 | * @date 2021/11/8 6:19 下午 12 | */ 13 | @Data 14 | public class OnlineResponse { 15 | /** 16 | * 全局唯一的任务ID 17 | */ 18 | private String id; 19 | 20 | /** 21 | * 项目ID 22 | */ 23 | private String project_id; 24 | 25 | /** 26 | * 子任务列表 27 | */ 28 | private List results; 29 | } 30 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/online/OnlineSubTask.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.online; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * json子任务对象【实时】 7 | * 8 | * 9 | * @date 2021/11/8 6:09 下午 10 | */ 11 | @Data 12 | public class OnlineSubTask { 13 | 14 | /** 15 | * 任务类型 16 | */ 17 | private String type; 18 | 19 | /** 20 | * 超时毫秒数 21 | */ 22 | private Integer timeout; 23 | 24 | /** 25 | * 远端回调地址 26 | */ 27 | private String url; 28 | 29 | /** 30 | * 调用方式 31 | */ 32 | private String method; 33 | 34 | /** 35 | * 请求header 36 | */ 37 | private String header; 38 | 39 | /** 40 | * 请求参数 41 | */ 42 | private String parameters; 43 | 44 | /** 45 | * redis 服务地址 46 | */ 47 | private String redis_server; 48 | 49 | /** 50 | * redis 服务密码 51 | */ 52 | private String redis_password; 53 | 54 | /** 55 | * redis 中存储任务信息的key 56 | */ 57 | private String redis_key; 58 | } 59 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/online/OnlineTask.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.online; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 全量任务对象【实时】 7 | * 8 | * 9 | * @date 2021/11/8 6:09 下午 10 | */ 11 | @Data 12 | public class OnlineTask { 13 | 14 | /** 15 | * 父任务id 16 | */ 17 | private String parentId; 18 | 19 | /** 20 | * 子任务序号 21 | */ 22 | private Integer taskIndex; 23 | 24 | /** 25 | * 任务状态 26 | */ 27 | private Integer status; 28 | 29 | /** 30 | * 任务类型 31 | */ 32 | private String type; 33 | 34 | /** 35 | * 超时毫秒数 36 | */ 37 | private Integer timeout; 38 | 39 | /** 40 | * 远端回调地址 41 | */ 42 | private String url; 43 | 44 | /** 45 | * 调用方式 46 | */ 47 | private String method; 48 | 49 | /** 50 | * 请求header 51 | */ 52 | private String header; 53 | 54 | /** 55 | * 请求参数 56 | */ 57 | private String parameters; 58 | 59 | /** 60 | * redis 服务地址 61 | */ 62 | private String redisServer; 63 | 64 | /** 65 | * redis 服务密码 66 | */ 67 | private String redisPassword; 68 | 69 | /** 70 | * redis 中存储任务信息的key 71 | */ 72 | private String redisKey; 73 | } 74 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/online/Result.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.online; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 返回结果【实时】 7 | * 8 | * 9 | * @date 2021/11/8 6:25 下午 10 | */ 11 | @Data 12 | public class Result { 13 | /** 14 | * 状态 15 | */ 16 | private Integer status; 17 | 18 | /** 19 | * 结果 20 | */ 21 | private String result; 22 | } 23 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/ExistParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @Description: 是否存在 9 | * 10 | * @Date: 2022/9/22 11 | */ 12 | @Data 13 | public class ExistParam { 14 | 15 | private String id; 16 | 17 | private String type; 18 | 19 | private String target; 20 | 21 | private Map parameters; 22 | } 23 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/GetConfigParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | @Data 7 | public class GetConfigParam { 8 | private List remoteTarget; 9 | private String coordinatorUrl; 10 | private List fileServiceList; 11 | } 12 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/GetCoordinatorLogParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class GetCoordinatorLogParam { 7 | private String logLevel; 8 | private Integer from; 9 | private Integer size; 10 | private String startTime; 11 | private String endTime; 12 | } 13 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/GetFileServiceLogParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class GetFileServiceLogParam { 7 | private Integer fileServiceType; 8 | private String bdpAccount; 9 | private String logLevel; 10 | private Integer from; 11 | private Integer size; 12 | private String startTime; 13 | private String endTime; 14 | } 15 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/GetNodeLogParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 获取联邦算子日志的方法入参 7 | * 8 | * @date 2023/06/27 9 | */ 10 | @Data 11 | public class GetNodeLogParam { 12 | private String coordinateTaskId; 13 | private String logLevel; 14 | private Integer nodeId; 15 | private Integer from; 16 | private Integer size; 17 | } 18 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/TaskInfoParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @Description: 任务信息 9 | * 10 | * @Date: 2022/9/19 11 | */ 12 | @Data 13 | public class TaskInfoParam { 14 | /** clusterId */ 15 | private String clusterID; 16 | /** nodeId */ 17 | private String nodeID; 18 | /** 本侧target */ 19 | private String localDomain; 20 | /** 对侧target */ 21 | private String remoteDomain; 22 | /** 额外的信息 */ 23 | private Map extra; 24 | } 25 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/param/WorkerInfoParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @Description: worker信息 7 | * 8 | * @Date: 2022/9/19 9 | */ 10 | @Data 11 | public class WorkerInfoParam { 12 | /** 随机数 */ 13 | private long random; 14 | /** 时间戳 */ 15 | private long timestamp; 16 | /** worker类型 */ 17 | private String workerType; 18 | /** worker版本号 */ 19 | private String workerVersion; 20 | /** 任务信息 */ 21 | private TaskInfoParam taskInfo; 22 | } 23 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/task/AuthInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.task; 2 | 3 | import com.baomidou.mybatisplus.annotation.IdType; 4 | import com.baomidou.mybatisplus.annotation.TableId; 5 | import com.baomidou.mybatisplus.annotation.TableName; 6 | import lombok.AllArgsConstructor; 7 | import lombok.Builder; 8 | import lombok.Data; 9 | import lombok.NoArgsConstructor; 10 | 11 | import java.util.Date; 12 | 13 | /** 14 | * @Description: cert pub/pri 15 | * domain 和 type 作为联合唯一索引 16 | * 17 | * @Date: 2022/9/18 18 | */ 19 | @Data 20 | @Builder 21 | @NoArgsConstructor 22 | @AllArgsConstructor 23 | @TableName("auth_info") 24 | public class AuthInfo { 25 | /** id */ 26 | @TableId(type = IdType.AUTO) 27 | private Long id; 28 | /** domain */ 29 | private String domain; 30 | /** 证书类型 */ 31 | private CertTypeEnum certType; 32 | /** 证书 */ 33 | private String cert; 34 | /** 公钥 */ 35 | private String pubKey; 36 | /** 私钥 */ 37 | private String priKey; 38 | /** 状态 */ 39 | private AuthStatusEnum status; 40 | /** 创建时间 */ 41 | private Date createTime; 42 | /** 更新时间 */ 43 | private Date updateTime; 44 | } 45 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/task/AuthStatusEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.task; 2 | 3 | /** 4 | * @Description: 认证信息状态 5 | * 6 | * @Date: 2022/10/9 7 | */ 8 | public enum AuthStatusEnum { 9 | SUBMIT("提交"), 10 | PASS("通过"), 11 | REJECT("拒绝"), 12 | ; 13 | 14 | private final String desc; 15 | 16 | AuthStatusEnum(String desc) { 17 | this.desc = desc; 18 | } 19 | 20 | public String getDesc() { 21 | return desc; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/task/CertTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.task; 2 | 3 | /** 4 | * @Description: 证书类型 5 | * 6 | * @Date: 2022/9/18 7 | */ 8 | public enum CertTypeEnum { 9 | ROOT("ROOT"), 10 | AUTH("AUTH"), 11 | WORKER("WORKER"), 12 | ; 13 | private final String desc; 14 | 15 | CertTypeEnum(String desc) { 16 | this.desc = desc; 17 | } 18 | 19 | public String getDesc() { 20 | return desc; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/AuthInfoDto.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import com.baomidou.mybatisplus.annotation.IdType; 4 | import com.baomidou.mybatisplus.annotation.TableId; 5 | import com.jd.mpc.domain.task.AuthStatusEnum; 6 | import com.jd.mpc.domain.task.CertTypeEnum; 7 | import lombok.Data; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.Date; 11 | 12 | /** 13 | * @Description: dto 14 | * 15 | * @Date: 2022/10/9 16 | */ 17 | @Data 18 | @Component 19 | public class AuthInfoDto { 20 | /** id */ 21 | private Long id; 22 | /** domain */ 23 | private String domain; 24 | /** 证书类型 */ 25 | private CertTypeEnum certType; 26 | /** 证书 */ 27 | private String cert; 28 | /** 公钥 */ 29 | private byte[] pubKey; 30 | /** 私钥 */ 31 | private byte[] priKey; 32 | /** 状态 */ 33 | private AuthStatusEnum status; 34 | /** 创建时间 */ 35 | private Date createTime; 36 | /** 更新时间 */ 37 | private Date updateTime; 38 | } 39 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/BlockVo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 项目名称： 7 | * 类名称： 8 | * 类描述： 9 | * 创建人：yuhaiyang47 10 | * 创建时间： 11 | */ 12 | @Data 13 | public class BlockVo { 14 | /**blockId */ 15 | private String blockId; 16 | /**数据名称 */ 17 | private String dataSourceName; 18 | /**样本数 */ 19 | private int exampleNum; 20 | } 21 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/CallbackBody.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 7 | * @date 2022/2/8 10:46 上午 8 | */ 9 | @Data 10 | public class CallbackBody { 11 | private Integer status; 12 | private String result; 13 | private String callbackUrl; 14 | } 15 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/Data.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | /** 4 | * 5 | * @date 2022/3/23 5:51 下午 6 | */ 7 | @lombok.Data 8 | public class Data { 9 | String data; 10 | } 11 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/EtlHeaderParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @Description: etl header param 7 | * 8 | * @Date: 2022/9/15 9 | */ 10 | @Data 11 | public class EtlHeaderParam { 12 | private String sql; 13 | private String sqlParams; 14 | private String customerId; 15 | 16 | private String target; 17 | } 18 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/FeatureDetail.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * 特征worker详细信息 9 | * 10 | * 11 | * @date 2021/12/9 5:01 下午 12 | */ 13 | @Data 14 | public class FeatureDetail { 15 | 16 | /** 17 | * 18 | */ 19 | private Map pearsonScore; 20 | 21 | /** 22 | * 23 | */ 24 | private Map ivScore; 25 | 26 | /** 27 | * 28 | */ 29 | private String msg; 30 | 31 | /** 32 | * 状态 0：未完成 1：已完成 33 | */ 34 | private Long status; 35 | } 36 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/FileSchemaInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Builder; 4 | import lombok.Data; 5 | 6 | /** 7 | * 文件信息对象 8 | * 9 | * 10 | * @date 2021/11/22 6:06 下午 11 | */ 12 | @Data 13 | @Builder 14 | public class FileSchemaInfo { 15 | 16 | /** 17 | * 列名 18 | */ 19 | private String colName; 20 | 21 | /** 22 | * 列类型 23 | */ 24 | private String dataType; 25 | 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/FileSizeInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 文件信息对象 7 | * 8 | * 9 | * @date 2021/11/22 6:06 下午 10 | */ 11 | @Data 12 | public class FileSizeInfo { 13 | 14 | /** 15 | * 文件行数 16 | */ 17 | private Long row; 18 | 19 | /** 20 | * 文件列数 21 | */ 22 | private Long col; 23 | 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/GrpcResourceLimitResult.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @Description: 资源限制结果对象 7 | * 8 | * @Date: 2022/4/6 9 | */ 10 | @Data 11 | public class GrpcResourceLimitResult { 12 | /** id */ 13 | private String id; 14 | /** subId */ 15 | private Integer subId; 16 | /** 是否pending */ 17 | private boolean pendFlag; 18 | /** 是否oom */ 19 | private boolean oomFlag; 20 | } 21 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/JobInfos.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @date 2022/1/26 2:55 下午 10 | */ 11 | @Data 12 | public class JobInfos { 13 | 14 | private String id; 15 | 16 | private String logs; 17 | 18 | private String results; 19 | } 20 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/JoinRawInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 项目名称： 7 | * 类名称： 8 | * 类描述： 9 | * 创建人： 10 | * 创建时间： 11 | */ 12 | @Data 13 | public class JoinRawInfo { 14 | String app_id ; 15 | String data_source_name; 16 | String train_data_start; 17 | String train_data_end ; 18 | int partition_num =0; 19 | float negative_sampling_rate = 0L; 20 | } 21 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/MailInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 邮件信息 9 | * 10 | * 11 | * @date 2022/1/5 10:36 上午 12 | */ 13 | @Data 14 | public class MailInfo { 15 | 16 | /** 17 | * 内容 18 | */ 19 | private String content; 20 | 21 | 22 | /** 23 | * 接收人 24 | */ 25 | private List mailTo; 26 | 27 | 28 | /** 29 | * 主题 30 | */ 31 | private String subject; 32 | } 33 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/PodInfos.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @date 2022/1/26 2:56 下午 10 | */ 11 | @Data 12 | public class PodInfos { 13 | 14 | private Integer podId; 15 | 16 | private List logsList; 17 | 18 | private List resultList; 19 | } 20 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/PredictQuery.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class PredictQuery { 7 | 8 | private String customerId; 9 | 10 | private String ids; 11 | 12 | private String target; 13 | } 14 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/PredictResult.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Builder; 4 | import lombok.Data; 5 | 6 | @Data 7 | @Builder 8 | public class PredictResult { 9 | 10 | private String id; 11 | 12 | private Long predictNum; 13 | } 14 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/ProxyInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Map; 6 | 7 | @Data 8 | public class ProxyInfo { 9 | 10 | private String customerId; 11 | 12 | private Map proxyMap; 13 | } 14 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/ResourcesInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 7 | * @date 2022/1/25 10:20 上午 8 | */ 9 | @Data 10 | public class ResourcesInfo { 11 | 12 | private String id; 13 | 14 | private Integer cpu; 15 | 16 | private Integer memory; 17 | } 18 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/SqlParserParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import java.util.List; 4 | import javax.validation.constraints.NotBlank; 5 | import javax.validation.constraints.NotNull; 6 | 7 | import lombok.Data; 8 | 9 | /** 10 | * 11 | * @Date: 2022/3/17 12 | */ 13 | @Data 14 | public class SqlParserParam { 15 | 16 | /** 17 | * 任务id 18 | */ 19 | @NotBlank(message = "任务id不能为空") 20 | private String id; 21 | 22 | /** 23 | * 任务类型 24 | */ 25 | @NotBlank(message = "任务类型不能为空") 26 | private String type; 27 | 28 | /** 29 | * sql 30 | */ 31 | @NotBlank(message = "sql不能为空") 32 | private String sql; 33 | 34 | /** 35 | * target信息 36 | */ 37 | @NotNull 38 | private List targetInfos; 39 | } 40 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/SyncInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * 9 | * @date 2022/1/12 8:28 下午 10 | */ 11 | @Data 12 | public class SyncInfo { 13 | 14 | /** 15 | * 16 | */ 17 | private String customerId; 18 | 19 | /** 20 | * 21 | */ 22 | private String target; 23 | 24 | /** 25 | * 26 | */ 27 | private Map dataInfo; 28 | } 29 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/SyncRequest.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @date 2022/1/12 8:28 下午 10 | */ 11 | @Data 12 | public class SyncRequest { 13 | 14 | private List infoList; 15 | } 16 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/SyncResInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 7 | * @date 2022/1/14 4:22 下午 8 | */ 9 | @Data 10 | public class SyncResInfo { 11 | 12 | private String customerId; 13 | 14 | private String target; 15 | 16 | private String result; 17 | } 18 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/SyncResponse.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @date 2022/1/12 8:28 下午 10 | */ 11 | @Data 12 | public class SyncResponse { 13 | 14 | List resInfoList; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/TableInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * table 信息 7 | * 8 | * 9 | * @Date: 2022/3/22 10 | */ 11 | @Data 12 | public class TableInfo { 13 | 14 | /** 15 | * 数据名称 16 | */ 17 | private String tableName; 18 | 19 | /** 20 | * 数据路径 21 | */ 22 | private String tablePath; 23 | 24 | /** 25 | * 格式 26 | */ 27 | private String format; 28 | } 29 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/TargetInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import java.util.List; 4 | 5 | import lombok.Data; 6 | 7 | /** 8 | * target 信息 9 | * 10 | * 11 | * @Date: 2022/3/22 12 | */ 13 | @Data 14 | public class TargetInfo { 15 | 16 | /** 17 | * customerId 18 | */ 19 | private String target; 20 | 21 | /** 22 | * leader / follower 23 | */ 24 | private String role; 25 | 26 | /** 27 | * 输出路径 28 | */ 29 | private String outputPath; 30 | 31 | /** 32 | * 表信息 33 | */ 34 | private List tableInfos; 35 | } 36 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/TaskIdList.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @date 2021/12/27 6:00 下午 10 | */ 11 | @Data 12 | public class TaskIdList { 13 | List idList; 14 | } 15 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/TaskInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import java.util.List; 4 | import javax.validation.constraints.NotBlank; 5 | import javax.validation.constraints.NotNull; 6 | 7 | import com.jd.mpc.domain.offline.commons.PreJob; 8 | import lombok.Data; 9 | 10 | /** 11 | * 12 | * @Date: 2022/3/17 13 | */ 14 | @Data 15 | public class TaskInfo { 16 | 17 | /** 18 | * 任务id, web端传过来的值 19 | */ 20 | @NotBlank 21 | private String id; 22 | 23 | /** 24 | * 任务类型 25 | */ 26 | @NotBlank(message = "任务类型不能为空") 27 | private String type; 28 | 29 | /** 30 | * 任务列表 31 | */ 32 | @NotNull(message = "任务列表不能为空") 33 | private List subTasks; 34 | } 35 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/TaskStatusInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Set; 6 | 7 | /** 8 | * 任务对象状态 9 | * 10 | * 11 | * @date 2021/11/23 7:30 下午 12 | */ 13 | @Data 14 | public class TaskStatusInfo { 15 | 16 | /** 17 | * 任务端id 18 | */ 19 | @Deprecated 20 | private String customerId; 21 | 22 | /** 23 | * target 24 | */ 25 | private String target; 26 | 27 | /** 28 | * 任务进度百分比 29 | */ 30 | private Integer percent; 31 | 32 | /** 33 | * 任务状态 34 | */ 35 | private Integer taskStatus; 36 | 37 | /** 38 | * 运行时间 39 | */ 40 | private Long runTime; 41 | 42 | /** 43 | * 任务错误msg 44 | * */ 45 | private String errorMsg; 46 | /** 47 | * 任务错误code 48 | * */ 49 | private Integer errorCode; 50 | 51 | private Set runningWorkers; 52 | 53 | private Integer workerNum; 54 | } 55 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/VerifyVo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @Description: TODO 7 | * 8 | * @Date: 2022/9/19 9 | */ 10 | @Data 11 | public class VerifyVo { 12 | private String cert; 13 | private String sig; 14 | private String data; 15 | } 16 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/domain/vo/WorkerInfo.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.domain.vo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * worker信息 7 | * 8 | * 9 | * @date 2021/12/21 4:35 下午 10 | */ 11 | @Data 12 | public class WorkerInfo { 13 | 14 | private String clusterId; 15 | 16 | private Integer nodeId; 17 | 18 | private Integer status; 19 | 20 | private String message; 21 | 22 | private String result; 23 | 24 | private Integer percent; 25 | } 26 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/grpc/GrpcPredictClient.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.grpc; 2 | 3 | import org.springframework.stereotype.Component; 4 | import predict.FeaturesListProto; 5 | import predict.Request; 6 | import predict.Response; 7 | 8 | import java.util.Base64; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | /** 12 | * 调用在线预测worker Grpc服务 13 | * 14 | * 15 | * @date 2021/9/26 10:30 下午 16 | */ 17 | @Component 18 | public class GrpcPredictClient { 19 | 20 | /** 21 | * 查询特征work信息 22 | * 23 | * @param host ip 24 | * @param port 端口 25 | * @return 特征work信息 26 | */ 27 | public String predict(String data, String host, int port) { 28 | try { 29 | GrpcClient client = new GrpcClient(host, port, "predict", ""); 30 | byte[] bytes = Base64.getDecoder().decode(data); 31 | Request request = Request.newBuilder().setExamples(FeaturesListProto.parseFrom(bytes)).build(); 32 | Response response = client.getPredictStub().predict(request); 33 | this.closeChannel(client); 34 | return response.toString(); 35 | } catch (Exception e) { 36 | e.printStackTrace(); 37 | } 38 | return "调用失败"; 39 | } 40 | 41 | private void closeChannel(GrpcClient client) { 42 | try { 43 | client.getChannel().shutdown().awaitTermination(5, TimeUnit.SECONDS); 44 | } catch (InterruptedException e) { 45 | e.printStackTrace(); 46 | } 47 | } 48 | 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/grpc/GrpcRetryExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.grpc; 2 | 3 | import io.grpc.Status; 4 | import io.grpc.StatusRuntimeException; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * @Description: grpc调用异常处理 10 | * 11 | * @Date: 2022/3/29 12 | */ 13 | @Slf4j 14 | @Component("grpcRetryExceptionHandler") 15 | public class GrpcRetryExceptionHandler { 16 | 17 | public boolean shouldRetry(Throwable t){ 18 | if (t instanceof StatusRuntimeException){ 19 | StatusRuntimeException statusRuntimeException = (StatusRuntimeException) t; 20 | boolean flag = statusRuntimeException.getStatus().getCode().equals(Status.UNAVAILABLE.getCode()); 21 | if (flag){ 22 | //打印重试的类和方法 23 | StackTraceElement element = null; 24 | for (StackTraceElement stackTraceElement : statusRuntimeException.getStackTrace()) { 25 | if (stackTraceElement.getClassName().equals("com.jd.mpc.grpc.GrpcOfflineClient") && !stackTraceElement.getMethodName().contains("$")){ 26 | element = stackTraceElement; 27 | break; 28 | } 29 | } 30 | if (element == null){ 31 | log.error("retry-method:unknown"); 32 | }else { 33 | log.error("retry-method:"+element.getMethodName()); 34 | } 35 | } 36 | return flag; 37 | } 38 | return false; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/grpc/GrpcSignClient.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.grpc; 2 | 3 | import com.google.protobuf.ByteString; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.springframework.stereotype.Component; 6 | import authprotocol.GrpcVo; 7 | import authprotocol.IssueGrpcParam; 8 | import authprotocol.VerifyGrpcParam; 9 | 10 | import jakarta.annotation.Resource; 11 | import java.util.concurrent.TimeUnit; 12 | 13 | /** 14 | * 调用jd Grpc服务 15 | * 16 | * 17 | * @date 2021/9/26 10:30 下午 18 | */ 19 | @Component 20 | @Slf4j 21 | public class GrpcSignClient { 22 | 23 | @Resource 24 | private GrpcClient grpcClient; 25 | 26 | public GrpcVo verify(String cert,String sig,String data){ 27 | VerifyGrpcParam verifySignReq = VerifyGrpcParam.newBuilder() 28 | .setCert(cert) 29 | .setData(data) 30 | .setSig(sig) 31 | .build(); 32 | GrpcClient client = grpcClient.getClient("sign", "9n_demo_1"); 33 | GrpcVo vo = client.getAuthStub().verify(verifySignReq); 34 | this.closeChannel(client); 35 | return vo; 36 | } 37 | 38 | public GrpcVo issueCert(String commonName,String organization,byte[] key){ 39 | IssueGrpcParam param = IssueGrpcParam.newBuilder() 40 | .setCommonName(commonName) 41 | .setOrganization(organization) 42 | .setKey(ByteString.copyFrom(key)) 43 | .build(); 44 | GrpcClient client = grpcClient.getClient("sign", "9n_demo_1"); 45 | GrpcVo vo = client.getAuthStub().issueCert(param); 46 | this.closeChannel(client); 47 | return vo; 48 | } 49 | 50 | private void closeChannel(GrpcClient client) { 51 | try { 52 | client.getChannel().shutdown().awaitTermination(60, TimeUnit.SECONDS); 53 | } 54 | catch (InterruptedException e) { 55 | e.printStackTrace(); 56 | } 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/mapper/AuthInfoMapper.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.jd.mpc.domain.task.AuthInfo; 5 | import com.jd.mpc.domain.task.CertTypeEnum; 6 | import org.apache.ibatis.annotations.Mapper; 7 | import org.apache.ibatis.annotations.Param; 8 | 9 | /** 10 | * @Description: mapper 11 | * 12 | * @Date: 2022/9/18 13 | */ 14 | @Mapper 15 | public interface AuthInfoMapper extends BaseMapper { 16 | 17 | /** 18 | * 通过domain和证书类型获得认证信息 19 | * @param domain 20 | * @param certType 21 | * @return 22 | */ 23 | AuthInfo selectByDomainCertType(@Param("domain")String domain,@Param("certType") CertTypeEnum certType); 24 | } 25 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/mapper/CertMapper.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.jd.mpc.domain.cert.CertInfo; 5 | 6 | public interface CertMapper extends BaseMapper { 7 | 8 | } -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/mapper/ChildrenTaskMapper.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.jd.mpc.domain.task.ChildrenTask; 5 | 6 | public interface ChildrenTaskMapper extends BaseMapper { 7 | 8 | } -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/mapper/JobTaskStubMapper.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.jd.mpc.domain.cert.JobTaskStub; 5 | 6 | public interface JobTaskStubMapper extends BaseMapper { 7 | 8 | } -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/mapper/ParentTaskMapper.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.jd.mpc.domain.task.ParentTask; 5 | import org.apache.ibatis.annotations.Mapper; 6 | 7 | @Mapper 8 | public interface ParentTaskMapper extends BaseMapper { 9 | 10 | } -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/quartz/QuartzConfig.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.quartz; 2 | 3 | import com.jd.mpc.quartz.job.FinishTaskJob; 4 | import com.jd.mpc.quartz.job.StartTaskJob; 5 | import org.quartz.*; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | 9 | @Configuration 10 | public class QuartzConfig { 11 | 12 | /** 13 | * for {@link StartTaskJob} 14 | * @return 15 | */ 16 | @Bean 17 | public JobDetail startTaskJobDetail(){ 18 | return JobBuilder 19 | .newJob(StartTaskJob.class) 20 | .withIdentity("StartTaskJobDetail") 21 | .storeDurably().build(); 22 | } 23 | 24 | /** 25 | * for {@link StartTaskJob} 26 | * @return 27 | */ 28 | @Bean 29 | public Trigger startTaskJobTrigger(){ 30 | return TriggerBuilder 31 | .newTrigger() 32 | .forJob(startTaskJobDetail()) 33 | .withIdentity("StartTaskJobTrigger") 34 | .withSchedule(CronScheduleBuilder.cronSchedule("0 0/1 * * * ? *")) 35 | .build(); 36 | } 37 | 38 | /** 39 | * for {@link FinishTaskJob} 40 | * @return 41 | */ 42 | @Bean 43 | public JobDetail finishTaskJobDetail(){ 44 | return JobBuilder 45 | .newJob(FinishTaskJob.class) 46 | .withIdentity("FinishTaskJobDetail") 47 | .storeDurably().build(); 48 | } 49 | 50 | /** 51 | * for {@link FinishTaskJob} 52 | * @return 53 | */ 54 | @Bean 55 | public Trigger finishTaskJobTrigger(){ 56 | return TriggerBuilder 57 | .newTrigger() 58 | .forJob(finishTaskJobDetail()) 59 | .withIdentity("FinishTaskJobTrigger") 60 | .withSchedule(CronScheduleBuilder.cronSchedule("0 0/1 * * * ? *")) 61 | .build(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/quartz/job/FinishTaskJob.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.quartz.job; 2 | 3 | import cn.hutool.extra.spring.SpringUtil; 4 | import com.jd.mpc.service.OfflineService; 5 | import org.quartz.JobExecutionContext; 6 | import org.quartz.JobExecutionException; 7 | import org.springframework.scheduling.quartz.QuartzJobBean; 8 | 9 | public class FinishTaskJob extends QuartzJobBean { 10 | 11 | @Override 12 | protected void executeInternal(JobExecutionContext context) throws JobExecutionException { 13 | OfflineService offlineService = SpringUtil.getBean(OfflineService.class); 14 | offlineService.finishTask(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/quartz/job/StartTaskJob.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.quartz.job; 2 | 3 | import cn.hutool.extra.spring.SpringUtil; 4 | import com.jd.mpc.service.OfflineService; 5 | import org.quartz.JobExecutionContext; 6 | import org.quartz.JobExecutionException; 7 | import org.springframework.scheduling.quartz.QuartzJobBean; 8 | 9 | public class StartTaskJob extends QuartzJobBean { 10 | 11 | @Override 12 | protected void executeInternal(JobExecutionContext context) throws JobExecutionException { 13 | OfflineService offlineService = SpringUtil.getBean(OfflineService.class); 14 | offlineService.startTask(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/redis/RedisConfig.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.redis; 2 | 3 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 4 | import com.fasterxml.jackson.annotation.PropertyAccessor; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.cache.annotation.EnableCaching; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.data.redis.connection.RedisConnectionFactory; 11 | import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; 12 | import org.springframework.data.redis.core.RedisTemplate; 13 | import org.springframework.data.redis.core.StringRedisTemplate; 14 | import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; 15 | 16 | @Configuration 17 | @EnableCaching 18 | public class RedisConfig { 19 | 20 | @Value("${spring.redis.host}") 21 | private String host; 22 | @Value("${spring.redis.port}") 23 | private int port; 24 | @Value("${spring.redis.password}") 25 | private String password; 26 | 27 | @Bean 28 | public JedisConnectionFactory redisConnectionFactory() { 29 | JedisConnectionFactory factory = new JedisConnectionFactory(); 30 | factory.setHostName(host); 31 | factory.setPort(port); 32 | factory.setPassword(password); 33 | factory.setDatabase(0); 34 | return factory; 35 | } 36 | 37 | 38 | @Bean 39 | public RedisTemplate redisTemplate(RedisConnectionFactory factory) { 40 | StringRedisTemplate template = new StringRedisTemplate(factory); 41 | setSerializer(template); //设置序列化工具，这样ReportBean不需要实现Serializable接口 42 | template.afterPropertiesSet(); 43 | return template; 44 | } 45 | 46 | 47 | private void setSerializer(StringRedisTemplate template) { 48 | Jackson2JsonRedisSerializer jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer(Object.class); 49 | ObjectMapper om = new ObjectMapper(); 50 | om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); 51 | om.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); 52 | jackson2JsonRedisSerializer.setObjectMapper(om); 53 | template.setValueSerializer(jackson2JsonRedisSerializer); 54 | } 55 | 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/redis/RedisLock.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.redis; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | import jakarta.annotation.Resource; 5 | 6 | import org.springframework.data.redis.core.StringRedisTemplate; 7 | import org.springframework.stereotype.Service; 8 | 9 | /** 10 | * redis 分布式锁 11 | * 12 | * 13 | * @Date: 2022/3/10 14 | */ 15 | @Service 16 | public class RedisLock { 17 | 18 | @Resource 19 | private StringRedisTemplate stringRedisTemplate; 20 | 21 | /** 22 | * 尝试获取时效锁 23 | * @param lockKey 24 | * @param value 25 | * @return 26 | */ 27 | public boolean tryGetLock(String lockKey, String value) { 28 | return stringRedisTemplate.opsForValue().setIfAbsent(lockKey, value); 29 | } 30 | 31 | /** 32 | * 尝试获取时效锁 33 | * @param lockKey 34 | * @param value 35 | * @param expire 36 | * @param timeUnit 37 | * @return 38 | */ 39 | public boolean tryGetDeadlineLock(String lockKey, String value, long expire, TimeUnit timeUnit) { 40 | return stringRedisTemplate.opsForValue().setIfAbsent(lockKey, value, expire, timeUnit); 41 | } 42 | 43 | /** 44 | * 释放锁 45 | * @param lockKey 46 | * @return 47 | */ 48 | public boolean unLock(String lockKey) { 49 | return stringRedisTemplate.delete(lockKey); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/redis/RedisServer.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.redis; 2 | 3 | import java.io.IOException; 4 | import java.net.InetAddress; 5 | import java.net.UnknownHostException; 6 | import java.security.InvalidAlgorithmParameterException; 7 | import java.security.InvalidKeyException; 8 | import java.security.NoSuchAlgorithmException; 9 | import java.security.NoSuchProviderException; 10 | import java.security.PrivateKey; 11 | import java.security.PublicKey; 12 | import java.security.cert.CertificateException; 13 | import java.security.spec.InvalidKeySpecException; 14 | import java.text.SimpleDateFormat; 15 | import java.time.LocalDateTime; 16 | import java.util.Calendar; 17 | import java.util.Date; 18 | 19 | import jakarta.annotation.Resource; 20 | 21 | import cn.hutool.core.codec.Base64; 22 | import com.jd.mpc.common.util.JNISigner; 23 | import org.bouncycastle.asn1.x500.X500Name; 24 | import org.bouncycastle.operator.OperatorCreationException; 25 | import org.springframework.beans.factory.annotation.Autowired; 26 | import org.springframework.beans.factory.annotation.Value; 27 | import org.springframework.boot.ApplicationArguments; 28 | import org.springframework.boot.ApplicationRunner; 29 | import org.springframework.data.redis.core.StringRedisTemplate; 30 | import org.springframework.stereotype.Component; 31 | 32 | import com.jd.mpc.aces.TdeService; 33 | import com.jd.mpc.cert.CSRUtil; 34 | import com.jd.mpc.cert.CertGenerator; 35 | import com.jd.mpc.cert.KeyGenerator; 36 | import com.jd.mpc.cert.KeyPairPojo; 37 | import com.jd.mpc.common.enums.IsDeletedEnum; 38 | import com.jd.mpc.common.enums.IsRootEnum; 39 | import com.jd.mpc.domain.cert.CertInfo; 40 | import com.jd.mpc.service.cert.CertPersistenceService; 41 | 42 | import lombok.extern.slf4j.Slf4j; 43 | 44 | /** 45 | * redis 客户端 46 | * 47 | * 48 | * @date 2021/9/28 3:29 下午 49 | */ 50 | @Component 51 | @Slf4j 52 | public class RedisServer implements ApplicationRunner { 53 | 54 | @Autowired 55 | private StringRedisTemplate stringRedisTemplate; 56 | 57 | @Value("${grpc.server.port}") 58 | private String grpcServerPort; 59 | 60 | @Value("${node.ip}") 61 | private String nodeIp; 62 | 63 | @Value("${node.port}") 64 | private String nodePort; 65 | 66 | @Value("${grpc.regist.port}") 67 | private String grpcRegistPort; 68 | 69 | @Autowired 70 | private CertPersistenceService certPersistenceService; 71 | 72 | @Autowired 73 | private TdeService tdeService; 74 | 75 | /** 76 | * 服务启动后执行此方法 77 | * 78 | * @param args 启动参数 79 | */ 80 | @Override 81 | public void run(ApplicationArguments args) { 82 | stringRedisTemplate.opsForValue().set("network:coordinator", 83 | nodeIp + ":" + grpcRegistPort); 84 | stringRedisTemplate.opsForValue().set("coordinator-portal-pk", 85 | nodeIp + ":" + nodePort); 86 | log.info("grpc host:port = {}:{}", nodeIp, nodePort); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/AuthInfoService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; 5 | import com.jd.mpc.domain.task.AuthInfo; 6 | import com.jd.mpc.domain.task.CertTypeEnum; 7 | import com.jd.mpc.mapper.AuthInfoMapper; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.springframework.stereotype.Service; 10 | 11 | 12 | /** 13 | * @Description: service 14 | * 15 | * @Date: 2022/9/18 16 | */ 17 | @Slf4j 18 | @Service 19 | public class AuthInfoService extends ServiceImpl { 20 | 21 | public AuthInfo getByCertInfo(String domain, CertTypeEnum certType){ 22 | return baseMapper.selectByDomainCertType(domain, certType); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/FileService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.alibaba.fastjson.TypeReference; 5 | import com.jd.mpc.common.enums.StoreTypeEnum; 6 | import com.jd.mpc.common.response.CommonResponse; 7 | import com.jd.mpc.common.response.ErrorStatus; 8 | import com.jd.mpc.common.util.HttpUtil; 9 | import com.jd.mpc.redis.RedisService; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.stereotype.Service; 12 | 13 | import jakarta.annotation.Resource; 14 | import java.util.Collections; 15 | import java.util.List; 16 | 17 | /** 18 | * @Description: 文件服务 19 | * 20 | * @Date: 2022/2/16 21 | */ 22 | @Slf4j 23 | @Service 24 | public class FileService { 25 | 26 | @Resource 27 | private RedisService redisService; 28 | 29 | /** 30 | * 批量创建文件 31 | * @param paths 32 | * @return 33 | */ 34 | public String mkdir(List paths){ 35 | if(!redisService.hasKey("file-service-addr")){ 36 | log.error("file-service-addr key of redis not exist!"); 37 | return ""; 38 | } 39 | String url = "http://" + redisService.get("file-service-addr") + "/api/file/mkdirs?path="+String.join(",",paths.toArray(new String[]{})); 40 | log.info("url:"+url); 41 | return HttpUtil.get(url); 42 | } 43 | public String mkdirs(List paths, String bdpAccount, StoreTypeEnum storeType){ 44 | String fileSvcKey = "file-service-addr"; 45 | if (storeType.equals(StoreTypeEnum.HDFS)){ 46 | fileSvcKey = "file-service-addr::"+bdpAccount; 47 | } 48 | if(!redisService.hasKey(fileSvcKey)){ 49 | log.error("file-service-addr key of redis not exist!"); 50 | return ""; 51 | } 52 | String url = "http://" + redisService.get(fileSvcKey) + "/api/file/mkdirs?path="+String.join(",",paths.toArray(new String[]{})); 53 | log.info("url:"+url); 54 | return HttpUtil.get(url); 55 | } 56 | 57 | /** 58 | * 59 | * @param paths 60 | * @param bdpAccount 61 | * @param storeType 62 | * @return 63 | */ 64 | public List mkListDirs(List paths, String bdpAccount, StoreTypeEnum storeType){ 65 | String rets = mkdirs(paths, bdpAccount, storeType); 66 | CommonResponse> retResp = JSONObject.parseObject(rets, new TypeReference>>() { 67 | }); 68 | if (!retResp.getStatus().equals(ErrorStatus.SUCCESS)) { 69 | return Collections.emptyList(); 70 | } 71 | return retResp.getResult(); 72 | } 73 | /** 74 | * 查询文件列表 75 | * @param path 76 | * @return 77 | */ 78 | public String ls(String path){ 79 | String url = "http://" + redisService.get("file-service-addr") + "/api/file/ls?path="+path; 80 | log.info("url:"+url); 81 | return HttpUtil.get(url); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/cert/JobTaskStubService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.cert; 2 | 3 | import com.jd.mpc.domain.cert.JobTaskStub; 4 | import com.jd.mpc.mapper.JobTaskStubMapper; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.stereotype.Component; 7 | 8 | import jakarta.annotation.Resource; 9 | 10 | /** 11 | * 12 | * @date 2022-04-02 11:07 13 | */ 14 | @Component 15 | @Slf4j 16 | public class JobTaskStubService { 17 | @Resource 18 | private JobTaskStubMapper jobTaskStubMapper; 19 | 20 | public int insert(JobTaskStub jobTaskStub){ 21 | return jobTaskStubMapper.insert(jobTaskStub); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/task/AbstractTaskService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.task; 2 | 3 | import com.jd.mpc.common.enums.TaskStatusEnum; 4 | import com.jd.mpc.domain.offline.commons.Job; 5 | import com.jd.mpc.domain.offline.commons.OfflineTask; 6 | import com.jd.mpc.domain.offline.commons.PreJob; 7 | import com.jd.mpc.service.FileService; 8 | import com.jd.mpc.service.TaskSupport; 9 | import org.springframework.beans.factory.annotation.Value; 10 | import org.springframework.stereotype.Service; 11 | 12 | import jakarta.annotation.Resource; 13 | import java.util.Map; 14 | 15 | /** 16 | * @Description: abstract task service, every taskService should inherit it 17 | * 18 | * @Date: 2022/6/28 19 | */ 20 | @Service 21 | public abstract class AbstractTaskService implements ITaskService{ 22 | 23 | @Value("${spring.redis.host}") 24 | String host; 25 | @Value("${spring.datasource.url}") 26 | String dbUrl; 27 | @Value("${spring.datasource.username}") 28 | String dbName; 29 | @Value("${spring.datasource.password}") 30 | String dbPwd; 31 | @Value("${spring.redis.port}") 32 | int port; 33 | @Value("${spring.redis.password}") 34 | String password; 35 | @Value("${grpc.proxy.host}") 36 | String proxyHost; 37 | @Value("${grpc.proxy.port}") 38 | String proxyPort; 39 | @Value("${grpc.proxy.local-port}") 40 | String proxyLocalPort; 41 | @Value("${target}") 42 | String localTarget; 43 | @Value("${mount.data.path}") 44 | String mountDataPath; 45 | @Value("${k8s.namespace}") 46 | String nameSpace; 47 | @Resource 48 | FileService fileService; 49 | @Resource 50 | TaskSupport taskSupport; 51 | @Value("${node.ip}") 52 | String nodeIp; 53 | @Value("${node.port}") 54 | String nodePort; 55 | @Value("${k8s.name.prefix}") 56 | String k8sPrefix; 57 | @Value("${portal.url}") 58 | String portalUrl; 59 | 60 | @Override 61 | public boolean match(PreJob preJob) { 62 | return false; 63 | } 64 | 65 | @Override 66 | public Map createTaskMap(PreJob preJob) { 67 | throw new UnsupportedOperationException(); 68 | } 69 | 70 | @Override 71 | public Job compile(PreJob preJob) { 72 | throw new UnsupportedOperationException(); 73 | } 74 | 75 | public OfflineTask initOriTask(PreJob preJob) { 76 | OfflineTask oriTask = preJob.getTasks().get(0); 77 | oriTask.setId(preJob.getId()); 78 | oriTask.setSubId(0); 79 | oriTask.setTaskIndex(0); 80 | oriTask.setStatus(TaskStatusEnum.NEW.getStatus()); 81 | /** k8s环境参数配置 */ 82 | oriTask.setRedis_server(host + ":" + port); 83 | oriTask.setRedis_password(password); 84 | oriTask.setProxy_remote(proxyHost + ":" + proxyPort); 85 | /** 算子侧参数配置 */ 86 | this.assembleParamMap(oriTask.getParameters()); 87 | return oriTask; 88 | } 89 | 90 | private void assembleParamMap(Map map) { 91 | map.put("redis-host", host); 92 | map.put("redis-port", String.valueOf(port)); 93 | map.put("redis-pwd", password); 94 | map.put("redis-server", host + ":" + port); 95 | map.put("redis-password", password); 96 | map.put("proxy-remote", proxyHost + ":" + proxyPort); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/task/FileSvcTaskService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.task; 2 | 3 | import cn.hutool.crypto.digest.DigestUtil; 4 | import com.google.common.collect.Maps; 5 | import com.jd.mpc.common.constant.DeploymentPathConstant; 6 | import com.jd.mpc.common.enums.TaskStatusEnum; 7 | import com.jd.mpc.common.enums.TaskTypeEnum; 8 | import com.jd.mpc.domain.offline.commons.Job; 9 | import com.jd.mpc.domain.offline.commons.OfflineTask; 10 | import com.jd.mpc.domain.offline.commons.PreJob; 11 | import com.jd.mpc.domain.offline.commons.SubTask; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.springframework.beans.BeanUtils; 14 | import org.springframework.stereotype.Service; 15 | 16 | import java.util.Collections; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.stream.Collectors; 21 | 22 | /** 23 | * @Description: create file-service 24 | * 25 | * @Date: 2022/9/22 26 | */ 27 | @Slf4j 28 | @Service 29 | public class FileSvcTaskService extends AbstractTaskService{ 30 | 31 | @Override 32 | public boolean match(PreJob preJob) { 33 | return TaskTypeEnum.FILE_SERVICE.equals(TaskTypeEnum.getByValue(preJob.getType())); 34 | } 35 | 36 | @Override 37 | public Map createTaskMap(PreJob preJob) { 38 | Map map = new HashMap<>(); 39 | Map> listMap = preJob.getTasks().stream() 40 | .collect(Collectors.groupingBy(OfflineTask::getTarget)); 41 | listMap.forEach((target, offlineTasks) -> { 42 | PreJob job = new PreJob(); 43 | BeanUtils.copyProperties(preJob,job); 44 | job.setTasks(offlineTasks); 45 | map.put(target, job); 46 | }); 47 | return map; 48 | } 49 | 50 | @Override 51 | public Job compile(PreJob preJob) { 52 | // 默认使用第一个解析,填充公共值 53 | OfflineTask oriTask = this.initOriTask(preJob); 54 | oriTask.setDeploymentPath(DeploymentPathConstant.FILE_SERVICE); 55 | oriTask.setTaskIndex(0); 56 | String name = k8sPrefix + "-" + oriTask.getTaskType() + "-"+ DigestUtil.md5Hex16(oriTask.getParameters().get("bdpAccount")); 57 | oriTask.setName(name); 58 | Map extParam = Maps.newHashMap(); 59 | extParam.put("INTERACTIVE_URL",portalUrl); 60 | extParam.put("INTERACTIVE_PROJECT_ID",oriTask.getParameters().get("projectId")); 61 | extParam.put("BDPACCOUNT",oriTask.getParameters().get("bdpAccount")); 62 | oriTask.setExtParameters(extParam); 63 | // 填充subTask 64 | SubTask subTask = SubTask.builder().id(preJob.getId()).subId(0) 65 | .status(TaskStatusEnum.NEW.getStatus()).tasks(Collections.singletonList(oriTask)).build(); 66 | // 填充job 67 | return Job.builder().id(preJob.getId()).type(preJob.getType()) 68 | .subTaskList(Collections.singletonList(subTask)).build(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/task/ITaskService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.task; 2 | 3 | import com.jd.mpc.domain.offline.commons.Job; 4 | import com.jd.mpc.domain.offline.commons.PreJob; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * @Description: taskService interface 10 | * 11 | * @Date: 2022/6/28 12 | */ 13 | public interface ITaskService { 14 | 15 | /** 16 | * match request's Prejob 17 | * @param preJob 18 | * @return 19 | */ 20 | boolean match(PreJob preJob); 21 | 22 | /** 23 | * create multi-party PreJob 24 | * @return 25 | */ 26 | Map createTaskMap(PreJob preJob); 27 | 28 | /** 29 | * compile local-party PreJob 30 | * @param preJob 31 | * @return 32 | */ 33 | Job compile(PreJob preJob); 34 | } 35 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/BuffaloZeebeService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe; 2 | 3 | import cn.hutool.core.date.DateUtil; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.jd.mpc.common.constant.DeploymentPathConstant; 6 | import com.jd.mpc.common.enums.TaskTypeEnum; 7 | import com.jd.mpc.common.util.CommonUtils; 8 | import com.jd.mpc.domain.offline.commons.OfflineTask; 9 | import io.camunda.zeebe.client.api.response.ActivatedJob; 10 | import io.camunda.zeebe.client.api.worker.JobClient; 11 | import org.springframework.stereotype.Service; 12 | 13 | import java.util.Collections; 14 | import java.util.Date; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | /** 19 | * @Description: buffalo 20 | * 21 | * @Date: 2022/10/17 22 | */ 23 | @Service 24 | public class BuffaloZeebeService extends AbstractZeebeService{ 25 | 26 | @Override 27 | public boolean match(TaskTypeEnum taskType) { 28 | return TaskTypeEnum.BUFFALO_WORKER.equals(taskType); 29 | } 30 | 31 | /** 32 | * @param client 33 | * @param job 34 | * @return 35 | */ 36 | @Override 37 | public List compile(JobClient client, ActivatedJob job) { 38 | //init 39 | OfflineTask offlineTask = this.initTask(TaskTypeEnum.BUFFALO_WORKER,DeploymentPathConstant.BUFFALO_WORKER, job.getVariablesAsMap()); 40 | //param 41 | Map param = offlineTask.getParameters(); 42 | Map varMap = job.getVariablesAsMap(); 43 | 44 | param.put("taskId",String.valueOf(varMap.get("InputVariable_task_id"))); 45 | param.put("appId",String.valueOf(varMap.get("InputVariable_app_id"))); 46 | param.put("userToken",String.valueOf(varMap.get("InputVariable_user_token"))); 47 | param.put("appToken",String.valueOf(varMap.get("InputVariable_app_token"))); 48 | param.put("scheTime", DateUtil.formatDateTime(new Date())); 49 | String args = JSONObject.toJSONString(new JSONObject()); 50 | if (varMap.containsKey("InputVariable_args") && varMap.get("InputVariable_args") != null){ 51 | args = JSONObject.toJSONString(varMap.get("InputVariable_args")); 52 | } 53 | param.put("args", args); 54 | //默认超时3天 55 | param.put("timeout", String.valueOf(CommonUtils.getPositiveIntegerOrDefault(varMap,"InputVariable_timeout",259200))); 56 | param.put("statusServer",nodeIp+":"+nodePort); 57 | 58 | return Collections.singletonList(offlineTask); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/FileTransferZeebeService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe; 2 | 3 | import com.jd.mpc.common.constant.DeploymentPathConstant; 4 | import com.jd.mpc.common.enums.TaskTypeEnum; 5 | import com.jd.mpc.common.util.CommonUtils; 6 | import com.jd.mpc.domain.offline.commons.OfflineTask; 7 | import io.camunda.zeebe.client.api.response.ActivatedJob; 8 | import io.camunda.zeebe.client.api.worker.JobClient; 9 | import org.springframework.stereotype.Service; 10 | 11 | import java.util.Collections; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /** 16 | * @Description: file-transfer 17 | * 18 | * @Date: 2022/10/18 19 | */ 20 | @Service 21 | public class FileTransferZeebeService extends AbstractZeebeService { 22 | 23 | @Override 24 | public boolean match(TaskTypeEnum taskType) { 25 | return TaskTypeEnum.FILE_TRANSFER.equals(taskType); 26 | } 27 | 28 | @Override 29 | public List compile(JobClient client, ActivatedJob job) { 30 | // init 31 | OfflineTask offlineTask = this.initTask(TaskTypeEnum.FILE_TRANSFER,DeploymentPathConstant.FILE_TRANSFER, job.getVariablesAsMap()); 32 | //param 33 | Map param = offlineTask.getParameters(); 34 | Map varMap = job.getVariablesAsMap(); 35 | 36 | String clusterId = offlineTask.getId() + "-" + offlineTask.getSubId() + "-" 37 | + offlineTask.getTaskIndex(); 38 | param.put("application-id", clusterId); 39 | param.put("status-server", nodeIp + ":" + nodePort); 40 | param.put("domain", localTarget); 41 | param.put("local-listen", "0.0.0.0:20000"); 42 | param.put("target", String.valueOf(varMap.get("InputVariable_target"))); 43 | if (CommonUtils.getStringOrDefault(varMap,"InputVariable_mode",null) != null){ 44 | param.put("mode", String.valueOf(varMap.get("InputVariable_mode"))); 45 | } 46 | param.put("path", String.valueOf(varMap.get("InputVariable_path"))); 47 | 48 | return Collections.singletonList(offlineTask); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/IZeebeService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe; 2 | 3 | import com.jd.mpc.common.enums.TaskTypeEnum; 4 | import com.jd.mpc.domain.offline.commons.OfflineTask; 5 | import io.camunda.zeebe.client.api.response.ActivatedJob; 6 | import io.camunda.zeebe.client.api.worker.JobClient; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @Description: interface 12 | * 13 | * @Date: 2022/10/17 14 | */ 15 | public interface IZeebeService { 16 | 17 | /** 18 | * match taskType 19 | * @param taskType 20 | * @return 21 | */ 22 | boolean match(TaskTypeEnum taskType); 23 | 24 | /** 25 | * compile task 26 | * @param client 27 | * @param job 28 | * @return 29 | */ 30 | List compile(final JobClient client, final ActivatedJob job); 31 | } 32 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/PsiZeebeService.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe; 2 | 3 | import com.google.common.collect.Maps; 4 | import com.jd.mpc.common.constant.DeploymentPathConstant; 5 | import com.jd.mpc.common.enums.TaskTypeEnum; 6 | import com.jd.mpc.common.util.CommonUtils; 7 | import com.jd.mpc.domain.offline.commons.OfflineTask; 8 | import io.camunda.zeebe.client.api.response.ActivatedJob; 9 | import io.camunda.zeebe.client.api.worker.JobClient; 10 | import org.springframework.stereotype.Service; 11 | 12 | import java.util.Collections; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | /** 17 | * @Description: psi 18 | * 19 | * @Date: 2022/10/17 20 | */ 21 | @Service 22 | public class PsiZeebeService extends AbstractZeebeService{ 23 | 24 | @Override 25 | public boolean match(TaskTypeEnum taskType) { 26 | return TaskTypeEnum.PSI.equals(taskType); 27 | } 28 | 29 | @Override 30 | public List compile(JobClient client, ActivatedJob job) { 31 | //init 32 | OfflineTask offlineTask = this.initTask(TaskTypeEnum.PSI,DeploymentPathConstant.PSI, job.getVariablesAsMap()); 33 | //param 34 | Map param = offlineTask.getParameters(); 35 | Map varMap = job.getVariablesAsMap(); 36 | 37 | param.put("output",(String) varMap.get("InputVariable_output_path")); 38 | param.put("input-col",(String) varMap.get("InputVariable_input_col")); 39 | param.put("input",(String) varMap.get("InputVariable_input_path")); 40 | param.put("target",(String) varMap.get("InputVariable_participants")); 41 | param.put("send-back",String.valueOf(CommonUtils.getBooleanOrDefault(varMap,"InputVariable_send_back",false))); 42 | param.put("status-server", nodeIp + ":" + nodePort); 43 | String clusterId = offlineTask.getId() + "-" + offlineTask.getSubId() + "-" 44 | + offlineTask.getTaskIndex(); 45 | param.put("cluster-id", clusterId); 46 | param.put("party-id", localTarget); 47 | if (varMap.containsKey("InputVariable_extra_configs") && varMap.get("InputVariable_extra_configs") != null){ 48 | Map extraMap =(Map) varMap.get("InputVariable_extra_configs"); 49 | for (Map.Entry entry : extraMap.entrySet()) { 50 | param.put(entry.getKey().replaceAll("_","-"),String.valueOf(entry.getValue())); 51 | } 52 | } 53 | 54 | return Collections.singletonList(offlineTask); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/ZeebeGateWay.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe; 2 | 3 | import com.jd.mpc.common.enums.TaskTypeEnum; 4 | import io.camunda.zeebe.client.api.response.ActivatedJob; 5 | import io.camunda.zeebe.client.api.worker.JobClient; 6 | import io.camunda.zeebe.spring.client.annotation.ZeebeWorker; 7 | import lombok.extern.slf4j.Slf4j; 8 | import org.springframework.stereotype.Service; 9 | 10 | import jakarta.annotation.Resource; 11 | 12 | /** 13 | * @Description: 融合分析算子化,类似与mvc中的controller层 14 | * ZeebeGateWay->ZeebeDispatcher->IZeebeService->***ZeebeService 15 | * 16 | * @Date: 2022/10/13 17 | */ 18 | @Slf4j 19 | @Service 20 | public class ZeebeGateWay { 21 | 22 | // @Resource 23 | // private ZeebeDispatcher dispatcher; 24 | // 25 | // /** 26 | // * - `InputVariable_id`: 任务ID 27 | // * - `InputVariable_input_path`：输入文件路径 28 | // * - `InputVariable_input_col`：求交列设置 29 | // * - `InputVariable_output_path` ：输出文件路径 30 | // * - `InputVariable_send_back` ：非leader是否可以得到输出结果 31 | // * - `InputVariable_participants` ：参与方列表，各方需要完全一致，第一个是leader 32 | // * - `InputVariable_extra_configs` ：其他可选输入参数 33 | // * @param client 34 | // * @param job 35 | // */ 36 | // @ZeebeWorker(type ="${target};psi") 37 | // public void handlePSI(final JobClient client, final ActivatedJob job) { 38 | // dispatcher.doDispatch(client,job,TaskTypeEnum.PSI); 39 | // } 40 | // 41 | // /** 42 | // * - `InputVariable_id` ：任务ID 43 | // * - `InputVariable_script_path`：任务脚本路径 44 | // * - `InputVariable_args`：任务脚本参数 45 | // * - `InputVariable_interpreter`：任务解释器 46 | // * @param client 47 | // * @param job 48 | // */ 49 | // @ZeebeWorker(type ="${target};local-worker") 50 | // public void handleLocalWorker(final JobClient client, final ActivatedJob job) { 51 | // dispatcher.doDispatch(client,job,TaskTypeEnum.LOCAL_WORKER); 52 | // } 53 | // 54 | // /** 55 | // * - `InputVariable_id` ：任务ID 56 | // * - `InputVariable_target`：对侧Domain名称 57 | // * - `InputVariable_mode`：收发模式[sender/receiver] 58 | // * - `InputVariable_path`：发送或接收文件路径 59 | // * @param client 60 | // * @param job 61 | // */ 62 | // @ZeebeWorker(type ="${target};transfer") 63 | // public void handleTransfer(final JobClient client, final ActivatedJob job) { 64 | // dispatcher.doDispatch(client,job,TaskTypeEnum.FILE_TRANSFER); 65 | // } 66 | // 67 | // /** 68 | // * call buffalo 69 | // * @param client 70 | // * @param job 71 | // */ 72 | // @ZeebeWorker(type = "${target};buffalo") 73 | // public void handleBuffalo(final JobClient client, final ActivatedJob job){ 74 | // dispatcher.doDispatch(client,job,TaskTypeEnum.BUFFALO_WORKER); 75 | // } 76 | // 77 | // /** 78 | // * - `InputVariable_runID`: process runID 79 | // * @param client 80 | // * @param job 81 | // */ 82 | // @ZeebeWorker(type = "${target};error-cleaner") 83 | // public void handleErrorCleaner(final JobClient client, final ActivatedJob job){ 84 | // dispatcher.handleErrorCleaner(client,job); 85 | // } 86 | // 87 | // /** 88 | // * send http request 89 | // * @param client 90 | // * @param job 91 | // */ 92 | // @ZeebeWorker(type = "${target};http-json-client") 93 | // public void handleHttpCall(final JobClient client, final ActivatedJob job){ 94 | // dispatcher.handleHttpCall(client,job); 95 | // } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/Zeebes.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe; 2 | 3 | import cn.hutool.core.lang.UUID; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.google.common.collect.Maps; 6 | import io.camunda.zeebe.client.ZeebeClient; 7 | import io.camunda.zeebe.client.api.response.ProcessInstanceEvent; 8 | import org.springframework.beans.factory.annotation.Value; 9 | import org.springframework.stereotype.Service; 10 | 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /** 16 | * @Description: zeebe 工具类 17 | * 18 | * @Date: 2022/10/17 19 | */ 20 | @Service 21 | public class Zeebes { 22 | 23 | private ZeebeClient client; 24 | @Value("${target}") 25 | private String target; 26 | 27 | public void sendDoneMsg(String taskId,String result,String msg) { 28 | HashMap map = Maps.newHashMap(); 29 | map.put("OutputVariable_result", JSONObject.parseObject(result)); 30 | map.put("OutputVariable_message",msg); 31 | String msgName = target; 32 | client.newPublishMessageCommand() 33 | .messageName(msgName) 34 | .correlationKey(taskId) 35 | .messageId(UUID.randomUUID().toString()) 36 | .variables(map) 37 | .send() 38 | .join(); 39 | } 40 | 41 | public void sendErrorMsg(String runID,String result,String msg){ 42 | HashMap map = Maps.newHashMap(); 43 | map.put("OutputVariable_result",JSONObject.parseObject(result)); 44 | map.put("OutputVariable_message",msg); 45 | client.newPublishMessageCommand() 46 | .messageName("Message_global_error") 47 | .correlationKey(runID) 48 | .messageId(UUID.randomUUID().toString()) 49 | .variables(map) 50 | .send() 51 | .join(); 52 | } 53 | 54 | public void sendResultMsg(String processID, String instanceID, String msgID, List data){ 55 | HashMap map = Maps.newHashMap(); 56 | map.put("OutputVariable_data",data); 57 | client.newPublishMessageCommand() 58 | .messageName("Message_"+processID+"_"+msgID) 59 | .correlationKey(instanceID) 60 | .messageId(UUID.randomUUID().toString()) 61 | .variables(map) 62 | .send() 63 | .join(); 64 | } 65 | 66 | public ProcessInstanceEvent createInstance(String processID,Integer version, Map map){ 67 | if (version != null){ 68 | return client.newCreateInstanceCommand() 69 | .bpmnProcessId(processID) 70 | .version(version) 71 | .variables(map) 72 | .send() 73 | .join(); 74 | } 75 | return client.newCreateInstanceCommand() 76 | .bpmnProcessId(processID) 77 | .latestVersion() 78 | .variables(map) 79 | .send() 80 | .join(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/service/zeebe/domain/param/ProcessResultParam.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.service.zeebe.domain.param; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @Description: 上报算子化结果 9 | * @Author: feiguodong 10 | * @Date: 2022/11/14 11 | */ 12 | @Data 13 | public class ProcessResultParam { 14 | private String processID; 15 | private String instanceID; 16 | private String msgID; 17 | /** 结果集合 */ 18 | private List data; 19 | } 20 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/java/com/jd/mpc/web/AuthController.java: -------------------------------------------------------------------------------- 1 | package com.jd.mpc.web; 2 | 3 | import cn.hutool.core.codec.Base64; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import com.jd.mpc.common.response.CommonResponse; 7 | import com.jd.mpc.common.util.JNISigner; 8 | import com.jd.mpc.domain.param.WorkerInfoParam; 9 | import com.jd.mpc.domain.vo.AuthInfoDto; 10 | import com.jd.mpc.domain.vo.VerifyVo; 11 | import com.jd.mpc.grpc.GrpcSignClient; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.springframework.web.bind.annotation.PostMapping; 14 | import org.springframework.web.bind.annotation.RequestBody; 15 | import org.springframework.web.bind.annotation.RequestMapping; 16 | import org.springframework.web.bind.annotation.RestController; 17 | import authprotocol.GrpcVo; 18 | import authprotocol.VerifyGrpcVo; 19 | 20 | import jakarta.annotation.Resource; 21 | 22 | /** 23 | * @Description: auth 24 | * 25 | * @Date: 2022/10/12 26 | */ 27 | @Slf4j 28 | @RestController 29 | @RequestMapping("auth") 30 | public class AuthController { 31 | 32 | private AuthInfoDto authInfoDto; 33 | @Resource 34 | private GrpcSignClient signClient; 35 | /** 36 | * 验签 37 | * @param workerInfo 38 | * @return 39 | */ 40 | @PostMapping("verify") 41 | public CommonResponse verify(@RequestBody WorkerInfoParam workerInfo) throws InvalidProtocolBufferException { 42 | //1.sign 43 | String dataStr = JSONObject.toJSONString(workerInfo); 44 | byte[] sig = JNISigner.sign(authInfoDto.getPriKey(), dataStr.getBytes()); 45 | GrpcVo grpcVo = signClient.verify(authInfoDto.getCert(), Base64.encode(sig), dataStr); 46 | VerifyVo verifyVo = new VerifyVo(); 47 | if (grpcVo.getStatus() == 0){ 48 | VerifyGrpcVo verifyGrpcVo = VerifyGrpcVo.parseFrom(grpcVo.getResult()); 49 | verifyVo.setCert(verifyGrpcVo.getCert()); 50 | verifyVo.setData(verifyGrpcVo.getData()); 51 | verifyVo.setSig(verifyGrpcVo.getSig()); 52 | return CommonResponse.ok(verifyVo); 53 | }else { 54 | return CommonResponse.fail(grpcVo.getStatus(),grpcVo.getErrMsg()); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/api.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pb; 4 | option java_multiple_files = true; 5 | option java_package = "predict"; 6 | 7 | message FeatureValueProto { 8 | message StringList { 9 | repeated string value = 1; 10 | } 11 | message DoubleList { 12 | repeated double value = 1; 13 | } 14 | message FloatList { 15 | repeated float value = 1; 16 | } 17 | message IntList { 18 | repeated int64 value = 1; 19 | } 20 | message ByteList { 21 | repeated bytes value = 1; 22 | } 23 | oneof value { 24 | string s_value = 1; 25 | double d_value = 2; 26 | int64 i_value = 3; 27 | bytes b_value = 4; 28 | StringList s_list = 5; 29 | DoubleList d_list = 6; 30 | IntList i_list = 7; 31 | ByteList b_list = 8; 32 | FloatList f_list = 9; 33 | } 34 | } 35 | 36 | message ValueMapProto { 37 | bytes id = 1; 38 | map value_map = 2; 39 | } 40 | 41 | message FeaturesListProto { 42 | repeated ValueMapProto FeaturesList = 1; 43 | } 44 | 45 | message Request { 46 | FeaturesListProto examples = 1; 47 | } 48 | 49 | // data type of of tensor 50 | enum DataType { 51 | DT_INVALID = 0; 52 | DT_DOUBLE = 1; 53 | DT_INT32 = 2; 54 | DT_STRING = 3; 55 | DT_INT64 = 4; 56 | }; 57 | 58 | // shape of a tensor 59 | message ShapeProto { 60 | message Dim { 61 | int64 size = 1; 62 | string name = 2; // Optional name of the tensor dimension. 63 | }; 64 | 65 | repeated Dim dim = 2; 66 | }; 67 | 68 | // representing a matrix tensor. 69 | message Matrix { 70 | //DataType dtype = 1; 71 | 72 | // Shape of the tensor. TODO(touts): sort out the 0-rank issues. 73 | //ShapeProto shape = 2; 74 | 75 | // DT_DOUBLE. 76 | repeated double double_val = 3 [packed = true]; 77 | 78 | // DT_INT32, DT_INT16, DT_INT8, DT_UINT8. 79 | //repeated int32 int_val = 4 [packed = true]; 80 | 81 | // DT_STRING. 82 | //repeated string str_val = 5; 83 | 84 | // DT_INT64. 85 | //repeated int64 int64_val = 6 [packed = true]; 86 | 87 | bytes id = 7; 88 | }; 89 | 90 | message Response { 91 | repeated bytes failed_examples = 1; 92 | repeated Matrix success_matrix = 2; 93 | } 94 | 95 | service ApiService { 96 | rpc Predict(Request) returns (Response); 97 | } -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/ast.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package app; 4 | 5 | // option go_package = "coding.jd.com/mpc/jasmine/models/pb"; 6 | option java_generic_services = true; 7 | option java_multiple_files = true; 8 | 9 | enum Operator { 10 | NOP = 0; 11 | NewInt = 1; 12 | NewBinary = 2; 13 | NewIntArray = 3; 14 | NewBinaryArray = 4; 15 | NewPackedBinary = 5; 16 | NewIntMatrix = 6; 17 | NewFloat = 7; 18 | Destroy = 9; 19 | 20 | LoadTxt = 10; 21 | LoadMySQL = 11; 22 | LoadCsv = 12; 23 | LoadStdinCsv = 13; 24 | LoadStdinOrc = 14; 25 | LoadOrc = 15; 26 | 27 | Share = 20; 28 | Move = 29; 29 | Reveal = 30; 30 | 31 | StoreTxt = 40; 32 | StoreMySQL = 41; 33 | StoreCsv = 42; 34 | StoreStdinCsv = 43; 35 | StoreStdinOrc = 44; 36 | StoreOrc = 45; 37 | 38 | ExportInt = 50; 39 | ExportIntArray = 51; 40 | ExportIntMatrix = 52; 41 | ExportSharedInt = 53; 42 | ExportSharedIntArray = 54; 43 | 44 | FilterCsv = 60; 45 | 46 | NewRandomInt = 70; 47 | NewRandomIntMatrix = 76; 48 | 49 | NewSharedInt = 80; 50 | NewSharedIntArray = 83; 51 | 52 | ConvertToInt = 90; 53 | ConvertToIntArray = 91; 54 | 55 | And = 100; 56 | Or = 101; 57 | Add = 102; 58 | Sub = 103; 59 | Mul = 104; 60 | DotProduct = 105; 61 | ImmSub = 106; 62 | Not = 107; 63 | 64 | MulXT = 110; 65 | MulYT = 111; 66 | MulXYT = 112; 67 | 68 | TestEQ = 120; 69 | TestNE = 121; 70 | TestGT = 122; 71 | TestGE = 123; 72 | TestLT = 124; 73 | TestLE = 125; 74 | } 75 | 76 | message IntMatrix { 77 | int64 Row = 1; 78 | int64 Col = 2; 79 | repeated int64 Values = 3; 80 | } 81 | 82 | message Instruction { 83 | string Src1 = 1; 84 | string Src2 = 2; 85 | Operator Op = 3; 86 | string Des = 4; 87 | int64 Imm = 5; 88 | repeated int64 IntArr = 6; 89 | IntMatrix Mat = 7; 90 | double Float = 8; 91 | } 92 | 93 | message Code { repeated Instruction Ins = 1; } 94 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/authprotocol.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package authprotocol; 4 | option java_multiple_files = true; 5 | option java_package = "authprotocol"; 6 | 7 | service AuthService{ 8 | rpc verify(VerifyGrpcParam)returns (GrpcVo){}; 9 | rpc issueCert(IssueGrpcParam) returns(GrpcVo){}; 10 | } 11 | 12 | message IssueGrpcParam{ 13 | //public key 14 | bytes key = 1; 15 | string commonName = 2; 16 | string organization = 3; 17 | } 18 | 19 | message VerifyGrpcVo{ 20 | string cert = 1; 21 | string sig = 2; 22 | string data = 3; 23 | } 24 | 25 | message VerifyGrpcParam{ 26 | string cert = 1; 27 | string sig = 2; 28 | string data = 3; 29 | } 30 | 31 | message GrpcVo{ 32 | // 是否校验通过 status 0 :成功 非0: 失败 33 | uint32 status = 1; 34 | // 错误信息 35 | string err_msg = 2; 36 | // 序列化信息 37 | bytes result = 3; 38 | } -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/external_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option java_package = "fedlearner.app"; 3 | package fedlearner.common; 4 | 5 | /****start****/ 6 | message ModelTrainMeta { 7 | string model_uri = 1; 8 | string version = 2; 9 | int32 batch_size = 3; 10 | int32 worker_num = 4; 11 | int32 checkpoint_interval = 5; 12 | int32 eval = 6; 13 | } 14 | 15 | message DataMeta { 16 | string data_source_name = 1; 17 | string train_data_start= 2; 18 | string train_data_end = 3; 19 | int32 data_num_epoch = 4; 20 | int32 total_block_num = 5; 21 | int32 total_example_num = 6; 22 | } 23 | message Block_info { 24 | string block_id = 1; 25 | int32 block_example_num = 2;; 26 | } 27 | message CheckJoinDataRequest { 28 | string app_id = 1; 29 | DataMeta data_meta = 2; 30 | repeated Block_info blocks = 3; 31 | } 32 | message TrainRequest { 33 | string app_id = 1; 34 | ModelTrainMeta model_train_mata = 2; 35 | DataMeta data_meta = 3; 36 | } 37 | 38 | message StopTrainRequest { 39 | string app_id = 1; 40 | uint32 type = 2; 41 | string msg = 3; 42 | } 43 | 44 | /****求交例行化 增加原始数据校验请求****/ 45 | message CheckRawDataRequest { 46 | string app_id = 1; 47 | string data_source_name = 2; 48 | string train_data_start= 3; 49 | string train_data_end = 4; 50 | int32 partition_num = 5; 51 | } 52 | /****求交例行化 增加提交求交请求****/ 53 | message SubmitJoinRequest { 54 | string app_id = 1; 55 | string data_source_name = 2; 56 | string train_data_start= 3; 57 | string train_data_end = 4; 58 | int32 partition_num = 5; 59 | float negative_sampling_rate = 6; 60 | } 61 | 62 | /****求交例行化 增加结束求交请求****/ 63 | message StopJoinRequest { 64 | string app_id = 1; 65 | string data_source_name = 2; 66 | string train_data_start= 3; 67 | string train_data_end = 4; 68 | } 69 | 70 | message Status { 71 | uint32 status = 1; 72 | string err_msg = 2; 73 | string app_id = 3; 74 | } 75 | 76 | enum Role { 77 | FOLLOWER = 0; 78 | LEADER = 1; 79 | } 80 | 81 | service Scheduler { 82 | /****求交例行化 增加提交求交请求****/ 83 | rpc SubmitJoin(SubmitJoinRequest) returns (Status); 84 | /****求交例行化 增加结束求交请求****/ 85 | rpc StopJoin(StopJoinRequest) returns (Status); 86 | rpc SubmitTrain(TrainRequest) returns (Status); 87 | rpc StopTrain(StopTrainRequest) returns (Status); 88 | } 89 | 90 | /****dataworker, trainer make pair****/ 91 | service StateSynService { 92 | rpc Syn(AppSynRequest) returns (Status); 93 | } 94 | service CheckJoinedDataService { 95 | /****求交例行化 增加原始数据检查，当指定数据的原始数据不存在或者有缺失时，应当返回失败****/ 96 | rpc CheckRawData(CheckRawDataRequest) returns (Status); 97 | rpc CheckJoinedData(CheckJoinDataRequest) returns (Status); 98 | } 99 | message AppSynRequest { 100 | string app_id = 1; 101 | repeated ServicePair service_pair = 2; 102 | AppCtrlFlag ctrl_flag = 3; 103 | } 104 | 105 | enum AppCtrlFlag { 106 | CREATE = 0; 107 | SHUTDOWN = 1; 108 | FINISH = 2; 109 | } 110 | 111 | message ServicePair { 112 | string leader_uuid = 1; 113 | string follower_uuid = 2; 114 | ServiceCtrlFlag ctrl_flag = 3; 115 | } 116 | 117 | enum ServiceCtrlFlag { 118 | RST = 0; 119 | SYN = 1; 120 | FIN = 2; 121 | } 122 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/feature.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | //package example; 3 | option java_multiple_files = true; 4 | option java_package = "feature"; 5 | 6 | 7 | service FeatureEngineering { //定义服务,用在rpc传输中 8 | rpc getDetail(GrpcDetailRequest) returns (GrpcDetailResponse); 9 | rpc sendEncryptedTarget (stream EncryptedTarget) returns (sendResponse); 10 | rpc sendIvMidValue (stream IvMidValue) returns (sendResponse); 11 | rpc sendPearsonMidValue (stream PearsonMidValue) returns (sendResponse); 12 | rpc sendBytesObject (bytesObject) returns (sendResponse); 13 | } 14 | 15 | 16 | message GrpcDetailRequest { 17 | } 18 | 19 | 20 | message featureList{ 21 | repeated string feature = 1; 22 | } 23 | 24 | message GrpcDetailResponse{ 25 | map pearson_score = 1; 26 | map iv_score = 2; 27 | string msg = 3; 28 | int64 status = 4; 29 | } 30 | 31 | message EncryptedTarget{ 32 | bytes encrypted_target = 1; 33 | string msg = 2; 34 | } 35 | 36 | message sendResponse{ 37 | string msg = 1; 38 | } 39 | 40 | message IvMidValue{ 41 | map sum_target = 1; 42 | string msg = 2; 43 | } 44 | 45 | message PearsonMidValue{ 46 | map mean_and_var = 1; 47 | string msg = 2; 48 | } 49 | 50 | message bytesObject{ 51 | bytes file = 1; 52 | string msg = 2; 53 | } 54 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/internal_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option java_package = "jdfl"; 3 | package jdfl; 4 | 5 | /****internal****/ 6 | enum AppStatus { 7 | CREATE = 0; 8 | SHUTDOWN = 1; 9 | FINISH = 2; 10 | } 11 | 12 | enum Role { 13 | FOLLOWER = 0; 14 | LEADER = 1; 15 | } 16 | 17 | message ConfInfo { 18 | string AppID = 1; 19 | string model_uri = 2; 20 | string version = 3; 21 | int32 worker_num = 4; 22 | int32 datacentor_num = 5; 23 | string data_source_name = 6; 24 | string train_data_start = 7; 25 | string train_data_end = 8; 26 | int32 data_num_epoch = 9; 27 | string git_url = 10; 28 | string branch = 11; 29 | Role role = 12; 30 | AppStatus local_status = 13; 31 | AppStatus remote_status = 14; 32 | string flconf = 15; 33 | string k8s_conf_string = 16; 34 | string idle_timeout = 17; 35 | string erp = 18; 36 | } 37 | 38 | message ServicePair { 39 | string local_uuid = 1; 40 | string remote_uuid = 2; 41 | } 42 | 43 | message PairInfo { 44 | string ip_port = 1; 45 | repeated ServicePair service_pair = 2; 46 | } 47 | 48 | message AppInfo { 49 | ConfInfo conf_info = 1; 50 | repeated PairInfo pair_infos = 2; 51 | } 52 | 53 | /****registered****/ 54 | service PairService { 55 | rpc RegisterUUID(Request) returns (Status); 56 | rpc GetPairInfo(Request) returns (PairInfoResponse); 57 | } 58 | 59 | message Status { 60 | uint32 status = 1; // 0:success, not 0:fail 61 | string err_msg = 2; 62 | } 63 | 64 | message Request { 65 | repeated string uuid = 1; // AppID_Service_index 66 | string ip_port = 2; // ip:port 67 | } 68 | 69 | message PairInfoResponse { 70 | Status status = 1; 71 | repeated ServicePair service_pair = 2; 72 | } 73 | 74 | message JoinResponse { 75 | uint64 project_id = 1; // project id, only valid at local 76 | Status status = 2; 77 | // JoinStatus join_status = 3; // join request result 78 | } 79 | 80 | /****leader start training****/ 81 | service StartApplication { 82 | rpc StartApplication(ModelURI) returns (Status); 83 | } 84 | message ModelURI { 85 | string model_uri = 1; 86 | string version = 2; 87 | } 88 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/online.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option java_multiple_files = true; 4 | option java_package = "online"; 5 | 6 | /** 7 | * 离线任务grpc服务定义 8 | */ 9 | service OnlineService { 10 | 11 | /** 12 | * 同步实时任务 13 | */ 14 | rpc syncTask (GrpcOnlineRequest) returns (GrpcOnlineResponse) { 15 | 16 | } 17 | 18 | /** 19 | * 同步任务状态 20 | */ 21 | rpc syncStatus (GrpcOnlineRequest) returns (GrpcOnlineResponse) { 22 | } 23 | 24 | /** 25 | * 测试 26 | */ 27 | rpc test (GrpcOnlineRequest) returns (GrpcOnlineResponse) { 28 | } 29 | 30 | } 31 | 32 | //定义消息类型 33 | message GrpcOnlineTask{ 34 | string parentId = 1; 35 | int32 taskIndex = 2; 36 | int32 status = 3; 37 | string type = 4; 38 | int32 timeout = 5; 39 | string url = 6; 40 | string method = 7; 41 | string header = 8; 42 | string parameters = 9; 43 | string redisServer = 10; 44 | string redisPassword = 11; 45 | string redisKey = 12; 46 | } 47 | 48 | message GrpcOnlineRequest { 49 | string test = 1; 50 | GrpcOnlineTask grpcOnlineTask = 2; 51 | string active = 3; 52 | } 53 | 54 | 55 | message GrpcOnlineResponse { 56 | bool isSucceed = 1; 57 | string test = 2; 58 | } 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/proto/table.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package database; 4 | 5 | //option go_package = "coding.jd.com/mpc/jasmine/models/pb"; 6 | option java_multiple_files = true; 7 | option java_package = "online"; 8 | 9 | enum ColumnType { 10 | Int = 0; 11 | Binary64 = 1; 12 | Float = 2; 13 | Binary = 3; 14 | String = 4; 15 | Blake = 5; 16 | ClearInt = 6; 17 | ClearBinary64 = 7; 18 | ClearFloat = 8; 19 | ClearBlake = 9; 20 | ClearBinary = 10; 21 | Valid = 11; 22 | PackedBinary = 12; 23 | ClearBool = 13; 24 | Bool = 14; 25 | } 26 | 27 | message Column { 28 | string Name = 1; 29 | ColumnType Type = 2; 30 | bytes Data = 3; 31 | } 32 | 33 | message Table { 34 | int64 Rows = 1; 35 | repeated Column Columns = 2; 36 | } 37 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/src/Coordinator/src/main/resources/application.properties -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/bootstrap.properties: -------------------------------------------------------------------------------- 1 | #spring.cloud.nacos.config.server-addr=11.136.250.17:32030 2 | #spring.cloud.nacos.config.namespace=public 3 | #spring.cloud.nacos.config.extension-configs[0].data-id=common 4 | #spring.cloud.nacos.config.extension-configs[0].refresh=true 5 | #spring.cloud.nacos.config.extension-configs[1].data-id=coordinator 6 | #spring.cloud.nacos.config.extension-configs[1].refresh=true 7 | #spring.cloud.nacos.config.extension-configs[2].data-id=coordinator-sync 8 | #spring.cloud.nacos.config.extension-configs[2].refresh=true -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /mnt/logs/json/coordinators.log 5 | 6 | /mnt/logs/json/coordinators.%d{yyyy-MM-dd}.log 7 | 3kB 8 | 9 | 10 | 12 | 13 | 14 | 15 | { 16 | "time": "%date{yyyy-MM-dd'T'HH:mm:ss.SSSZ}", 17 | "level": "%level", 18 | "msg": "%thread %msg", 19 | "platform_id": "9n-mpc", 20 | "task_id": "coordinator" 21 | } 22 | 23 | 24 | 25 | 26 | 27 | WARN 28 | ACCEPT 29 | NEUTRAL 30 | 31 | 32 | ERROR 33 | ACCEPT 34 | DENY 35 | 36 | 37 | 38 | 39 | 40 | 41 | %d{MM/dd/yyyy HH:mm:ss} %-5level [%thread%X{sourceThread}]%logger{24} - %msg%n 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/mapper/AuthInfoMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 8 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/mapper/CertMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/mapper/JobTaskStubMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/Coordinator/src/main/resources/nacos-application.properties: -------------------------------------------------------------------------------- 1 | target=9n_demo_1 2 | server.port=8080 3 | 4 | # k8s set 5 | k8s.namespace=jcloud-jd 6 | k8s.name.prefix=pk 7 | 8 | mount.data.path=/mnt/data 9 | train.server.port=8083 10 | mail.url= 11 | mail.receivers= 12 | 13 | # ext 14 | spring.servlet.multipart.max-file-size=100MB 15 | spring.servlet.multipart.max-request-size=1000MB 16 | grpc.server.maxInboundMessageSize=104857600 17 | grpc.server.maxOutboundMessageSize=104857600 18 | 19 | grpc.regist.port=32026 -------------------------------------------------------------------------------- /src/FileService/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM silverlogic/python3.8:latest 2 | LABEL maintainer="chenghekai1 " 3 | USER root 4 | 5 | RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple python-snappy==0.6.1 \ 6 | flask redis protobuf pyarrow && \ 7 | mkdir /app && \ 8 | mkdir -p /mnt/transfile 9 | 10 | COPY /app/redis_client.py /app/redis_client.py 11 | COPY start.sh /tmp/start.sh 12 | COPY /app/file_service.py /app/file_service.py 13 | 14 | ENTRYPOINT [ "sh","/tmp/start.sh" ] 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/FileService/README.md: -------------------------------------------------------------------------------- 1 | 文件服务 2 | 启动自行注册 3 | 提供文件行列数查询 4 | 提供创建文件夹功能 5 | 提供文件夹的文件列表功能 6 | 提供文件的表头读取功能 7 | 8 | 镜像构建：直接使用9nm-mpc/src/FileService路径下Dockerfile即可 9 | 10 | -------------------------------------------------------------------------------- /src/FileService/fileservice.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: file-service 6 | name: file-service 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: file-service 12 | template: 13 | metadata: 14 | labels: 15 | app: file-service 16 | spec: 17 | imagePullSecrets: 18 | - name: jd-cloud-secret 19 | containers: 20 | - image: 21 | imagePullPolicy: Always 22 | name: file-service 23 | ports: 24 | - containerPort: 8800 25 | name: http 26 | protocol: TCP 27 | resources: 28 | limits: 29 | cpu: "1" 30 | memory: 4Gi 31 | requests: 32 | cpu: "1" 33 | memory: 4Gi 34 | terminationMessagePath: /dev/termination-log 35 | terminationMessagePolicy: File 36 | volumeMounts: 37 | - mountPath: /mnt/data 38 | name: data 39 | - mountPath: /mnt/logs 40 | name: logs 41 | dnsPolicy: ClusterFirst 42 | restartPolicy: Always 43 | volumes: 44 | - name: data 45 | hostPath: 46 | path: /mnt/cfs/mpc-test/test 47 | type: DirectoryOrCreate 48 | - name: logs 49 | hostPath: 50 | path: /mnt/cfs/mpc-test/test 51 | type: DirectoryOrCreate 52 | tolerations: 53 | - key: "cpu-server" 54 | operator: "Equal" 55 | value: "pre1" 56 | effect: "NoSchedule" 57 | nodeSelector: 58 | mpc: pre1 59 | -------------------------------------------------------------------------------- /src/FileService/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /app && python /app/file_service.py && sleep 99999999d -------------------------------------------------------------------------------- /src/PSI/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM buildpack-deps:22.04 16 | 17 | RUN set -ex && \ 18 | sed -i 's/archive.ubuntu.com/mirrors.jd.com/g' /etc/apt/sources.list && \ 19 | sed -i 's/security.ubuntu.com/mirrors.jd.com/g' /etc/apt/sources.list && \ 20 | apt-get update && \ 21 | apt-get upgrade -y --no-install-recommends && \ 22 | apt-get install -y --no-install-recommends \ 23 | cmake \ 24 | python3 \ 25 | bison \ 26 | python3-dev \ 27 | python3-pip \ 28 | ninja-build \ 29 | flex && \ 30 | rm -rf /var/lib/apt/lists/* 31 | 32 | RUN set -ex && \ 33 | wget https://github.com/bazelbuild/bazel/releases/download/5.1.1/bazel-5.1.1-installer-linux-x86_64.sh && \ 34 | bash bazel-5.1.1-installer-linux-x86_64.sh 35 | 36 | RUN set -ex && \ 37 | pip3 install maturin && \ 38 | pip3 install ray && \ 39 | pip3 install pyarrow && \ 40 | pip3 install grpcio-tools && \ 41 | pip3 install pyOpenSSL && \ 42 | pip3 install pandas==2.1.4 && \ 43 | pip3 install redis 44 | 45 | RUN set -ex && \ 46 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >rustup.sh && \ 47 | chmod +x rustup.sh && \ 48 | ./rustup.sh -y && \ 49 | cp ~/.cargo/bin/* /usr/local/bin/ 50 | 51 | COPY crypto /App/crypto 52 | WORKDIR /App/crypto 53 | RUN maturin build --release && \ 54 | pip3 install --force-reinstall target/wheels/crypto-0.1.0-cp310-cp310-manylinux_2_34_x86_64.whl 55 | 56 | COPY link_py /App/link_py 57 | WORKDIR /App/link_py 58 | RUN bazel build wheel --compilation_mode=opt --host_compilation_mode=opt && \ 59 | pip3 install --force-reinstall bazel-bin/link_py-v0.1-py3-none-any.whl 60 | 61 | COPY psi /App/psi 62 | WORKDIR /App/psi 63 | RUN python3 setup.py bdist_wheel && \ 64 | pip3 install --force-reinstall dist/interconnection_psi-0.0.0-py3-none-any.whl 65 | 66 | COPY psi_actors.py /App/psi_actors.py 67 | COPY start.sh /App/start.sh 68 | 69 | WORKDIR /App/ 70 | 71 | RUN chmod 777 ./start.sh 72 | 73 | ENV RAY_DEDUP_LOGS=0 74 | ENV RAY_OBJECT_STORE_ALLOW_SLOW_STORAGE=1 75 | 76 | ENTRYPOINT [ "/App/start.sh" ] 77 | -------------------------------------------------------------------------------- /src/PSI/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Interoperable PSI Operator 3 | 4 | Implementation of the Interoperable PSI standard according to the [ECDH-PSI specification](https://www.secretflow.org.cn/static/ECDH-PSI.579428f2.pdf). 5 | 6 | ![PSI Process Diagram](./image/psi.png) 7 | 8 | ## Crypto Library 9 | 10 | The encryption algorithm library. 11 | 12 | ### Build Instructions 13 | 14 | ```bash 15 | maturin build --release && \ 16 | pip3 install --force-reinstall target/wheels/crypto-0.1.0-cp310-cp310-manylinux_2_34_x86_64.whl 17 | ``` 18 | 19 | ## Link_py 20 | 21 | Implementation of the interoperable transport layer protocol, using the implementation from [secretflow/yacl](https://github.com/secretflow/yacl), with Python interface wrapping. 22 | 23 | ### Build Instructions 24 | 25 | ```bash 26 | bazel build wheel --compilation_mode=opt --host_compilation_mode=opt && \ 27 | pip3 install --force-reinstall bazel-bin/link_py-v0.1-py3-none-any.whl 28 | ``` 29 | 30 | ## PSI 31 | 32 | The main body of the interoperable PSI algorithm flow. 33 | 34 | ### Build Instructions 35 | 36 | ```bash 37 | python3 setup.py bdist_wheel && \ 38 | pip3 install --force-reinstall dist/interconnection_psi-0.0.0-py3-none-any.whl 39 | ``` 40 | 41 | ## PSI Actors 42 | 43 | Main PSI program. 44 | 45 | ## Build Image 46 | 47 | To build the operator into an image: 48 | 49 | ```bash 50 | docker build -t interconnection_psi_v1 . 51 | ``` -------------------------------------------------------------------------------- /src/PSI/crypto/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | .pytest_cache/ 6 | *.py[cod] 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | .venv/ 14 | env/ 15 | bin/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | include/ 26 | man/ 27 | venv/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | pip-selfcheck.json 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Translations 46 | *.mo 47 | 48 | # Mr Developer 49 | .mr.developer.cfg 50 | .project 51 | .pydevproject 52 | 53 | # Rope 54 | .ropeproject 55 | 56 | # Django stuff: 57 | *.log 58 | *.pot 59 | 60 | .DS_Store 61 | 62 | # Sphinx documentation 63 | docs/_build/ 64 | 65 | # PyCharm 66 | .idea/ 67 | 68 | # VSCode 69 | .vscode/ 70 | 71 | # Pyenv 72 | .python-version 73 | -------------------------------------------------------------------------------- /src/PSI/crypto/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crypto" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | name = "crypto" 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | arrow = { version = "49.0.0", features = ["pyarrow"] } 13 | curve25519-dalek = "4.1.1" 14 | pyo3 = { version = "0.20.0", features = ["extension-module"] } 15 | rand = "0.8.5" 16 | sha3 = "0.10.8" 17 | -------------------------------------------------------------------------------- /src/PSI/crypto/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1.3,<2.0"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "crypto" 7 | requires-python = ">=3.8" 8 | classifiers = [ 9 | "Programming Language :: Rust", 10 | "Programming Language :: Python :: Implementation :: CPython", 11 | "Programming Language :: Python :: Implementation :: PyPy", 12 | ] 13 | dynamic = ["version"] 14 | 15 | [tool.maturin] 16 | features = ["pyo3/extension-module"] 17 | -------------------------------------------------------------------------------- /src/PSI/crypto/src/hash.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use arrow::array::{make_array, Array, ArrayData, BinaryArray, BinaryBuilder}; 16 | use arrow::pyarrow::PyArrowType; 17 | use pyo3::exceptions::PyValueError; 18 | use std::sync::Arc; 19 | 20 | use pyo3::prelude::*; 21 | 22 | use sha3::{ 23 | digest::{ExtendableOutput, Update, XofReader}, 24 | Shake256, 25 | }; 26 | 27 | fn shake256_hash(data: &[u8], build: &mut BinaryBuilder) { 28 | let mut hasher = Shake256::default(); 29 | hasher.update(data); 30 | let mut reader = hasher.finalize_xof(); 31 | let mut hash_output = [0u8; 32]; 32 | reader.read(&mut hash_output); 33 | 34 | build.append_value(&hash_output) 35 | } 36 | 37 | fn hash_impl(typ: &str, arr: &BinaryArray) -> BinaryArray { 38 | let fnc = match typ { 39 | "HASH_TYPE_SHAKE_256" => shake256_hash, 40 | &_ => panic!("no support hash type {}", typ), 41 | }; 42 | 43 | let mut build = BinaryBuilder::new(); 44 | for i in arr.iter() { 45 | fnc(i.unwrap(), &mut build); 46 | } 47 | 48 | return build.finish(); 49 | } 50 | 51 | #[pyfunction] 52 | pub(crate) fn hash(typ: &str, array: PyArrowType) -> PyResult> { 53 | let array: ArrayData = array.0; 54 | let array: Arc = make_array(array); 55 | let array: &BinaryArray = array 56 | .as_any() 57 | .downcast_ref() 58 | .ok_or_else(|| PyValueError::new_err("expected binary array"))?; 59 | 60 | let array: arrow::array::GenericByteArray> = 61 | hash_impl(typ, &array); 62 | Ok(PyArrowType(array.into_data())) 63 | } 64 | 65 | pub(crate) static SUPPORT_HASH: [&str; 1] = ["HASH_TYPE_SHAKE_256"]; 66 | -------------------------------------------------------------------------------- /src/PSI/crypto/src/hash_set.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use pyo3::prelude::*; 16 | use std::collections::HashSet; 17 | 18 | #[pyclass] 19 | pub(crate) struct BytesHashSet { 20 | inner: HashSet>, 21 | } 22 | 23 | #[pymethods] 24 | impl BytesHashSet { 25 | #[new] 26 | fn new() -> Self { 27 | BytesHashSet { 28 | inner: HashSet::new(), 29 | } 30 | } 31 | 32 | fn insert(&mut self, value: Vec) { 33 | self.inner.insert(value); 34 | } 35 | 36 | fn contains(&self, value: &[u8]) -> bool { 37 | self.inner.contains(value) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/PSI/crypto/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use pyo3::prelude::*; 16 | 17 | mod curve; 18 | mod hash; 19 | mod hash_set; 20 | 21 | #[pymodule] 22 | fn crypto(_py: Python, m: &PyModule) -> PyResult<()> { 23 | m.add("SUPPORT_HASH", hash::SUPPORT_HASH)?; 24 | m.add_function(wrap_pyfunction!(hash::hash, m)?)?; 25 | 26 | m.add("SUPPORT_HASHTOCURVE", curve::SUPPORT_HASHTOCURVE)?; 27 | m.add_function(wrap_pyfunction!(curve::hash_to_curve, m)?)?; 28 | 29 | m.add("SUPPORT_CURVE", curve::SUPPORT_CURVE)?; 30 | m.add_class::()?; 31 | 32 | m.add( 33 | "SUPPORT_CURVE_POINT_OCTET", 34 | curve::SUPPORT_CURVE_POINT_OCTET, 35 | )?; 36 | m.add_function(wrap_pyfunction!(curve::point_octet_marshal, m)?)?; 37 | m.add_function(wrap_pyfunction!(curve::point_octet_unmarshal, m)?)?; 38 | 39 | m.add_class::()?; 40 | 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /src/PSI/image/psi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jd-opensource/9n-mpc/277595e8b1174e2e7c697cc2678118cb18de8557/src/PSI/image/psi.png -------------------------------------------------------------------------------- /src/PSI/link_py/.bazelrc: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | build --incompatible_new_actions_api=false 16 | build --copt=-fdiagnostics-color=always 17 | build --enable_platform_specific_config 18 | build --cxxopt=-std=c++17 19 | build --host_cxxopt=-std=c++17 20 | build --host_copt=-fPIE 21 | build --host_copt=-fstack-protector-strong 22 | build:linux --host_copt=-Wl,-z,noexecstack 23 | -------------------------------------------------------------------------------- /src/PSI/link_py/.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /src/PSI/link_py/BUILD.bazel: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") 16 | load("@rules_python//python:packaging.bzl", "py_package", "py_wheel") 17 | 18 | pybind_extension( 19 | name = "link_py", 20 | srcs = ["link_py.cc"], 21 | deps = [ 22 | "@yacl//yacl/link", 23 | ], 24 | ) 25 | 26 | py_library( 27 | name = "lib_link_py", 28 | data = [ 29 | ":link_py.so", 30 | ], 31 | ) 32 | 33 | py_package( 34 | name = "link_py_package", 35 | deps = [":lib_link_py"], 36 | ) 37 | 38 | py_wheel( 39 | name = "wheel", 40 | python_tag = "py3", 41 | version = "v0.1", 42 | distribution = "link_py", 43 | deps = [":link_py_package"] 44 | ) 45 | -------------------------------------------------------------------------------- /src/PSI/link_py/WORKSPACE: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | workspace(name="link_py") 16 | 17 | load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") 18 | load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") 19 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 20 | 21 | http_archive( 22 | name="pybind11_bazel", 23 | sha256="6426567481ee345eb48661e7db86adc053881cb4dd39fbf527c8986316b682b9", 24 | strip_prefix="pybind11_bazel-fc56ce8a8b51e3dd941139d329b63ccfea1d304b", 25 | urls=[ 26 | "https://github.com/pybind/pybind11_bazel/archive/fc56ce8a8b51e3dd941139d329b63ccfea1d304b.zip", 27 | ], 28 | ) 29 | 30 | http_archive( 31 | name="pybind11", 32 | build_file="@pybind11_bazel//:pybind11.BUILD", 33 | urls=["https://github.com/pybind/pybind11/archive/refs/tags/v2.10.4.tar.gz"], 34 | type="tar.gz", 35 | sha256="832e2f309c57da9c1e6d4542dedd34b24e4192ecb4d62f6f4866a737454c9970", 36 | strip_prefix="pybind11-2.10.4", 37 | ) 38 | 39 | load("@pybind11_bazel//:python_configure.bzl", "python_configure") 40 | python_configure( 41 | name="local_config_python", 42 | python_version="3", 43 | ) 44 | 45 | maybe( 46 | git_repository, 47 | name="yacl", 48 | branch="a1", 49 | remote="https://github.com/982945902/yacl.git", 50 | ) 51 | 52 | load("@yacl//bazel:repositories.bzl", "yacl_deps") 53 | 54 | yacl_deps() 55 | 56 | load( 57 | "@rules_foreign_cc//foreign_cc:repositories.bzl", 58 | "rules_foreign_cc_dependencies", 59 | ) 60 | 61 | rules_foreign_cc_dependencies( 62 | register_built_tools=False, 63 | register_default_tools=False, 64 | register_preinstalled_tools=True, 65 | ) 66 | 67 | load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") 68 | 69 | protobuf_deps() 70 | 71 | load("@pybind11_bazel//:python_configure.bzl", "python_configure") 72 | 73 | python_configure( 74 | name="local_config_python", 75 | python_version="3", 76 | ) 77 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/cipher_store.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .interconnection.runtime import ecdh_psi_pb2 16 | 17 | from .base import lctx_send_proto 18 | 19 | import crypto 20 | import queue 21 | from threading import Event 22 | 23 | 24 | class CipherStore(): 25 | def __init__(self) -> None: 26 | self.peer_cipher = queue.Queue() 27 | self.peer_cipher_set = crypto.BytesHashSet() 28 | self.peer_cipher_set_done = Event() 29 | 30 | self.local_index_record = 0 31 | self.local_take_index = [] 32 | 33 | def calcu_add_peer_cipher(self, array, ctx, done): 34 | array = ctx.curve.diffie_hellman(array) 35 | 36 | if done: 37 | self.peer_cipher.put(None) 38 | self.peer_cipher.put(array) 39 | 40 | if ctx.need_recv_cipher: 41 | for cipher in array.to_pylist(): 42 | self.peer_cipher_set.insert(cipher) 43 | if done: 44 | self.peer_cipher_set_done.set() 45 | 46 | def send_dualenc(self, ctx, lctx): 47 | batch_index = 0 48 | is_last_batch = False 49 | 50 | while not is_last_batch: 51 | arr = self.peer_cipher.get() 52 | 53 | if arr is None: 54 | arr = self.peer_cipher.get() 55 | is_last_batch = True 56 | 57 | arr = ctx.point_octet_marshal(arr) 58 | count = len(arr) 59 | 60 | protomsg = ecdh_psi_pb2.EcdhPsiCipherBatch( 61 | type="dual.enc", 62 | batch_index=batch_index, 63 | is_last_batch=is_last_batch, 64 | count=count, 65 | ciphertext=b''.join(arr.to_pylist()) 66 | ) 67 | 68 | lctx_send_proto(lctx, protomsg) 69 | 70 | batch_index += 1 71 | 72 | def recv_duaenc_local_cipher(self, array, ctx, done): 73 | self.peer_cipher_set_done.wait() 74 | for cipher in array.to_pylist(): 75 | if self.peer_cipher_set.contains(cipher): 76 | self.local_take_index.append(self.local_index_record) 77 | 78 | self.local_index_record += 1 79 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/common/header.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Ant Group Co., Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // [Sphinx doc begin anchor: ResponseHeader] 16 | syntax = "proto3"; 17 | 18 | package org.interconnection; 19 | 20 | // 31100xxx is the white box interconnection code segment 21 | // 31100xxx 为引擎白盒互联互通号段 22 | enum ErrorCode { 23 | OK = 0; 24 | 25 | GENERIC_ERROR = 31100000; 26 | UNEXPECTED_ERROR = 31100001; 27 | NETWORK_ERROR = 31100002; 28 | 29 | INVALID_REQUEST = 31100100; 30 | INVALID_RESOURCE = 31100101; 31 | 32 | HANDSHAKE_REFUSED = 31100200; 33 | UNSUPPORTED_VERSION = 31100201; 34 | UNSUPPORTED_ALGO = 31100202; 35 | UNSUPPORTED_PARAMS = 31100203; 36 | } 37 | 38 | message ResponseHeader { 39 | int32 error_code = 1; 40 | string error_msg = 2; 41 | } 42 | // [Sphinx doc end anchor: ResponseHeader] 43 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/handshake/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/handshake/algos/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/handshake/algos/psi.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Ant Group Co., Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package org.interconnection.v2.algos; 18 | 19 | // [Sphinx doc begin anchor: PsiDataIoProposal] 20 | message PsiDataInfoProposal { 21 | repeated int32 supported_versions = 1; 22 | 23 | // How many items do I've. 24 | // 25 | // 待求交的 PSI 数据总量 26 | int64 item_num = 2; 27 | 28 | // Which rank can receive the psi results. 29 | // 30 | // 确定 PSI 结果获取方。 31 | // 32 | // NOTES: 33 | // `-1`: all parties (所有机构都可以拿到交集结果) 34 | // `>= 0`: corresponding rank can get the results (指定机构拿到交集结果) 35 | int32 result_to_rank = 3; 36 | } 37 | // [Sphinx doc end anchor: PsiDataIoProposal] 38 | 39 | // [Sphinx doc begin anchor: PsiDataIoResult] 40 | message PsiDataInfoResult { 41 | int32 version = 1; 42 | 43 | // 确定 PSI 结果获取方。 44 | // 45 | // NOTES: 46 | // `-1`: all parties (所有机构都可以拿到交集结果) 47 | // `>= 0`: corresponding rank can get the results (指定机构拿到交集结果) 48 | int32 result_to_rank = 2; 49 | } 50 | // [Sphinx doc end anchor: PsiDataIoResult] 51 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/handshake/protocol_family/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/interconnection/runtime/ecdh_psi.proto: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2023 Ant Group Co., Ltd. 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | syntax = "proto3"; 17 | 18 | package org.interconnection.v2.runtime; 19 | 20 | // [Sphinx doc begin anchor: EcdhPsiCipherBatch] 21 | // ECDH PSI 密文传输 22 | message EcdhPsiCipherBatch { 23 | // The type hint for each message. (密文类型) 24 | // 25 | // "enc": the first stage ciphertext 26 | // 27 | // "dual.enc": the second stage ciphertext 28 | // 29 | // ECDH PSI 密文阶段类型，主要用来区分一阶段和二阶段的密文. 30 | string type = 1; 31 | 32 | // The batch index. Start from 0. 33 | // 34 | // Batch 索引，从 0 开始 35 | int32 batch_index = 3; 36 | 37 | // Is last batch flag 38 | bool is_last_batch = 4; 39 | 40 | // Count of items in this batch. 41 | // count == 0 is allowed for last batch 42 | int32 count = 6; 43 | 44 | // The packed all in one ciphertext for this batch. 45 | // 46 | // The first stage ciphertext takes 256 bits for each ciphertext element. 47 | // However, the second stage ciphertext takes 96 bits each. According to PSI 48 | // papers, we do not need to send all 256 bit for the final ciphertext. The 49 | // number of bits needed to compare is `Log(MN) + 40` given a 40 bits 50 | // statistical security parameter. TODO (add paper link here). 51 | // 52 | // We define each bucket has less than 2^28 items, i.e. about 270 million 53 | // (单桶最多 2.7亿) items, which is general enough for various psi algorithms. 54 | // 55 | // NOTE: we do not use `repeated`` here to save overhead of metadata. 56 | bytes ciphertext = 7; 57 | } 58 | // [Sphinx doc end anchor: EcdhPsiCipherBatch] 59 | -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/log.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | logger = logging.getLogger() -------------------------------------------------------------------------------- /src/PSI/psi/interconnection_psi/register_uuid.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import socket 16 | from .log import logger 17 | import redis 18 | 19 | 20 | class Register(object): 21 | 22 | def __init__(self, redis_address: str, redis_password: str, role_port: int, app_id: str): 23 | (host, port) = redis_address.split(':') 24 | self._redis_cli = redis.Redis( 25 | host=host, port=int(port), 26 | password=redis_password if len(redis_password) > 0 else None, 27 | retry_on_timeout=True) 28 | self._role_port = role_port 29 | self.app_id = app_id 30 | 31 | self.register_uuid() 32 | 33 | def __del__(self): 34 | self.unregister_uuid() 35 | 36 | def register_uuid(self): 37 | uuid = "network:" + self.app_id 38 | host_ip = self.get_host_ip() 39 | address = "{0}:{1}".format(host_ip, self._role_port) 40 | self._redis_cli.set(uuid, address) 41 | logger.info("Register key %s, host_address %s ", uuid, address) 42 | 43 | def unregister_uuid(self): 44 | uuid = "network:" + self.app_id 45 | self._redis_cli.delete(uuid) 46 | logger.info("UnRegister key %s", uuid) 47 | 48 | def get_host_ip(self): 49 | """ 50 | 查询本机ip地址 51 | :return: ip 52 | """ 53 | try: 54 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 55 | s.connect(('8.8.8.8', 80)) 56 | ip = s.getsockname()[0] 57 | finally: 58 | s.close() 59 | return ip 60 | -------------------------------------------------------------------------------- /src/PSI/psi/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from setuptools import setup, find_packages 16 | from setuptools.command.build_py import build_py 17 | import os 18 | 19 | 20 | class CustomBuildPy(build_py): 21 | def run(self): 22 | self.generate_proto() 23 | super().run() 24 | 25 | def generate_proto(self): 26 | os.system( 27 | "python3 -m grpc_tools.protoc -I. --python_out=. $( find . \\( -name \"*.proto\" \\) )") 28 | 29 | 30 | setup( 31 | name="interconnection_psi", 32 | packages=find_packages(), 33 | cmdclass={ 34 | 'build_py': CustomBuildPy, 35 | }, 36 | ) 37 | -------------------------------------------------------------------------------- /src/PSI/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2020 The 9nFL Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | echo $* 18 | 19 | source ~/.bashrc 20 | 21 | # default log root dir 22 | LOGDIR=/mnt/logs 23 | 24 | # obtain log files 25 | for key in "$@"; do 26 | case $key in 27 | --log-path=*) 28 | LOGDIR="${key#*=}" 29 | shift # past argument=value 30 | ;; 31 | --default) 32 | DEFAULT=YES 33 | shift # past argument with no value 34 | ;; 35 | *) ;; 36 | esac 37 | done 38 | 39 | LOGFILENAME="psi.log" 40 | 41 | echo "environment node name ${MY_NODE_NAME}" 42 | 43 | if [[ -z ${MY_NODE_NAME} ]]; then 44 | LOGFILE=${LOGDIR}/${TASK_ID}_${APP_ID}_${NODE_ID}_${LOGFILENAME} 45 | else 46 | LOGFILE=${LOGDIR}/${MY_NODE_NAME}_${LOGFILENAME} 47 | fi 48 | 49 | echo "write logs to ${LOGFILE}" 50 | 51 | source /root/.bashrd 52 | python3 ./psi_actors.py "$@" 2>&1 | tee ${LOGFILE} 53 | 54 | sleep 1d 55 | --------------------------------------------------------------------------------