├── .dockerignore ├── .editorconfig ├── .github └── workflows │ ├── build-apiserver.yaml │ ├── build-operator.yaml │ ├── ci.yaml │ ├── release-apiserver.yaml │ ├── release-fedlearner.yaml │ ├── release-operator.yaml │ ├── tag-apiserver.yaml │ ├── tag-fedlearner.yaml │ └── tag-operator.yaml ├── .gitignore ├── AUTHORS ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── cc ├── build.sh └── operators │ ├── kernels │ ├── embedding_pooling.cc │ └── embedding_pooling.h │ ├── ops │ └── embedding_pooling.cc │ └── utils │ └── common.h ├── ci ├── ci_test.sh └── pylintrc ├── config.mk.template ├── deploy ├── README.md ├── charts │ ├── fedlearner-add-on │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── configuration-snippet.txt │ │ ├── server-snippet.txt │ │ ├── templates │ │ │ ├── _helpers.tpl │ │ │ ├── ingress-v1.yaml │ │ │ ├── ingress-v1beta1.yaml │ │ │ ├── secrets.yaml │ │ │ └── service.yaml │ │ └── values.yaml │ ├── fedlearner-pvc │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── charts │ │ │ ├── nfs-client-provisioner-1.2.8.tgz │ │ │ └── nfs-server-provisioner-1.0.0.tgz │ │ ├── templates │ │ │ ├── pv.yaml │ │ │ └── pvc.yaml │ │ └── values.yaml │ ├── fedlearner-stack │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── charts │ │ │ ├── elastic-stack │ │ │ │ ├── .helmignore │ │ │ │ ├── Chart.yaml │ │ │ │ ├── OWNERS │ │ │ │ ├── README.md │ │ │ │ ├── charts │ │ │ │ │ ├── elasticsearch-curator │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── ci │ │ │ │ │ │ │ └── initcontainer-values.yaml │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── configmap.yaml │ │ │ │ │ │ │ ├── cronjob.yaml │ │ │ │ │ │ │ ├── hooks │ │ │ │ │ │ │ │ └── job.install.yaml │ │ │ │ │ │ │ ├── psp.yml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ └── serviceaccount.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── elasticsearch-exporter │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── ci │ │ │ │ │ │ │ ├── default-values.yaml │ │ │ │ │ │ │ └── security-context.yaml │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── cert-secret.yaml │ │ │ │ │ │ │ ├── deployment.yaml │ │ │ │ │ │ │ ├── podsecuritypolicies.yaml │ │ │ │ │ │ │ ├── prometheusrule.yaml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ ├── service.yaml │ │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ │ └── servicemonitor.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── elasticsearch │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── ci │ │ │ │ │ │ │ ├── expose-transport-port-on-service-values.yaml │ │ │ │ │ │ │ ├── extrainitcontainers-values.yaml │ │ │ │ │ │ │ ├── hooks-values.yaml │ │ │ │ │ │ │ ├── nonroot-deployment-values.yaml │ │ │ │ │ │ │ ├── plugin-initcontainer-values.yaml │ │ │ │ │ │ │ └── updatestrategy-values.yaml │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── client-auth.yaml │ │ │ │ │ │ │ ├── client-deployment.yaml │ │ │ │ │ │ │ ├── client-ingress.yaml │ │ │ │ │ │ │ ├── client-pdb.yaml │ │ │ │ │ │ │ ├── client-serviceaccount.yaml │ │ │ │ │ │ │ ├── client-svc.yaml │ │ │ │ │ │ │ ├── configmap.yaml │ │ │ │ │ │ │ ├── data-pdb.yaml │ │ │ │ │ │ │ ├── data-serviceaccount.yaml │ │ │ │ │ │ │ ├── data-statefulset.yaml │ │ │ │ │ │ │ ├── job.yaml │ │ │ │ │ │ │ ├── master-pdb.yaml │ │ │ │ │ │ │ ├── master-serviceaccount.yaml │ │ │ │ │ │ │ ├── master-statefulset.yaml │ │ │ │ │ │ │ ├── master-svc.yaml │ │ │ │ │ │ │ ├── podsecuritypolicy.yaml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ └── tests │ │ │ │ │ │ │ │ ├── test-configmap.yaml │ │ │ │ │ │ │ │ └── test.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── filebeat │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── clusterrole.yaml │ │ │ │ │ │ │ ├── clusterrolebinding.yaml │ │ │ │ │ │ │ ├── daemonset.yaml │ │ │ │ │ │ │ ├── podsecuritypolicy.yaml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ ├── secret.yaml │ │ │ │ │ │ │ ├── service.yaml │ │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ │ └── servicemonitor.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── fluent-bit │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── cluster-role.yaml │ │ │ │ │ │ │ ├── cluster-rolebinding.yaml │ │ │ │ │ │ │ ├── config.yaml │ │ │ │ │ │ │ ├── daemonset.yaml │ │ │ │ │ │ │ ├── psp.yaml │ │ │ │ │ │ │ ├── secret.yaml │ │ │ │ │ │ │ ├── service.yaml │ │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ │ ├── servicemonitor.yaml │ │ │ │ │ │ │ └── tests │ │ │ │ │ │ │ │ ├── test-configmap.yaml │ │ │ │ │ │ │ │ └── test.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── fluentd-elasticsearch │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── clusterrole.yaml │ │ │ │ │ │ │ ├── clusterrolebinding.yaml │ │ │ │ │ │ │ ├── configmap.yaml │ │ │ │ │ │ │ ├── daemonset.yaml │ │ │ │ │ │ │ ├── pod-security-policy.yaml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ ├── service-account.yaml │ │ │ │ │ │ │ └── service.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── fluentd │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── clusterrole.yaml │ │ │ │ │ │ │ ├── clusterrolebinding.yaml │ │ │ │ │ │ │ ├── configmap.yaml │ │ │ │ │ │ │ ├── deployment.yaml │ │ │ │ │ │ │ ├── hpa.yaml │ │ │ │ │ │ │ ├── ingress.yaml │ │ │ │ │ │ │ ├── pvc.yaml │ │ │ │ │ │ │ ├── pvc.yml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ ├── service.yaml │ │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ │ └── servicemonitor.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── kibana │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── ci │ │ │ │ │ │ │ ├── authproxy-enabled.yaml │ │ │ │ │ │ │ ├── dashboard-values.yaml │ │ │ │ │ │ │ ├── extra-configmap-mounts.yaml │ │ │ │ │ │ │ ├── ingress-hosts-paths.yaml │ │ │ │ │ │ │ ├── ingress-hosts.yaml │ │ │ │ │ │ │ ├── initcontainers-all-values.yaml │ │ │ │ │ │ │ ├── initcontainers-values.yaml │ │ │ │ │ │ │ ├── plugin-install.yaml │ │ │ │ │ │ │ ├── pvc.yaml │ │ │ │ │ │ │ ├── security-context.yaml │ │ │ │ │ │ │ ├── service-values.yaml │ │ │ │ │ │ │ └── url_dashboard-values.yaml │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── configmap-dashboardimport.yaml │ │ │ │ │ │ │ ├── configmap.yaml │ │ │ │ │ │ │ ├── deployment.yaml │ │ │ │ │ │ │ ├── ingress-v1.yaml │ │ │ │ │ │ │ ├── ingress-v1beta1.yaml │ │ │ │ │ │ │ ├── service.yaml │ │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ │ ├── tests │ │ │ │ │ │ │ │ ├── test-configmap.yaml │ │ │ │ │ │ │ │ └── test.yaml │ │ │ │ │ │ │ └── volume-claim.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ ├── logstash │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── OWNERS │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ ├── templates │ │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ │ ├── files-config.yaml │ │ │ │ │ │ │ ├── ingress.yaml │ │ │ │ │ │ │ ├── patterns-config.yaml │ │ │ │ │ │ │ ├── pipeline-config.yaml │ │ │ │ │ │ │ ├── poddisruptionbudget.yaml │ │ │ │ │ │ │ ├── service.yaml │ │ │ │ │ │ │ ├── serviceaccount.yaml │ │ │ │ │ │ │ ├── servicemonitor.yaml │ │ │ │ │ │ │ └── statefulset.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ │ └── nginx-ldapauth-proxy │ │ │ │ │ │ ├── .helmignore │ │ │ │ │ │ ├── Chart.yaml │ │ │ │ │ │ ├── templates │ │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ │ ├── configmap.yaml │ │ │ │ │ │ ├── deployment.yaml │ │ │ │ │ │ ├── ingress.yaml │ │ │ │ │ │ ├── secrets.yaml │ │ │ │ │ │ └── service.yaml │ │ │ │ │ │ └── values.yaml │ │ │ │ ├── requirements.lock │ │ │ │ ├── requirements.yaml │ │ │ │ ├── templates │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ └── _helpers.tpl │ │ │ │ └── values.yaml │ │ │ ├── etcd-4.8.2.tgz │ │ │ ├── ingress-nginx │ │ │ │ ├── .helmignore │ │ │ │ ├── Chart.yaml │ │ │ │ ├── OWNERS │ │ │ │ ├── README.md │ │ │ │ ├── ci │ │ │ │ │ ├── daemonset-customconfig-values.yaml │ │ │ │ │ ├── daemonset-customnodeport-values.yaml │ │ │ │ │ ├── daemonset-headers-values.yaml │ │ │ │ │ ├── daemonset-nodeport-values.yaml │ │ │ │ │ ├── daemonset-tcp-udp-configMapNamespace-values.yaml │ │ │ │ │ ├── daemonset-tcp-udp-values.yaml │ │ │ │ │ ├── daemonset-tcp-values.yaml │ │ │ │ │ ├── deamonset-default-values.yaml │ │ │ │ │ ├── deamonset-metrics-values.yaml │ │ │ │ │ ├── deamonset-psp-values.yaml │ │ │ │ │ ├── deamonset-webhook-and-psp-values.yaml │ │ │ │ │ ├── deamonset-webhook-values.yaml │ │ │ │ │ ├── deployment-autoscaling-values.yaml │ │ │ │ │ ├── deployment-customconfig-values.yaml │ │ │ │ │ ├── deployment-customnodeport-values.yaml │ │ │ │ │ ├── deployment-default-values.yaml │ │ │ │ │ ├── deployment-headers-values.yaml │ │ │ │ │ ├── deployment-metrics-values.yaml │ │ │ │ │ ├── deployment-nodeport-values.yaml │ │ │ │ │ ├── deployment-psp-values.yaml │ │ │ │ │ ├── deployment-tcp-udp-configMapNamespace-values.yaml │ │ │ │ │ ├── deployment-tcp-udp-values.yaml │ │ │ │ │ ├── deployment-tcp-values.yaml │ │ │ │ │ ├── deployment-webhook-and-psp-values.yaml │ │ │ │ │ └── deployment-webhook-values.yaml │ │ │ │ ├── templates │ │ │ │ │ ├── NOTES.txt │ │ │ │ │ ├── _helpers.tpl │ │ │ │ │ ├── admission-webhooks │ │ │ │ │ │ ├── job-patch │ │ │ │ │ │ │ ├── clusterrole.yaml │ │ │ │ │ │ │ ├── clusterrolebinding.yaml │ │ │ │ │ │ │ ├── job-createSecret.yaml │ │ │ │ │ │ │ ├── job-patchWebhook.yaml │ │ │ │ │ │ │ ├── psp.yaml │ │ │ │ │ │ │ ├── role.yaml │ │ │ │ │ │ │ ├── rolebinding.yaml │ │ │ │ │ │ │ └── serviceaccount.yaml │ │ │ │ │ │ └── validating-webhook.yaml │ │ │ │ │ ├── clusterrole.yaml │ │ │ │ │ ├── clusterrolebinding.yaml │ │ │ │ │ ├── controller-configmap-addheaders.yaml │ │ │ │ │ ├── controller-configmap-proxyheaders.yaml │ │ │ │ │ ├── controller-configmap-tcp.yaml │ │ │ │ │ ├── controller-configmap-udp.yaml │ │ │ │ │ ├── controller-configmap.yaml │ │ │ │ │ ├── controller-daemonset.yaml │ │ │ │ │ ├── controller-deployment.yaml │ │ │ │ │ ├── controller-hpa.yaml │ │ │ │ │ ├── controller-poddisruptionbudget.yaml │ │ │ │ │ ├── controller-prometheusrules.yaml │ │ │ │ │ ├── controller-psp.yaml │ │ │ │ │ ├── controller-role.yaml │ │ │ │ │ ├── controller-rolebinding.yaml │ │ │ │ │ ├── controller-service-metrics.yaml │ │ │ │ │ ├── controller-service-webhook.yaml │ │ │ │ │ ├── controller-service.yaml │ │ │ │ │ ├── controller-serviceaccount.yaml │ │ │ │ │ ├── controller-servicemonitor.yaml │ │ │ │ │ ├── default-backend-deployment.yaml │ │ │ │ │ ├── default-backend-poddisruptionbudget.yaml │ │ │ │ │ ├── default-backend-psp.yaml │ │ │ │ │ ├── default-backend-role.yaml │ │ │ │ │ ├── default-backend-rolebinding.yaml │ │ │ │ │ ├── default-backend-service.yaml │ │ │ │ │ └── default-backend-serviceaccount.yaml │ │ │ │ └── values.yaml │ │ │ ├── mysql-8.0.22.tgz │ │ │ └── sparkoperator │ │ │ │ ├── .helmignore │ │ │ │ ├── Chart.yaml │ │ │ │ ├── README.md │ │ │ │ ├── ci │ │ │ │ └── test-values.yaml │ │ │ │ ├── hack │ │ │ │ └── update-ci.sh │ │ │ │ ├── templates │ │ │ │ ├── _helpers.tpl │ │ │ │ ├── crd-cleanup-job.yaml │ │ │ │ ├── crds-v1.yaml │ │ │ │ ├── crds-v1beta1.yaml │ │ │ │ ├── spark-operator-deployment.yaml │ │ │ │ ├── spark-operator-rbac.yaml │ │ │ │ ├── spark-operator-serviceaccount.yaml │ │ │ │ ├── spark-rbac.yaml │ │ │ │ ├── spark-serviceaccount.yaml │ │ │ │ ├── webhook-cleanup-job.yaml │ │ │ │ ├── webhook-init-job.yaml │ │ │ │ └── webhook-service.yaml │ │ │ │ └── values.yaml │ │ └── values.yaml │ ├── fedlearner-test │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ │ ├── NOTES.txt │ │ │ ├── _helpers.tpl │ │ │ └── tests │ │ │ │ ├── egress.yaml │ │ │ │ └── ingress.yaml │ │ └── values.yaml │ └── fedlearner │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── charts │ │ ├── fedlearner-apiserver │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ │ ├── _helpers.tpl │ │ │ │ ├── cluster_role.yaml │ │ │ │ ├── cluster_role_binding.yaml │ │ │ │ ├── deployment.yaml │ │ │ │ ├── service.yaml │ │ │ │ └── serviceaccount.yaml │ │ │ └── values.yaml │ │ ├── fedlearner-operator │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ │ ├── _helpers.tpl │ │ │ │ ├── deployment.yaml │ │ │ │ ├── fedlearner.k8s.io_flapps_v1.yaml │ │ │ │ ├── fedlearner.k8s.io_flapps_v1beta1.yaml │ │ │ │ ├── ingress-v1.yaml │ │ │ │ ├── ingress-v1beta1.yaml │ │ │ │ ├── role.yaml │ │ │ │ ├── role_binding.yaml │ │ │ │ ├── service.yaml │ │ │ │ └── serviceaccount.yaml │ │ │ └── values.yaml │ │ ├── fedlearner-web-console-v2 │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ │ ├── _helpers.tpl │ │ │ │ ├── cluster_role.yaml │ │ │ │ ├── cluster_role_binding.yaml │ │ │ │ ├── deployment.yaml │ │ │ │ ├── hook_config_map.yaml │ │ │ │ ├── ingress-v1.yaml │ │ │ │ ├── ingress-v1beta1.yaml │ │ │ │ ├── public_service.yaml │ │ │ │ ├── service.yaml │ │ │ │ └── serviceaccount.yaml │ │ │ └── values.yaml │ │ └── fedlearner-web-console │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ ├── _helpers.tpl │ │ │ ├── deployment.yaml │ │ │ ├── ingress.yaml │ │ │ ├── service.yaml │ │ │ └── serviceaccount.yaml │ │ │ └── values.yaml │ │ └── values.yaml ├── examples │ ├── abnormal_follower.yaml │ ├── abnormal_leader.yaml │ ├── netshoot.yaml │ ├── normal_follower.yaml │ └── normal_leader.yaml ├── integrated_test │ ├── client_integrated_test.py │ ├── code_key │ │ └── criteo-train-2.tar.gz │ ├── credit_default │ │ ├── default_credit_hetero_guest.csv │ │ └── default_credit_hetero_host.csv │ ├── rsa_private.key │ ├── template_json │ │ ├── template_client_federation.json │ │ ├── template_nn_ticket.json │ │ ├── template_psi_join_ticket.json │ │ ├── template_raw_data.json │ │ ├── template_streaming_join_ticket.json │ │ └── template_tree_ticket.json │ ├── tfrecord_raw_data │ │ ├── raw_data_partition_0000.rd │ │ ├── raw_data_partition_0001.rd │ │ ├── raw_data_partition_0002.rd │ │ └── raw_data_partition_0003.rd │ └── tools.py ├── kubernetes_operator │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── cmd │ │ ├── apiserver │ │ │ └── main.go │ │ └── operator │ │ │ └── main.go │ ├── go.mod │ ├── go.sum │ ├── hack │ │ ├── boilerplate.go.txt │ │ └── update-codegen.sh │ ├── pkg │ │ ├── apis │ │ │ └── fedlearner.k8s.io │ │ │ │ └── v1alpha1 │ │ │ │ ├── constants.go │ │ │ │ ├── defaults.go │ │ │ │ ├── doc.go │ │ │ │ ├── register.go │ │ │ │ ├── types.go │ │ │ │ ├── zz_generated.deepcopy.go │ │ │ │ └── zz_generated.defaults.go │ │ ├── apiserver │ │ │ └── handler │ │ │ │ ├── handler.go │ │ │ │ └── terminal.go │ │ ├── client │ │ │ ├── clientset │ │ │ │ └── versioned │ │ │ │ │ ├── clientset.go │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake │ │ │ │ │ ├── clientset_generated.go │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ │ ├── scheme │ │ │ │ │ ├── doc.go │ │ │ │ │ └── register.go │ │ │ │ │ └── typed │ │ │ │ │ └── fedlearner.k8s.io │ │ │ │ │ └── v1alpha1 │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake │ │ │ │ │ ├── doc.go │ │ │ │ │ ├── fake_fedlearner.k8s.io_client.go │ │ │ │ │ └── fake_flapp.go │ │ │ │ │ ├── fedlearner.k8s.io_client.go │ │ │ │ │ ├── flapp.go │ │ │ │ │ └── generated_expansion.go │ │ │ ├── informers │ │ │ │ └── externalversions │ │ │ │ │ ├── factory.go │ │ │ │ │ ├── fedlearner.k8s.io │ │ │ │ │ ├── interface.go │ │ │ │ │ └── v1alpha1 │ │ │ │ │ │ ├── flapp.go │ │ │ │ │ │ └── interface.go │ │ │ │ │ ├── generic.go │ │ │ │ │ └── internalinterfaces │ │ │ │ │ └── factory_interfaces.go │ │ │ └── listers │ │ │ │ └── fedlearner.k8s.io │ │ │ │ └── v1alpha1 │ │ │ │ ├── expansion_generated.go │ │ │ │ └── flapp.go │ │ ├── operator │ │ │ ├── app_manager.go │ │ │ ├── cluster_spec.go │ │ │ ├── controller.go │ │ │ ├── controller_ref_manager.go │ │ │ ├── event_handler.go │ │ │ ├── pod.go │ │ │ ├── pod_cache.go │ │ │ ├── pod_control.go │ │ │ ├── service.go │ │ │ ├── service_control.go │ │ │ ├── status.go │ │ │ ├── status_updater.go │ │ │ └── util.go │ │ ├── server │ │ │ └── server.go │ │ └── util │ │ │ └── train │ │ │ └── train_util.go │ ├── proto │ │ ├── pairing_service.pb.go │ │ └── pairing_service.proto │ └── tools.go ├── proxy │ ├── README.md │ ├── fl_router │ │ ├── access_router.lua │ │ ├── etcd_resolver.lua │ │ └── router.lua │ └── patches │ │ └── ngx_http_grpc_module │ │ └── nginx__grpc_pass_variable_1.16.1+.patch └── scripts │ ├── aliyun │ ├── clear.sh │ ├── install-add-on.sh │ ├── install.sh │ ├── scale.sh │ └── upgrade-add-on.sh │ ├── data_join │ ├── run_data_join_follower_master.sh │ ├── run_data_join_leader_master.sh │ ├── run_data_join_master.sh │ ├── run_data_join_worker.sh │ ├── run_psi_data_join_follower_worker.sh │ ├── run_psi_data_join_follower_worker_v2.sh │ ├── run_psi_data_join_leader_worker.sh │ └── run_psi_data_join_leader_worker_v2.sh │ ├── data_portal │ ├── run_data_portal_master.sh │ └── run_data_portal_worker.sh │ ├── data_sensor.py │ ├── env_to_args.sh │ ├── etcd2mysql │ └── etcd_to_mysql.py │ ├── hdfs_common.sh │ ├── pre_start_hook.sh │ ├── rsa_psi │ ├── run_psi_data_join_master.sh │ ├── run_psi_data_join_worker.sh │ ├── run_psi_preprocessor.sh │ ├── run_raw_data_partitioner.sh │ └── run_rsa_psi_signer.sh │ ├── trainer │ ├── run_fedavg.sh │ ├── run_merge_scores.sh │ ├── run_trainer_local_worker.sh │ ├── run_trainer_master.sh │ ├── run_trainer_ps.sh │ ├── run_trainer_worker.sh │ └── run_tree_worker.sh │ ├── wait4pair_wrapper.sh │ ├── xxx.sh │ └── zip.py ├── docker └── dataflow │ ├── Dockerfile │ └── requirements.txt ├── docs └── tutorials │ ├── deploy.md │ └── quick_start.md ├── example ├── data_portal │ ├── make_data.py │ └── run.sh ├── fedavg │ └── mnist │ │ ├── follower.py │ │ ├── leader.py │ │ └── model.py ├── mnist │ ├── follower.py │ ├── leader.py │ ├── make_data.py │ └── test.sh ├── privacy │ ├── DPAUC │ │ ├── README.md │ │ ├── figures │ │ │ └── roc_auc │ │ │ │ └── 20220618_04_01_49_sampled_clients_ratio_1.0_numThresholds_200_Laplace_eps_2.0.pdf │ │ ├── label_on_device_auc_computation_main.py │ │ ├── label_on_device_auc_computation_util.py │ │ ├── outputs │ │ │ ├── sample_0.1_of_test │ │ │ │ ├── Laplace │ │ │ │ │ ├── numClients_1 │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.01_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.01_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.02_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_12_56_07_Laplace_eps_0.02_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.01_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.02_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.04_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220621_17_18_00_Laplace_eps_0.000625_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220621_17_18_00_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220621_17_18_00_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ └── 20220621_17_18_00_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ ├── numClients_10 │ │ │ │ │ │ ├── 20220618_11_18_47_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_14_54_14_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_14_54_14_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_14_54_14_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_14_54_14_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_14_55_20_Laplace_eps_0.05_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_14_55_20_Laplace_eps_0.1_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_11_30_15_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_11_30_15_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_11_35_31_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_11_35_31_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_19_57_17_None_eps_0.0_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_19_57_17_None_eps_0.0_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_19_57_17_None_eps_0.0_numClients_10_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.04_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.025_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.05_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.1_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.2_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_22_23_55_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_22_23_55_Laplace_eps_0.04_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_22_23_56_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ └── 20220809_22_23_56_Laplace_eps_0.08_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ └── numClients_1000 │ │ │ │ │ │ ├── 20220618_20_44_13_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_20_44_14_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_20_44_14_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220618_22_18_12_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_11_30_15_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_11_30_15_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_11_30_15_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_11_36_52_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_31_44_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220619_17_38_08_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220620_15_00_02_Laplace_eps_0.04_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.025_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.05_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.1_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_11_56_31_Laplace_eps_0.2_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_22_23_55_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_22_23_55_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ ├── 20220809_22_23_55_Laplace_eps_0.04_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ │ └── 20220809_22_23_55_Laplace_eps_0.08_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ └── randomized_responses │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_1.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_1.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_2.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_2.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_4.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_4.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_8.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220616_18_33_36_RR_eps_8.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv │ │ │ │ │ ├── 20220618_11_19_51_RR_eps_0.5_numClients_1000_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ │ │ └── 20220618_11_19_51_RR_eps_10.0_numClients_1000_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv │ │ │ └── stdout_logs │ │ │ │ ├── 20220617_20_03_34_numberClients_10Laplace_eps_2.0numThreshold_100_repeatTimes_50.txt │ │ │ │ ├── 20220617_20_03_34_numberClients_10Laplace_eps_2.0numThreshold_200_repeatTimes_50.txt │ │ │ │ ├── 20220618_11_18_42_numberClients_10Laplace_eps_0.01numThreshold_200_repeatTimes_50.txt │ │ │ │ ├── 20220618_11_19_46_numberClients_1000RR_eps_0.5numThreshold_400_repeatTimes_50.txt │ │ │ │ ├── 20220618_11_19_46_numberClients_1000RR_eps_10numThreshold_400_repeatTimes_50.txt │ │ │ │ ├── 20220618_14_55_15_numberClients_10Laplace_eps_0.05numThreshold_200_repeatTimes_50.txt │ │ │ │ ├── 20220618_14_55_15_numberClients_10Laplace_eps_0.1numThreshold_200_repeatTimes_50.txt │ │ │ │ ├── 20220619_11_35_26_numberClients_10Laplace_eps_0.00125numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt │ │ │ │ ├── 20220619_11_35_26_numberClients_10Laplace_eps_0.01numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt │ │ │ │ ├── 20220619_11_36_48_numberClients_1000Laplace_eps_0.005numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt │ │ │ │ ├── 20221126_13_45_41_numberClients_10Laplace_eps_1.0numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt │ │ │ │ ├── 20221126_13_46_37_numberClients_10Laplace_eps_1.0numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt │ │ │ │ ├── 20221126_21_32_37_numberClients_10RR_eps_0.01numThreshold_100_repeatTimes_50_ClientAssignedSkewed.txt │ │ │ │ └── 20221126_21_33_33_numberClients_10RR_eps_0.01numThreshold_100_repeatTimes_50.txt │ │ ├── resource_setup.py │ │ ├── run_script_label_on_device_auc_cal.py │ │ └── visualization_util.py │ ├── embedding_protection │ │ ├── FMNIST_Embedding_Protection_Framework_Demo.py │ │ ├── README.md │ │ └── Technical_Report_of_Embedding_Protection_in_FL.pdf │ └── label_protection │ │ ├── FL_Label_Protection_FMNIST_Demo.py │ │ ├── NeurIPS2020_SpicyFL │ │ ├── 2mins_video.mp4 │ │ ├── Neurips SpicyFL Camera Ready Paper_ Label Leakage & Protection.pdf │ │ └── Neurips SpicyFL Poster Label Leakage & Protection.pdf │ │ ├── README.md │ │ ├── Technical_Report_of_Label_Leakage_and_Protection_in_Federated_Learning.pdf │ │ ├── shared_var.py │ │ └── solver.py ├── sparse_model │ ├── follower.py │ ├── leader.py │ ├── make_data.py │ ├── test.sh │ └── test_local_worker.sh ├── tree_model │ ├── README.md │ ├── baseline.py │ ├── make_data.py │ ├── merge_scores.py │ └── test.sh └── wide_n_deep │ ├── follower.py │ ├── leader.py │ ├── make_data.py │ └── test.sh ├── fedlearner ├── __init__.py ├── channel │ ├── __init__.py │ ├── channel.py │ ├── client_interceptor.py │ └── server_interceptor.py ├── common │ ├── __init__.py │ ├── argparse_util.py │ ├── common.py │ ├── db_client.py │ ├── dfs_client.py │ ├── etcd_client.py │ ├── fl_logging.py │ ├── grpc_utils.py │ ├── hooks.py │ ├── leveldb.py │ ├── metric_collector.py │ ├── metrics.py │ ├── mock_kvstore.py │ ├── mysql_client.py │ ├── stats.py │ └── summary_hook.py ├── data_join │ ├── __init__.py │ ├── cmd │ │ ├── __init__.py │ │ ├── data_join_master_service.py │ │ ├── data_join_worker_service.py │ │ ├── generate_csv_raw_data.py │ │ ├── prepare_launch_data_join_cli.py │ │ ├── raw_data_cli.py │ │ ├── raw_data_partitioner_cli.py │ │ ├── rsa_key_generator_cli.py │ │ ├── rsa_psi_preprocessor_cli.py │ │ └── rsa_psi_signer_service.py │ ├── common.py │ ├── csv_dict_writer.py │ ├── data_block_dumper.py │ ├── data_block_manager.py │ ├── data_block_visitor.py │ ├── data_join_master.py │ ├── data_join_worker.py │ ├── example_id_batch_fetcher.py │ ├── example_id_dumper.py │ ├── example_id_sync_follower.py │ ├── example_id_sync_leader.py │ ├── example_id_visitor.py │ ├── example_join_follower.py │ ├── example_join_leader.py │ ├── example_validate_impl │ │ ├── __init__.py │ │ └── example_validator.py │ ├── item_batch_seq_processor.py │ ├── join_expr │ │ ├── __init__.py │ │ └── expression.py │ ├── joiner_impl │ │ ├── __init__.py │ │ ├── example_joiner.py │ │ ├── joiner_stats.py │ │ ├── optional_stats.py │ │ ├── sort_run_joiner.py │ │ ├── stream_joiner.py │ │ └── universal_joiner.py │ ├── key_mapper │ │ ├── __init__.py │ │ ├── impl │ │ │ ├── __init__.py │ │ │ └── default.py │ │ └── key_mapping.py │ ├── negative_example_generator.py │ ├── output_writer_impl │ │ ├── __init__.py │ │ ├── csv_dict_builder.py │ │ ├── output_writer.py │ │ └── tf_record_builder.py │ ├── raw_data │ │ ├── __init__.py │ │ ├── common.py │ │ ├── input_data_manager.py │ │ ├── raw_data.py │ │ ├── raw_data_config.py │ │ ├── raw_data_job.py │ │ ├── raw_data_meta.py │ │ ├── spark_application.py │ │ └── webconsole_client.py │ ├── raw_data_iter_impl │ │ ├── __init__.py │ │ ├── csv_dict_iter.py │ │ ├── metric_stats.py │ │ ├── raw_data_iter.py │ │ ├── tf_record_iter.py │ │ └── validator.py │ ├── raw_data_manifest_manager.py │ ├── raw_data_partitioner.py │ ├── raw_data_publisher.py │ ├── raw_data_visitor.py │ ├── routine_worker.py │ ├── rsa_psi │ │ ├── __init__.py │ │ ├── rsa_psi_component.py │ │ ├── rsa_psi_preprocessor.py │ │ └── rsa_psi_signer.py │ ├── sort_run_dumper.py │ ├── sort_run_merger.py │ ├── transmit_follower.py │ ├── transmit_leader.py │ └── visitor.py ├── fedavg │ ├── __init__.py │ ├── _global_context.py │ ├── cluster │ │ ├── __init__.py │ │ ├── cluster.proto │ │ └── cluster_spec.py │ ├── fedavg.py │ ├── master.py │ └── training_service.proto ├── model │ ├── __init__.py │ ├── crypto │ │ ├── __init__.py │ │ ├── fixed_point_number.py │ │ ├── gmpy_math.py │ │ ├── paillier.py │ │ └── secret_sharing.py │ └── tree │ │ ├── __init__.py │ │ ├── loss.py │ │ ├── packing.py │ │ ├── trainer.py │ │ ├── trainer_master_client.py │ │ ├── tree.py │ │ └── utils.py ├── proxy │ ├── __init__.py │ └── channel.py └── trainer │ ├── __init__.py │ ├── _global_context.py │ ├── bridge.py │ ├── cluster_server.py │ ├── data │ ├── __init__.py │ └── data_block_loader.py │ ├── data_visitor.py │ ├── embedding.py │ ├── estimator.py │ ├── feature.py │ ├── operator.py │ ├── parameter_server.py │ ├── run_hooks.py │ ├── sparse_estimator.py │ ├── trainer_master.py │ ├── trainer_master_client.py │ ├── trainer_worker.py │ └── utils.py ├── integration_tests.sh ├── metrics.ndjson ├── protocols └── fedlearner │ ├── channel │ └── channel.proto │ └── common │ ├── common.proto │ ├── data_join_service.proto │ ├── scheduler_service.proto │ ├── trainer_master_service.proto │ ├── trainer_worker_service.proto │ └── tree_model.proto ├── requirements.txt ├── setup.py ├── test ├── __init__.py ├── channel │ ├── __init__.py │ ├── example.py │ ├── greeter.proto │ ├── greeter_pb2.py │ ├── greeter_pb2_grpc.py │ └── test_channel.py ├── common │ ├── for_test_hooks.py │ ├── test_hooks.py │ └── test_stats.py ├── compressed_raw_data │ └── partition_0000 │ │ └── 0-0.idx ├── csv_raw_data │ └── partition_0000 │ │ └── test_raw_data.csv ├── data_join │ ├── datasource_producer.py │ ├── jars │ │ ├── spark-tensorflow-connector_2.12-1.15.0.jar │ │ └── tensorflow-hadoop-1.15.0.jar │ ├── test_data_block.py │ ├── test_data_block_dumper.py │ ├── test_data_block_manager.py │ ├── test_data_block_visitor.py │ ├── test_data_join_master.py │ ├── test_data_join_worker.py │ ├── test_dfs_client.py │ ├── test_dumped_example_id.py │ ├── test_etcd_client.py │ ├── test_example_join.py │ ├── test_expression.py │ ├── test_input_data_manager.py │ ├── test_input_data_validator.py │ ├── test_mysql_client.py │ ├── test_raw_data.py │ ├── test_raw_data_manifest_manager.py │ ├── test_raw_data_visitor.py │ ├── test_rsa_psi.py │ ├── test_sliding_window.py │ ├── test_special_raw_data_visitor.py │ ├── test_time_diff.py │ └── test_universal_joiner.py ├── fedavg │ └── test_fedavg.py ├── rsa_key │ ├── rsa_psi │ └── rsa_psi.pub ├── trainer │ ├── __init__.py │ ├── graph_def │ │ ├── __init__.py │ │ ├── follower.py │ │ ├── horizontal_follower.py │ │ ├── horizontal_leader.py │ │ └── leader.py │ ├── test_bridge.py │ ├── test_data_visitor.py │ ├── test_horizontal_nn_trainer.py │ ├── test_nn_trainer.py │ └── test_step_trainer.py └── tree_model │ ├── test_crypto.py │ ├── test_filter_files.py │ ├── test_pack.py │ ├── test_secret_sharing.py │ ├── test_trainer.py │ └── test_tree_model.py ├── thirdparty └── tensorflow │ └── tensorflow │ └── core │ ├── example │ └── feature.proto │ ├── framework │ ├── allocation_description.proto │ ├── api_def.proto │ ├── attr_value.proto │ ├── cost_graph.proto │ ├── device_attributes.proto │ ├── function.proto │ ├── graph.proto │ ├── graph_transfer_info.proto │ ├── kernel_def.proto │ ├── log_memory.proto │ ├── node_def.proto │ ├── op_def.proto │ ├── reader_base.proto │ ├── remote_fused_graph_execute_info.proto │ ├── resource_handle.proto │ ├── step_stats.proto │ ├── summary.proto │ ├── tensor.proto │ ├── tensor_description.proto │ ├── tensor_shape.proto │ ├── tensor_slice.proto │ ├── types.proto │ ├── variable.proto │ └── versions.proto │ └── protobuf │ └── cluster.proto ├── web_console ├── .dockerignore ├── .eslintignore ├── .eslintrc ├── .npmrc ├── Dockerfile ├── README.md ├── api │ ├── activity.js │ ├── deployment.js │ ├── federation.js │ ├── job.js │ ├── login.js │ ├── logout.js │ ├── raw_data.js │ ├── ticket.js │ └── user.js ├── bootstrap.js ├── components │ ├── ActivityListItem.jsx │ ├── BooleanSelect.jsx │ ├── ClientTicketSelect.jsx │ ├── CommonJobList.jsx │ ├── CommonTicket.jsx │ ├── DataPortalTypeSelect.jsx │ ├── DataSourceSelect.jsx │ ├── Dot.jsx │ ├── Empty.jsx │ ├── FederationList.jsx │ ├── FederationSelect.jsx │ ├── Footer.jsx │ ├── Form.jsx │ ├── Header.jsx │ ├── Job.jsx │ ├── JobCard.jsx │ ├── JobCommonInfo.jsx │ ├── JobRoleSelect.jsx │ ├── JobTypeSelect.jsx │ ├── Layout.jsx │ ├── NameValueGroup.jsx │ ├── PopConfirm.jsx │ ├── RawDataSelect.jsx │ ├── ServerTicketSelect.jsx │ ├── Shell.jsx │ ├── ToastContainer.jsx │ └── UserList.jsx ├── constants │ ├── form-default.js │ ├── index.js │ └── job.js ├── libs │ ├── cache.js │ ├── docker.js │ ├── es.js │ ├── http.js │ └── k8s.js ├── middlewares │ ├── admin.js │ ├── find_options.js │ ├── pagination.js │ └── session.js ├── migrations │ ├── 20200903051922-create_tables_v_1_0.js │ ├── 20200904051922-modify_job_model_v_1_5.js │ └── 20201012072548-datasource_meta_v_1_5.js ├── models │ ├── federation.js │ ├── index.js │ ├── job.js │ ├── raw_data.js │ ├── ticket.js │ └── user.js ├── next.config.js ├── package-lock.json ├── package.json ├── pages │ ├── _app.js │ ├── _document.js │ ├── activity.jsx │ ├── admin │ │ ├── federation.jsx │ │ └── user.jsx │ ├── charts │ │ └── [id].jsx │ ├── datasource │ │ ├── job │ │ │ ├── [id].jsx │ │ │ └── index.jsx │ │ └── tickets │ │ │ └── index.jsx │ ├── index.jsx │ ├── job │ │ ├── [id].jsx │ │ ├── index.jsx │ │ ├── pod-log.jsx │ │ └── pod-shell.jsx │ ├── login.jsx │ ├── raw_data │ │ ├── [id].jsx │ │ └── index.jsx │ ├── setting.jsx │ ├── store.js │ ├── ticket │ │ └── index.jsx │ └── training │ │ ├── job │ │ ├── [id].jsx │ │ └── index.jsx │ │ └── tickets │ │ └── index.jsx ├── public │ └── favicon.ico ├── rpc │ ├── client.js │ ├── meta.proto │ └── server.js ├── server.js ├── services │ ├── deployment.js │ ├── index.js │ ├── job.js │ ├── raw_data.js │ └── ticket.js ├── startup.sh ├── tests │ ├── .eslintrc │ ├── api │ │ ├── activity.test.js │ │ ├── federation.test.js │ │ ├── job.test.js │ │ ├── login.test.js │ │ ├── logout.test.js │ │ ├── raw_data.test.js │ │ ├── ticket.test.js │ │ └── user.test.js │ ├── fixtures │ │ ├── activity.js │ │ ├── federation.js │ │ ├── images.js │ │ ├── job.js │ │ ├── raw_data.js │ │ ├── server.js │ │ ├── test.json │ │ ├── test.yaml │ │ ├── test_data_join.yaml │ │ ├── test_data_portal.yaml │ │ ├── test_role.json │ │ ├── test_train.yaml │ │ ├── ticket.js │ │ └── user.js │ ├── libs │ │ ├── cache.test.js │ │ ├── docker.test.js │ │ ├── es.test.js │ │ └── k8s.test.js │ ├── middlewares │ │ ├── admin.test.js │ │ ├── find_options.test.js │ │ └── session.test.js │ ├── rpc │ │ └── service.test.js │ └── utils │ │ ├── check_parse_json.test.js │ │ ├── encrypt.test.js │ │ ├── form_utils.test.js │ │ ├── get_config.test.js │ │ ├── job_builder.test.js │ │ ├── kibana.test.js │ │ ├── time.test.js │ │ └── yaml.test.js └── utils │ ├── check_parse_json.js │ ├── encrypt.js │ ├── form_utils.js │ ├── get_confg.js │ ├── index.js │ ├── job.js │ ├── job_builder.js │ ├── kibana.js │ ├── parse_stream.js │ ├── time.js │ └── yaml.js └── web_console_v2 ├── .dockerignore ├── .gitignore ├── Dockerfile ├── Makefile ├── README.md ├── api ├── .coveragerc ├── .gitignore ├── Makefile ├── README.md ├── ci │ └── pylintrc ├── command.py ├── config.py ├── docs │ ├── best_practices.md │ └── style_guide.md ├── envs.py ├── es_configuration.py ├── fedlearner_webconsole │ ├── __init__.py │ ├── app.py │ ├── auth │ │ ├── __init__.py │ │ ├── apis.py │ │ └── models.py │ ├── composer │ │ ├── __init__.py │ │ ├── composer.py │ │ ├── interface.py │ │ ├── models.py │ │ ├── op_locker.py │ │ ├── runner.py │ │ ├── runner_cache.py │ │ └── thread_reaper.py │ ├── dataset │ │ ├── __init__.py │ │ ├── apis.py │ │ ├── data_pipeline.py │ │ ├── import_handler.py │ │ ├── models.py │ │ ├── services.py │ │ └── sparkapp │ │ │ └── pipeline │ │ │ ├── analyzer.py │ │ │ ├── converter.py │ │ │ ├── transformer.py │ │ │ └── util.py │ ├── db.py │ ├── debug │ │ ├── __init__.py │ │ └── apis.py │ ├── exceptions.py │ ├── initial_db.py │ ├── job │ │ ├── __init__.py │ │ ├── apis.py │ │ ├── metrics.py │ │ ├── models.py │ │ ├── service.py │ │ └── yaml_formatter.py │ ├── k8s │ │ ├── __init__.py │ │ └── models.py │ ├── mmgr │ │ ├── apis.py │ │ ├── models.py │ │ └── service.py │ ├── project │ │ ├── __init__.py │ │ ├── add_on.py │ │ ├── apis.py │ │ └── models.py │ ├── proto │ │ └── __init__.py │ ├── rpc │ │ ├── __init__.py │ │ ├── client.py │ │ └── server.py │ ├── scheduler │ │ ├── __init__.py │ │ ├── scheduler.py │ │ └── transaction.py │ ├── setting │ │ ├── __init__.py │ │ ├── apis.py │ │ └── models.py │ ├── sparkapp │ │ ├── __init__.py │ │ ├── apis.py │ │ ├── schema.py │ │ └── service.py │ ├── utils │ │ ├── __init__.py │ │ ├── base64.py │ │ ├── certificate.py │ │ ├── decorators.py │ │ ├── es.py │ │ ├── es_misc.py │ │ ├── fake_k8s_client.py │ │ ├── file_manager.py │ │ ├── hooks.py │ │ ├── k8s_cache.py │ │ ├── k8s_client.py │ │ ├── k8s_watcher.py │ │ ├── kibana.py │ │ ├── metrics.py │ │ ├── middlewares.py │ │ ├── mixins.py │ │ ├── system_envs.py │ │ └── tars.py │ ├── workflow │ │ ├── __init__.py │ │ ├── apis.py │ │ ├── cronjob.py │ │ └── models.py │ └── workflow_template │ │ ├── __init__.py │ │ ├── apis.py │ │ ├── models.py │ │ ├── slots_formatter.py │ │ └── template_validaor.py ├── gunicorn_config.py ├── logging_config.py ├── migrations │ ├── README │ ├── alembic.ini │ ├── env.py │ ├── script.py.mako │ └── versions │ │ ├── 1b5c86643811_add_editor_info.py │ │ ├── 20d2e65b1fd2_add_scheduler_related_tables.py │ │ ├── 27a868485bf7_add_kind_in_workflow_templates.py │ │ ├── 5a580877595d_user_add_role_and_state.py │ │ ├── 7efe1ea47186_add_project_id_in_datatset.py │ │ ├── 80b0a98ce379_make_config_long.py │ │ ├── 8f4d3ac26935_rename_interval_in_scheduler_item.py │ │ ├── 96788629305d_job_add_error_rm_yaml.py │ │ ├── b3290c1bf67a_add_completed_failed_jobstate.py │ │ ├── b3512a6ce912_initial_comment.py │ │ ├── ecfc6488a6c8_session.py │ │ ├── fe30109949aa_add_extra_field_to_workflow.py │ │ └── fefa06dd93cc_job_snapshot_long.py ├── protocols │ └── fedlearner_webconsole │ │ └── proto │ │ ├── common.proto │ │ ├── dataset.proto │ │ ├── project.proto │ │ ├── service.proto │ │ └── workflow_definition.proto ├── requirements.txt ├── run_coverage.sh ├── run_dev.sh ├── run_prod.sh ├── server.py ├── test │ ├── __init__.py │ ├── auth_test.py │ └── fedlearner_webconsole │ │ ├── app_test.py │ │ ├── composer │ │ ├── common.py │ │ ├── composer_test.py │ │ ├── op_locker_test.py │ │ ├── runner_cache_test.py │ │ └── thread_reaper_test.py │ │ ├── dataset │ │ └── apis_test.py │ │ ├── db_test.py │ │ ├── exceptions_test.py │ │ ├── job │ │ ├── metrics_test.py │ │ ├── service_test.py │ │ └── yaml_formatter_test.py │ │ ├── k8s │ │ └── models_test.py │ │ ├── mmgr │ │ └── model_test.py │ │ ├── project │ │ ├── add_on_test.py │ │ ├── apis_test.py │ │ ├── models_test.py │ │ └── test.tar.gz │ │ ├── rpc │ │ └── client_test.py │ │ ├── scheduler │ │ ├── scheduler_func_test.py │ │ ├── scheduler_test.py │ │ ├── workflow_commit_test.py │ │ └── workflow_template_test.py │ │ ├── setting │ │ └── apis_test.py │ │ ├── sparkapp │ │ ├── __init__.py │ │ ├── apis_test.py │ │ ├── schema_test.py │ │ └── service_test.py │ │ ├── test_data │ │ ├── code.tar.gz │ │ ├── dataset_metainfo │ │ │ ├── _FEATURES │ │ │ ├── _HIST │ │ │ └── _META │ │ ├── sparkapp.tar │ │ ├── workflow_config.json │ │ └── workflow_config_right.json │ │ ├── utils │ │ ├── base64_test.py │ │ ├── decorators_test.py │ │ ├── file_manager_test.py │ │ ├── k8s_client_test.py │ │ ├── kibana_test.py │ │ ├── metrics_test.py │ │ ├── mixins_test.py │ │ └── system_envs_test.py │ │ ├── workflow │ │ ├── apis_test.py │ │ └── cronjob_test.py │ │ └── workflow_template │ │ ├── apis_test.py │ │ ├── slots_formater_test.py │ │ ├── template_validator_test.py │ │ ├── test_template_left.py │ │ └── test_template_right.py ├── testing │ ├── __init__.py │ ├── common.py │ ├── fake_file_manager.py │ └── workflow_template │ │ ├── __init__.py │ │ ├── psi_join_tree_model_no_label.py │ │ └── psi_join_tree_model_with_label.py └── tools │ └── local_runner │ ├── app_a.py │ ├── app_b.py │ ├── initial_db.py │ ├── run_a.sh │ └── run_b.sh ├── client ├── .editorconfig ├── .eslintignore ├── .eslintrc.js ├── .npmrc ├── .prettierrc.js ├── README.md ├── config │ ├── env.js │ ├── getHttpsConfig.js │ ├── jest │ │ ├── cssTransform.js │ │ └── fileTransform.js │ ├── modules.js │ ├── paths.js │ ├── pnpTs.js │ ├── webpack.config.js │ └── webpackDevServer.config.js ├── docs │ └── KNOWN_ISSUES.md ├── jest.config.js ├── package.json ├── pnpm-lock.yaml ├── public │ ├── fed-favicon.ico │ ├── index.html │ └── robots.txt ├── scripts │ ├── build.js │ ├── lessVarsToJs.js │ ├── lessVarsTransform.js │ ├── start.js │ └── test.js ├── src │ ├── App.tsx │ ├── assets │ │ ├── fonts │ │ │ └── ClarityMono │ │ │ │ ├── ClarityMono-Bold.otf │ │ │ │ ├── ClarityMono-Medium.otf │ │ │ │ ├── ClarityMono-Regular.otf │ │ │ │ ├── ClarityMono-Thin.otf │ │ │ │ └── index.less │ │ ├── icons │ │ │ ├── python.svg │ │ │ ├── workflow-completed.svg │ │ │ ├── workflow-error.svg │ │ │ ├── workflow-pending.svg │ │ │ └── workflow-warning.svg │ │ └── images │ │ │ ├── avatar.jpg │ │ │ ├── close-icon.svg │ │ │ ├── empty.svg │ │ │ ├── file.svg │ │ │ ├── get-metrics.svg │ │ │ ├── hacker-codes.jpg │ │ │ ├── login-illustration.png │ │ │ ├── logo-colorful.svg │ │ │ ├── logo-white.svg │ │ │ ├── logo.svg │ │ │ ├── no-result.svg │ │ │ ├── project-action.svg │ │ │ └── settings.svg │ ├── components │ │ ├── BackButton │ │ │ └── index.tsx │ │ ├── BreadcrumbLink │ │ │ ├── Slash.tsx │ │ │ └── index.tsx │ │ ├── ClickToCopy │ │ │ └── index.tsx │ │ ├── CodeEditor │ │ │ └── index.tsx │ │ ├── CountTime │ │ │ └── index.tsx │ │ ├── DatasetSelect │ │ │ └── index.tsx │ │ ├── Footer │ │ │ └── index.tsx │ │ ├── FormLabel │ │ │ └── index.tsx │ │ ├── Header │ │ │ ├── Account.tsx │ │ │ ├── LanguageSwitch.tsx │ │ │ ├── ProjectSelect.tsx │ │ │ └── index.tsx │ │ ├── IconButton │ │ │ └── index.tsx │ │ ├── IconPark │ │ │ ├── icons │ │ │ │ ├── AdProduct.tsx │ │ │ │ ├── Apps.tsx │ │ │ │ ├── ArrowDown.tsx │ │ │ │ ├── ArrowFall.tsx │ │ │ │ ├── ArrowLeft.tsx │ │ │ │ ├── ArrowRight.tsx │ │ │ │ ├── ArrowRise.tsx │ │ │ │ ├── ArrowUp.tsx │ │ │ │ ├── Book.tsx │ │ │ │ ├── Branch.tsx │ │ │ │ ├── Bug.tsx │ │ │ │ ├── Bulb.tsx │ │ │ │ ├── Calendar.tsx │ │ │ │ ├── CaretDown.tsx │ │ │ │ ├── CaretLeft.tsx │ │ │ │ ├── CaretRight.tsx │ │ │ │ ├── CaretUp.tsx │ │ │ │ ├── Check.tsx │ │ │ │ ├── CheckCircle.tsx │ │ │ │ ├── ClockCircle.tsx │ │ │ │ ├── Close.tsx │ │ │ │ ├── CloseCircle.tsx │ │ │ │ ├── Cloud.tsx │ │ │ │ ├── CloudDownload.tsx │ │ │ │ ├── Command.tsx │ │ │ │ ├── Common.tsx │ │ │ │ ├── Compass.tsx │ │ │ │ ├── Copy.tsx │ │ │ │ ├── Crown.tsx │ │ │ │ ├── Dashboard.tsx │ │ │ │ ├── DataServer.tsx │ │ │ │ ├── Delete.tsx │ │ │ │ ├── Desktop.tsx │ │ │ │ ├── DoubleDown.tsx │ │ │ │ ├── DoubleLeft.tsx │ │ │ │ ├── DoubleRight.tsx │ │ │ │ ├── DoubleUp.tsx │ │ │ │ ├── Down.tsx │ │ │ │ ├── DownCircle.tsx │ │ │ │ ├── Download.tsx │ │ │ │ ├── DragArrow.tsx │ │ │ │ ├── DriveFile.tsx │ │ │ │ ├── Edit.tsx │ │ │ │ ├── Email.tsx │ │ │ │ ├── Empty.tsx │ │ │ │ ├── ExclamationCircle.tsx │ │ │ │ ├── Expand.tsx │ │ │ │ ├── Experiment.tsx │ │ │ │ ├── Export.tsx │ │ │ │ ├── Eye.tsx │ │ │ │ ├── EyeInvisible.tsx │ │ │ │ ├── File.tsx │ │ │ │ ├── FileAudio.tsx │ │ │ │ ├── FileImage.tsx │ │ │ │ ├── FilePdf.tsx │ │ │ │ ├── FileVideo.tsx │ │ │ │ ├── FindReplace.tsx │ │ │ │ ├── Folder.tsx │ │ │ │ ├── FolderAdd.tsx │ │ │ │ ├── FolderDelete.tsx │ │ │ │ ├── Fullscreen.tsx │ │ │ │ ├── FullscreenExit.tsx │ │ │ │ ├── Heart.tsx │ │ │ │ ├── History.tsx │ │ │ │ ├── Home.tsx │ │ │ │ ├── Idcard.tsx │ │ │ │ ├── Image.tsx │ │ │ │ ├── ImageClose.tsx │ │ │ │ ├── Import.tsx │ │ │ │ ├── InfoCircle.tsx │ │ │ │ ├── Interaction.tsx │ │ │ │ ├── Layout.tsx │ │ │ │ ├── Left.tsx │ │ │ │ ├── LeftCircle.tsx │ │ │ │ ├── Link.tsx │ │ │ │ ├── Location.tsx │ │ │ │ ├── Lock.tsx │ │ │ │ ├── Log.tsx │ │ │ │ ├── Loop.tsx │ │ │ │ ├── Man.tsx │ │ │ │ ├── Menu.tsx │ │ │ │ ├── MenuFold.tsx │ │ │ │ ├── MenuUnfold.tsx │ │ │ │ ├── Message.tsx │ │ │ │ ├── MindMapping.tsx │ │ │ │ ├── Minus.tsx │ │ │ │ ├── MinusCircle.tsx │ │ │ │ ├── Mobile.tsx │ │ │ │ ├── More.tsx │ │ │ │ ├── MoreVertical.tsx │ │ │ │ ├── Mosaic.tsx │ │ │ │ ├── Nav.tsx │ │ │ │ ├── Notes.tsx │ │ │ │ ├── Notification.tsx │ │ │ │ ├── Paste.tsx │ │ │ │ ├── Pause.tsx │ │ │ │ ├── PauseCircle.tsx │ │ │ │ ├── Pen.tsx │ │ │ │ ├── Phone.tsx │ │ │ │ ├── PlayArrow.tsx │ │ │ │ ├── PlayCircle.tsx │ │ │ │ ├── Plus.tsx │ │ │ │ ├── PlusCircle.tsx │ │ │ │ ├── Poweroff.tsx │ │ │ │ ├── Public.tsx │ │ │ │ ├── Pushpin.tsx │ │ │ │ ├── Qrcode.tsx │ │ │ │ ├── QuestionCircle.tsx │ │ │ │ ├── Quote.tsx │ │ │ │ ├── Redo.tsx │ │ │ │ ├── Refresh.tsx │ │ │ │ ├── Reply.tsx │ │ │ │ ├── Right.tsx │ │ │ │ ├── RightCircle.tsx │ │ │ │ ├── Robot.tsx │ │ │ │ ├── RobotAdd.tsx │ │ │ │ ├── RotateLeft.tsx │ │ │ │ ├── RotateRight.tsx │ │ │ │ ├── Safe.tsx │ │ │ │ ├── Save.tsx │ │ │ │ ├── Scan.tsx │ │ │ │ ├── Schedule.tsx │ │ │ │ ├── Search.tsx │ │ │ │ ├── SelectAll.tsx │ │ │ │ ├── Send.tsx │ │ │ │ ├── SendOne.tsx │ │ │ │ ├── SettingConfig.tsx │ │ │ │ ├── Settings.tsx │ │ │ │ ├── ShareAlt.tsx │ │ │ │ ├── ShareExternal.tsx │ │ │ │ ├── ShareInternal.tsx │ │ │ │ ├── Shrink.tsx │ │ │ │ ├── Sort.tsx │ │ │ │ ├── SortAscending.tsx │ │ │ │ ├── SortDescending.tsx │ │ │ │ ├── Star.tsx │ │ │ │ ├── Stop.tsx │ │ │ │ ├── Storage.tsx │ │ │ │ ├── Struct.tsx │ │ │ │ ├── Subscribe.tsx │ │ │ │ ├── SubscribeAdd.tsx │ │ │ │ ├── Subscribed.tsx │ │ │ │ ├── Swap.tsx │ │ │ │ ├── Sync.tsx │ │ │ │ ├── TableReport.tsx │ │ │ │ ├── Tag.tsx │ │ │ │ ├── Tags.tsx │ │ │ │ ├── Thunderbolt.tsx │ │ │ │ ├── ToBottom.tsx │ │ │ │ ├── ToLeft.tsx │ │ │ │ ├── ToRight.tsx │ │ │ │ ├── ToTop.tsx │ │ │ │ ├── Tool.tsx │ │ │ │ ├── Undo.tsx │ │ │ │ ├── Unlock.tsx │ │ │ │ ├── UnorderedList.tsx │ │ │ │ ├── Up.tsx │ │ │ │ ├── UpCircle.tsx │ │ │ │ ├── Usb.tsx │ │ │ │ ├── User.tsx │ │ │ │ ├── UserAdd.tsx │ │ │ │ ├── UserGroup.tsx │ │ │ │ ├── Workbench.tsx │ │ │ │ ├── ZoomIn.tsx │ │ │ │ └── ZoomOut.tsx │ │ │ ├── index.ts │ │ │ └── runtime │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ ├── ModelCodesEditorButton │ │ │ ├── FileExplorer.tsx │ │ │ └── index.tsx │ │ ├── NoResult │ │ │ └── index.tsx │ │ ├── PrettyMenu │ │ │ └── index.tsx │ │ ├── PrintLogs │ │ │ └── index.tsx │ │ ├── PropertyList │ │ │ └── index.tsx │ │ ├── ReadFile │ │ │ └── index.tsx │ │ ├── SharedPageLayout │ │ │ └── index.tsx │ │ ├── Sidebar │ │ │ └── index.tsx │ │ ├── StateIndicator │ │ │ └── index.tsx │ │ ├── UserRoleBadge │ │ │ └── index.tsx │ │ ├── Username │ │ │ └── index.tsx │ │ ├── VariableLabel │ │ │ └── index.tsx │ │ ├── VariableSchemaForm │ │ │ └── index.tsx │ │ ├── VariblePermission │ │ │ └── index.tsx │ │ ├── WhichProject │ │ │ └── index.tsx │ │ ├── WorkflowJobsCanvas │ │ │ ├── JobNodes │ │ │ │ ├── ConfigNode.tsx │ │ │ │ ├── DisabledSwitch.tsx │ │ │ │ ├── EditConfigNode.tsx │ │ │ │ ├── ExecutionNode.tsx │ │ │ │ ├── ForkNode.tsx │ │ │ │ ├── GlobalConfigNode.tsx │ │ │ │ ├── elements.ts │ │ │ │ ├── index.ts │ │ │ │ └── shared.ts │ │ │ ├── elements.ts │ │ │ ├── helpers.test.ts │ │ │ ├── helpers.ts │ │ │ ├── hooks.ts │ │ │ ├── index.tsx │ │ │ └── types.ts │ │ ├── YAMLTemplateEditorButton │ │ │ └── index.tsx │ │ └── _base │ │ │ ├── BlockRadio │ │ │ └── index.tsx │ │ │ ├── GridRow │ │ │ └── index.tsx │ │ │ └── MockDevtools │ │ │ ├── MockControlPanel.tsx │ │ │ ├── index.js │ │ │ └── utils.ts │ ├── hooks │ │ ├── dataset.ts │ │ ├── index.ts │ │ ├── project.ts │ │ ├── recoil.ts │ │ ├── template.ts │ │ └── workflow.ts │ ├── i18n │ │ ├── helper.test.ts │ │ ├── helpers.ts │ │ ├── index.ts │ │ └── resources │ │ │ ├── en.ts │ │ │ ├── modules │ │ │ ├── app.ts │ │ │ ├── dataset.ts │ │ │ ├── error.ts │ │ │ ├── login.ts │ │ │ ├── menu.ts │ │ │ ├── project.ts │ │ │ ├── settings.ts │ │ │ ├── term.ts │ │ │ ├── upload.ts │ │ │ ├── users.ts │ │ │ └── workflow.ts │ │ │ └── zh_CN.ts │ ├── index.tsx │ ├── libs │ │ ├── mockAdapter.ts │ │ └── request.ts │ ├── services │ │ ├── dataset.ts │ │ ├── mocks │ │ │ └── v2 │ │ │ │ ├── auth │ │ │ │ ├── signin.ts │ │ │ │ ├── signout.ts │ │ │ │ └── users │ │ │ │ │ └── :id.ts │ │ │ │ ├── datasets │ │ │ │ ├── :id │ │ │ │ │ ├── batches.ts │ │ │ │ │ └── index.ts │ │ │ │ ├── examples.ts │ │ │ │ └── index.ts │ │ │ │ ├── files │ │ │ │ └── index.ts │ │ │ │ ├── jobs │ │ │ │ └── :id │ │ │ │ │ ├── events.ts │ │ │ │ │ ├── log.ts │ │ │ │ │ └── metrics.ts │ │ │ │ ├── projects │ │ │ │ ├── :id │ │ │ │ │ ├── connection_checks.ts │ │ │ │ │ └── index.ts │ │ │ │ └── index.ts │ │ │ │ ├── settings │ │ │ │ └── index.ts │ │ │ │ ├── variables │ │ │ │ └── examples.ts │ │ │ │ ├── workflow_templates │ │ │ │ ├── :id │ │ │ │ │ └── index.ts │ │ │ │ ├── examples.ts │ │ │ │ └── index.ts │ │ │ │ └── workflows │ │ │ │ ├── :id │ │ │ │ ├── index.ts │ │ │ │ └── peer_workflows.ts │ │ │ │ ├── examples.ts │ │ │ │ └── index.ts │ │ ├── project.ts │ │ ├── settings.ts │ │ ├── system.ts │ │ ├── user.ts │ │ └── workflow.ts │ ├── shared │ │ ├── array.test.ts │ │ ├── array.ts │ │ ├── base64.test.ts │ │ ├── base64.ts │ │ ├── dataset.test.ts │ │ ├── dataset.ts │ │ ├── date.test.ts │ │ ├── date.ts │ │ ├── file.ts │ │ ├── formSchema.test.ts │ │ ├── formSchema.tsx │ │ ├── helpers.ts │ │ ├── localStorageKeys.ts │ │ ├── object.test.ts │ │ ├── object.ts │ │ ├── project.test.ts │ │ ├── project.ts │ │ ├── queryClient.ts │ │ ├── validator.test.ts │ │ ├── validator.ts │ │ ├── variablePresets.ts │ │ ├── workflow.test.ts │ │ └── workflow.ts │ ├── stores │ │ ├── dataset.ts │ │ ├── project.ts │ │ ├── template.ts │ │ ├── user.ts │ │ └── workflow.ts │ ├── styles │ │ ├── _theme.ts │ │ ├── _variables.css │ │ ├── animations.ts │ │ ├── antd-overrides.less │ │ ├── elements.ts │ │ ├── mixins.test.ts │ │ ├── mixins.ts │ │ └── variables.less │ ├── typings │ │ ├── app.ts │ │ ├── auth.ts │ │ ├── component.ts │ │ ├── dataset.ts │ │ ├── formily.ts │ │ ├── global.d.ts │ │ ├── job.ts │ │ ├── kibana.ts │ │ ├── project.ts │ │ ├── settings.ts │ │ ├── variable.ts │ │ └── workflow.ts │ └── views │ │ ├── Dashboard │ │ └── index.tsx │ │ ├── Datasets │ │ ├── AddBatchForm │ │ │ ├── FileToImportList.tsx │ │ │ └── index.tsx │ │ ├── CreateDataset │ │ │ ├── StepOneBasic │ │ │ │ └── index.tsx │ │ │ ├── StepTwoAddBatch │ │ │ │ └── index.tsx │ │ │ └── index.tsx │ │ ├── DatasetList │ │ │ ├── AddBatchModal.tsx │ │ │ ├── BatchImportRecordsModal.tsx │ │ │ ├── DatasetActions.tsx │ │ │ ├── ImportProgress.tsx │ │ │ └── index.tsx │ │ └── index.tsx │ │ ├── Login │ │ └── index.tsx │ │ ├── LogsViewer │ │ ├── JobEvents │ │ │ └── index.tsx │ │ ├── JobLogs │ │ │ └── index.tsx │ │ ├── PodLogs │ │ │ └── index.tsx │ │ ├── SystemLogs │ │ │ └── index.tsx │ │ └── index.tsx │ │ ├── Projects │ │ ├── ConnectionStatus.tsx │ │ ├── CreateProject │ │ │ └── index.tsx │ │ ├── CreateTime.tsx │ │ ├── EditProject │ │ │ └── index.tsx │ │ ├── ProjectDetailDrawer │ │ │ ├── DetailBody.tsx │ │ │ ├── DetailHeader.tsx │ │ │ └── index.tsx │ │ ├── ProjectForm │ │ │ ├── Certificate.tsx │ │ │ ├── EnvVariablesForm.tsx │ │ │ ├── SecondaryForm.tsx │ │ │ └── index.tsx │ │ ├── ProjectList │ │ │ ├── CardView │ │ │ │ ├── ProjectCard.tsx │ │ │ │ ├── ProjectCardProp.tsx │ │ │ │ └── index.tsx │ │ │ ├── ProjectListFilters.tsx │ │ │ ├── TableView │ │ │ │ └── index.tsx │ │ │ └── index.tsx │ │ ├── ProjectMoreActions.tsx │ │ ├── ProjectName.tsx │ │ └── index.tsx │ │ ├── ProtectedRoute.tsx │ │ ├── Settings │ │ └── index.tsx │ │ ├── Users │ │ ├── UserCreate │ │ │ └── index.tsx │ │ ├── UserEdit │ │ │ └── index.tsx │ │ ├── UserForm │ │ │ └── index.tsx │ │ ├── UserList │ │ │ └── index.tsx │ │ └── index.tsx │ │ ├── WorkflowTemplates │ │ ├── CreateTemplate │ │ │ └── index.tsx │ │ ├── EditTemplate │ │ │ └── index.tsx │ │ ├── TemplateForm │ │ │ ├── StepOneBasic │ │ │ │ └── index.tsx │ │ │ ├── StepTwoJobs │ │ │ │ ├── JobComposeDrawer │ │ │ │ │ ├── VariableForm │ │ │ │ │ │ ├── WidgetSchema.tsx │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── VariableList.tsx │ │ │ │ │ └── index.tsx │ │ │ │ ├── TemplateCanvas.tsx │ │ │ │ ├── TemplateConfigNode.tsx │ │ │ │ └── index.tsx │ │ │ ├── index.tsx │ │ │ └── store.ts │ │ ├── TemplateList │ │ │ ├── TemplateUploadDialog.tsx │ │ │ └── index.tsx │ │ └── index.tsx │ │ ├── Workflows │ │ ├── CreateWorkflow │ │ │ ├── StepOneBasic │ │ │ │ └── index.tsx │ │ │ ├── SteptTwoConfig │ │ │ │ └── index.tsx │ │ │ └── index.tsx │ │ ├── EditWorkflow │ │ │ ├── StepOneBasic │ │ │ │ └── index.tsx │ │ │ ├── SteptTwoConfig │ │ │ │ └── index.tsx │ │ │ └── index.tsx │ │ ├── ForkWorkflow │ │ │ ├── StepOneBasic │ │ │ │ └── index.tsx │ │ │ ├── StepTwoConfig │ │ │ │ └── index.tsx │ │ │ └── index.tsx │ │ ├── InspectPeerConfig.tsx │ │ ├── JobFormDrawer.tsx │ │ ├── ScheduledWorkflowRunning │ │ │ └── index.tsx │ │ ├── WorkflowAccessControl │ │ │ ├── AccessSwitch.tsx │ │ │ └── index.tsx │ │ ├── WorkflowActions.tsx │ │ ├── WorkflowDetail │ │ │ ├── GlobalConfigDrawer.tsx │ │ │ ├── JobExecutionDetailsDrawer.tsx │ │ │ ├── JobExecutionLogs.tsx │ │ │ ├── JobExecutionMetrics.tsx │ │ │ ├── JobExecutionPods.tsx │ │ │ ├── JobKibanaMetrics │ │ │ │ ├── FieldComponents │ │ │ │ │ ├── AggregatorSelect.tsx │ │ │ │ │ ├── IntervalInput.tsx │ │ │ │ │ ├── JsonStringInput.tsx │ │ │ │ │ ├── TimerNameInput.tsx │ │ │ │ │ ├── UnixTimePicker.tsx │ │ │ │ │ └── XAxisInput.tsx │ │ │ │ ├── KibanaChart │ │ │ │ │ ├── EmbeddedChart.tsx │ │ │ │ │ └── LineChart.tsx │ │ │ │ ├── KibanaItem.tsx │ │ │ │ ├── KibanaParamsForm.tsx │ │ │ │ ├── elements.ts │ │ │ │ └── index.tsx.tsx │ │ │ └── index.tsx │ │ ├── WorkflowList │ │ │ ├── WorkflowStage.tsx │ │ │ └── index.tsx │ │ ├── index.tsx │ │ └── pubsub.ts │ │ ├── index.tsx │ │ └── routes.tsx ├── tests │ ├── e2e │ │ └── .gitkeep │ └── setup.ts └── tsconfig.json ├── docker └── spark │ ├── Dockerfile │ └── requirements.txt ├── nginx.conf ├── run_prod.sh └── tools └── start_db.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .github 3 | deploy/proxy 4 | test/ 5 | thirdparty/tensorflow/.git 6 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | 9 | [*.{js,py}] 10 | charset = utf-8 11 | 12 | [*.{babelrc,eslintrc,js,json,jsx}] 13 | indent_style = space 14 | indent_size = 2 15 | charset = utf-8 16 | trim_trailing_whitespace = true 17 | insert_final_newline = true 18 | 19 | [*.js] 20 | quote_type = single 21 | 22 | [*.py] 23 | indent_style = space 24 | indent_size = 4 25 | 26 | [*.md] 27 | trim_trailing_whitespace = false 28 | 29 | [Makefile] 30 | indent_size = 4 31 | indent_style = tab 32 | 33 | [{package.json,*.yml}] 34 | indent_style = space 35 | indent_size = 2 36 | -------------------------------------------------------------------------------- /.github/workflows/build-apiserver.yaml: -------------------------------------------------------------------------------- 1 | name: Build Fedlearner APIServer 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Build 11 | run: | 12 | cd ./deploy/kubernetes_operator/ 13 | go build ./cmd/apiserver/main.go 14 | -------------------------------------------------------------------------------- /.github/workflows/build-operator.yaml: -------------------------------------------------------------------------------- 1 | name: Build Kubernetes Operator 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Build 11 | run: | 12 | cd ./deploy/kubernetes_operator/ 13 | go build ./cmd/operator/main.go 14 | -------------------------------------------------------------------------------- /.github/workflows/release-fedlearner.yaml: -------------------------------------------------------------------------------- 1 | name: Release Fedlearner 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Set Short Commit Sha 11 | id: vars 12 | run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" 13 | - uses: whoan/docker-build-with-cache-action@v5 14 | with: 15 | username: fedlearner 16 | password: "${{ secrets.DOCKER_REGISTRY }}" 17 | image_name: fedlearner 18 | image_tag: ${{ steps.vars.outputs.sha_short }} 19 | context: ./ 20 | -------------------------------------------------------------------------------- /.github/workflows/tag-fedlearner.yaml: -------------------------------------------------------------------------------- 1 | name: Tag Fedlearner 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set Short Commit Sha 13 | id: vars 14 | run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" 15 | - uses: whoan/docker-build-with-cache-action@v5 16 | with: 17 | username: fedlearner 18 | password: "${{ secrets.DOCKER_REGISTRY }}" 19 | image_name: fedlearner 20 | image_tag: ${{ github.event.release.tag_name }} 21 | context: ./ 22 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of FedLearner authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | Bytedance Inc. 10 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | FedLearner 2 | Copyright 2020 The FedLearner Authors 3 | 4 | This product includes software developed by 5 | The FedLearner Authors. 6 | -------------------------------------------------------------------------------- /cc/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | TF_CFLAGS=$(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') 5 | TF_LFLAGS=$(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') 6 | 7 | SRC_DIR=$(pwd)/cc 8 | 9 | g++ -std=c++11 -shared ${SRC_DIR}/operators/kernels/*.cc ${SRC_DIR}/operators/ops/*.cc -o ${SRC_DIR}/embedding.so -fPIC ${TF_CFLAGS} ${TF_LFLAGS} -O2 10 | -------------------------------------------------------------------------------- /ci/ci_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | export PYTHONPATH=${PWD}:${PYTHONPATH} 5 | 6 | make op 7 | make protobuf 8 | make lint 9 | make test 10 | -------------------------------------------------------------------------------- /config.mk.template: -------------------------------------------------------------------------------- 1 | TF_PATH=thirdparty/tensorflow 2 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-add-on/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-add-on/configuration-snippet.txt: -------------------------------------------------------------------------------- 1 | grpc_next_upstream_tries 5; 2 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-add-on/server-snippet.txt: -------------------------------------------------------------------------------- 1 | grpc_ssl_verify on; 2 | grpc_ssl_server_name on; 3 | grpc_ssl_trusted_certificate /etc/ingress-nginx/client/all.pem; 4 | grpc_ssl_certificate /etc/ingress-nginx/client/client.pem; 5 | grpc_ssl_certificate_key /etc/ingress-nginx/client/client.key; 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-add-on/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.service.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: fedlearner-proxy 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "fedlearner-add-on.labels" . | nindent 4 }} 9 | spec: 10 | type: {{ .Values.service.type }} 11 | externalName: {{ .Values.service.externalName }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-pvc/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-pvc/charts/nfs-client-provisioner-1.2.8.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/charts/fedlearner-pvc/charts/nfs-client-provisioner-1.2.8.tgz -------------------------------------------------------------------------------- /deploy/charts/fedlearner-pvc/charts/nfs-server-provisioner-1.0.0.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/charts/fedlearner-pvc/charts/nfs-server-provisioner-1.0.0.tgz -------------------------------------------------------------------------------- /deploy/charts/fedlearner-pvc/templates/pv.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolume 3 | metadata: 4 | name: pv-fedlearner-default 5 | spec: 6 | accessModes: 7 | - ReadWriteMany 8 | capacity: 9 | storage: {{ .Values.nfs.capacity }} 10 | mountOptions: 11 | - vers=3 12 | - nolock,tcp,noresvport 13 | nfs: 14 | path: {{ .Values.nfs.path }} 15 | server: {{ .Values.nfs.server }} 16 | persistentVolumeReclaimPolicy: Retain 17 | storageClassName: nfs 18 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-pvc/templates/pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: pvc-fedlearner-default 5 | spec: 6 | accessModes: 7 | - ReadWriteMany 8 | resources: 9 | requests: 10 | storage: {{ .Values.nfs.capacity }} 11 | volumeName: pv-fedlearner-default 12 | storageClassName: nfs 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "6" 3 | description: A Helm chart for ELK 4 | home: https://www.elastic.co/products 5 | icon: https://www.elastic.co/assets/bltb35193323e8f1770/logo-elastic-stack-lt.svg 6 | maintainers: 7 | - email: pete.brown@powerhrg.com 8 | name: rendhalver 9 | - email: jrodgers@powerhrg.com 10 | name: jar361 11 | - email: christian.roggia@gmail.com 12 | name: christian-roggia 13 | name: elastic-stack 14 | version: 2.0.0 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - christian-roggia 3 | - rendhalver 4 | reviewers: 5 | - christian-roggia 6 | - rendhalver 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-curator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-curator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 5.7.6 3 | description: A Helm chart for Elasticsearch Curator 4 | home: https://github.com/elastic/curator 5 | keywords: 6 | - curator 7 | - elasticsearch 8 | - elasticsearch-curator 9 | maintainers: 10 | - email: cedric.dsm@gmail.com 11 | name: desaintmartin 12 | - email: gianrubio@gmail.com 13 | name: gianrubio 14 | name: elasticsearch-curator 15 | sources: 16 | - https://github.com/kubernetes/charts/elasticsearch-curator 17 | - https://github.com/pires/docker-elasticsearch-curator 18 | version: 2.1.0 19 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-curator/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - desaintmartin 3 | - gianrubio 4 | reviewers: 5 | - desaintmartin 6 | - gianrubio 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-curator/ci/initcontainer-values.yaml: -------------------------------------------------------------------------------- 1 | extraInitContainers: 2 | test: 3 | image: alpine:latest 4 | command: 5 | - "/bin/sh" 6 | - "-c" 7 | args: 8 | - | 9 | true 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-curator/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | A CronJob will run with schedule {{ .Values.cronjob.schedule }}. 2 | 3 | The Jobs will not be removed automagically when deleting this Helm chart. 4 | To remove these jobs, run the following : 5 | 6 | kubectl -n {{ .Release.Namespace }} delete job -l app={{ template "elasticsearch-curator.name" . }},release={{ .Release.Name }} -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-curator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.serviceAccount.create .Values.rbac.enabled }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "elasticsearch-curator.serviceAccountName" .}} 6 | labels: 7 | app: {{ template "elasticsearch-curator.fullname" . }} 8 | chart: {{ template "elasticsearch-curator.chart" . }} 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | {{- end }} 12 | 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-exporter/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | # OWNERS file for Kubernetes 5 | OWNERS 6 | .DS_Store 7 | # Common VCS dirs 8 | .git/ 9 | .gitignore 10 | .bzr/ 11 | .bzrignore 12 | .hg/ 13 | .hgignore 14 | .svn/ 15 | # Common backup files 16 | *.swp 17 | *.bak 18 | *.tmp 19 | *~ 20 | # Various IDEs 21 | .project 22 | .idea/ 23 | *.tmproj 24 | .vscode/ 25 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-exporter/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 1.1.0 3 | description: Elasticsearch stats exporter for Prometheus 4 | home: https://github.com/justwatchcom/elasticsearch_exporter 5 | keywords: 6 | - metrics 7 | - elasticsearch 8 | - monitoring 9 | kubeVersion: '>=1.10.0-0' 10 | maintainers: 11 | - email: sven.mueller@commercetools.com 12 | name: svenmueller 13 | - email: carlos@carlosbecker.com 14 | name: caarlos0 15 | name: elasticsearch-exporter 16 | sources: 17 | - https://github.com/justwatchcom/elasticsearch_exporter 18 | version: 2.1.0 19 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-exporter/ci/default-values.yaml: -------------------------------------------------------------------------------- 1 | # Leave this file empty to ensure that CI runs builds against the default configuration in values.yaml. 2 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-exporter/ci/security-context.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Set default security context for kubernetes 3 | 4 | securityContext: 5 | disable: true 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch-exporter/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "elasticsearch-exporter.fullname" . }} 6 | labels: 7 | chart: {{ template "elasticsearch-exporter.chart" . }} 8 | app: {{ template "elasticsearch-exporter.name" . }} 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/.helmignore: -------------------------------------------------------------------------------- 1 | .git 2 | # OWNERS file for Kubernetes 3 | OWNERS -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/ci/expose-transport-port-on-service-values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Expose transport port on ClusterIP service 3 | 4 | client: 5 | exposeTransportPort: true 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/ci/extrainitcontainers-values.yaml: -------------------------------------------------------------------------------- 1 | extraInitContainers: | 2 | - name: "plugin-install-ingest-attachment" 3 | image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.1" 4 | command: ["/bin/bash"] 5 | args: ["-c", "yes | /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-attachment"] 6 | - name: "plugin-install-mapper-size" 7 | image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.1" 8 | command: ["/bin/bash"] 9 | args: ["-c", "yes | /usr/share/elasticsearch/bin/elasticsearch-plugin install mapper-size"] 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/ci/nonroot-deployment-values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Deploy Chart as non-root and unprivileged 3 | 4 | chownInitContainer: 5 | enabled: false 6 | 7 | securityContext: 8 | enabled: true 9 | runAsUser: 1000 10 | 11 | sysctlInitContainer: 12 | enabled: false 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/ci/plugin-initcontainer-values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Enable init container for installing plugins 3 | 4 | cluster: 5 | plugins: 6 | - ingest-attachment 7 | - mapper-size 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/ci/updatestrategy-values.yaml: -------------------------------------------------------------------------------- 1 | data: 2 | updateStrategy: 3 | type: RollingUpdate 4 | 5 | master: 6 | updateStrategy: 7 | type: RollingUpdate 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/templates/client-auth.yaml: -------------------------------------------------------------------------------- 1 | {{- if and ( .Values.client.ingress.user ) ( .Values.client.ingress.password ) }} 2 | --- 3 | apiVersion: v1 4 | kind: Secret 5 | metadata: 6 | name: '{{ include "elasticsearch.client.fullname" . }}-auth' 7 | type: Opaque 8 | data: 9 | auth: {{ printf "%s:{PLAIN}%s\n" .Values.client.ingress.user .Values.client.ingress.password | b64enc | quote }} 10 | {{- end }} 11 | 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/templates/client-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccounts.client.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app: {{ template "elasticsearch.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | component: "{{ .Values.client.name }}" 9 | heritage: {{ .Release.Service }} 10 | release: {{ .Release.Name }} 11 | name: {{ template "elasticsearch.client.fullname" . }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/templates/data-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccounts.data.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app: {{ template "elasticsearch.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | component: "{{ .Values.data.name }}" 9 | heritage: {{ .Release.Service }} 10 | release: {{ .Release.Name }} 11 | name: {{ template "elasticsearch.data.fullname" . }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/templates/master-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccounts.master.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app: {{ template "elasticsearch.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | component: "{{ .Values.master.name }}" 9 | heritage: {{ .Release.Service }} 10 | release: {{ .Release.Name }} 11 | name: {{ template "elasticsearch.master.fullname" . }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/elasticsearch/templates/role.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.podSecurityPolicy.enabled }} 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | kind: Role 4 | metadata: 5 | name: {{ template "elasticsearch.fullname" . }} 6 | labels: 7 | app: {{ template "elasticsearch.name" . }} 8 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | rules: 12 | - apiGroups: ['extensions'] 13 | resources: ['podsecuritypolicies'] 14 | verbs: ['use'] 15 | resourceNames: 16 | - {{ template "elasticsearch.fullname" . }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/filebeat/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/filebeat/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 7.0.1 3 | description: A Helm chart to collect Kubernetes logs with filebeat 4 | home: https://www.elastic.co/products/beats/filebeat 5 | icon: https://www.elastic.co/assets/blt47799dcdcf08438d/logo-elastic-beats-lt.svg 6 | maintainers: 7 | - email: pete.brown@powerhrg.com 8 | name: rendhalver 9 | - email: shane.starcher@gmail.com 10 | name: sstarcher 11 | name: filebeat 12 | sources: 13 | - https://www.elastic.co/guide/en/beats/filebeat/current/index.html 14 | version: 3.1.1 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/filebeat/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - rendhalver 3 | - sstarcher 4 | reviewers: 5 | - rendhalver 6 | - sstarcher 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/filebeat/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | To verify that Filebeat has started, run: 2 | 3 | kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "filebeat.name" . }},release={{ .Release.Name }}" 4 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/filebeat/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: {{ template "filebeat.fullname" . }} 6 | labels: 7 | app.kubernetes.io/name: {{ template "filebeat.name" . }} 8 | helm.sh/chart: {{ template "filebeat.chart" . }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | app.kubernetes.io/managed-by: {{ .Release.Service }} 11 | rules: 12 | - apiGroups: [""] 13 | resources: 14 | - namespaces 15 | - pods 16 | verbs: ["get", "list", "watch"] 17 | {{- end -}} 18 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/filebeat/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "filebeat.serviceAccountName" . }} 6 | labels: 7 | app.kubernetes.io/name: {{ template "filebeat.name" . }} 8 | helm.sh/chart: {{ template "filebeat.chart" . }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | app.kubernetes.io/managed-by: {{ .Release.Service }} 11 | {{- end -}} 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluent-bit/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 1.3.2 3 | description: Fast and Lightweight Log/Data Forwarder for Linux, BSD and OSX 4 | home: https://fluentbit.io 5 | icon: https://fluentbit.io/assets/img/logo1-default.png 6 | keywords: 7 | - logging 8 | - monitoring 9 | - fluent 10 | - fluentd 11 | maintainers: 12 | - email: Kevin.Fox@pnnl.gov 13 | name: kfox1111 14 | - email: eduardo@treasure-data.com 15 | name: edsiper 16 | - email: hfernandez@mesosphere.com 17 | name: hectorj2f 18 | name: fluent-bit 19 | sources: 20 | - https://fluentbit.io 21 | version: 2.8.0 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluent-bit/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - kfox1111 3 | - edsiper 4 | - hectorj2f 5 | - Towmeykaw 6 | reviewers: 7 | - kfox1111 8 | - edsiper 9 | - hectorj2f 10 | - Towmeykaw 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluent-bit/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | app: {{ template "fluent-bit.name" . }} 7 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 8 | heritage: {{ .Release.Service }} 9 | release: {{ .Release.Name }} 10 | name: {{ template "fluent-bit.serviceAccountName" . }} 11 | {{- end -}} 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd-elasticsearch/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - axdotl 3 | - monotek 4 | reviewers: 5 | - axdotl 6 | - monotek 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd-elasticsearch/templates/service-account.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "fluentd-elasticsearch.fullname" . }} 6 | labels: 7 | app.kubernetes.io/name: {{ include "fluentd-elasticsearch.name" . }} 8 | helm.sh/chart: {{ include "fluentd-elasticsearch.chart" . }} 9 | app.kubernetes.io/instance: {{ .Release.Name }} 10 | app.kubernetes.io/managed-by: {{ .Release.Service }} 11 | kubernetes.io/cluster-service: "true" 12 | addonmanager.kubernetes.io/mode: Reconcile 13 | {{- end -}} 14 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - rendhalver 3 | - miouge1 4 | - hectorj2f 5 | reviewers: 6 | - rendhalver 7 | - miouge1 8 | - hectorj2f 9 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | To verify that Fluentd Elasticsearch has started, run: 2 | 3 | kubectl --namespace={{ .Release.Namespace }} get all -l "app={{ template "fluentd.name" . }},release={{ .Release.Name }}" 4 | 5 | THIS APPLICATION CAPTURES ALL CONSOLE OUTPUT AND FORWARDS IT TO Elasticsearch. Anything that might be identifying, 6 | including things like IP addresses, container images, and object names will NOT be anonymized. 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | kind: ClusterRole 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: {{ include "fluentd.fullname" . }} 6 | labels: 7 | app: {{ template "fluentd.name" . }} 8 | chart: {{ template "fluentd.chart" . }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | rules: 12 | - apiGroups: 13 | - "" 14 | resources: 15 | - "namespaces" 16 | - "pods" 17 | verbs: 18 | - "get" 19 | - "watch" 20 | - "list" 21 | {{- end -}} 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd/templates/role.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create }} 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | kind: Role 4 | metadata: 5 | name: {{ template "fluentd.fullname" . }} 6 | labels: 7 | app: {{ template "fluentd.name" . }} 8 | chart: {{ template "fluentd.chart" . }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | rules: 12 | - apiGroups: ['extensions'] 13 | resources: ['podsecuritypolicies'] 14 | verbs: ['use'] 15 | resourceNames: 16 | - {{ template "fluentd.fullname" . }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/fluentd/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "fluentd.serviceAccountName" . }} 6 | labels: 7 | app: {{ template "fluentd.name" . }} 8 | chart: {{ template "fluentd.chart" . }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- end -}} 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 6.7.0 3 | description: Kibana is an open source data visualization plugin for Elasticsearch 4 | engine: gotpl 5 | home: https://www.elastic.co/products/kibana 6 | icon: https://raw.githubusercontent.com/elastic/kibana/master/src/ui/public/icons/kibana-color.svg 7 | keywords: 8 | - elasticsearch 9 | - kibana 10 | maintainers: 11 | - email: casey@monax.io 12 | name: compleatang 13 | - email: monotek23@gmail.com 14 | name: monotek 15 | name: kibana 16 | sources: 17 | - https://github.com/elastic/kibana 18 | version: 3.2.4 19 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - compleatang 3 | - monotek 4 | reviewers: 5 | - compleatang 6 | - monotek 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/authproxy-enabled.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # disable internal port by setting authProxyEnabled 3 | authProxyEnabled: true 4 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/extra-configmap-mounts.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | extraConfigMapMounts: 3 | - name: logtrail-configs 4 | configMap: kibana-logtrail 5 | mountPath: /usr/share/kibana/plugins/logtrail/logtrail.json 6 | subPath: logtrail.json 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/ingress-hosts-paths.yaml: -------------------------------------------------------------------------------- 1 | ingress: 2 | hosts: 3 | - localhost.localdomain/kibana 4 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/ingress-hosts.yaml: -------------------------------------------------------------------------------- 1 | ingress: 2 | hosts: 3 | - kibana.localhost.localdomain 4 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/initcontainers-values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # enable user-defined init containers 3 | 4 | initContainers: 5 | numbers-container: 6 | image: "busybox" 7 | imagePullPolicy: "IfNotPresent" 8 | command: 9 | - "/bin/sh" 10 | - "-c" 11 | - | 12 | for i in $(seq 1 10); do 13 | echo $i 14 | done 15 | 16 | echo-container: 17 | image: "busybox" 18 | command: ['sh', '-c', 'echo Hello from init container! && sleep 3'] 19 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/plugin-install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # enable the plugin init container with plugins retrieved from an URL 3 | plugins: 4 | enabled: true 5 | reset: false 6 | # Use to add/upgrade plugin 7 | values: 8 | - analyze-api-ui-plugin,6.7.0,https://github.com/johtani/analyze-api-ui-plugin/releases/download/6.7.0/analyze-api-ui-plugin-6.7.0.zip 9 | # - other_plugin 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/pvc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | persistentVolumeClaim: 3 | # set to true to use pvc 4 | enabled: true 5 | # set to true to use you own pvc 6 | existingClaim: false 7 | annotations: {} 8 | 9 | accessModes: 10 | - ReadWriteOnce 11 | size: "5Gi" -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/security-context.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | securityContext: 3 | enabled: true 4 | allowPrivilegeEscalation: false 5 | runAsUser: 1000 6 | fsGroup: 2000 -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/service-values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | service: 3 | selector: 4 | foo: bar 5 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/ci/url_dashboard-values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # enable the dashboard init container with dashboard retrieved from an URL 3 | 4 | dashboardImport: 5 | enabled: true 6 | dashboards: 7 | k8s: https://raw.githubusercontent.com/monotek/kibana-dashboards/master/k8s-fluentd-elasticsearch.json 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ template "kibana.fullname" . }} 5 | labels: 6 | app: {{ template "kibana.name" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | data: 11 | {{- range $key, $value := .Values.files }} 12 | {{ $key }}: | 13 | {{ toYaml $value | default "{}" | indent 4 }} 14 | {{- end -}} 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/kibana/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "kibana.serviceAccountName" . }} 6 | labels: 7 | app: {{ template "kibana.name" . }} 8 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 9 | heritage: {{ .Release.Service }} 10 | release: {{ .Release.Name }} 11 | {{- end -}} 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - christian-roggia 3 | - rendhalver 4 | reviewers: 5 | - christian-roggia 6 | - rendhalver 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/serviceaccount.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/serviceaccount.yaml -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/templates/files-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ template "logstash.fullname" . }}-files 5 | labels: 6 | app: {{ template "logstash.name" . }} 7 | chart: {{ template "logstash.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | data: 11 | {{- range $key, $value := .Values.files }} 12 | {{ $key }}: |- 13 | {{ $value | indent 4 }} 14 | {{- end }} 15 | binaryData: 16 | {{- range $key, $value := .Values.binaryFiles }} 17 | {{ $key }}: |- 18 | {{ $value | indent 4 }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/templates/patterns-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ template "logstash.fullname" . }}-patterns 5 | labels: 6 | app: {{ template "logstash.name" . }} 7 | chart: {{ template "logstash.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | data: 11 | {{- range $key, $value := .Values.patterns }} 12 | {{ $key }}: |- 13 | {{ $value | indent 4 }} 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/templates/poddisruptionbudget.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1beta1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: {{ template "logstash.fullname" . }} 5 | labels: 6 | app: {{ template "logstash.name" . }} 7 | chart: {{ template "logstash.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: {{ template "logstash.name" . }} 14 | release: {{ .Release.Name }} 15 | {{ toYaml .Values.podDisruptionBudget | indent 2 }} 16 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/logstash/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "logstash.serviceAccountName" . }} 6 | labels: 7 | app: {{ template "logstash.name" . }} 8 | chart: {{ template "logstash.chart" . }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | {{- end -}} 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/nginx-ldapauth-proxy/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/nginx-ldapauth-proxy/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 1.13.5 3 | description: nginx proxy with ldapauth 4 | home: https://github.com/nginxinc/nginx-ldap-auth 5 | icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png 6 | maintainers: 7 | - email: pete.brown@powerhrg.com 8 | name: rendhalver 9 | - email: jrodgers@powerhrg.com 10 | name: jar361 11 | name: nginx-ldapauth-proxy 12 | sources: 13 | - https://github.com/dweomer/dockerfiles-nginx-auth-ldap 14 | - https://github.com/kvspb/nginx-auth-ldap 15 | version: 0.1.3 16 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/elastic-stack/charts/nginx-ldapauth-proxy/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets.ldapBindPassword }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "nginx-ldapauth-proxy.fullname" . }} 6 | labels: 7 | app: {{ template "nginx-ldapauth-proxy.name" . }} 8 | chart: {{ template "nginx-ldapauth-proxy.chart" . }} 9 | release: {{ .Release.Name }} 10 | heritage: {{ .Release.Service }} 11 | type: Opaque 12 | data: 13 | ldapBindPassword: {{ .Values.secrets.ldapBindPassword | b64enc | quote }} 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/etcd-4.8.2.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/charts/fedlearner-stack/charts/etcd-4.8.2.tgz -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: ingress-nginx 3 | version: 2.1.0 4 | appVersion: 0.32.0 5 | home: https://github.com/kubernetes/ingress-nginx 6 | description: Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer 7 | icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png 8 | keywords: 9 | - ingress 10 | - nginx 11 | sources: 12 | - https://github.com/kubernetes/ingress-nginx 13 | maintainers: 14 | - name: ChiefAlexander 15 | engine: gotpl 16 | kubeVersion: ">=1.10.0-0" 17 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - ChiefAlexander 3 | 4 | reviewers: 5 | - ChiefAlexander 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: ClusterIP 7 | 8 | config: 9 | use-proxy-protocol: "true" 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | 6 | service: 7 | type: NodePort 8 | nodePorts: 9 | tcp: 10 | 9000: 30090 11 | udp: 12 | 9001: 30091 13 | 14 | tcp: 15 | 9000: "default/test:8080" 16 | 17 | udp: 18 | 9001: "default/test:8080" 19 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-headers-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | addHeaders: 6 | X-Frame-Options: deny 7 | proxySetHeaders: 8 | X-Forwarded-Proto: https 9 | service: 10 | type: ClusterIP 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: NodePort 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: ClusterIP 7 | tcp: 8 | configMapNamespace: default 9 | udp: 10 | configMapNamespace: default 11 | 12 | tcp: 13 | 9000: "default/test:8080" 14 | 15 | udp: 16 | 9001: "default/test:8080" 17 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: ClusterIP 7 | 8 | tcp: 9 | 9000: "default/test:8080" 10 | 11 | udp: 12 | 9001: "default/test:8080" 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/daemonset-tcp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: ClusterIP 7 | 8 | tcp: 9 | 9000: "default/test:8080" 10 | 9001: "default/test:8080" 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deamonset-default-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: ClusterIP 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deamonset-metrics-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | metrics: 6 | enabled: true 7 | service: 8 | type: ClusterIP 9 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deamonset-psp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: false 5 | service: 6 | type: ClusterIP 7 | 8 | podSecurityPolicy: 9 | enabled: true 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: true 5 | service: 6 | type: ClusterIP 7 | 8 | podSecurityPolicy: 9 | enabled: true 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deamonset-webhook-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | kind: DaemonSet 3 | admissionWebhooks: 4 | enabled: true 5 | service: 6 | type: ClusterIP 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | autoscaling: 3 | enabled: true 4 | admissionWebhooks: 5 | enabled: false 6 | service: 7 | type: ClusterIP 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-customconfig-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | config: 3 | use-proxy-protocol: "true" 4 | admissionWebhooks: 5 | enabled: false 6 | service: 7 | type: ClusterIP 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: false 4 | service: 5 | type: NodePort 6 | nodePorts: 7 | tcp: 8 | 9000: 30090 9 | udp: 10 | 9001: 30091 11 | 12 | tcp: 13 | 9000: "default/test:8080" 14 | 15 | udp: 16 | 9001: "default/test:8080" 17 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-default-values.yaml: -------------------------------------------------------------------------------- 1 | # Left blank to test default values 2 | controller: 3 | service: 4 | type: ClusterIP 5 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-headers-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: false 4 | addHeaders: 5 | X-Frame-Options: deny 6 | proxySetHeaders: 7 | X-Forwarded-Proto: https 8 | service: 9 | type: ClusterIP 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-metrics-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: false 4 | metrics: 5 | enabled: true 6 | service: 7 | type: ClusterIP 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-nodeport-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: false 4 | service: 5 | type: NodePort 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-psp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | service: 3 | type: ClusterIP 4 | 5 | podSecurityPolicy: 6 | enabled: true 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: false 4 | service: 5 | type: ClusterIP 6 | tcp: 7 | configMapNamespace: default 8 | udp: 9 | configMapNamespace: default 10 | 11 | tcp: 12 | 9000: "default/test:8080" 13 | 14 | udp: 15 | 9001: "default/test:8080" 16 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: false 4 | service: 5 | type: ClusterIP 6 | 7 | tcp: 8 | 9000: "default/test:8080" 9 | 10 | udp: 11 | 9001: "default/test:8080" 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-tcp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | service: 3 | type: ClusterIP 4 | 5 | tcp: 6 | 9000: "default/test:8080" 7 | 9001: "default/test:8080" 8 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: true 4 | service: 5 | type: ClusterIP 6 | 7 | podSecurityPolicy: 8 | enabled: true 9 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/ci/deployment-webhook-values.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | admissionWebhooks: 3 | enabled: true 4 | service: 5 | type: ClusterIP 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "ingress-nginx.fullname" . }}-admission 6 | annotations: 7 | "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade 8 | "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded 9 | labels: 10 | {{- include "ingress-nginx.labels" . | nindent 4 }} 11 | app.kubernetes.io/component: admission-webhook 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.rbac.create (not .Values.rbac.scope) -}} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | name: {{ include "ingress-nginx.fullname" . }} 8 | roleRef: 9 | apiGroup: rbac.authorization.k8s.io 10 | kind: ClusterRole 11 | name: {{ include "ingress-nginx.fullname" . }} 12 | subjects: 13 | - kind: ServiceAccount 14 | name: {{ template "ingress-nginx.serviceAccountName" . }} 15 | namespace: {{ .Release.Namespace }} 16 | {{- end }} 17 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.controller.addHeaders -}} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | app.kubernetes.io/component: controller 8 | name: {{ include "ingress-nginx.fullname" . }}-custom-add-headers 9 | data: {{ toYaml .Values.controller.addHeaders | nindent 2 }} 10 | {{- end }} 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/controller-configmap-tcp.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.tcp -}} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | app.kubernetes.io/component: controller 8 | {{- if .Values.controller.tcp.annotations }} 9 | annotations: {{ toYaml .Values.controller.tcp.annotations | nindent 4 }} 10 | {{- end }} 11 | name: {{ include "ingress-nginx.fullname" . }}-tcp 12 | data: {{ tpl (toYaml .Values.tcp) . | nindent 2 }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/controller-configmap-udp.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.udp -}} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | app.kubernetes.io/component: controller 8 | {{- if .Values.controller.udp.annotations }} 9 | annotations: {{ toYaml .Values.controller.udp.annotations | nindent 4 }} 10 | {{- end }} 11 | name: {{ include "ingress-nginx.fullname" . }}-udp 12 | data: {{ tpl (toYaml .Values.udp) . | nindent 2 }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/controller-rolebinding.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbac.create -}} 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: RoleBinding 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | app.kubernetes.io/component: controller 8 | name: {{ include "ingress-nginx.fullname" . }} 9 | roleRef: 10 | apiGroup: rbac.authorization.k8s.io 11 | kind: Role 12 | name: {{ include "ingress-nginx.fullname" . }} 13 | subjects: 14 | - kind: ServiceAccount 15 | name: {{ template "ingress-nginx.serviceAccountName" . }} 16 | namespace: {{ .Release.Namespace }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/controller-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if or .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | app.kubernetes.io/component: controller 8 | name: {{ template "ingress-nginx.serviceAccountName" . }} 9 | {{- end }} 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.defaultBackend.enabled .Values.defaultBackend.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | labels: 6 | {{- include "ingress-nginx.labels" . | nindent 4 }} 7 | app.kubernetes.io/component: default-backend 8 | name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} 9 | {{- end }} 10 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/mysql-8.0.22.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/charts/fedlearner-stack/charts/mysql-8.0.22.tgz -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/sparkoperator/.helmignore: -------------------------------------------------------------------------------- 1 | OWNERS 2 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/sparkoperator/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: v1beta2-1.2.0-3.0.0 3 | description: A Helm chart for Spark on Kubernetes operator 4 | home: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator 5 | keywords: 6 | - spark 7 | maintainers: 8 | - email: yuchaoran2011@gmail.com 9 | name: yuchaoran2011 10 | name: sparkoperator 11 | version: 0.8.4 12 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/sparkoperator/hack/update-ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | sed \ 4 | -e 's/cleanupCrdsBeforeInstall: false/cleanupCrdsBeforeInstall: true/' \ 5 | values.yaml > ci/test-values.yaml 6 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/sparkoperator/templates/spark-operator-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccounts.sparkoperator.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "sparkoperator.serviceAccountName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | app.kubernetes.io/name: {{ include "sparkoperator.name" . }} 9 | helm.sh/chart: {{ include "sparkoperator.chart" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-stack/charts/sparkoperator/templates/spark-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccounts.spark.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "spark.serviceAccountName" . }} 6 | namespace: {{ .Values.sparkJobNamespace }} 7 | labels: 8 | app.kubernetes.io/name: {{ include "sparkoperator.name" . }} 9 | helm.sh/chart: {{ include "sparkoperator.chart" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-test/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner-test/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | The fedlearner/fedlearner-add-on charts have been installed successfully. 2 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: fedlearner 3 | description: A Helm chart for Fedleaner 4 | type: application 5 | version: 0.1.0 6 | appVersion: 2.1.2 7 | dependencies: 8 | - name: fedlearner-operator 9 | version: 0.1.0 10 | condition: fedlearner-operator.enabled 11 | - name: fedlearner-apiserver 12 | version: 0.1.0 13 | condition: fedlearner-apiserver.enabled 14 | - name: fedlearner-web-console 15 | version: 0.1.0 16 | condition: fedlearner-web-console.enabled 17 | - name: fedlearner-web-console-v2 18 | version: 0.1.0 19 | condition: fedlearner-web-console-v2.enabled 20 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-apiserver/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-apiserver/templates/cluster_role.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRole 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: fedlearner-apiserver 5 | namespace: {{ .Release.Namespace }} 6 | rules: 7 | - apiGroups: ["*"] 8 | resources: ["*"] 9 | verbs: ["*"] 10 | --- 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-apiserver/templates/cluster_role_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: fedlearner-apiserver 5 | namespace: {{ .Release.Namespace }} 6 | subjects: 7 | - kind: ServiceAccount 8 | name: fedlearner-apiserver 9 | namespace: {{ .Release.Namespace }} 10 | roleRef: 11 | kind: ClusterRole 12 | name: fedlearner-apiserver 13 | apiGroup: rbac.authorization.k8s.io 14 | --- 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-apiserver/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: fedlearner-apiserver 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | selector: 8 | app: fedlearner-apiserver 9 | ports: 10 | - protocol: TCP 11 | port: 8101 12 | targetPort: 8101 13 | --- 14 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-apiserver/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: fedlearner-apiserver 5 | namespace: {{ .Release.Namespace }} 6 | --- 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-operator/templates/role.yaml: -------------------------------------------------------------------------------- 1 | kind: Role 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: fedlearner-operator 5 | namespace: {{ .Release.Namespace }} 6 | rules: 7 | - apiGroups: ["*"] 8 | resources: ["*"] 9 | verbs: ["*"] 10 | --- 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-operator/templates/role_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: RoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: fedlearner-operator 5 | namespace: {{ .Release.Namespace }} 6 | subjects: 7 | - kind: ServiceAccount 8 | name: fedlearner-operator 9 | namespace: {{ .Release.Namespace }} 10 | roleRef: 11 | kind: Role 12 | name: fedlearner-operator 13 | apiGroup: rbac.authorization.k8s.io 14 | --- 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-operator/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: fedlearner-operator 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | selector: 8 | app: fedlearner-operator 9 | ports: 10 | - protocol: TCP 11 | port: 8100 12 | targetPort: 8100 13 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-operator/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: fedlearner-operator 5 | namespace: {{ .Release.Namespace }} 6 | --- 7 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console-v2/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console-v2/templates/cluster_role.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRole 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: fedlearner-web-console-v2 5 | namespace: {{ .Release.Namespace }} 6 | rules: 7 | - apiGroups: ["*"] 8 | resources: ["*"] 9 | verbs: ["*"] 10 | --- 11 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console-v2/templates/cluster_role_binding.yaml: -------------------------------------------------------------------------------- 1 | kind: ClusterRoleBinding 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | metadata: 4 | name: fedlearner-web-console-v2 5 | namespace: {{ .Release.Namespace }} 6 | subjects: 7 | - kind: ServiceAccount 8 | name: {{ include "fedlearner-web-console-v2.serviceAccountName" . }} 9 | namespace: {{ .Release.Namespace }} 10 | roleRef: 11 | kind: ClusterRole 12 | name: fedlearner-web-console-v2 13 | apiGroup: rbac.authorization.k8s.io 14 | --- 15 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console-v2/templates/hook_config_map.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.hook }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: "{{ .Release.Name }}-hook" 6 | data: 7 | hook: | 8 | {{ .Files.Get .Values.hook | nindent 4 }} 9 | {{- end }} -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console-v2/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "fedlearner-web-console-v2.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "fedlearner-web-console-v2.labels" . | nindent 4 }} 8 | spec: 9 | type: {{ .Values.service.type }} 10 | ports: 11 | - port: {{ .Values.service.port }} 12 | targetPort: 1990 13 | protocol: TCP 14 | name: grpc 15 | selector: 16 | {{- include "fedlearner-web-console-v2.selectorLabels" . | nindent 4 }} 17 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console-v2/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "fedlearner-web-console-v2.serviceAccountName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "fedlearner-web-console-v2.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "fedlearner-web-console.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{- include "fedlearner-web-console.labels" . | nindent 4 }} 8 | spec: 9 | type: {{ .Values.service.type }} 10 | ports: 11 | - port: {{ .Values.service.port }} 12 | targetPort: 1990 13 | protocol: TCP 14 | name: grpc 15 | selector: 16 | {{- include "fedlearner-web-console.selectorLabels" . | nindent 4 }} 17 | -------------------------------------------------------------------------------- /deploy/charts/fedlearner/charts/fedlearner-web-console/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "fedlearner-web-console.serviceAccountName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "fedlearner-web-console.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /deploy/integrated_test/code_key/criteo-train-2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/integrated_test/code_key/criteo-train-2.tar.gz -------------------------------------------------------------------------------- /deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0000.rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0000.rd -------------------------------------------------------------------------------- /deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0001.rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0001.rd -------------------------------------------------------------------------------- /deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0002.rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0002.rd -------------------------------------------------------------------------------- /deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0003.rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/deploy/integrated_test/tfrecord_raw_data/raw_data_partition_0003.rd -------------------------------------------------------------------------------- /deploy/kubernetes_operator/.dockerignore: -------------------------------------------------------------------------------- 1 | .dockerignore 2 | .gitignore 3 | hack 4 | Dockerfile -------------------------------------------------------------------------------- /deploy/kubernetes_operator/.gitignore: -------------------------------------------------------------------------------- 1 | vendor -------------------------------------------------------------------------------- /deploy/kubernetes_operator/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: docker-build 2 | 3 | docker-build: 4 | docker build -t ${IMG} . 5 | 6 | docker-push: 7 | docker push ${IMG} 8 | 9 | deploy: 10 | kubectl apply -f manifest/controller.yaml 11 | 12 | update-codegen: 13 | ./hack/update-codegen.sh 14 | 15 | update-defaulter: 16 | defaulter-gen --input-dirs github.com/bytedance/fedlearner/deploy/kubernetes_operator/pkg/apis/fedlearner.k8s.io/v1alpha1 --go-header-file ./hack/boilerplate.go.txt 17 | 18 | vendor: 19 | go mod vendor 20 | 21 | crd: 22 | controller-gen paths=./pkg/apis/... output:dir=../charts/fedlearner/charts/fedlearner-operator/templates/ crd:trivialVersions=true 23 | -------------------------------------------------------------------------------- /deploy/kubernetes_operator/tools.go: -------------------------------------------------------------------------------- 1 | // +build codegen 2 | 3 | package main 4 | 5 | import _ "k8s.io/code-generator" 6 | -------------------------------------------------------------------------------- /deploy/proxy/README.md: -------------------------------------------------------------------------------- 1 | * Depends: 2 | ** nginx-1.16.1 release code; 3 | ** nginx_lua release code; 4 | ** luajit 5 | -------------------------------------------------------------------------------- /deploy/scripts/pre_start_hook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python -c 'from fedlearner.common.hooks import pre_start_hook; pre_start_hook()' 4 | -------------------------------------------------------------------------------- /deploy/scripts/trainer/run_fedavg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | source /app/deploy/scripts/hdfs_common.sh 5 | source /app/deploy/scripts/env_to_args.sh 6 | 7 | if [[ -n "${CODE_KEY}" ]]; then 8 | pull_code ${CODE_KEY} $PWD 9 | fi 10 | 11 | LISTEN_PORT=50051 12 | if [[ -n "${PORT0}" ]]; then 13 | LISTEN_PORT=${PORT0} 14 | fi 15 | 16 | if [[ $ROLE == "leader" ]]; then 17 | export FL_LEADER_ADDRESS="0.0.0.0:${LISTEN_PORT}" 18 | elif [[ -n $PEER_ADDR ]]; then 19 | export FL_LEADER_ADDRESS=$PEER_ADDR 20 | fi 21 | 22 | python $ROLE.py 23 | -------------------------------------------------------------------------------- /deploy/scripts/wait4pair_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cmd=$1 4 | if [[ -z $WORKER_ID ]]; then 5 | pair="/etc/master/${MASTER_ID}" 6 | else 7 | pair="/etc/worker/${WORKER_ID}" 8 | fi 9 | 10 | while [[ true ]]; do 11 | if [[ -f ${pair} ]] && [[ -n "$(cat ${pair})" ]]; then 12 | export PEER_ADDR=`cat ${pair}` 13 | break 14 | else 15 | echo "still waiting for peer addr" 16 | sleep 1 17 | fi 18 | done 19 | exec ${cmd} 20 | -------------------------------------------------------------------------------- /deploy/scripts/xxx.sh: -------------------------------------------------------------------------------- 1 | source ./env_checker.sh 2 | 3 | OOO=111 4 | xxx=$(normalize_env_to_args "--xxx" $OOO) 5 | echo $xxx 6 | -------------------------------------------------------------------------------- /docker/dataflow/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow==1.15.3 2 | cityhash 3 | psutil==5.8.0 4 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/figures/roc_auc/20220618_04_01_49_sampled_clients_ratio_1.0_numThresholds_200_Laplace_eps_2.0.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/DPAUC/figures/roc_auc/20220618_04_01_49_sampled_clients_ratio_1.0_numThresholds_200_Laplace_eps_2.0.pdf -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7490864984341428,0.0012365534341790313 3 | 1,0.7664340734481812,0.7664771312694274,0.7661786955468604,0.0011361112112047587 4 | 2,0.7701894640922546,0.7702185466609048,0.7695845457837901,0.0011586575785696807 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7491549978340831,0.0007141863005305501 3 | 1,0.7664340734481812,0.7664771312694274,0.7660957345269941,0.0008109945729075044 4 | 2,0.7701894640922546,0.7702185466609048,0.7698178846760253,0.0006306505620342503 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.749245345377578,0.0005120904179416405 3 | 1,0.7664340734481812,0.7664771312694274,0.7662184821882646,0.0005087807270743277 4 | 2,0.7701894640922546,0.7702185466609048,0.7699403573468965,0.0005117222449113793 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7491556372400723,0.00036337904030412884 3 | 1,0.7664340734481812,0.7664771312694274,0.7663377765762687,0.00031106132897057665 4 | 2,0.7701894640922546,0.7702185466609048,0.770024714792177,0.00037148629407595215 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7492644033867041,0.00021036893478395353 3 | 1,0.7664340734481812,0.7664771312694274,0.7663794359664178,0.00029102655446489154 4 | 2,0.7701894640922546,0.7702185466609048,0.7701074813119518,0.00027389747558106785 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493371799318683,0.00020538089920872226 3 | 1,0.7664340734481812,0.7664771312694274,0.7663673714694884,0.0001909704961757539 4 | 2,0.7701894640922546,0.7702185466609048,0.7701182048179708,0.00017555293000663406 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.01_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493374834891686,0.00012622955436230105 3 | 1,0.7664340734481812,0.7664771312694274,0.7663599766908891,0.00020464407535214453 4 | 2,0.7701894640922546,0.7702185466609048,0.770131564048489,0.00014587397134585613 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.01_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493497972418678,9.58820307435546e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7664410608340779,7.273906399760201e-05 4 | 2,0.7701894640922546,0.7702185466609048,0.7701725697857187,8.995378777835118e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.02_numClients_1_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493160871917407,7.367398986539217e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7663881300938917,8.613022851480706e-05 4 | 2,0.7701894640922546,0.7702185466609048,0.770156140318612,7.410512990412792e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_12_56_07_Laplace_eps_0.02_numClients_1_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493701049512963,4.945030333155125e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7664588720978063,5.092127768893481e-05 4 | 2,0.7701894640922546,0.7702185466609048,0.7701836046780408,4.517542795313456e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_15_00_02_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7484392515156825,0.0017701108319654899 3 | 1,0.7664340734481812,0.7664771312694274,0.7655288114425599,0.0014595230956988986 4 | 2,0.7701894640922546,0.7702185466609048,0.769324064713913,0.0017828361471528846 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_15_00_02_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489679463125392,0.0008247289543801151 3 | 1,0.7664340734481812,0.7664771312694274,0.7658266671201752,0.0009221410795575566 4 | 2,0.7701894640922546,0.7702185466609048,0.7694094879274097,0.0007930413314124923 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_15_00_02_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7488577755367913,0.00041659232977861145 3 | 1,0.7664340734481812,0.7664771312694274,0.7660280757334702,0.0005421425287984757 4 | 2,0.7701894640922546,0.7702185466609048,0.7697929492017254,0.00041401300946507875 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_15_00_02_Laplace_eps_0.01_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489555421645331,0.00021634555220567344 3 | 1,0.7664340734481812,0.7664771312694274,0.7661928047376284,0.0002132653437997858 4 | 2,0.7701894640922546,0.7702185466609048,0.7697736935442953,0.00022783247046302322 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_15_00_02_Laplace_eps_0.02_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489892500095832,9.563527847681958e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7661952023880336,0.00011522857921515394 4 | 2,0.7701894640922546,0.7702185466609048,0.7698215230979689,0.0001269670162862026 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220620_15_00_02_Laplace_eps_0.04_numClients_1_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489776347678954,4.877745781519459e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7661740349866271,7.80584996977885e-05 4 | 2,0.7701894640922546,0.7702185466609048,0.7698764156547826,7.608944666646652e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220621_17_18_00_Laplace_eps_0.000625_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7483728547156245,0.0011576408168081182 3 | 1,0.7664340734481812,0.7664771312694274,0.7653611940115748,0.0012513797010898125 4 | 2,0.7701894640922546,0.7702185466609048,0.7689043335563194,0.0010297667462607638 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220621_17_18_00_Laplace_eps_0.00125_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7490508749193749,0.0005865454451660085 3 | 1,0.7664340734481812,0.7664771312694274,0.7660704634327464,0.0004849649727969218 4 | 2,0.7701894640922546,0.7702185466609048,0.7698670094238662,0.0005344632648838307 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220621_17_18_00_Laplace_eps_0.0025_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7491705202179822,0.00020864728864888567 3 | 1,0.7664340734481812,0.7664771312694274,0.7663324529024307,0.00025801465023111395 4 | 2,0.7701894640922546,0.7702185466609048,0.7700924077858069,0.0002433988205052466 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1/20220621_17_18_00_Laplace_eps_0.005_numClients_1_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493526542772164,0.0001359668228893367 3 | 1,0.7664340734481812,0.7664771312694274,0.7664082536474958,0.00012188108578675745 4 | 2,0.7701894640922546,0.7702185466609048,0.7701801896108753,0.00012574032657684524 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_11_18_47_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7492950730487096,0.000319911880818886 3 | 1,0.7664340734481812,0.7664771312694274,0.7663180869925703,0.0002746323074260759 4 | 2,0.7701895236968994,0.7702185466609048,0.7700262557504394,0.0002876859068016942 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_14_54_14_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7471172670172279,0.0022239334110612523 3 | 1,0.76643413,0.7664771312694274,0.7643990350617783,0.0022638148628411946 4 | 2,0.7701895,0.7702185466609048,0.7676941227856494,0.0027462609432046302 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_14_54_14_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7484907750516292,0.001204029541355474 3 | 1,0.76643413,0.7664771312694274,0.7656937297128947,0.001367427990504751 4 | 2,0.7701895,0.7702185466609048,0.7690655043401999,0.001085318268047186 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_14_54_14_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7491602702237384,0.0005727254230964415 3 | 1,0.76643413,0.7664771312694274,0.7662349691119849,0.0006768237724827029 4 | 2,0.7701895,0.7702185466609048,0.7697294630569858,0.0006328126948487662 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_14_54_14_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7492899825493304,0.0002887338623426797 3 | 1,0.76643413,0.7664771312694274,0.7663797813477217,0.0003114079898909809 4 | 2,0.7701895,0.7702185466609048,0.7701365005450892,0.00031900285376222604 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_14_55_20_Laplace_eps_0.05_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7493885014435907,5.789762371478911e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7664366507606598,6.674089874209686e-05 4 | 2,0.7701895236968994,0.7702185466609048,0.7701891197135151,6.094653268302645e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220618_14_55_20_Laplace_eps_0.1_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7493721562169893,2.811476816583884e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7664507916242788,2.8764294083465792e-05 4 | 2,0.7701895236968994,0.7702185466609048,0.7702071809887931,3.647442370847441e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_11_30_15_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.748562313282484,0.0012162253782936638 3 | 1,0.76643413,0.7664771312694274,0.7658116417256358,0.0014046556324751964 4 | 2,0.7701895,0.7702185466609048,0.7694403356304887,0.001104847670628167 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_11_30_15_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7491335038796462,0.000672228935151469 3 | 1,0.76643413,0.7664771312694274,0.766139121559466,0.0005756591635269422 4 | 2,0.7701895,0.7702185466609048,0.7697904650413101,0.0005448490910429008 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_11_35_31_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7470728828816674,0.0020915539996795296 3 | 1,0.7664340734481812,0.7664771312694274,0.7640870850149455,0.00224056763197147 4 | 2,0.7701895236968994,0.7702185466609048,0.7681056779843007,0.0020583713468528835 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_11_35_31_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7492971968248578,0.0002753135130038797 3 | 1,0.7664340734481812,0.7664771312694274,0.7663835704779244,0.0003134918618140742 4 | 2,0.7701895236968994,0.7702185466609048,0.7701619674762271,0.00030960841098911655 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_31_44_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7470215141442524,0.0029150610129904833 3 | 1,0.7664340734481812,0.7664771312694274,0.7637963748961155,0.0038151510396188513 4 | 2,0.7701894640922546,0.7702185466609048,0.7672442834051786,0.0032722995726908667 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_31_44_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7482063155108201,0.0016494225571066173 3 | 1,0.7664340734481812,0.7664771312694274,0.7656838667656668,0.0016774585762831431 4 | 2,0.7701894640922546,0.7702185466609048,0.7691431952520628,0.0017198685388018142 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_31_44_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489370584968629,0.0007553942279893401 3 | 1,0.7664340734481812,0.7664771312694274,0.7661096359009205,0.0008940355724306189 4 | 2,0.7701894640922546,0.7702185466609048,0.7700276293480028,0.0007883345565454775 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_31_44_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7491775980783606,0.0004841326709068503 3 | 1,0.7664340734481812,0.7664771312694274,0.7663093203528092,0.00040175783862169225 4 | 2,0.7701894640922546,0.7702185466609048,0.7699945071338767,0.0004983971272676795 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_31_44_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.749346669072678,0.00021580706031998275 3 | 1,0.7664340734481812,0.7664771312694274,0.7663727561571371,0.0002385693352626017 4 | 2,0.7701894640922546,0.7702185466609048,0.7700911357548381,0.00025769547107331594 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_38_08_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7467646393124484,0.003759946316874297 3 | 1,0.7664340734481812,0.7664771312694274,0.763648406030224,0.0037629214356497223 4 | 2,0.7701894640922546,0.7702185466609048,0.7671629846635066,0.003422922142395169 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_38_08_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489616411645863,0.0017796312208357464 3 | 1,0.7664340734481812,0.7664771312694274,0.7654470999495662,0.001804681432136088 4 | 2,0.7701894640922546,0.7702185466609048,0.7692647719931263,0.0015021438791011866 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_38_08_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7491917405036037,0.000719656635860551 3 | 1,0.7664340734481812,0.7664771312694274,0.7661200355025609,0.0007553829601412577 4 | 2,0.7701894640922546,0.7702185466609048,0.7698101881465805,0.0009026964045466816 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_38_08_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7492603992653958,0.0004740758168134068 3 | 1,0.7664340734481812,0.7664771312694274,0.7661971005256631,0.0004713513793020965 4 | 2,0.7701894640922546,0.7702185466609048,0.7701146906937102,0.00043364990827292945 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_17_38_08_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493185503995486,0.00024155722790823248 3 | 1,0.7664340734481812,0.7664771312694274,0.7663776274964941,0.00023146635997050005 4 | 2,0.7701894640922546,0.7702185466609048,0.7700948744807528,0.00021330387422370338 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_19_57_17_None_eps_0.0_numClients_10_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493208810991583,1.1102230246251565e-16 3 | 1,0.7664340734481812,0.7664771312694274,0.7664168536238896,1.1102230246251565e-16 4 | 2,0.7701894640922546,0.7702185466609048,0.7701717688067073,0.0 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_19_57_17_None_eps_0.0_numClients_10_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.749377193659084,1.1102230246251565e-16 3 | 1,0.7664340734481812,0.7664771312694274,0.7664553572915065,1.1102230246251565e-16 4 | 2,0.7701894640922546,0.7702185466609048,0.7702029317598936,1.1102230246251565e-16 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220619_19_57_17_None_eps_0.0_numClients_10_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7493852211547164,0.0 3 | 1,0.7664340734481812,0.7664771312694274,0.766476513873569,0.0 4 | 2,0.7701894640922546,0.7702185466609048,0.7702135903882318,0.0 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220620_15_00_02_Laplace_eps_0.00125_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7460049996421085,0.004170724015526678 3 | 1,0.7664340734481812,0.7664771312694274,0.7638751446752821,0.004665668806746559 4 | 2,0.7701894640922546,0.7702185466609048,0.7673917624658508,0.003920572548242524 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220620_15_00_02_Laplace_eps_0.0025_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7487561047954859,0.002394534167973158 3 | 1,0.7664340734481812,0.7664771312694274,0.7646054231859999,0.0025345512168104627 4 | 2,0.7701894640922546,0.7702185466609048,0.7692269019339251,0.002556490801533411 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220620_15_00_02_Laplace_eps_0.005_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7488157520097333,0.0012790756832293997 3 | 1,0.7664340734481812,0.7664771312694274,0.7660086260671554,0.0012163815981963727 4 | 2,0.7701894640922546,0.7702185466609048,0.7693900056383183,0.0010340831503868814 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220620_15_00_02_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489879811078144,0.0006570138106297717 3 | 1,0.7664340734481812,0.7664771312694274,0.7659725242152399,0.0007234482890477635 4 | 2,0.7701894640922546,0.7702185466609048,0.7695485592596428,0.0008054724044492771 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220620_15_00_02_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7488837133415488,0.0003434393426776603 3 | 1,0.7664340734481812,0.7664771312694274,0.7661807937823423,0.0003288013971106177 4 | 2,0.7701894640922546,0.7702185466609048,0.7697560198247193,0.0004998099089755664 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220620_15_00_02_Laplace_eps_0.04_numClients_10_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7489142114567957,0.0001608496252508366 3 | 1,0.7664340734481812,0.7664771312694274,0.7661982141091657,0.00018696293248855205 4 | 2,0.7701894640922546,0.7702185466609048,0.7698060919072209,0.00022172950534593336 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_11_56_31_Laplace_eps_0.025_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7248994068549988,0.000607081542130315 3 | 1,0.7664340734481812,0.7664771312694274,0.7440576547730009,0.0006387824183131745 4 | 2,0.7701894640922546,0.7702185466609048,0.7486257690936275,0.0005907853025604357 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_11_56_31_Laplace_eps_0.05_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7250083683787841,0.0003227942152073478 3 | 1,0.7664340734481812,0.7664771312694274,0.7442153990998311,0.0003275174426368852 4 | 2,0.7701894640922546,0.7702185466609048,0.7488213376800447,0.0003307320044024094 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_11_56_31_Laplace_eps_0.1_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7250636926810049,0.0001665032689452729 3 | 1,0.7664340734481812,0.7664771312694274,0.7443171960909926,0.0001439458433423622 4 | 2,0.7701894640922546,0.7702185466609048,0.7487983100588452,0.00012278548680289556 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_11_56_31_Laplace_eps_0.2_numClients_10_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7251052112609664,7.393264733265014e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7443480244097677,8.400641161225239e-05 4 | 2,0.7701894640922546,0.7702185466609048,0.7488576779993784,7.336435000710954e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_22_23_55_Laplace_eps_0.01_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7472776067420734,0.0008889855322799361 3 | 1,0.7664340734481812,0.7664771312694274,0.7649677344899528,0.001281544865763959 4 | 2,0.7701894640922546,0.7702185466609048,0.7683570545385828,0.0013083762635100163 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_22_23_55_Laplace_eps_0.04_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7478951275540564,0.00027544673007919544 3 | 1,0.7664340734481812,0.7664771312694274,0.7650454895206305,0.00025238086624457744 4 | 2,0.7701894640922546,0.7702185466609048,0.7689917978273176,0.00032311656114494987 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_22_23_56_Laplace_eps_0.02_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7476687236558105,0.0006249149312025853 3 | 1,0.7664340734481812,0.7664771312694274,0.7649144824507401,0.0005069683378136706 4 | 2,0.7701894640922546,0.7702185466609048,0.7687047714605388,0.0005081711291563749 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_10/20220809_22_23_56_Laplace_eps_0.08_numClients_10_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7478788417584596,0.00014710262598840621 3 | 1,0.7664340734481812,0.7664771312694274,0.7651780360241556,0.0001591045578145918 4 | 2,0.7701894640922546,0.7702185466609048,0.7690055131219201,0.00016228641378326106 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220618_20_44_13_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7472859437961402,0.0033809034747838964 3 | 1,0.76643413,0.7664771312694274,0.7635882121009985,0.002464523188776586 4 | 2,0.7701895,0.7702185466609048,0.7662457875798523,0.0033539346883607636 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220618_20_44_14_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7262096396181399,0.01110386418351576 3 | 1,0.76643413,0.7664771312694274,0.7441803560214226,0.009724966758596475 4 | 2,0.7701895,0.7702185466609048,0.7449127871541257,0.011228329254112032 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220618_20_44_14_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7411613626702163,0.006277961764138028 3 | 1,0.76643413,0.7664771312694274,0.7581417414045799,0.005686720033882445 4 | 2,0.7701895,0.7702185466609048,0.7608070721376278,0.006807470799744551 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220618_22_18_12_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.6900679243967356,0.019574913033665736 3 | 1,0.76643413,0.7664771312694274,0.7027434260290072,0.014594712737633467 4 | 2,0.7701895,0.7702185466609048,0.7093074055684153,0.018684155908874164 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_11_30_15_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.6951086249334593,0.018655170635369832 3 | 1,0.76643413,0.7664771312694274,0.7062089726297427,0.014563666671875027 4 | 2,0.7701895,0.7702185466609048,0.7126703607697255,0.018564779739082844 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_11_30_15_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7254592242087642,0.012307234466288513 3 | 1,0.76643413,0.7664771312694274,0.74381664404883,0.010322767381497828 4 | 2,0.7701895,0.7702185466609048,0.7469110363384598,0.010126038204297354 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_11_30_15_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574,0.7493831215889618,0.7459324568026342,0.0033552212593783532 3 | 1,0.76643413,0.7664771312694274,0.7623731645920203,0.0027742329404534423 4 | 2,0.7701895,0.7702185466609048,0.7667252486601477,0.0033671471321969283 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_11_36_52_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7400273760775704,0.006304363544091946 3 | 1,0.7664340734481812,0.7664771312694274,0.7561205245884365,0.0055106037520865295 4 | 2,0.7701895236968994,0.7702185466609048,0.7599551852684495,0.005241907740752647 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_31_44_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.6930171298218367,0.025835842054161267 3 | 1,0.7664340734481812,0.7664771312694274,0.7054161437843806,0.028573427446877634 4 | 2,0.7701894640922546,0.7702185466609048,0.7084791165568243,0.02773977847892722 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_31_44_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7261873489241103,0.014703528174579235 3 | 1,0.7664340734481812,0.7664771312694274,0.7406191518209723,0.01745712888070726 4 | 2,0.7701894640922546,0.7702185466609048,0.7458116047023384,0.014595697856349947 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_31_44_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7389440143200133,0.00800760567237764 3 | 1,0.7664340734481812,0.7664771312694274,0.7569375109710472,0.00809579246771157 4 | 2,0.7701894640922546,0.7702185466609048,0.7610738296161224,0.007947452583595091 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_31_44_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7455574187487716,0.004497902456019528 3 | 1,0.7664340734481812,0.7664771312694274,0.7630117496577129,0.004377678169164721 4 | 2,0.7701894640922546,0.7702185466609048,0.7664819526511596,0.004387402338974191 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_31_44_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7481103818528944,0.0023354691105621833 3 | 1,0.7664340734481812,0.7664771312694274,0.7648865031895297,0.0022414706178778125 4 | 2,0.7701894640922546,0.7702185466609048,0.7691796455705212,0.0025185806808043524 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_38_08_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.6920434678384564,0.02161145156646243 3 | 1,0.7664340734481812,0.7664771312694274,0.7017552775855945,0.024027694528729153 4 | 2,0.7701894640922546,0.7702185466609048,0.7103666189733369,0.025589254458515304 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_38_08_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7268845425951445,0.016539028608569886 3 | 1,0.7664340734481812,0.7664771312694274,0.7449784230870721,0.018234134206908777 4 | 2,0.7701894640922546,0.7702185466609048,0.7468146256535602,0.01651083017908516 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_38_08_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7417112473280693,0.009246306237752844 3 | 1,0.7664340734481812,0.7664771312694274,0.7567149520248941,0.009104736538646407 4 | 2,0.7701894640922546,0.7702185466609048,0.7599156676701027,0.007251129292936545 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_38_08_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7461054828086009,0.004464581659040133 3 | 1,0.7664340734481812,0.7664771312694274,0.7632457664817829,0.004022467459897699 4 | 2,0.7701894640922546,0.7702185466609048,0.7668191404118668,0.0038020284568576774 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220619_17_38_08_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_100_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7480616057390658,0.0019808896045675514 3 | 1,0.7664340734481812,0.7664771312694274,0.7649230563155431,0.0020358532427562407 4 | 2,0.7701894640922546,0.7702185466609048,0.7688940491702086,0.002104239287852919 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220620_15_00_02_Laplace_eps_0.00125_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7000978033910894,0.03947031703437214 3 | 1,0.7664340734481812,0.7664771312694274,0.705460099962939,0.033148012119304786 4 | 2,0.7701894640922546,0.7702185466609048,0.703643402319741,0.03272116090121208 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220620_15_00_02_Laplace_eps_0.0025_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7295102420676598,0.023246421531521913 3 | 1,0.7664340734481812,0.7664771312694274,0.7439812920677409,0.021997789608282158 4 | 2,0.7701894640922546,0.7702185466609048,0.7419910429280878,0.02184600207128123 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220620_15_00_02_Laplace_eps_0.005_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7396880631699618,0.012265943842244331 3 | 1,0.7664340734481812,0.7664771312694274,0.7559005323310969,0.010381316286536862 4 | 2,0.7701894640922546,0.7702185466609048,0.7578472618230717,0.010772755703692186 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220620_15_00_02_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7430630762595092,0.005342807368402494 3 | 1,0.7664340734481812,0.7664771312694274,0.7608958768631352,0.006080717548474137 4 | 2,0.7701894640922546,0.7702185466609048,0.7651661270945185,0.006494263905196648 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220620_15_00_02_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7479075908154554,0.003165296094560561 3 | 1,0.7664340734481812,0.7664771312694274,0.7645220805242456,0.0030091924252177212 4 | 2,0.7701894640922546,0.7702185466609048,0.7681586713509878,0.003311976721349531 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220620_15_00_02_Laplace_eps_0.04_numClients_1000_repeat_times_50_numThresholds_50_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7482655118811279,0.0016689171663377429 3 | 1,0.7664340734481812,0.7664771312694274,0.7656750338598982,0.0017527768586152533 4 | 2,0.7701894640922546,0.7702185466609048,0.7691390685956104,0.0015873050189622148 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_11_56_31_Laplace_eps_0.025_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7216721539067094,0.006905835687819979 3 | 1,0.7664340734481812,0.7664771312694274,0.7422464994511182,0.005897034624496511 4 | 2,0.7701894640922546,0.7702185466609048,0.7477163866266828,0.006317814948987051 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_11_56_31_Laplace_eps_0.05_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7235959606604362,0.0025023400499974424 3 | 1,0.7664340734481812,0.7664771312694274,0.7436526000033054,0.0030465045628864855 4 | 2,0.7701894640922546,0.7702185466609048,0.7474555804834033,0.0032602336683363914 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_11_56_31_Laplace_eps_0.1_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.724370050754919,0.00156133521423379 3 | 1,0.7664340734481812,0.7664771312694274,0.7441505104521186,0.0017254027470567893 4 | 2,0.7701894640922546,0.7702185466609048,0.7483240603648634,0.001435266658324914 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_11_56_31_Laplace_eps_0.2_numClients_1000_repeat_times_50_numThresholds_10_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7247587716881347,0.0008300991582102435 3 | 1,0.7664340734481812,0.7664771312694274,0.7439942362169307,0.0006493927431825696 4 | 2,0.7701894640922546,0.7702185466609048,0.7485283529268023,0.0008044706233783658 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_22_23_55_Laplace_eps_0.01_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7443989022163018,0.009958599231668223 3 | 1,0.7664340734481812,0.7664771312694274,0.7617174707317766,0.008990140348980163 4 | 2,0.7701894640922546,0.7702185466609048,0.7623883512101303,0.010546096691884714 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_22_23_55_Laplace_eps_0.02_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7457684219196156,0.004863693245141473 3 | 1,0.7664340734481812,0.7664771312694274,0.7618567892453723,0.004508049286836041 4 | 2,0.7701894640922546,0.7702185466609048,0.7668263004235187,0.004923247327790038 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_22_23_55_Laplace_eps_0.04_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7471773105039539,0.002225243143183333 3 | 1,0.7664340734481812,0.7664771312694274,0.7643517460696228,0.002488758106232984 4 | 2,0.7701894640922546,0.7702185466609048,0.767099685673912,0.003361665831389378 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/Laplace/numClients_1000/20220809_22_23_55_Laplace_eps_0.08_numClients_1000_repeat_times_50_numThresholds_25_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493573427200317,0.7493831215889618,0.7476140086780134,0.0013619002648965432 3 | 1,0.7664340734481812,0.7664771312694274,0.7647592972986723,0.0013884364602388879 4 | 2,0.7701894640922546,0.7702185466609048,0.7685332241343044,0.0014357091857544685 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_1.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7502389944779708,0.0017663580992345895 3 | 1,0.7664783000946045,0.7664771312694274,0.7663924581497257,0.0020980661285662635 4 | 2,0.7702188491821289,0.7702185466609048,0.7710983643185262,0.0018913152966873047 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_1.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7492782721610458,0.0023150634293915254 3 | 1,0.7664783000946045,0.7664771312694274,0.7665233607785048,0.0024648756721491554 4 | 2,0.7702188491821289,0.7702185466609048,0.7706943839843049,0.002651834414479606 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_2.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7491075082591847,0.0009317289289354084 3 | 1,0.7664783000946045,0.7664771312694274,0.7665580192958494,0.0011089754380812207 4 | 2,0.7702188491821289,0.7702185466609048,0.7702042740189887,0.0007953422859975229 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_2.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7493367721413742,0.001272311079760135 3 | 1,0.7664783000946045,0.7664771312694274,0.766282083462223,0.0011434421602409524 4 | 2,0.7702188491821289,0.7702185466609048,0.770004445094487,0.0012000917321013738 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_4.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7493976670247522,0.0004003951221627852 3 | 1,0.7664783000946045,0.7664771312694274,0.7663379129490904,0.000338037273851815 4 | 2,0.7702188491821289,0.7702185466609048,0.7701116052998958,0.0003749708573776893 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_4.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7493788101943664,0.00033091524639949655 3 | 1,0.7664783000946045,0.7664771312694274,0.766587835054247,0.0003550507004974835 4 | 2,0.7702188491821289,0.7702185466609048,0.7702071915164133,0.00033004446916164724 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_8.0_numClients_10_repeat_times_20_numThresholds_200_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7493798686038214,3.9595165052262076e-05 3 | 1,0.7664783000946045,0.7664771312694274,0.766446740657986,5.4372747190499306e-05 4 | 2,0.7702188491821289,0.7702185466609048,0.7702038254929227,4.9081107699671094e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220616_18_33_36_RR_eps_8.0_numClients_10_repeat_times_20_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedSkewed.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493821382522583,0.7493831215889618,0.7493856869870072,4.928232331082365e-05 3 | 1,0.7664783000946045,0.7664771312694274,0.7664936711461384,3.90609198676194e-05 4 | 2,0.7702188491821289,0.7702185466609048,0.7702069533611244,4.09626200896776e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220618_11_19_51_RR_eps_0.5_numClients_1000_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7498784023076334,0.005013612576178735 3 | 1,0.7664340734481812,0.7664771312694274,0.7666667481800893,0.004397031659175423 4 | 2,0.7701895236968994,0.7702185466609048,0.7707450062251757,0.004415037050393732 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/sample_0.1_of_test/randomized_responses/20220618_11_19_51_RR_eps_10.0_numClients_1000_repeat_times_50_numThresholds_400_ClientsSampledRatio_1.0_ClientsAssignedUniformly.csv: -------------------------------------------------------------------------------- 1 | epoch,auc_gt_tf,auc_gt_sl,roc_auc_mean,roc_auc_std 2 | 0,0.7493574619293213,0.7493831215889618,0.7493874489100262,1.6337842444418495e-05 3 | 1,0.7664340734481812,0.7664771312694274,0.7664777838691426,1.314151237753813e-05 4 | 2,0.7701895236968994,0.7702185466609048,0.7702138094446298,1.727848915795719e-05 5 | -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/stdout_logs/20221126_13_45_41_numberClients_10Laplace_eps_1.0numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/DPAUC/outputs/stdout_logs/20221126_13_45_41_numberClients_10Laplace_eps_1.0numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt -------------------------------------------------------------------------------- /example/privacy/DPAUC/outputs/stdout_logs/20221126_13_46_37_numberClients_10Laplace_eps_1.0numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/DPAUC/outputs/stdout_logs/20221126_13_46_37_numberClients_10Laplace_eps_1.0numThreshold_200_repeatTimes_50_ClientAssignedSkewed.txt -------------------------------------------------------------------------------- /example/privacy/embedding_protection/Technical_Report_of_Embedding_Protection_in_FL.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/embedding_protection/Technical_Report_of_Embedding_Protection_in_FL.pdf -------------------------------------------------------------------------------- /example/privacy/label_protection/NeurIPS2020_SpicyFL/2mins_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/label_protection/NeurIPS2020_SpicyFL/2mins_video.mp4 -------------------------------------------------------------------------------- /example/privacy/label_protection/NeurIPS2020_SpicyFL/Neurips SpicyFL Camera Ready Paper_ Label Leakage & Protection.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/label_protection/NeurIPS2020_SpicyFL/Neurips SpicyFL Camera Ready Paper_ Label Leakage & Protection.pdf -------------------------------------------------------------------------------- /example/privacy/label_protection/NeurIPS2020_SpicyFL/Neurips SpicyFL Poster Label Leakage & Protection.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/label_protection/NeurIPS2020_SpicyFL/Neurips SpicyFL Poster Label Leakage & Protection.pdf -------------------------------------------------------------------------------- /example/privacy/label_protection/Technical_Report_of_Label_Leakage_and_Protection_in_Federated_Learning.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/example/privacy/label_protection/Technical_Report_of_Label_Leakage_and_Protection_in_Federated_Learning.pdf -------------------------------------------------------------------------------- /example/privacy/label_protection/shared_var.py: -------------------------------------------------------------------------------- 1 | G_Batch_Labels, G_Batch_Positive_Predicted_Probabilities = None, None 2 | -------------------------------------------------------------------------------- /example/sparse_model/test_local_worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd "$( dirname "${BASH_SOURCE[0]}" )" 4 | rm -rf data model 5 | 6 | export CUDA_VISIBLE_DEVICES="" 7 | set -e 8 | 9 | rm -rf data model 10 | python make_data.py --fid_version=2 11 | python leader.py --local-addr=localhost:50011 \ 12 | --local-worker \ 13 | --data-path=data/leader/ \ 14 | --checkpoint-path=model/leader \ 15 | --save-checkpoint-steps=100 \ 16 | --export-path=model/leader/saved_model \ 17 | --sparse-estimator=True \ 18 | --fid_version=2 19 | rm -rf data model 20 | -------------------------------------------------------------------------------- /fedlearner/data_join/join_expr/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/fedlearner/data_join/join_expr/__init__.py -------------------------------------------------------------------------------- /fedlearner/data_join/key_mapper/impl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/fedlearner/data_join/key_mapper/impl/__init__.py -------------------------------------------------------------------------------- /fedlearner/data_join/key_mapper/impl/default.py: -------------------------------------------------------------------------------- 1 | from fedlearner.data_join.key_mapper.key_mapping import BaseKeyMapper 2 | class DefaultKeyMapper(BaseKeyMapper): 3 | def leader_mapping(self, item) -> dict: 4 | return dict() 5 | 6 | def follower_mapping(self, item) -> dict: 7 | return dict() 8 | 9 | @classmethod 10 | def name(cls): 11 | return "DEFAULT" 12 | -------------------------------------------------------------------------------- /fedlearner/data_join/key_mapper/key_mapping.py: -------------------------------------------------------------------------------- 1 | class BaseKeyMapper(object): 2 | def leader_mapping(self, item) -> dict: 3 | raise NotImplementedError 4 | 5 | def follower_mapping(self, item) -> dict: 6 | raise NotImplementedError 7 | 8 | @classmethod 9 | def name(cls): 10 | return "BASE_KEY_MAPPER" 11 | -------------------------------------------------------------------------------- /fedlearner/data_join/raw_data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/fedlearner/data_join/raw_data/__init__.py -------------------------------------------------------------------------------- /fedlearner/fedavg/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .fedavg import train_from_keras_model 3 | -------------------------------------------------------------------------------- /fedlearner/fedavg/cluster/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .cluster_pb2 import FLNodeDef, FLClusterDef 3 | from .cluster_spec import FLClusterSpec 4 | -------------------------------------------------------------------------------- /fedlearner/fedavg/cluster/cluster.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package fedlearner.cluster; 4 | 5 | message FLNodeDef { 6 | string name = 1; 7 | string address = 2; 8 | } 9 | 10 | message FLClusterDef { 11 | FLNodeDef leader = 1; 12 | repeated FLNodeDef followers = 2; 13 | } -------------------------------------------------------------------------------- /fedlearner/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/fedlearner/model/__init__.py -------------------------------------------------------------------------------- /fedlearner/model/crypto/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/fedlearner/model/crypto/__init__.py -------------------------------------------------------------------------------- /fedlearner/model/tree/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/fedlearner/model/tree/__init__.py -------------------------------------------------------------------------------- /integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bash example/tree_model/test.sh 4 | bash example/sparse_model/test.sh -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools==41.0.0 2 | tensorflow==1.15.2 3 | cityhash 4 | pylint==2.4.4 5 | jinja2 6 | grpcio-tools 7 | etcd3 8 | influxdb 9 | peewee 10 | apsw 11 | configparser 12 | prettytable 13 | kubernetes 14 | scipy 15 | gmpy2 16 | cityhash 17 | scikit-learn 18 | pycryptodomex 19 | rsa 20 | elasticsearch==7.11.0 21 | elasticsearch6 22 | guppy3 23 | tensorflow-io==0.8.1 24 | psutil 25 | sqlalchemy==1.2.19 26 | mysqlclient 27 | leveldb 28 | prison==0.1.3 29 | matplotlib 30 | flatten_dict 31 | pyspark==3.0.2 32 | pandas==1.1.5 33 | opentelemetry-api==1.10.0 34 | opentelemetry-sdk==1.10.0 35 | opentelemetry-exporter-otlp==1.10.0 36 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The FedLearner Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/channel/greeter.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package test.channel; 4 | 5 | message Request { 6 | string name = 1; 7 | } 8 | 9 | message Response { 10 | string message = 1; 11 | } 12 | 13 | service Greeter { 14 | rpc HelloUnaryUnary (Request) returns (Response) {} 15 | rpc HelloUnaryStream (Request) returns (stream Response) {} 16 | rpc HelloStreamUnary (stream Request) returns (Response) {} 17 | rpc HelloStreamStream (stream Request) returns (stream Response) {} 18 | } -------------------------------------------------------------------------------- /test/compressed_raw_data/partition_0000/0-0.idx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/test/compressed_raw_data/partition_0000/0-0.idx -------------------------------------------------------------------------------- /test/data_join/jars/spark-tensorflow-connector_2.12-1.15.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/test/data_join/jars/spark-tensorflow-connector_2.12-1.15.0.jar -------------------------------------------------------------------------------- /test/data_join/jars/tensorflow-hadoop-1.15.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/test/data_join/jars/tensorflow-hadoop-1.15.0.jar -------------------------------------------------------------------------------- /test/rsa_key/rsa_psi: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIBOwIBAAJBAL/AQLios2Q0sN/S+EseP4OXrSyUacGF87Zop+yl03xWGDuKEGNM 3 | d8vwXaIdssg7W/dfKHDRsNCOGsIdU9c0eGECAwEAAQJATih1+Q7CeFDF26WpkSzW 4 | JLQ5YRdpd/k6mppUADCjChddWdjI6nOAzec+Shzwrv+k2XWpWdNswWRhKuYF1acM 5 | MQIjANO9/jo19CZbhV2wCi7oEFq8lfUfUbCrREYA0Sdoai6rrA0CHwDn1I/aMZS/ 6 | Q+IdizyRrtDrQg9s+wmsedgf4nbf5KUCIhu9hnMF4oir8NsHVy4yBQJ+Nn84rTEi 7 | Hv202bH6Af8/SnkCHnh+i7I92PE8iDcQ2o9hzGaqtDfRluqVfda8r0qZnQIiOyQj 8 | DBNeHFUq822lfiQQcuGqPjvwDzP7YHbR0J0NCVXuEA== 9 | -----END RSA PRIVATE KEY----- 10 | -------------------------------------------------------------------------------- /test/rsa_key/rsa_psi.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PUBLIC KEY----- 2 | MEgCQQC/wEC4qLNkNLDf0vhLHj+Dl60slGnBhfO2aKfspdN8Vhg7ihBjTHfL8F2i 3 | HbLIO1v3Xyhw0bDQjhrCHVPXNHhhAgMBAAE= 4 | -----END RSA PUBLIC KEY----- 5 | -------------------------------------------------------------------------------- /test/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/test/trainer/__init__.py -------------------------------------------------------------------------------- /thirdparty/tensorflow/tensorflow/core/framework/reader_base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package tensorflow; 4 | option cc_enable_arenas = true; 5 | option java_outer_classname = "ReaderBaseProtos"; 6 | option java_multiple_files = true; 7 | option java_package = "org.tensorflow.framework"; 8 | option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; 9 | 10 | // For serializing and restoring the state of ReaderBase, see 11 | // reader_base.h for details. 12 | message ReaderBaseState { 13 | int64 work_started = 1; 14 | int64 work_finished = 2; 15 | int64 num_records_produced = 3; 16 | bytes current_work = 4; 17 | }; 18 | -------------------------------------------------------------------------------- /web_console/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .github 3 | .next 4 | node_modules 5 | tests 6 | Dockerfile 7 | -------------------------------------------------------------------------------- /web_console/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | **/.next/** 3 | **/_next/** 4 | **/dist/** 5 | **/__tmp__/** 6 | coverage 7 | *.config.js 8 | bootstrap.js 9 | server.js 10 | _app.js 11 | -------------------------------------------------------------------------------- /web_console/.npmrc: -------------------------------------------------------------------------------- 1 | registry=https://registry.npmjs.org 2 | 3 | -------------------------------------------------------------------------------- /web_console/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mhart/alpine-node:12 2 | 3 | WORKDIR /app 4 | 5 | COPY . . 6 | 7 | # add command below if native addons were required 8 | # RUN apk add --no-cache make gcc g++ python 9 | 10 | RUN npm ci 11 | 12 | RUN npm run build 13 | 14 | EXPOSE 1989 15 | 16 | CMD ["startup.sh"] 17 | -------------------------------------------------------------------------------- /web_console/api/logout.js: -------------------------------------------------------------------------------- 1 | const router = require('@koa/router')(); 2 | 3 | router.post('/api/v1/logout', async (ctx) => { 4 | if (ctx.session.isNew || !ctx.session.user) { 5 | ctx.status = 404; 6 | ctx.body = { 7 | error: 'Session not found', 8 | }; 9 | } else { 10 | ctx.session = null; 11 | ctx.cookies.set('user.session', null); 12 | ctx.cookies.set('user.session.sig', null); 13 | ctx.body = { 14 | message: 'Logout successfully', 15 | }; 16 | } 17 | }); 18 | 19 | module.exports = router; 20 | -------------------------------------------------------------------------------- /web_console/components/DataPortalTypeSelect.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Select } from '@zeit-ui/react'; 3 | 4 | export default function DataPortalTypeSelect(props) { 5 | const options = [ 6 | { label: 'Streaming', value: 'Streaming' }, 7 | { label: 'PSI', value: 'PSI' }, 8 | ]; 9 | return ( 10 | 13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /web_console/components/Dot.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export default function Dot({ color = 'currentColor', size = '8px', style }) { 4 | return ( 5 | <> 6 | 7 | 18 | 19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /web_console/components/JobRoleSelect.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Select } from '@zeit-ui/react'; 3 | 4 | export default function JobRoleSelect(props) { 5 | const options = [ 6 | { label: 'Leader', value: 'Leader' }, 7 | { label: 'Follower', value: 'Follower' }, 8 | ]; 9 | return ( 10 | 13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /web_console/components/JobTypeSelect.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Select } from '@zeit-ui/react'; 3 | import { JOB_TYPE_CLASS } from '../constants/job'; 4 | 5 | export default function JobTypeSelect(props) { 6 | const types = props.type ? JOB_TYPE_CLASS[props.type] : JOB_TYPE_CLASS.all 7 | const options = types.map((x) => ({ label: x, value: x })); 8 | return ( 9 | 12 | ); 13 | } 14 | -------------------------------------------------------------------------------- /web_console/constants/job.js: -------------------------------------------------------------------------------- 1 | const JOB_TYPE = { 2 | data_join: 'data_join', 3 | psi_data_join: 'psi_data_join', 4 | tree_model: 'tree_model', 5 | nn_model: 'nn_model', 6 | } 7 | 8 | const JOB_TYPE_CLASS = { 9 | all: [ 10 | JOB_TYPE.data_join, 11 | JOB_TYPE.psi_data_join, 12 | JOB_TYPE.nn_model, 13 | JOB_TYPE.tree_model 14 | ], 15 | datasource: [ 16 | JOB_TYPE.data_join, 17 | JOB_TYPE.psi_data_join, 18 | ], 19 | training: [ 20 | JOB_TYPE.nn_model, 21 | JOB_TYPE.tree_model 22 | ], 23 | } 24 | 25 | 26 | module.exports = { 27 | JOB_TYPE_CLASS, 28 | JOB_TYPE, 29 | }; 30 | -------------------------------------------------------------------------------- /web_console/libs/http.js: -------------------------------------------------------------------------------- 1 | import ky from 'ky-universal'; 2 | 3 | export const client = ky.create({ 4 | prefixUrl: '/api/v1', 5 | throwHttpErrors: false, 6 | }); 7 | 8 | export const fetcher = (url, options = {}) => { 9 | const federationID = localStorage.getItem('federationID'); 10 | if (federationID && federationID > 0) { 11 | options.headers = { 12 | ...options.headers, 13 | 'X-Federation-Id': federationID, 14 | }; 15 | } 16 | return client.get(url, options).json(); 17 | }; 18 | -------------------------------------------------------------------------------- /web_console/middlewares/admin.js: -------------------------------------------------------------------------------- 1 | module.exports = async function AdminMiddleware(ctx, next) { 2 | if (!ctx.session.user.is_admin) { 3 | ctx.status = 403; 4 | ctx.body = { 5 | error: 'Permission denied', 6 | }; 7 | return; 8 | } 9 | 10 | await next(); 11 | }; 12 | -------------------------------------------------------------------------------- /web_console/middlewares/find_options.js: -------------------------------------------------------------------------------- 1 | // Common middleware for Sequelize FindOptions generation 2 | 3 | module.exports = async function FindOptionsMiddleware(ctx, next) { 4 | ctx.findOptions = {}; 5 | 6 | if (ctx.headers['x-federation-id']) { 7 | const federation_id = parseInt(ctx.headers['x-federation-id'], 10); 8 | if (Number.isNaN(federation_id) || federation_id < 1) { 9 | ctx.status = 400; 10 | ctx.body = { 11 | error: 'Invalid header: X-Federation-Id', 12 | }; 13 | return; 14 | } 15 | ctx.findOptions.where = { federation_id }; 16 | } 17 | 18 | await next(); 19 | }; 20 | -------------------------------------------------------------------------------- /web_console/next.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | pageExtensions: ['jsx'], 3 | poweredByHeader: false, 4 | experimental: { 5 | rewrites() { 6 | return [ 7 | { 8 | source: '/admin', 9 | destination: '/admin/federation', 10 | }, 11 | { 12 | source: '/datasource', 13 | destination: '/datasource/job', 14 | }, 15 | { 16 | source: '/training', 17 | destination: '/training/job', 18 | }, 19 | ]; 20 | }, 21 | }, 22 | }; 23 | -------------------------------------------------------------------------------- /web_console/pages/_app.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { ZeitProvider, CssBaseline } from '@zeit-ui/react'; 3 | 4 | export default function FLApp({ Component, pageProps }) { 5 | return ( 6 | 7 | 8 | 9 | 10 | ); 11 | } 12 | -------------------------------------------------------------------------------- /web_console/pages/datasource/job/[id].jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { useRouter } from 'next/router'; 3 | 4 | import Job from '../../../components/Job'; 5 | 6 | export default function Jobs() { 7 | const router = useRouter(); 8 | return 9 | } 10 | -------------------------------------------------------------------------------- /web_console/pages/datasource/job/index.jsx: -------------------------------------------------------------------------------- 1 | import CommonJobList from '../../../components/CommonJobList' 2 | 3 | export default function JobList() { 4 | return 5 | } -------------------------------------------------------------------------------- /web_console/pages/datasource/tickets/index.jsx: -------------------------------------------------------------------------------- 1 | import CommonTicket from '../../../components/CommonTicket' 2 | 3 | export default function Ticket() { 4 | return 5 | } -------------------------------------------------------------------------------- /web_console/pages/training/job/[id].jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { useRouter } from 'next/router'; 3 | 4 | import Job from '../../../components/Job'; 5 | 6 | export default function Jobs() { 7 | const router = useRouter(); 8 | return 9 | } 10 | -------------------------------------------------------------------------------- /web_console/pages/training/job/index.jsx: -------------------------------------------------------------------------------- 1 | import CommonJobList from '../../../components/CommonJobList'; 2 | 3 | export default function JobList_() { 4 | return 5 | } 6 | -------------------------------------------------------------------------------- /web_console/pages/training/tickets/index.jsx: -------------------------------------------------------------------------------- 1 | import CommonTicket from '../../../components/CommonTicket' 2 | 3 | export default function Ticket() { 4 | return 5 | } -------------------------------------------------------------------------------- /web_console/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console/public/favicon.ico -------------------------------------------------------------------------------- /web_console/services/deployment.js: -------------------------------------------------------------------------------- 1 | import { client } from '../libs/http'; 2 | 3 | export async function updateDeployment(deployment) { 4 | return client.put(`deployments/${deployment.metadata.name}`, { json: deployment }).json(); 5 | } 6 | 7 | export async function foo() { } 8 | -------------------------------------------------------------------------------- /web_console/services/job.js: -------------------------------------------------------------------------------- 1 | import { client } from '../libs/http'; 2 | 3 | export async function deleteJob(name) { 4 | return client.delete(`job/${name}`).json(); 5 | } 6 | 7 | export async function createJob(json) { 8 | return client.post('job', { json }).json(); 9 | } 10 | 11 | export async function updateJobStatus(id, json) { 12 | return client.post(`job/${id}/update`, { json }).json(); 13 | } -------------------------------------------------------------------------------- /web_console/services/ticket.js: -------------------------------------------------------------------------------- 1 | import { client } from '../libs/http'; 2 | 3 | export async function getTickets() { 4 | return client.get('tickets').json(); 5 | } 6 | 7 | export async function createTicket(json) { 8 | return client.post('tickets', { json }).json(); 9 | } 10 | 11 | export async function updateTicket(id, json) { 12 | return client.put(`tickets/${id}`, { json }).json(); 13 | } 14 | 15 | export async function enableTicket(id) { 16 | return client.post(`tickets/${id}/enable`).json(); 17 | } 18 | 19 | export async function revokeTicket(id) { 20 | return client.post(`tickets/${id}/revoke`).json(); 21 | } 22 | -------------------------------------------------------------------------------- /web_console/tests/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /web_console/tests/fixtures/raw_data.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: 'raw-data-test', 3 | input: 'hdfs://open.aliyun.com/foo/bar', 4 | output: 'hdfs://open.aliyun.com/foo/bar', 5 | context: '{}', 6 | remark: 'raw-data-test comment', 7 | federation_id: 1, 8 | output_partition_num: 8, 9 | data_portal_type: 'Streaming', 10 | }; 11 | -------------------------------------------------------------------------------- /web_console/tests/fixtures/user.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | admin: { 3 | username: 'ada', 4 | password: 'fdee430d40bd57de', 5 | name: 'Ada', 6 | tel: null, 7 | is_admin: true, 8 | }, 9 | user: { 10 | username: 'ruby', 11 | password: 'b9138194ffe9e7c8', 12 | name: 'Ruby', 13 | tel: null, 14 | is_admin: false, 15 | }, 16 | }; 17 | -------------------------------------------------------------------------------- /web_console/tests/libs/es.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const es = require('../../libs/es'); 3 | 4 | describe('ElasticSearch Client', () => { 5 | describe('query log from elastic search', () => { 6 | it('should query log from elastic search', async () => { 7 | const logs = await es.queryLog( 8 | 'filebeat-7.0.1-*', 9 | 'test-produce-psi-preprocessor', 10 | 'flapp-operator-6845d4d9d5-wpdqx', 11 | 1592627446768, 12 | 1592713846768, 13 | ); 14 | assert.ok(Array.isArray(logs)); 15 | }); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /web_console/utils/encrypt.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | 3 | module.exports = function encrypt(str, size = 16) { 4 | if (!str || typeof str !== 'string') { 5 | throw new Error(`Illegal input: ${str}`); 6 | } 7 | 8 | if (typeof size !== 'number' || size <= 0 || size > 64) { 9 | throw new Error(`Illegal size: ${size}`); 10 | } 11 | 12 | return crypto.createHash('sha256').update(str).digest('hex').substring(0, size); 13 | }; 14 | -------------------------------------------------------------------------------- /web_console/utils/get_confg.js: -------------------------------------------------------------------------------- 1 | const isPlainObject = require('lodash/isPlainObject'); 2 | 3 | let config = {}; 4 | try { 5 | config = require('../server.config'); 6 | } catch (err) { /* */ } 7 | 8 | const defaultConfig = require('../constants').DEFAULT_SERVER_CONFIG; 9 | 10 | module.exports = function getConfig(customConfig) { 11 | const override = isPlainObject(customConfig) 12 | ? Object.keys(customConfig).reduce((prev, key) => { 13 | if (customConfig[key]) { 14 | prev[key] = customConfig[key]; 15 | } 16 | return prev; 17 | }, {}) 18 | : null; 19 | return { ...defaultConfig, ...config, ...override }; 20 | }; 21 | -------------------------------------------------------------------------------- /web_console/utils/index.js: -------------------------------------------------------------------------------- 1 | const { readFile, readFileSync, writeFile, writeFileSync, readdir, readdirSync, unlink, unlinkSync } = require('fs'); 2 | const { exec, execSync } = require('child_process'); 3 | const { promisify } = require('util'); 4 | 5 | module.exports = { 6 | exec: promisify(exec), 7 | execSync: (cmd, options) => execSync(cmd, { encoding: 'utf-8', ...options }), 8 | readdir: promisify(readdir), 9 | readdirSync, 10 | readFile: promisify(readFile), 11 | readFileSync, 12 | writeFile: promisify(writeFile), 13 | writeFileSync, 14 | unlink: promisify(unlink), 15 | unlinkSync, 16 | }; 17 | -------------------------------------------------------------------------------- /web_console_v2/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | **/node_modules 3 | Dockerfile 4 | 5 | # Tests 6 | client/tests 7 | api/test 8 | api/testing 9 | -------------------------------------------------------------------------------- /web_console_v2/Makefile: -------------------------------------------------------------------------------- 1 | api-test: 2 | cd api && \ 3 | make protobuf && \ 4 | make lint && \ 5 | make test 6 | 7 | docker-spark: 8 | cd ./docker/spark && docker build . -t spark-tfrecord:latest -------------------------------------------------------------------------------- /web_console_v2/api/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | concurrency = multiprocessing 3 | include = fedlearner_webconsole/* 4 | omit = fedlearner_webconsole/proto/* 5 | data_file = .coverage 6 | 7 | [report] 8 | show_missing = True 9 | 10 | [html] 11 | directory = .coverage_html_report 12 | -------------------------------------------------------------------------------- /web_console_v2/api/.gitignore: -------------------------------------------------------------------------------- 1 | *.db 2 | *.log 3 | 4 | # Generated proto python code 5 | fedlearner_webconsole/proto/*.py 6 | fedlearner_webconsole/proto/*.pyi 7 | 8 | # Coverage generated 9 | .coverage_html_report/ 10 | .coverage* 11 | 12 | root.log.* -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/auth/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/composer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/composer/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/dataset/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/job/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/job/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/k8s/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/k8s/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/project/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/proto/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/proto/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/rpc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/rpc/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/scheduler/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/setting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/setting/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/sparkapp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/sparkapp/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/utils/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/workflow/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/fedlearner_webconsole/workflow_template/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/fedlearner_webconsole/workflow_template/__init__.py -------------------------------------------------------------------------------- /web_console_v2/api/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /web_console_v2/api/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /web_console_v2/api/migrations/versions/fe30109949aa_add_extra_field_to_workflow.py: -------------------------------------------------------------------------------- 1 | """add extra field to workflow 2 | 3 | Revision ID: fe30109949aa 4 | Revises: 27a868485bf7 5 | Create Date: 2021-06-06 14:31:29.833200 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'fe30109949aa' 14 | down_revision = '27a868485bf7' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('workflow_v2', sa.Column('extra', sa.Text(), nullable=True, comment='extra')) 21 | 22 | 23 | def downgrade(): 24 | op.drop_column('workflow_v2', 'extra') 25 | -------------------------------------------------------------------------------- /web_console_v2/api/test/fedlearner_webconsole/project/test.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/test/fedlearner_webconsole/project/test.tar.gz -------------------------------------------------------------------------------- /web_console_v2/api/test/fedlearner_webconsole/test_data/code.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/api/test/fedlearner_webconsole/test_data/code.tar.gz -------------------------------------------------------------------------------- /web_console_v2/api/tools/local_runner/run_a.sh: -------------------------------------------------------------------------------- 1 | export PYTHONPATH=$PYTHONPATH:"../../" 2 | export FLASK_APP=app_a:app 3 | export FLASK_ENV=development 4 | flask create-db 5 | export K8S_CONFIG_PATH=$1 6 | export FEDLEARNER_WEBCONSOLE_POLLING_INTERVAL=10 7 | export SQLALCHEMY_DATABASE_URI="sqlite:///app_a.db" 8 | export FEATURE_MODEL_WORKFLOW_HOOK=True 9 | export FEATURE_MODEL_K8S_HOOK=True 10 | export ES_READ_HOST=172.21.8.76 # aliyun-demo1 fedlearner-stack-elasticsearch-client 11 | flask run --host=0.0.0.0 --no-reload --eager-loading -p 9001 12 | -------------------------------------------------------------------------------- /web_console_v2/api/tools/local_runner/run_b.sh: -------------------------------------------------------------------------------- 1 | export PYTHONPATH=$PYTHONPATH:"../../" 2 | export FLASK_APP=app_b:app 3 | export FLASK_ENV=development 4 | flask create-db 5 | export K8S_CONFIG_PATH=$1 6 | export FEDLEARNER_WEBCONSOLE_POLLING_INTERVAL=1 7 | export SQLALCHEMY_DATABASE_URI="sqlite:///app_b.db" 8 | export FEATURE_MODEL_WORKFLOW_HOOK=True 9 | export FEATURE_MODEL_K8S_HOOK=True 10 | export ES_READ_HOST=172.21.14.199 # aliyun-demo2 fedlearner-stack-elasticsearch-client 11 | flask run --host=0.0.0.0 --no-reload --eager-loading -p 9002 12 | -------------------------------------------------------------------------------- /web_console_v2/client/.editorconfig: -------------------------------------------------------------------------------- 1 | 2 | # http://editorconfig.org 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 2 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /web_console_v2/client/.eslintignore: -------------------------------------------------------------------------------- 1 | .vscode/* 2 | config/* 3 | scripts/* 4 | -------------------------------------------------------------------------------- /web_console_v2/client/.npmrc: -------------------------------------------------------------------------------- 1 | ; Force to use official registry 2 | registry=https://registry.npmjs.org/ 3 | -------------------------------------------------------------------------------- /web_console_v2/client/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | trailingComma: 'all', 3 | semi: true, 4 | singleQuote: true, 5 | printWidth: 100, 6 | useTabs: false, 7 | tabWidth: 2, 8 | }; 9 | -------------------------------------------------------------------------------- /web_console_v2/client/README.md: -------------------------------------------------------------------------------- 1 | # Code for UI 2 | 3 | ## Prerequisites 4 | 5 | - Node.js >= 14.0 && Node.js <= 15.0 6 | 7 | ## Get started 8 | 9 | we using [pnpm](https://pnpm.js.org/) as package manager 10 | 11 | ```bash 12 | $ npm i -g pnpm 13 | 14 | $ pnpm install 15 | 16 | # start developing with hot reload 17 | $ pnpm start 18 | 19 | # build for production 20 | $ pnpm build 21 | 22 | # build for fully mocked version 23 | $ REACT_APP_ENABLE_FULLY_MOCK=true pnpm build 24 | 25 | # run tests 26 | $ pnpm test 27 | ``` 28 | -------------------------------------------------------------------------------- /web_console_v2/client/config/jest/cssTransform.js: -------------------------------------------------------------------------------- 1 | // This is a custom Jest transformer turning style imports into empty objects. 2 | // http://facebook.github.io/jest/docs/en/webpack.html 3 | 4 | module.exports = { 5 | process() { 6 | return 'module.exports = {};'; 7 | }, 8 | getCacheKey() { 9 | // The output is always the same. 10 | return 'cssTransform'; 11 | }, 12 | }; 13 | -------------------------------------------------------------------------------- /web_console_v2/client/docs/KNOWN_ISSUES.md: -------------------------------------------------------------------------------- 1 | # Known issues 2 | 3 | ## Development 4 | 5 | 1. Build stuck at `Compiling...` 6 | 7 | Sometime you changed files' structure will lead this happen, just ctrl+c quit process and restart building should get it works. 8 | -------------------------------------------------------------------------------- /web_console_v2/client/public/fed-favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/public/fed-favicon.ico -------------------------------------------------------------------------------- /web_console_v2/client/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Bold.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Bold.otf -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Medium.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Medium.otf -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Regular.otf -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Thin.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/fonts/ClarityMono/ClarityMono-Thin.otf -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/icons/workflow-pending.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/images/avatar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/images/avatar.jpg -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/images/close-icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/images/hacker-codes.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/images/hacker-codes.jpg -------------------------------------------------------------------------------- /web_console_v2/client/src/assets/images/login-illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/src/assets/images/login-illustration.png -------------------------------------------------------------------------------- /web_console_v2/client/src/components/BreadcrumbLink/Slash.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactElement } from 'react'; 2 | import styled from 'styled-components'; 3 | 4 | const Slash = styled.div` 5 | display: inline-block; 6 | width: 1.75px; 7 | height: 9px; 8 | transform: matrix(0.87, 0.5, 0.5, -0.87, 0, 0); 9 | border-radius: 0.5px; 10 | margin: 7px 2px 0; 11 | background-color: var(--gray4); 12 | `; 13 | 14 | function BreadcrumbSlash(): ReactElement { 15 | return ; 16 | } 17 | 18 | export default BreadcrumbSlash; 19 | -------------------------------------------------------------------------------- /web_console_v2/client/src/components/Footer/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import styled from 'styled-components'; 3 | import { StyledComponetProps } from 'typings/component'; 4 | 5 | const Container = styled.footer` 6 | text-align: center; 7 | background-color: #f0f0f0; 8 | `; 9 | 10 | function Footer({ className }: StyledComponetProps) { 11 | return @fl; 12 | } 13 | 14 | export default Footer; 15 | -------------------------------------------------------------------------------- /web_console_v2/client/src/components/IconPark/icons/More.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * @file More more 3 | * @author Auto Generated by IconPark 4 | */ 5 | 6 | /* tslint:disable: max-line-length */ 7 | /* eslint-disable max-len */ 8 | import React from 'react'; 9 | import { ISvgIconProps, IconWrapper } from '../runtime'; 10 | 11 | export default IconWrapper( 12 | 'more', 13 | false, 14 | (props: ISvgIconProps) => ( 15 | 16 | 17 | 18 | 19 | ), 20 | (props: ISvgIconProps) => ` 21 | `, 22 | ); 23 | -------------------------------------------------------------------------------- /web_console_v2/client/src/components/IconPark/runtime/index.less: -------------------------------------------------------------------------------- 1 | // Pretending our custom icons are anticons :) 2 | .anticon { 3 | display: inline-block; 4 | color: inherit; 5 | font-style: normal; 6 | line-height: 0; 7 | text-align: center; 8 | text-transform: none; 9 | text-rendering: optimizeLegibility; 10 | -webkit-font-smoothing: antialiased; 11 | -moz-osx-font-smoothing: grayscale; 12 | 13 | &-spin svg { 14 | animation: anticon-spin 1s infinite linear; 15 | } 16 | } 17 | 18 | @keyframes anticon-spin { 19 | 100% { 20 | -webkit-transform: rotate(360deg); 21 | transform: rotate(360deg); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /web_console_v2/client/src/components/Username/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import { useRecoilQuery } from 'hooks/recoil'; 3 | import { userInfoQuery } from 'stores/user'; 4 | 5 | const Username: FC = () => { 6 | const query = useRecoilQuery(userInfoQuery); 7 | 8 | return {query.data?.username || '-'}; 9 | }; 10 | 11 | export default Username; 12 | -------------------------------------------------------------------------------- /web_console_v2/client/src/components/WorkflowJobsCanvas/JobNodes/index.ts: -------------------------------------------------------------------------------- 1 | export { default as config } from './ConfigNode'; 2 | export { default as edit } from './EditConfigNode'; 3 | export { default as execution } from './ExecutionNode'; 4 | export { default as global } from './GlobalConfigNode'; 5 | export { default as fork } from './ForkNode'; 6 | -------------------------------------------------------------------------------- /web_console_v2/client/src/components/_base/MockDevtools/index.js: -------------------------------------------------------------------------------- 1 | if (process.env.NODE_ENV === 'production') { 2 | module.exports = function () { 3 | return null; 4 | }; 5 | } else { 6 | module.exports = require('./MockControlPanel'); 7 | } 8 | -------------------------------------------------------------------------------- /web_console_v2/client/src/hooks/dataset.ts: -------------------------------------------------------------------------------- 1 | import { useResetRecoilState } from 'recoil'; 2 | import { datasetBasicForm } from 'stores/dataset'; 3 | 4 | export function useResetCreateForm() { 5 | const resetBasicForm = useResetRecoilState(datasetBasicForm); 6 | 7 | return function () { 8 | resetBasicForm(); 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /web_console_v2/client/src/hooks/recoil.ts: -------------------------------------------------------------------------------- 1 | import { ServerError } from 'libs/request'; 2 | import { RecoilValue, useRecoilValueLoadable } from 'recoil'; 3 | 4 | export function useRecoilQuery(recoilValue: RecoilValue) { 5 | const loadable = useRecoilValueLoadable(recoilValue); 6 | return { 7 | data: loadable.state === 'hasValue' ? loadable.contents : (undefined as never), 8 | isLoading: loadable.state === 'loading', 9 | error: loadable.state === 'hasError' ? (loadable.errorOrThrow() as ServerError) : null, 10 | }; 11 | } 12 | -------------------------------------------------------------------------------- /web_console_v2/client/src/hooks/template.ts: -------------------------------------------------------------------------------- 1 | import { useResetRecoilState } from 'recoil'; 2 | import { templateForm } from 'stores/template'; 3 | 4 | export function useResetCreateForm() { 5 | const resetForm = useResetRecoilState(templateForm); 6 | 7 | return function () { 8 | resetForm(); 9 | }; 10 | } 11 | -------------------------------------------------------------------------------- /web_console_v2/client/src/i18n/helper.test.ts: -------------------------------------------------------------------------------- 1 | import { I18nMessageModule, separateLng } from './helpers'; 2 | 3 | const mockModule: I18nMessageModule = { 4 | a: { zh: '1', en: '2' }, 5 | b: { zh: '3' }, 6 | }; 7 | 8 | describe('Separate ZH and EN messages from module', () => { 9 | it('Should works fine', () => { 10 | expect(separateLng(mockModule)).toEqual({ 11 | zh: { 12 | a: '1', 13 | b: '3', 14 | }, 15 | en: { 16 | a: '2', 17 | b: null, 18 | }, 19 | }); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /web_console_v2/client/src/i18n/helpers.ts: -------------------------------------------------------------------------------- 1 | export type I18nMessage = { zh: string; en?: string | null }; 2 | export type I18nMessageModule = { [key: string]: I18nMessage }; 3 | 4 | export function separateLng(msgModule: I18nMessageModule) { 5 | const ret: { [key in 'zh' | 'en']: { [key: string]: string | null } } = { 6 | zh: {}, 7 | en: {}, 8 | }; 9 | 10 | Object.entries(msgModule).forEach(([key, values]) => { 11 | ret.zh[key] = values.zh; 12 | ret.en[key] = values.en || null; 13 | }); 14 | 15 | return ret; 16 | } 17 | -------------------------------------------------------------------------------- /web_console_v2/client/src/i18n/resources/modules/app.ts: -------------------------------------------------------------------------------- 1 | import { separateLng } from 'i18n/helpers'; 2 | 3 | const error = { 4 | go_create: { zh: '去创建' }, 5 | help: { zh: '[WIP]查看帮助，快速了解联邦学习' }, 6 | switch_lng: { zh: '切换语言', en: 'Language' }, 7 | logout: { zh: '退出登录', en: 'Logout' }, 8 | login_success: { zh: '登录成功', en: 'Login successfully' }, 9 | copy_success: { zh: '复制成功', en: 'Copied!' }, 10 | system_settings: { zh: '系统配置', en: 'Sttings' }, 11 | }; 12 | 13 | export default separateLng(error); 14 | -------------------------------------------------------------------------------- /web_console_v2/client/src/i18n/resources/modules/error.ts: -------------------------------------------------------------------------------- 1 | import { separateLng } from 'i18n/helpers'; 2 | 3 | const error = { 4 | please_sign_in: { zh: '请登录账号', en: 'Pelase Sign in' }, 5 | token_expired: { 6 | zh: '登录状态已过期，请重新登录', 7 | en: 'Login status has been expired, please sign in again', 8 | }, 9 | no_result: { zh: '' }, 10 | }; 11 | 12 | export default separateLng(error); 13 | -------------------------------------------------------------------------------- /web_console_v2/client/src/i18n/resources/modules/term.ts: -------------------------------------------------------------------------------- 1 | import { separateLng } from 'i18n/helpers'; 2 | 3 | const term = { 4 | project: { zh: '项目', en: 'Project' }, 5 | coordinator: { zh: '发起方', en: 'Coordinator' }, 6 | participant: { zh: '合作方', en: 'Participant' }, 7 | workflow: { zh: '工作流', en: 'Workflow' }, 8 | dataset: { zh: '数据集', en: 'Datasets' }, 9 | workflow_template: { zh: ' 工作流模板', en: 'Workflow template' }, 10 | certificate: { zh: ' 证书', en: 'Certificate' }, 11 | }; 12 | 13 | export default separateLng(term); 14 | -------------------------------------------------------------------------------- /web_console_v2/client/src/i18n/resources/modules/upload.ts: -------------------------------------------------------------------------------- 1 | import { separateLng } from 'i18n/helpers'; 2 | 3 | const upload = { 4 | placeholder: { zh: '点击或拖拽文件到此处上传' }, 5 | hint: { zh: '请上传{{fileTypes}}格式文件，大小不超过{{maxSize}}MB' }, 6 | }; 7 | 8 | export default separateLng(upload); 9 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/auth/signin.ts: -------------------------------------------------------------------------------- 1 | export const post = { 2 | data: { 3 | data: { 4 | access_token: 'token', 5 | user: { 6 | id: 1, 7 | name: '', 8 | username: '', 9 | role: '', 10 | }, 11 | }, 12 | }, 13 | status: 204, 14 | }; 15 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/auth/signout.ts: -------------------------------------------------------------------------------- 1 | export const post = { 2 | data: { success: true }, 3 | status: 204, 4 | }; 5 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/auth/users/:id.ts: -------------------------------------------------------------------------------- 1 | import { FedRoles } from 'typings/auth'; 2 | 3 | const fakeUserInfo = { 4 | data: { 5 | data: { id: 1, username: 'Mocked Admin', email: 'fl@mocked.com', role: FedRoles.Admin }, 6 | }, 7 | // to mock server error, just tweak the status code below 8 | status: 200, 9 | }; 10 | 11 | export default fakeUserInfo; 12 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/datasets/:id/batches.ts: -------------------------------------------------------------------------------- 1 | export const post = () => ({ 2 | data: { 3 | data: { success: true }, 4 | }, 5 | status: 200, 6 | }); 7 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/datasets/:id/index.ts: -------------------------------------------------------------------------------- 1 | import { unfinishedImporting, importFailed } from '../examples'; 2 | 3 | const get = { 4 | data: { 5 | data: [unfinishedImporting, importFailed], 6 | }, 7 | status: 200, 8 | }; 9 | 10 | export const DELETE = { 11 | data: { 12 | data: unfinishedImporting, 13 | }, 14 | status: 200, 15 | }; 16 | 17 | export default get; 18 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/datasets/index.ts: -------------------------------------------------------------------------------- 1 | import { unfinishedImporting, importFailed, successfullyImport } from './examples'; 2 | 3 | const get = { 4 | data: { 5 | data: [unfinishedImporting, importFailed, successfullyImport], 6 | }, 7 | status: 200, 8 | }; 9 | 10 | export const post = { 11 | data: { 12 | data: unfinishedImporting, 13 | }, 14 | status: 200, 15 | }; 16 | 17 | export default get; 18 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/files/index.ts: -------------------------------------------------------------------------------- 1 | import { FileToImport } from 'typings/dataset'; 2 | 3 | const get = { 4 | data: { 5 | data: [ 6 | { 7 | path: '/root/admin/rw_datas/f_1.db', 8 | size: 123456, 9 | mtime: 1601937685, 10 | }, 11 | { 12 | path: '/root/admin/rw_datas/f_2.db', 13 | size: 445678, 14 | mtime: 1678937685, 15 | }, 16 | { 17 | path: '/root/admin/fl_datas/f_3.db', 18 | size: 30340, 19 | mtime: 1678937685, 20 | }, 21 | ] as FileToImport[], 22 | }, 23 | status: 200, 24 | }; 25 | 26 | export default get; 27 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/jobs/:id/events.ts: -------------------------------------------------------------------------------- 1 | let offset = 0; 2 | 3 | const get = (config: any) => { 4 | offset += 1; 5 | return { 6 | data: { 7 | data: Array(config.params.max_lines) 8 | .fill(null) 9 | .map((_, index) => 'Events:' + index + offset), 10 | }, 11 | status: 200, 12 | }; 13 | }; 14 | 15 | export default get; 16 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/jobs/:id/log.ts: -------------------------------------------------------------------------------- 1 | let offset = 0; 2 | 3 | const get = (config: any) => { 4 | offset += 1; 5 | return { 6 | data: { 7 | data: Array(config.params.max_lines) 8 | .fill(null) 9 | .map((_, index) => index + offset), 10 | }, 11 | status: 200, 12 | }; 13 | }; 14 | 15 | export default get; 16 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/projects/:id/connection_checks.ts: -------------------------------------------------------------------------------- 1 | export const post = () => ({ 2 | data: { 3 | data: { success: Math.random() < 0.6, details: [] }, 4 | }, 5 | status: 200, 6 | }); 7 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/settings/index.ts: -------------------------------------------------------------------------------- 1 | import { SettingOptions } from 'typings/settings'; 2 | 3 | const get = { 4 | data: { 5 | data: { webconsole_image: '2.0.0-rc.3' } as SettingOptions, 6 | }, 7 | status: 200, 8 | }; 9 | 10 | export const patch = { 11 | data: { 12 | data: { success: true }, 13 | }, 14 | status: 200, 15 | }; 16 | 17 | export default get; 18 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/workflow_templates/:id/index.ts: -------------------------------------------------------------------------------- 1 | import { AxiosRequestConfig } from 'axios'; 2 | import { normalTemplate, complexDepsTemplate, xShapeTemplate } from '../examples'; 3 | 4 | const get = (config: AxiosRequestConfig) => { 5 | const rets: Record = { 6 | 1: normalTemplate, 7 | 2: complexDepsTemplate, 8 | 3: xShapeTemplate, 9 | }; 10 | 11 | return { 12 | data: { data: rets[config._id!] }, 13 | status: 200, 14 | }; 15 | }; 16 | 17 | export default get; 18 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/workflows/:id/peer_workflows.ts: -------------------------------------------------------------------------------- 1 | import { cloneDeep } from 'lodash'; 2 | import { JobState } from 'typings/job'; 3 | import { newlyCreated, withExecutionDetail } from '../examples'; 4 | 5 | const get = () => { 6 | const modified = cloneDeep(withExecutionDetail); 7 | 8 | modified.jobs[1].state = JobState.COMPLETED; 9 | modified.jobs[2].state = JobState.STARTED; 10 | 11 | return { 12 | data: { 13 | data: { peer_1: modified || newlyCreated }, 14 | }, 15 | status: 200, 16 | }; 17 | }; 18 | 19 | export default get; 20 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/mocks/v2/workflows/index.ts: -------------------------------------------------------------------------------- 1 | import { AxiosRequestConfig } from 'axios'; 2 | import { Workflow } from 'typings/workflow'; 3 | import { newlyCreated, pendingAcceptAndConfig, completed } from './examples'; 4 | 5 | const list: Workflow[] = [pendingAcceptAndConfig, newlyCreated, completed]; 6 | 7 | export const get = { 8 | data: { 9 | data: list, 10 | }, 11 | status: 200, 12 | }; 13 | 14 | export const post = (config: AxiosRequestConfig) => { 15 | return { 16 | data: { 17 | data: config.data, 18 | }, 19 | status: 200, 20 | }; 21 | }; 22 | 23 | export default get; 24 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/settings.ts: -------------------------------------------------------------------------------- 1 | import request from 'libs/request'; 2 | import { SettingOptions } from 'typings/settings'; 3 | 4 | export function fetchSettings(): Promise<{ data: SettingOptions }> { 5 | return request('/v2/settings'); 6 | } 7 | 8 | export function updateSettings(payload: SettingOptions): Promise<{ data: any }> { 9 | return request.patch('/v2/settings', payload); 10 | } 11 | -------------------------------------------------------------------------------- /web_console_v2/client/src/services/system.ts: -------------------------------------------------------------------------------- 1 | import request from 'libs/request'; 2 | 3 | export function fetchPodNameList(): Promise<{ data: string[] }> { 4 | return request('/v2/system_pods/name'); 5 | } 6 | 7 | export function fetchSystemLogs(tailLines: number, podName: string): Promise<{ data: string[] }> { 8 | return request(`/v2/system_pods/${podName}/logs`, { params: { tailLines }, snake_case: true }); 9 | } 10 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/array.ts: -------------------------------------------------------------------------------- 1 | export function isHead(target: any, source: any[]) { 2 | return source.indexOf(target) === 0; 3 | } 4 | 5 | export function isLast(target: any, source: any[]) { 6 | return source.lastIndexOf(target) === source.length - 1; 7 | } 8 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/base64.ts: -------------------------------------------------------------------------------- 1 | export function decodeBase64(str?: string) { 2 | if (!str) return ''; 3 | let temp = ''; 4 | try { 5 | temp = decodeURIComponent(atob(str)); 6 | } catch (e) { 7 | // the base 64 was invalid, then check for 'e.code === 5'. 8 | // (because 'DOMException.INVALID_CHARACTER_ERR === 5') 9 | if (e.code === 5) return str; 10 | } 11 | return temp; 12 | } 13 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/localStorageKeys.ts: -------------------------------------------------------------------------------- 1 | const LOCAL_STORAGE_KEYS = { 2 | language: 'language', 3 | current_user: 'current_user', 4 | mock_configs: 'mock_configs', 5 | sidebar_folded: 'sidebar_folded', 6 | projects_display: 'projects_display', 7 | current_project: 'current_project', 8 | }; 9 | 10 | export default LOCAL_STORAGE_KEYS; 11 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/project.test.ts: -------------------------------------------------------------------------------- 1 | import { wrapWithDomainName, unwrapDomainName, DOMAIN_PREFIX, DOMAIN_SUFFIX } from './project'; 2 | 3 | describe("Domain's wrap/unwrap", () => { 4 | it('Wrap should works fine', () => { 5 | expect(wrapWithDomainName('case1')).toBe(`${DOMAIN_PREFIX}case1${DOMAIN_SUFFIX}`); 6 | }); 7 | 8 | it('Unwrap should works fine', () => { 9 | expect(unwrapDomainName('fl-case2.com')).toBe('case2'); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/project.ts: -------------------------------------------------------------------------------- 1 | export const DOMAIN_PREFIX = 'fl-'; 2 | export const DOMAIN_SUFFIX = '.com'; 3 | 4 | /** Turn user input 'xxx' to 'fl-xxx.com' */ 5 | export function wrapWithDomainName(input: string) { 6 | return `${DOMAIN_PREFIX}${input}${DOMAIN_SUFFIX}`; 7 | } 8 | /** Remove prefix and suffix of 'fl-xxx.com' */ 9 | export function unwrapDomainName(input: string) { 10 | return input.replace(DOMAIN_PREFIX, '').replace(DOMAIN_SUFFIX, ''); 11 | } 12 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/queryClient.ts: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | 3 | import { QueryClient } from 'react-query'; 4 | 5 | const queryClient = new QueryClient(); 6 | 7 | /** 8 | * Force to trigger one or multiple query's refetch 9 | * by invalid there's result 10 | */ 11 | export function forceToRefreshQuery(queryKey: string | string[]) { 12 | return queryClient.invalidateQueries(queryKey); 13 | } 14 | 15 | export default queryClient; 16 | -------------------------------------------------------------------------------- /web_console_v2/client/src/shared/validator.ts: -------------------------------------------------------------------------------- 1 | import i18n from 'i18n'; 2 | 3 | const PASSWORD_REGX = /^(?=.*[A-Za-z])(?=.*\d)(?=.*[!@#$%^&*()_=+|{};:'",<.>/?~])[A-Za-z\d!@#$%^&*()_=+|{};:'",<.>/?~]{8,20}$/i; 4 | export async function validatePassword( 5 | value: string, 6 | options: { message: string } = { message: i18n.t('users.placeholder_password_message') }, 7 | ) { 8 | if (PASSWORD_REGX.test(value)) { 9 | return true; 10 | } 11 | 12 | throw new Error(options.message); 13 | } 14 | -------------------------------------------------------------------------------- /web_console_v2/client/src/stores/dataset.ts: -------------------------------------------------------------------------------- 1 | import { atom } from 'recoil'; 2 | import { DatasetCreatePayload, DatasetType } from 'typings/dataset'; 3 | 4 | export const datasetBasicForm = atom({ 5 | key: 'DatasetBasicForm', 6 | default: { 7 | name: '', 8 | project_id: (undefined as unknown) as ID, 9 | dataset_type: DatasetType.PSI, 10 | comment: '', 11 | }, 12 | }); 13 | -------------------------------------------------------------------------------- /web_console_v2/client/src/styles/animations.ts: -------------------------------------------------------------------------------- 1 | import { keyframes } from 'styled-components'; 2 | 3 | export const ScrollDown = keyframes` 4 | 0% { 5 | transform: translateY(0); 6 | } 7 | 25% { 8 | transform: translateY(13%); 9 | } 10 | 75% { 11 | transform: translateY(-13%); 12 | } 13 | 100% { 14 | transform: translateY(0); 15 | } 16 | `; 17 | -------------------------------------------------------------------------------- /web_console_v2/client/src/typings/app.ts: -------------------------------------------------------------------------------- 1 | /** Federation Learner global types */ 2 | import { FedRoles } from 'typings/auth'; 3 | export interface FedRouteConfig { 4 | path: string; 5 | component: React.FunctionComponent; 6 | exact?: boolean; 7 | auth?: boolean; // whether require logged in 8 | roles?: FedRoles[]; 9 | } 10 | 11 | export enum FedLanguages { 12 | Chinese = 'zh', 13 | English = 'en', 14 | } 15 | 16 | export enum ErrorCodes { 17 | TokenExpired = 422, 18 | } 19 | 20 | export type Side = 'self' | 'peer'; 21 | -------------------------------------------------------------------------------- /web_console_v2/client/src/typings/auth.ts: -------------------------------------------------------------------------------- 1 | export interface FedUserInfo { 2 | id: string; 3 | username: string; 4 | name?: string; 5 | email?: string; 6 | role: FedRoles; 7 | } 8 | 9 | export interface FedLoginFormData { 10 | username: string; 11 | password: string; 12 | } 13 | 14 | export enum FedRoles { 15 | Admin = 'ADMIN', 16 | User = 'USER', 17 | } 18 | -------------------------------------------------------------------------------- /web_console_v2/client/src/typings/component.ts: -------------------------------------------------------------------------------- 1 | export type ComponentSize = 'small' | 'medium' | 'large' | 'default'; 2 | 3 | export interface StyledComponetProps { 4 | className?: string; 5 | [key: string]: any; 6 | } 7 | export interface PaginationConfig { 8 | total: number; 9 | page_size: number; 10 | page: number; 11 | } 12 | 13 | export enum DisplayType { 14 | Card = 1, 15 | Table = 2, 16 | } 17 | -------------------------------------------------------------------------------- /web_console_v2/client/src/typings/settings.ts: -------------------------------------------------------------------------------- 1 | export type SettingOptions = { 2 | webconsole_image: string; 3 | }; 4 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/Dashboard/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactElement } from 'react'; 2 | 3 | function DashboardPage(): ReactElement { 4 | return
dashboard
; 5 | } 6 | 7 | export default DashboardPage; 8 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/Datasets/index.tsx: -------------------------------------------------------------------------------- 1 | import ErrorBoundary from 'antd/lib/alert/ErrorBoundary'; 2 | import React, { FC } from 'react'; 3 | import { Route } from 'react-router-dom'; 4 | import CreateDataset from './CreateDataset'; 5 | import DatasetList from './DatasetList'; 6 | 7 | const DatasetsPage: FC = () => { 8 | return ( 9 | 10 | 11 | 12 | 13 | ); 14 | }; 15 | 16 | export default DatasetsPage; 17 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/Users/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import ErrorBoundary from 'antd/lib/alert/ErrorBoundary'; 3 | import { Route } from 'react-router-dom'; 4 | import UserList from './UserList'; 5 | import UserCreate from './UserCreate'; 6 | import UserEdit from './UserEdit'; 7 | 8 | const UsersPage: FC = () => { 9 | return ( 10 | 11 | 12 | 13 | 14 | 15 | ); 16 | }; 17 | 18 | export default UsersPage; 19 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/WorkflowTemplates/CreateTemplate/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import TemplateForm from '../TemplateForm'; 3 | 4 | const CreateTemplate: FC = () => { 5 | return ; 6 | }; 7 | 8 | export default CreateTemplate; 9 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/WorkflowTemplates/EditTemplate/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC, useRef } from 'react'; 2 | import TemplateForm from '../TemplateForm'; 3 | 4 | const EditTemplate: FC = () => { 5 | const isHydrated = useRef(false); 6 | 7 | return ; 8 | }; 9 | 10 | export default EditTemplate; 11 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/Workflows/WorkflowDetail/JobKibanaMetrics/FieldComponents/XAxisInput.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import { useTranslation } from 'react-i18next'; 3 | import { Input } from 'antd'; 4 | 5 | type Props = { 6 | value?: string; 7 | onChange?: any; 8 | }; 9 | 10 | const XAxisInput: FC = ({ value, onChange }) => { 11 | const { t } = useTranslation(); 12 | 13 | return ; 14 | }; 15 | 16 | export default XAxisInput; 17 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/Workflows/WorkflowList/WorkflowStage.tsx: -------------------------------------------------------------------------------- 1 | import StateIndicator from 'components/StateIndicator'; 2 | import React, { FC } from 'react'; 3 | import { getWorkflowStage } from 'shared/workflow'; 4 | import { Workflow } from 'typings/workflow'; 5 | 6 | const WorkflowStage: FC<{ workflow: Workflow; tag?: boolean }> = ({ workflow, tag }) => { 7 | return ; 8 | }; 9 | 10 | export default WorkflowStage; 11 | -------------------------------------------------------------------------------- /web_console_v2/client/src/views/Workflows/pubsub.ts: -------------------------------------------------------------------------------- 1 | /* istanbul ignore file */ 2 | 3 | import PubSub from 'pubsub-js'; 4 | 5 | export const workflowPubsub = PubSub; 6 | 7 | const WORKFLOW_CHANNELS = { 8 | create_new_tpl: 'workflow.create_new_tpl', 9 | }; 10 | 11 | export default WORKFLOW_CHANNELS; 12 | -------------------------------------------------------------------------------- /web_console_v2/client/tests/e2e/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/fedlearner/e88f72c1ce3638c6133c51e36239b42ac59dd725/web_console_v2/client/tests/e2e/.gitkeep -------------------------------------------------------------------------------- /web_console_v2/client/tests/setup.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | 7 | /** 8 | * Since i18next doesn't support jest environment 9 | * we did a mock here for unit tests 10 | */ 11 | jest.mock('i18next', () => ({ 12 | use: () => { 13 | return { 14 | init() { 15 | return { t: (k: any) => k, on: () => {} }; 16 | }, 17 | }; 18 | }, 19 | t: (k: any) => k, 20 | })); 21 | -------------------------------------------------------------------------------- /web_console_v2/docker/spark/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==1.1.5 2 | fsspec==2021.4.0 -------------------------------------------------------------------------------- /web_console_v2/run_prod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | service nginx restart 6 | 7 | code-server & 8 | 9 | # Starts API server 10 | cd /app/api 11 | sh run_prod.sh --migrate 12 | --------------------------------------------------------------------------------