├── FATE_plugin ├── arch │ └── api │ │ ├── transfer │ │ └── cluster.py │ │ └── utils │ │ └── file_utils.py ├── examples │ └── federatedml-1.x-examples │ │ ├── cifar │ │ ├── conf.json │ │ ├── dsl.json │ │ ├── upload_guest.json │ │ └── upload_host.json │ │ ├── cifar_batch │ │ ├── conf.json │ │ ├── dsl.json │ │ ├── upload_guest.json │ │ └── upload_host.json │ │ ├── fmnist │ │ ├── conf.json │ │ ├── dsl.json │ │ ├── upload_guest.json │ │ └── upload_host.json │ │ ├── fmnist_batch │ │ ├── conf.json │ │ ├── dsl.json │ │ ├── upload_guest.json │ │ └── upload_host.json │ │ ├── lstm │ │ ├── conf.json │ │ ├── dsl.json │ │ ├── upload_guest.json │ │ └── upload_host.json │ │ ├── lstm_batch │ │ ├── conf.json │ │ ├── dsl.json │ │ ├── upload_guest.json │ │ └── upload_host.json │ │ ├── quick_run_fmnist.py │ │ ├── quick_run_fmnist_batch.py │ │ ├── test_predict_conf.json │ │ ├── upload_data.json │ │ ├── upload_data_guest.json │ │ └── upload_data_host.json ├── federatedml │ ├── __init__.py │ ├── conf │ │ ├── default_runtime_conf │ │ │ ├── data_io_param.json │ │ │ ├── download_param.json │ │ │ ├── evaluation_param.json │ │ │ ├── sample_param.json │ │ │ └── upload_param.json │ │ └── setting_conf │ │ │ ├── DataIO.json │ │ │ ├── Download.json │ │ │ ├── Evaluation.json │ │ │ ├── HeteroLR.json │ │ │ ├── HeteroLinR.json │ │ │ ├── HomoLRCifar.json │ │ │ ├── HomoLRCifarBatch.json │ │ │ ├── HomoLRFmnist.json │ │ │ ├── HomoLRFmnistBatch.json │ │ │ ├── HomoLRLstm.json │ │ │ ├── HomoLRLstmBatch.json │ │ │ ├── HomoLRLstmDummy.json │ │ │ └── Upload.json │ ├── framework │ │ ├── __init__.py │ │ ├── gradients.py │ │ ├── hetero │ │ │ ├── __init__.py │ │ │ ├── procedure │ │ │ │ ├── __init__.py │ │ │ │ ├── batch_generator.py │ │ │ │ ├── convergence.py │ │ │ │ └── paillier_cipher.py │ │ │ ├── sync │ │ │ │ ├── __init__.py │ │ │ │ ├── batch_info_sync.py │ │ │ │ ├── converge_sync.py │ │ │ │ ├── gradient_sync.py │ │ │ │ ├── loss_sync.py │ │ │ │ └── paillier_keygen_sync.py │ │ │ └── util │ │ │ │ └── __init__.py │ │ ├── homo │ │ │ ├── __init__.py │ │ │ ├── model.py │ │ │ ├── procedure │ │ │ │ ├── __init__.py │ │ │ │ ├── aggregator.py │ │ │ │ ├── paillier_cipher.py │ │ │ │ └── random_padding_cipher.py │ │ │ ├── sync │ │ │ │ ├── __init__.py │ │ │ │ ├── dh_keys_exchange_sync.py │ │ │ │ ├── identify_uuid_sync.py │ │ │ │ ├── is_converge_sync.py │ │ │ │ ├── loss_transfer_sync.py │ │ │ │ ├── model_broadcast_sync.py │ │ │ │ ├── model_scatter_sync.py │ │ │ │ ├── paillier_keygen_sync.py │ │ │ │ └── paillier_re_cipher_sync.py │ │ │ └── util │ │ │ │ ├── __init__.py │ │ │ │ └── scatter.py │ │ ├── test │ │ │ ├── __init__.py │ │ │ └── homo │ │ │ │ ├── __init__.py │ │ │ │ ├── aggregator_test.py │ │ │ │ ├── dh_keys_exchange_sync_test.py │ │ │ │ ├── homo_test_sync_base.py │ │ │ │ ├── identify_uuid_sync_test.py │ │ │ │ ├── is_converge_sync_test.py │ │ │ │ ├── loss_transfer_test.py │ │ │ │ ├── model_broadcast_test.py │ │ │ │ ├── model_scatter_test.py │ │ │ │ ├── paillier_cipher_test.py │ │ │ │ ├── paillier_keygen_sync_test.py │ │ │ │ ├── paillier_re_cipher_sync_test.py │ │ │ │ └── random_padding_cipher_test.py │ │ └── weights.py │ ├── linear_model │ │ ├── __init__.py │ │ ├── base_linear_model_arbiter.py │ │ ├── linear_model_base.py │ │ ├── linear_model_weight.py │ │ └── logistic_regression │ │ │ ├── __init__.py │ │ │ ├── base_logistic_regression.py │ │ │ ├── hetero_logistic_regression │ │ │ ├── __init__.py │ │ │ ├── hetero_lr_arbiter.py │ │ │ ├── hetero_lr_base.py │ │ │ ├── hetero_lr_guest.py │ │ │ └── hetero_lr_host.py │ │ │ ├── homo_logsitic_regression │ │ │ ├── __init__.py │ │ │ ├── homo_lr_arbiter.py │ │ │ ├── homo_lr_base.py │ │ │ ├── homo_lr_guest.py │ │ │ └── homo_lr_host.py │ │ │ ├── homo_zcl_fmnist │ │ │ ├── __init__.py │ │ │ ├── fmnist_init.h5 │ │ │ ├── fmnist_init.json │ │ │ ├── homo_lr_arbiter.py │ │ │ ├── homo_lr_base.py │ │ │ ├── homo_lr_guest.py │ │ │ └── homo_lr_host.py │ │ │ ├── homo_zcl_fmnist_batch │ │ │ ├── __init__.py │ │ │ ├── fmnist_init.h5 │ │ │ ├── fmnist_init.json │ │ │ ├── homo_lr_arbiter.py │ │ │ ├── homo_lr_base.py │ │ │ ├── homo_lr_guest.py │ │ │ └── homo_lr_host.py │ │ │ └── images │ │ │ └── __init__.py │ ├── loss │ │ ├── __init__.py │ │ ├── cross_entropy.py │ │ ├── regression_loss.py │ │ └── test │ │ │ ├── __init__.py │ │ │ ├── cross_entropy_test.py │ │ │ └── regression_loss_test.py │ ├── model_base.py │ ├── optim │ │ ├── __init__.py │ │ ├── activation.py │ │ ├── convergence.py │ │ ├── gradient │ │ │ ├── __init__.py │ │ │ ├── hetero_linear_model_gradient.py │ │ │ ├── hetero_linr_gradient_and_loss.py │ │ │ ├── hetero_lr_gradient_and_loss.py │ │ │ ├── hetero_poisson_gradient_and_loss.py │ │ │ ├── homo_lr_gradient.py │ │ │ └── test │ │ │ │ ├── __init__.py │ │ │ │ ├── gradient_method_test.py │ │ │ │ ├── hetero_lr_gradient_test.py │ │ │ │ └── homo_lr_gradient_test.py │ │ ├── initialize.py │ │ ├── optimizer.py │ │ └── test │ │ │ ├── __init__.py │ │ │ ├── activation_test.py │ │ │ ├── convergence_test.py │ │ │ └── initialize_test.py │ ├── param │ │ ├── __init__.py │ │ ├── base_param.py │ │ ├── boosting_tree_param.py │ │ ├── cross_validation_param.py │ │ ├── dataio_param.py │ │ ├── encrypt_param.py │ │ ├── encrypted_mode_calculation_param.py │ │ ├── evaluation_param.py │ │ ├── feature_binning_param.py │ │ ├── feature_selection_param.py │ │ ├── ftl_param.py │ │ ├── homo_nn_param.py │ │ ├── init_model_param.py │ │ ├── intersect_param.py │ │ ├── linear_regression_param.py │ │ ├── logistic_regression_param.py │ │ ├── one_vs_rest_param.py │ │ ├── onehot_encoder_param.py │ │ ├── poisson_regression_param.py │ │ ├── predict_param.py │ │ ├── rsa_param.py │ │ ├── sample_param.py │ │ ├── scale_param.py │ │ ├── secure_add_example_param.py │ │ ├── test │ │ │ ├── __init__.py │ │ │ ├── param.json │ │ │ ├── param_feature_binning.json │ │ │ ├── param_feature_selection.json │ │ │ ├── param_json_test.py │ │ │ └── param_one_hot.json │ │ └── workflow_param.py │ ├── protobuf │ │ ├── __init__.py │ │ └── generated │ │ │ ├── __init__.py │ │ │ ├── boosting_tree_model_meta_pb2.py │ │ │ ├── boosting_tree_model_param_pb2.py │ │ │ ├── data_io_meta_pb2.py │ │ │ ├── data_io_param_pb2.py │ │ │ ├── feature_binning_meta_pb2.py │ │ │ ├── feature_binning_param_pb2.py │ │ │ ├── feature_scale_meta_pb2.py │ │ │ ├── feature_scale_param_pb2.py │ │ │ ├── feature_selection_meta_pb2.py │ │ │ ├── feature_selection_param_pb2.py │ │ │ ├── linr_model_meta_pb2.py │ │ │ ├── linr_model_param_pb2.py │ │ │ ├── lr_model_meta_pb2.py │ │ │ ├── lr_model_param_pb2.py │ │ │ ├── nn_model_meta_pb2.py │ │ │ ├── nn_model_param_pb2.py │ │ │ ├── one_vs_rest_param_pb2.py │ │ │ ├── onehot_meta_pb2.py │ │ │ ├── onehot_param_pb2.py │ │ │ ├── pipeline_pb2.py │ │ │ ├── poisson_model_meta_pb2.py │ │ │ └── poisson_model_param_pb2.py │ ├── secureprotol │ │ ├── __init__.py │ │ ├── aciq.py │ │ ├── affine.py │ │ ├── affine_encoder.py │ │ ├── batch_encryption.py │ │ ├── diffie_hellman.py │ │ ├── encode.py │ │ ├── encrypt.py │ │ ├── encrypt_mode.py │ │ ├── fate_paillier.py │ │ ├── fixedpoint.py │ │ ├── gmpy_math.py │ │ ├── iterative_affine.py │ │ ├── random.py │ │ └── test │ │ │ ├── __init__.py │ │ │ ├── affine_test.py │ │ │ ├── encode_test.py │ │ │ ├── encrypt_mode_test.py │ │ │ ├── fate_paillier_test.py │ │ │ ├── fixedpoin_test.py │ │ │ └── iterative_affine_test.py │ ├── transfer_variable │ │ ├── __init__.py │ │ ├── definition │ │ │ ├── __init__.py │ │ │ ├── cross_validation.json │ │ │ ├── hetero_decision_tree.json │ │ │ ├── hetero_dnn_lr.json │ │ │ ├── hetero_feature_binning.json │ │ │ ├── hetero_feature_selection.json │ │ │ ├── hetero_ftl.json │ │ │ ├── hetero_linr.json │ │ │ ├── hetero_lr.json │ │ │ ├── hetero_poisson.json │ │ │ ├── hetero_secure_boost.json │ │ │ ├── homo.json │ │ │ ├── homo_lr.json │ │ │ ├── homo_zcl_alex.json │ │ │ ├── one_vs_rest.json │ │ │ ├── raw_intersect.json │ │ │ ├── rsa_intersect.json │ │ │ ├── sample.json │ │ │ ├── secure_add_example.json │ │ │ └── transfer_conf.json │ │ ├── transfer_class │ │ │ ├── __init__.py │ │ │ ├── base_transfer_variable.py │ │ │ ├── cross_validation_transfer_variable.py │ │ │ ├── hetero_decision_tree_transfer_variable.py │ │ │ ├── hetero_dnn_lr_transfer_variable.py │ │ │ ├── hetero_feature_binning_transfer_variable.py │ │ │ ├── hetero_feature_selection_transfer_variable.py │ │ │ ├── hetero_ftl_transfer_variable.py │ │ │ ├── hetero_linr_transfer_variable.py │ │ │ ├── hetero_lr_transfer_variable.py │ │ │ ├── hetero_poisson_transfer_variable.py │ │ │ ├── hetero_secure_boost_transfer_variable.py │ │ │ ├── homo_lr_transfer_variable.py │ │ │ ├── homo_transfer_variable.py │ │ │ ├── homo_zcl_alex_transfer_variable.py │ │ │ ├── one_vs_rest_transfer_variable.py │ │ │ ├── raw_intersect_transfer_variable.py │ │ │ ├── rsa_intersect_transfer_variable.py │ │ │ ├── sample_transfer_variable.py │ │ │ └── secure_add_example_transfer_variable.py │ │ ├── transfer_variable_generator.py │ │ └── transfer_variable_template.py │ └── util │ │ ├── __init__.py │ │ ├── abnormal_detection.py │ │ ├── classify_label_checker.py │ │ ├── consts.py │ │ ├── data_io.py │ │ ├── fate_operator.py │ │ ├── param_extract.py │ │ ├── rsa.py │ │ ├── test │ │ ├── __init__.py │ │ ├── classify_label_checker_test.py │ │ ├── data_io_test.py │ │ └── param_extract_test.py │ │ └── validation_strategy.py └── profile │ ├── __init__.py │ ├── mem_stat.py │ └── net_tools.py ├── README.md └── accuracy_eval ├── augmentation.py ├── encryption ├── __init__.py ├── aciq.py ├── encoding.py ├── encryption.py ├── fixedpoint.py ├── gmpy_math.py ├── paillier.py └── util.py ├── regularization.py └── test_10parties ├── federated_lr_alex_experiments.py ├── federated_lr_fmnist_experiments.py ├── federated_lr_lstm_experiments.py └── utils.py /FATE_plugin/arch/api/utils/file_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import json 18 | import os 19 | 20 | from cachetools import LRUCache 21 | from cachetools import cached 22 | 23 | PROJECT_BASE = None 24 | 25 | 26 | def get_project_base_directory(): 27 | global PROJECT_BASE 28 | if PROJECT_BASE is None: 29 | PROJECT_BASE = os.path.abspath( 30 | os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)) 31 | return PROJECT_BASE 32 | 33 | 34 | @cached(cache=LRUCache(maxsize=10)) 35 | def load_json_conf(conf_path): 36 | if os.path.isabs(conf_path): 37 | json_conf_path = conf_path 38 | else: 39 | json_conf_path = os.path.join(get_project_base_directory(), conf_path) 40 | try: 41 | with open(json_conf_path) as f: 42 | return json.load(f) 43 | except: 44 | raise EnvironmentError("loading json file config from '{}' failed!".format(json_conf_path)) 45 | 46 | 47 | def dump_json_conf(config_data, conf_path): 48 | if os.path.isabs(conf_path): 49 | json_conf_path = conf_path 50 | else: 51 | json_conf_path = os.path.join(get_project_base_directory(), conf_path) 52 | try: 53 | with open(json_conf_path, "w") as f: 54 | json.dump(config_data, f, indent=4) 55 | except: 56 | raise EnvironmentError("loading json file config from '{}' failed!".format(json_conf_path)) 57 | 58 | 59 | if __name__ == "__main__": 60 | print(get_project_base_directory()) 61 | print(load_json_conf('federatedml/transfer_variable_conf/transfer_conf.json')) 62 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "initiator": { 3 | "role": "guest", 4 | "party_id": 9999 5 | }, 6 | "job_parameters": { 7 | "work_mode": 1 8 | }, 9 | "role": { 10 | "arbiter": [10000], 11 | "guest": [9999], 12 | "host": [9998, 9997, 9996, 9995, 9994, 9993, 9992, 9991] 13 | }, 14 | "role_parameters": { 15 | "guest": { 16 | "args": { 17 | "data": { 18 | "train_data": [{"name": "homo_breast_guest", "namespace": "homo_breast_guest"}] 19 | } 20 | } 21 | }, 22 | "host": { 23 | "args": { 24 | "data": { 25 | "train_data": [ 26 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 27 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 28 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 29 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 30 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 31 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 32 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 33 | {"name": "homo_breast_host", "namespace": "homo_breast_host"} 34 | ] 35 | } 36 | }, 37 | "evaluation_0": { 38 | "need_run": [ 39 | false, 40 | false, 41 | false, 42 | false, 43 | false, 44 | false, 45 | false, 46 | false 47 | ] 48 | } 49 | } 50 | }, 51 | "algorithm_parameters": { 52 | "dataio_0":{ 53 | "with_label": true, 54 | "label_name": "y", 55 | "label_type": "int", 56 | "output_format": "dense" 57 | }, 58 | "homo_lr_0": { 59 | "penalty": "L2", 60 | "optimizer": "sgd", 61 | "eps": 1e-5, 62 | "alpha": 0.01, 63 | "max_iter": 10, 64 | "converge_func": "diff", 65 | "batch_size": 500, 66 | "learning_rate": 0.15, 67 | "decay": 1, 68 | "decay_sqrt": true, 69 | "init_param": { 70 | "init_method": "zeros" 71 | }, 72 | "encrypt_param": { 73 | "method": "Paillier" 74 | }, 75 | "cv_param": { 76 | "n_splits": 4, 77 | "shuffle": true, 78 | "random_seed": 33, 79 | "need_cv": false 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar/dsl.json: -------------------------------------------------------------------------------- 1 | { 2 | "components" : { 3 | "dataio_0": { 4 | "module": "DataIO", 5 | "input": { 6 | "data": { 7 | "data": [ 8 | "args.train_data" 9 | ] 10 | } 11 | }, 12 | "output": { 13 | "data": ["train"], 14 | "model": ["dataio"] 15 | } 16 | }, 17 | "homo_lr_0": { 18 | "module": "HomoLRCifar", 19 | "input": { 20 | "data": { 21 | "train_data": [ 22 | "dataio_0.train" 23 | ] 24 | } 25 | }, 26 | "output": { 27 | "data": ["train"], 28 | "model": ["homolr"] 29 | } 30 | }, 31 | "evaluation_0": { 32 | "module": "Evaluation", 33 | "input": { 34 | "data": { 35 | "data": [ 36 | "homo_lr_0.train" 37 | ] 38 | } 39 | }, 40 | "output": { 41 | "data": ["evaluate"] 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar_batch/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "initiator": { 3 | "role": "guest", 4 | "party_id": 9999 5 | }, 6 | "job_parameters": { 7 | "work_mode": 1 8 | }, 9 | "role": { 10 | "arbiter": [10000], 11 | "guest": [9999], 12 | "host": [9998, 9997, 9996, 9995, 9994, 9993, 9992, 9991] 13 | }, 14 | "role_parameters": { 15 | "guest": { 16 | "args": { 17 | "data": { 18 | "train_data": [{"name": "homo_breast_guest", "namespace": "homo_breast_guest"}] 19 | } 20 | } 21 | }, 22 | "host": { 23 | "args": { 24 | "data": { 25 | "train_data": [ 26 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 27 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 28 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 29 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 30 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 31 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 32 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 33 | {"name": "homo_breast_host", "namespace": "homo_breast_host"} 34 | ] 35 | } 36 | }, 37 | "evaluation_0": { 38 | "need_run": [ 39 | false, 40 | false, 41 | false, 42 | false, 43 | false, 44 | false, 45 | false, 46 | false 47 | ] 48 | } 49 | } 50 | }, 51 | "algorithm_parameters": { 52 | "dataio_0":{ 53 | "with_label": true, 54 | "label_name": "y", 55 | "label_type": "int", 56 | "output_format": "dense" 57 | }, 58 | "homo_lr_0": { 59 | "penalty": "L2", 60 | "optimizer": "sgd", 61 | "eps": 1e-5, 62 | "alpha": 0.01, 63 | "max_iter": 10, 64 | "converge_func": "diff", 65 | "batch_size": 500, 66 | "learning_rate": 0.15, 67 | "decay": 1, 68 | "decay_sqrt": true, 69 | "init_param": { 70 | "init_method": "zeros" 71 | }, 72 | "encrypt_param": { 73 | "method": "Paillier" 74 | }, 75 | "cv_param": { 76 | "n_splits": 4, 77 | "shuffle": true, 78 | "random_seed": 33, 79 | "need_cv": false 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar_batch/dsl.json: -------------------------------------------------------------------------------- 1 | { 2 | "components" : { 3 | "dataio_0": { 4 | "module": "DataIO", 5 | "input": { 6 | "data": { 7 | "data": [ 8 | "args.train_data" 9 | ] 10 | } 11 | }, 12 | "output": { 13 | "data": ["train"], 14 | "model": ["dataio"] 15 | } 16 | }, 17 | "homo_lr_0": { 18 | "module": "HomoLRCifarBatch", 19 | "input": { 20 | "data": { 21 | "train_data": [ 22 | "dataio_0.train" 23 | ] 24 | } 25 | }, 26 | "output": { 27 | "data": ["train"], 28 | "model": ["homolr"] 29 | } 30 | }, 31 | "evaluation_0": { 32 | "module": "Evaluation", 33 | "input": { 34 | "data": { 35 | "data": [ 36 | "homo_lr_0.train" 37 | ] 38 | } 39 | }, 40 | "output": { 41 | "data": ["evaluate"] 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar_batch/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/cifar_batch/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "initiator": { 3 | "role": "guest", 4 | "party_id": 9999 5 | }, 6 | "job_parameters": { 7 | "work_mode": 1 8 | }, 9 | "role": { 10 | "arbiter": [10000], 11 | "guest": [9999], 12 | "host": [9998, 9997, 9996, 9995, 9994, 9993, 9992, 9991] 13 | }, 14 | "role_parameters": { 15 | "guest": { 16 | "args": { 17 | "data": { 18 | "train_data": [{"name": "homo_breast_guest", "namespace": "homo_breast_guest"}] 19 | } 20 | } 21 | }, 22 | "host": { 23 | "args": { 24 | "data": { 25 | "train_data": [ 26 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 27 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 28 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 29 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 30 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 31 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 32 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 33 | {"name": "homo_breast_host", "namespace": "homo_breast_host"} 34 | ] 35 | } 36 | }, 37 | "evaluation_0": { 38 | "need_run": [ 39 | false, 40 | false, 41 | false, 42 | false, 43 | false, 44 | false, 45 | false, 46 | false 47 | ] 48 | } 49 | } 50 | }, 51 | "algorithm_parameters": { 52 | "dataio_0":{ 53 | "with_label": true, 54 | "label_name": "y", 55 | "label_type": "int", 56 | "output_format": "dense" 57 | }, 58 | "homo_lr_0": { 59 | "penalty": "L2", 60 | "optimizer": "sgd", 61 | "eps": 1e-5, 62 | "alpha": 0.01, 63 | "max_iter": 10, 64 | "converge_func": "diff", 65 | "batch_size": 500, 66 | "learning_rate": 0.15, 67 | "decay": 1, 68 | "decay_sqrt": true, 69 | "init_param": { 70 | "init_method": "zeros" 71 | }, 72 | "encrypt_param": { 73 | "method": "Paillier" 74 | }, 75 | "cv_param": { 76 | "n_splits": 4, 77 | "shuffle": true, 78 | "random_seed": 33, 79 | "need_cv": false 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist/dsl.json: -------------------------------------------------------------------------------- 1 | { 2 | "components" : { 3 | "dataio_0": { 4 | "module": "DataIO", 5 | "input": { 6 | "data": { 7 | "data": [ 8 | "args.train_data" 9 | ] 10 | } 11 | }, 12 | "output": { 13 | "data": ["train"], 14 | "model": ["dataio"] 15 | } 16 | }, 17 | "homo_lr_0": { 18 | "module": "HomoLRFmnist", 19 | "input": { 20 | "data": { 21 | "train_data": [ 22 | "dataio_0.train" 23 | ] 24 | } 25 | }, 26 | "output": { 27 | "data": ["train"], 28 | "model": ["homolr"] 29 | } 30 | }, 31 | "evaluation_0": { 32 | "module": "Evaluation", 33 | "input": { 34 | "data": { 35 | "data": [ 36 | "homo_lr_0.train" 37 | ] 38 | } 39 | }, 40 | "output": { 41 | "data": ["evaluate"] 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist_batch/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "initiator": { 3 | "role": "guest", 4 | "party_id": 9999 5 | }, 6 | "job_parameters": { 7 | "work_mode": 1 8 | }, 9 | "role": { 10 | "arbiter": [10000], 11 | "guest": [9999], 12 | "host": [9998, 9997, 9996, 9995, 9994, 9993, 9992, 9991] 13 | }, 14 | "role_parameters": { 15 | "guest": { 16 | "args": { 17 | "data": { 18 | "train_data": [{"name": "homo_breast_guest", "namespace": "homo_breast_guest"}] 19 | } 20 | } 21 | }, 22 | "host": { 23 | "args": { 24 | "data": { 25 | "train_data": [ 26 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 27 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 28 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 29 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 30 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 31 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 32 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 33 | {"name": "homo_breast_host", "namespace": "homo_breast_host"} 34 | ] 35 | } 36 | }, 37 | "evaluation_0": { 38 | "need_run": [ 39 | false, 40 | false, 41 | false, 42 | false, 43 | false, 44 | false, 45 | false, 46 | false 47 | ] 48 | } 49 | } 50 | }, 51 | "algorithm_parameters": { 52 | "dataio_0":{ 53 | "with_label": true, 54 | "label_name": "y", 55 | "label_type": "int", 56 | "output_format": "dense" 57 | }, 58 | "homo_lr_0": { 59 | "penalty": "L2", 60 | "optimizer": "sgd", 61 | "eps": 1e-5, 62 | "alpha": 0.01, 63 | "max_iter": 10, 64 | "converge_func": "diff", 65 | "batch_size": 500, 66 | "learning_rate": 0.15, 67 | "decay": 1, 68 | "decay_sqrt": true, 69 | "init_param": { 70 | "init_method": "zeros" 71 | }, 72 | "encrypt_param": { 73 | "method": "Paillier" 74 | }, 75 | "cv_param": { 76 | "n_splits": 4, 77 | "shuffle": true, 78 | "random_seed": 33, 79 | "need_cv": false 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist_batch/dsl.json: -------------------------------------------------------------------------------- 1 | { 2 | "components" : { 3 | "dataio_0": { 4 | "module": "DataIO", 5 | "input": { 6 | "data": { 7 | "data": [ 8 | "args.train_data" 9 | ] 10 | } 11 | }, 12 | "output": { 13 | "data": ["train"], 14 | "model": ["dataio"] 15 | } 16 | }, 17 | "homo_lr_0": { 18 | "module": "HomoLRFmnistBatch", 19 | "input": { 20 | "data": { 21 | "train_data": [ 22 | "dataio_0.train" 23 | ] 24 | } 25 | }, 26 | "output": { 27 | "data": ["train"], 28 | "model": ["homolr"] 29 | } 30 | }, 31 | "evaluation_0": { 32 | "module": "Evaluation", 33 | "input": { 34 | "data": { 35 | "data": [ 36 | "homo_lr_0.train" 37 | ] 38 | } 39 | }, 40 | "output": { 41 | "data": ["evaluate"] 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist_batch/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/fmnist_batch/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "initiator": { 3 | "role": "guest", 4 | "party_id": 9999 5 | }, 6 | "job_parameters": { 7 | "work_mode": 1 8 | }, 9 | "role": { 10 | "arbiter": [10000], 11 | "guest": [9999], 12 | "host": [9998, 9997, 9996, 9995, 9994, 9993, 9992, 9991] 13 | }, 14 | "role_parameters": { 15 | "guest": { 16 | "args": { 17 | "data": { 18 | "train_data": [{"name": "homo_breast_guest", "namespace": "homo_breast_guest"}] 19 | } 20 | } 21 | }, 22 | "host": { 23 | "args": { 24 | "data": { 25 | "train_data": [ 26 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 27 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 28 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 29 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 30 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 31 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 32 | {"name": "homo_breast_host", "namespace": "homo_breast_host"}, 33 | {"name": "homo_breast_host", "namespace": "homo_breast_host"} 34 | ] 35 | } 36 | }, 37 | "evaluation_0": { 38 | "need_run": [ 39 | false, 40 | false, 41 | false, 42 | false, 43 | false, 44 | false, 45 | false, 46 | false 47 | ] 48 | } 49 | } 50 | }, 51 | "algorithm_parameters": { 52 | "dataio_0":{ 53 | "with_label": true, 54 | "label_name": "y", 55 | "label_type": "int", 56 | "output_format": "dense" 57 | }, 58 | "homo_lr_0": { 59 | "penalty": "L2", 60 | "optimizer": "sgd", 61 | "eps": 1e-5, 62 | "alpha": 0.01, 63 | "max_iter": 10, 64 | "converge_func": "diff", 65 | "batch_size": 500, 66 | "learning_rate": 0.15, 67 | "decay": 1, 68 | "decay_sqrt": true, 69 | "init_param": { 70 | "init_method": "zeros" 71 | }, 72 | "encrypt_param": { 73 | "method": "Paillier" 74 | }, 75 | "cv_param": { 76 | "n_splits": 4, 77 | "shuffle": true, 78 | "random_seed": 33, 79 | "need_cv": false 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm/dsl.json: -------------------------------------------------------------------------------- 1 | { 2 | "components" : { 3 | "dataio_0": { 4 | "module": "DataIO", 5 | "input": { 6 | "data": { 7 | "data": [ 8 | "args.train_data" 9 | ] 10 | } 11 | }, 12 | "output": { 13 | "data": ["train"], 14 | "model": ["dataio"] 15 | } 16 | }, 17 | "homo_lr_0": { 18 | "module": "HomoLRLstm", 19 | "input": { 20 | "data": { 21 | "train_data": [ 22 | "dataio_0.train" 23 | ] 24 | } 25 | }, 26 | "output": { 27 | "data": ["train"], 28 | "model": ["homolr"] 29 | } 30 | }, 31 | "evaluation_0": { 32 | "module": "Evaluation", 33 | "input": { 34 | "data": { 35 | "data": [ 36 | "homo_lr_0.train" 37 | ] 38 | } 39 | }, 40 | "output": { 41 | "data": ["evaluate"] 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm_batch/dsl.json: -------------------------------------------------------------------------------- 1 | { 2 | "components" : { 3 | "dataio_0": { 4 | "module": "DataIO", 5 | "input": { 6 | "data": { 7 | "data": [ 8 | "args.train_data" 9 | ] 10 | } 11 | }, 12 | "output": { 13 | "data": ["train"], 14 | "model": ["dataio"] 15 | } 16 | }, 17 | "homo_lr_0": { 18 | "module": "HomoLRLstmBatch", 19 | "input": { 20 | "data": { 21 | "train_data": [ 22 | "dataio_0.train" 23 | ] 24 | } 25 | }, 26 | "output": { 27 | "data": ["train"], 28 | "model": ["homolr"] 29 | } 30 | }, 31 | "evaluation_0": { 32 | "module": "Evaluation", 33 | "input": { 34 | "data": { 35 | "data": [ 36 | "homo_lr_0.train" 37 | ] 38 | } 39 | }, 40 | "output": { 41 | "data": ["evaluate"] 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm_batch/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/lstm_batch/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 1, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/test_predict_conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "initiator": { 3 | "role": "guest", 4 | "party_id": 10000 5 | }, 6 | "job_parameters": { 7 | "work_mode": 0, 8 | "job_type": "predict", 9 | "model_id": "arbiter-10000#guest-10000#host-10000#model", 10 | "model_version": "20190810154805024303_1" 11 | }, 12 | "role": { 13 | "guest": [10000], 14 | "host": [10000], 15 | "arbiter": [10000] 16 | }, 17 | "role_parameters": { 18 | "guest": { 19 | "args": { 20 | "data": { 21 | "eval_data": [{"name": "breast_b", "namespace": "fate_flow_test_breast"}] 22 | } 23 | } 24 | }, 25 | "host": { 26 | "args": { 27 | "data": { 28 | "eval_data": [{"name": "breast_a", "namespace": "fate_flow_test_breast"}] 29 | } 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/upload_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_b.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 0, 6 | "table_name": "hetero_breast_b", 7 | "namespace": "hetero_guest_breast" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/upload_data_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_guest.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 0, 6 | "table_name": "homo_breast_guest", 7 | "namespace": "homo_breast_guest" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/examples/federatedml-1.x-examples/upload_data_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_homo_host.csv", 3 | "head": 1, 4 | "partition": 10, 5 | "work_mode": 0, 6 | "table_name": "homo_breast_host", 7 | "namespace": "homo_breast_host" 8 | } 9 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/default_runtime_conf/data_io_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "input_format": "dense", 3 | "missing_fill": false, 4 | "outlier_replace": false, 5 | "with_label": true, 6 | "label_idx": 0, 7 | "label_type": "int", 8 | "output_format": "dense" 9 | } 10 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/default_runtime_conf/download_param.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/default_runtime_conf/evaluation_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "eval_type": "binary", 3 | "pos_label": 1 4 | } 5 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/default_runtime_conf/sample_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "mode": "random", 3 | "method": "downsample", 4 | "fractions": 0.9, 5 | "need_run": true 6 | } 7 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/default_runtime_conf/upload_param.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/DataIO.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/util", 3 | "default_runtime_conf": "data_io_param.json", 4 | "param_class" : "federatedml/param/dataio_param.py/DataIOParam", 5 | "role": 6 | { 7 | "guest|host": 8 | { 9 | "program": "data_io.py/DataIO" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/Download.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "fate_flow/utils", 3 | "default_runtime_conf": "download_param.json", 4 | "param_class" : "fate_flow/param/download_param.py/DownloadParam", 5 | "role": 6 | { 7 | "local": 8 | { 9 | "program": "download.py/Download" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/Evaluation.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/evaluation", 3 | "default_runtime_conf": "evaluation_param.json", 4 | "param_class" : "federatedml/param/evaluation_param.py/EvaluateParam", 5 | "need_deploy": false, 6 | "role": 7 | { 8 | "guest|host": 9 | { 10 | "program": "evaluation.py/Evaluation" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HeteroLR.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/hetero_logistic_regression", 3 | "default_runtime_conf": "logistic_regression_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HeteroLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "hetero_lr_guest.py/HeteroLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "hetero_lr_host.py/HeteroLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "hetero_lr_arbiter.py/HeteroLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HeteroLinR.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/linear_regression/hetero_linear_regression", 3 | "default_runtime_conf": "linear_regression_param.json", 4 | "param_class" : "federatedml/param/linear_regression_param.py/LinearParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "hetero_linr_guest.py/HeteroLinRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "hetero_linr_host.py/HeteroLinRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "hetero_linr_arbiter.py/HeteroLinRArbiter" 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRCifar.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_cifar", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRCifarBatch.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_cifar_batch", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRFmnist.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_fmnist", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRFmnistBatch.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_fmnist_batch", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRLstm.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_lstm", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRLstmBatch.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_lstm_batch", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/HomoLRLstmDummy.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "federatedml/linear_model/logistic_regression/homo_zcl_lstm_dummy", 3 | "default_runtime_conf": "logistic_regression_test_param.json", 4 | "param_class" : "federatedml/param/logistic_regression_param.py/HomoLogisticParam", 5 | "role": 6 | { 7 | "guest": 8 | { 9 | "program": "homo_lr_guest.py/HomoLRGuest" 10 | }, 11 | "host": 12 | { 13 | "program": "homo_lr_host.py/HomoLRHost" 14 | }, 15 | "arbiter": 16 | { 17 | "program": "homo_lr_arbiter.py/HomoLRArbiter" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/conf/setting_conf/Upload.json: -------------------------------------------------------------------------------- 1 | { 2 | "module_path": "fate_flow/utils", 3 | "default_runtime_conf": "upload_param.json", 4 | "param_class" : "fate_flow/param/upload_param.py/UploadParam", 5 | "role": 6 | { 7 | "local": 8 | { 9 | "program": "upload.py/Upload" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/gradients.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import abc 18 | import operator 19 | from arch.api.utils import log_utils 20 | 21 | import numpy as np 22 | 23 | from arch.api.utils.splitable import segment_transfer_enabled 24 | from federatedml.secureprotol.encrypt import Encrypt 25 | LOGGER = log_utils.getLogger() 26 | 27 | FRAGMENT_16M = 0x1000000 28 | FRAGMENT_24M = 0x1100000 29 | 30 | 31 | class TransferableGradients(metaclass=segment_transfer_enabled(max_part_size=FRAGMENT_24M)): 32 | def __init__(self, gradients, cls, *args, **kwargs): 33 | self._gradients = gradients 34 | self._cls = cls 35 | if args: 36 | self._args = args 37 | if kwargs: 38 | self._kwargs = kwargs 39 | 40 | @property 41 | def unboxed(self): 42 | return self._gradients 43 | 44 | @property 45 | def gradients(self): 46 | if not hasattr(self, "_args") and not hasattr(self, "_kwargs"): 47 | return self._cls(self._gradients) 48 | else: 49 | args = self._args if hasattr(self, "_args") else () 50 | kwargs = self._kwargs if hasattr(self, "_kwargs") else {} 51 | return self._cls(self._gradients, *args, **kwargs) 52 | 53 | 54 | class Gradients(object): 55 | 56 | def __init__(self, g): 57 | self._gradients = g 58 | 59 | def for_remote(self): 60 | return TransferableGradients(self._gradients, self.__class__) 61 | 62 | @property 63 | def unboxed(self): 64 | return self._gradients 65 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/procedure/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/procedure/convergence.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from arch.api.utils import log_utils 18 | from federatedml.framework.hetero.sync import converge_sync 19 | 20 | LOGGER = log_utils.getLogger() 21 | 22 | 23 | class Host(converge_sync.Host): 24 | 25 | def register_convergence(self, transfer_variables): 26 | self._register_convergence(is_stopped_transfer=transfer_variables.converge_flag) 27 | 28 | 29 | class Guest(converge_sync.Guest): 30 | 31 | def register_convergence(self, transfer_variables): 32 | self._register_convergence(is_stopped_transfer=transfer_variables.converge_flag) 33 | 34 | 35 | class Arbiter(converge_sync.Arbiter): 36 | 37 | def register_convergence(self, transfer_variables): 38 | self._register_convergence(is_stopped_transfer=transfer_variables.converge_flag) 39 | 40 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/procedure/paillier_cipher.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from arch.api.utils import log_utils 18 | from federatedml.framework.hetero.sync import paillier_keygen_sync 19 | 20 | LOGGER = log_utils.getLogger() 21 | 22 | 23 | class Host(paillier_keygen_sync.Host): 24 | 25 | def register_paillier_cipher(self, transfer_variables): 26 | self._register_paillier_keygen(pubkey_transfer=transfer_variables.paillier_pubkey) 27 | 28 | 29 | class Guest(paillier_keygen_sync.Guest): 30 | 31 | def register_paillier_cipher(self, transfer_variables): 32 | self._register_paillier_keygen(pubkey_transfer=transfer_variables.paillier_pubkey) 33 | 34 | 35 | class Arbiter(paillier_keygen_sync.Arbiter): 36 | 37 | def register_paillier_cipher(self, transfer_variables): 38 | self._register_paillier_keygen(pubkey_transfer=transfer_variables.paillier_pubkey) 39 | 40 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/sync/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/sync/converge_sync.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The FATE Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | 17 | from federatedml.util import consts 18 | 19 | 20 | class Arbiter(object): 21 | # noinspection PyAttributeOutsideInit 22 | def _register_convergence(self, is_stopped_transfer): 23 | self._is_stopped_transfer = is_stopped_transfer 24 | 25 | def sync_converge_info(self, is_converged, suffix=tuple()): 26 | self._is_stopped_transfer.remote(obj=is_converged, role=consts.HOST, idx=-1, suffix=suffix) 27 | self._is_stopped_transfer.remote(obj=is_converged, role=consts.GUEST, idx=-1, suffix=suffix) 28 | 29 | 30 | class _Client(object): 31 | # noinspection PyAttributeOutsideInit 32 | def _register_convergence(self, is_stopped_transfer): 33 | self._is_stopped_transfer = is_stopped_transfer 34 | 35 | def sync_converge_info(self, suffix=tuple()): 36 | is_converged = self._is_stopped_transfer.get(idx=0, suffix=suffix) 37 | return is_converged 38 | 39 | 40 | Host = _Client 41 | Guest = _Client 42 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/sync/gradient_sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | import abc 20 | 21 | 22 | class Guest(object): 23 | @abc.abstractclassmethod 24 | def compute_intermediate(self, *args, **kwargs): 25 | raise NotImplementedError("This method should be be called here") 26 | 27 | @abc.abstractclassmethod 28 | def aggregate_host_result(self, *args, **kwargs): 29 | raise NotImplementedError("This method should be be called here") 30 | 31 | def compute_gradient_procedure(self, *args, **kwargs): 32 | raise NotImplementedError("This method should be be called here") 33 | 34 | 35 | class Host(object): 36 | @abc.abstractclassmethod 37 | def compute_intermediate(self, *args, **kwargs): 38 | raise NotImplementedError("This method should be be called here") 39 | 40 | def compute_gradient_procedure(self, *args, **kwargs): 41 | raise NotImplementedError("This method should be be called here") 42 | 43 | 44 | class Arbiter(object): 45 | def compute_gradient_procedure(self, *args, **kwargs): 46 | raise NotImplementedError("This method should be be called here") 47 | 48 | 49 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/sync/loss_sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from federatedml.util import consts 20 | 21 | 22 | class Arbiter(object): 23 | def _register_loss_sync(self, loss_transfer): 24 | self.loss_transfer = loss_transfer 25 | 26 | def sync_loss_info(self, suffix=tuple()): 27 | loss = self.loss_transfer.get(idx=0, suffix=suffix) 28 | return loss 29 | 30 | 31 | class Guest(object): 32 | def _register_loss_sync(self, host_loss_regular_transfer, loss_transfer, loss_intermediate_transfer): 33 | self.host_loss_regular_transfer = host_loss_regular_transfer 34 | self.loss_transfer = loss_transfer 35 | self.loss_intermediate_transfer = loss_intermediate_transfer 36 | 37 | def sync_loss_info(self, loss, suffix=tuple()): 38 | self.loss_transfer.remote(loss, role=consts.ARBITER, idx=0, suffix=suffix) 39 | 40 | def get_host_loss_intermediate(self, suffix=tuple()): 41 | loss_intermediate = self.loss_intermediate_transfer.get(idx=-1, suffix=suffix) 42 | return loss_intermediate 43 | 44 | def get_host_loss_regular(self, suffix=tuple()): 45 | losses = self.host_loss_regular_transfer.get(idx=-1, suffix=suffix) 46 | return losses 47 | 48 | 49 | class Host(object): 50 | def _register_loss_sync(self, host_loss_regular_transfer, loss_transfer, loss_intermediate_transfer): 51 | self.host_loss_regular_transfer = host_loss_regular_transfer 52 | self.loss_transfer = loss_transfer 53 | self.loss_intermediate_transfer = loss_intermediate_transfer 54 | 55 | def remote_loss_intermediate(self, loss_intermediate, suffix=tuple()): 56 | self.loss_intermediate_transfer.remote(obj=loss_intermediate, role=consts.GUEST, idx=0, suffix=suffix) 57 | 58 | def remote_loss_regular(self, loss_regular, suffix=tuple()): 59 | self.host_loss_regular_transfer.remote(obj=loss_regular, role=consts.GUEST, idx=0, suffix=suffix) 60 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/sync/paillier_keygen_sync.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The FATE Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | 17 | from federatedml.secureprotol.encrypt import PaillierEncrypt 18 | from federatedml.util import consts 19 | 20 | 21 | class Arbiter(object): 22 | # noinspection PyAttributeOutsideInit 23 | def _register_paillier_keygen(self, pubkey_transfer): 24 | self._pubkey_transfer = pubkey_transfer 25 | 26 | def paillier_keygen(self, key_length, suffix=tuple()): 27 | cipher = PaillierEncrypt() 28 | cipher.generate_key(key_length) 29 | pub_key = cipher.get_public_key() 30 | self._pubkey_transfer.remote(obj=pub_key, role=consts.HOST, idx=-1, suffix=suffix) 31 | self._pubkey_transfer.remote(obj=pub_key, role=consts.GUEST, idx=-1, suffix=suffix) 32 | return cipher 33 | 34 | 35 | class _Client(object): 36 | # noinspection PyAttributeOutsideInit 37 | def _register_paillier_keygen(self, pubkey_transfer): 38 | self._pubkey_transfer = pubkey_transfer 39 | 40 | def gen_paillier_cipher_operator(self, suffix=tuple()): 41 | pubkey = self._pubkey_transfer.get(idx=0, suffix=suffix) 42 | cipher = PaillierEncrypt() 43 | cipher.set_public_key(pubkey) 44 | return cipher 45 | 46 | 47 | Host = _Client 48 | Guest = _Client 49 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/hetero/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/procedure/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/procedure/paillier_cipher.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.homo.sync import paillier_keygen_sync 18 | 19 | from arch.api.utils import log_utils 20 | from federatedml.framework.homo.sync import paillier_re_cipher_sync 21 | 22 | LOGGER = log_utils.getLogger() 23 | 24 | 25 | class Host(paillier_keygen_sync.Host, paillier_re_cipher_sync.Host): 26 | 27 | def register_paillier_cipher(self, transfer_variables): 28 | self._register_paillier_keygen(use_encrypt_transfer=transfer_variables.use_encrypt, 29 | pubkey_transfer=transfer_variables.paillier_pubkey) 30 | self._register_paillier_re_cipher(re_encrypt_times_transfer=transfer_variables.re_encrypt_times, 31 | model_to_re_encrypt_transfer=transfer_variables.to_encrypt_model, 32 | model_re_encrypted_transfer=transfer_variables.re_encrypted_model) 33 | 34 | 35 | class Arbiter(paillier_keygen_sync.Arbiter, paillier_re_cipher_sync.Arbiter): 36 | 37 | def register_paillier_cipher(self, transfer_variables): 38 | self._register_paillier_keygen(use_encrypt_transfer=transfer_variables.use_encrypt, 39 | pubkey_transfer=transfer_variables.paillier_pubkey) 40 | self._register_paillier_re_cipher(re_encrypt_times_transfer=transfer_variables.re_encrypt_times, 41 | model_to_re_encrypt_transfer=transfer_variables.to_encrypt_model, 42 | model_re_encrypted_transfer=transfer_variables.re_encrypted_model) 43 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/identify_uuid_sync.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import uuid 18 | 19 | from federatedml.framework.homo.util.scatter import Scatter 20 | from federatedml.util import consts 21 | 22 | 23 | class Arbiter(object): 24 | 25 | # noinspection PyAttributeOutsideInit 26 | def register_identify_uuid(self, guest_uuid_trv, host_uuid_trv, conflict_flag_trv): 27 | self._conflict_flag_trv = conflict_flag_trv 28 | self._scatter = Scatter(host_uuid_trv, guest_uuid_trv) 29 | return self 30 | 31 | def validate_uuid(self): 32 | ind = 0 33 | while True: 34 | uuid_set = set() 35 | for uid in self._scatter.get(suffix=ind): 36 | if uid in uuid_set: 37 | self._conflict_flag_trv.remote(obj=False, role=None, idx=-1, suffix=ind) 38 | ind += 1 39 | break 40 | uuid_set.add(uid) 41 | else: 42 | self._conflict_flag_trv.remote(obj=True, role=None, idx=-1, suffix=ind) 43 | break 44 | return uuid_set 45 | 46 | 47 | class Client(object): 48 | 49 | # noinspection PyAttributeOutsideInit 50 | def register_identify_uuid(self, uuid_transfer_variable, conflict_flag_transfer_variable): 51 | self._conflict_flag_transfer_variable = conflict_flag_transfer_variable 52 | self._uuid_transfer_variable = uuid_transfer_variable 53 | return self 54 | 55 | def generate_uuid(self): 56 | ind = -1 57 | while True: 58 | ind = ind + 1 59 | _uid = uuid.uuid1() 60 | self._uuid_transfer_variable.remote(obj=_uid, role=consts.ARBITER, idx=0, suffix=ind) 61 | if self._conflict_flag_transfer_variable.get(idx=0, suffix=ind): 62 | break 63 | return _uid 64 | 65 | 66 | Host = Client 67 | Guest = Client 68 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/is_converge_sync.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import Variable 18 | import types 19 | 20 | 21 | class Arbiter(object): 22 | 23 | # noinspection PyAttributeOutsideInit 24 | def register_is_converge(self, is_converge_variable: Variable): 25 | self._is_converge_variable = is_converge_variable 26 | return self 27 | 28 | def check_converge_status(self, converge_func: types.FunctionType, converge_args, suffix=tuple()): 29 | is_converge = converge_func(*converge_args) 30 | self._is_converge_variable.remote(is_converge, role=None, idx=-1, suffix=suffix) 31 | return is_converge 32 | 33 | 34 | class _Client(object): 35 | 36 | # noinspection PyAttributeOutsideInit 37 | def register_is_converge(self, is_converge_variable: Variable): 38 | self._is_converge_variable = is_converge_variable 39 | return self 40 | 41 | def get_converge_status(self, suffix=tuple()): 42 | return self._is_converge_variable.get(idx=0, suffix=suffix) 43 | 44 | 45 | Guest = _Client 46 | Host = _Client 47 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/loss_transfer_sync.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.homo.util import scatter 18 | from federatedml.util import consts 19 | 20 | 21 | class Arbiter(object): 22 | 23 | # noinspection PyAttributeOutsideInit 24 | def register_loss_transfer(self, host_loss_transfer, guest_loss_transfer): 25 | self._loss_sync = scatter.Scatter(host_loss_transfer, guest_loss_transfer) 26 | return self 27 | 28 | def get_losses(self, idx=None, suffix=tuple()): 29 | return self._loss_sync.get(host_ids=idx, suffix=suffix) 30 | 31 | 32 | class _Client(object): 33 | 34 | # noinspection PyAttributeOutsideInit 35 | def register_loss_transfer(self, loss_transfer): 36 | self._loss_sync = loss_transfer 37 | return self 38 | 39 | def send_loss(self, loss, suffix=tuple()): 40 | self._loss_sync.remote(obj=loss, role=consts.ARBITER, idx=0, suffix=suffix) 41 | return loss 42 | 43 | 44 | Guest = _Client 45 | Host = _Client 46 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/model_broadcast_sync.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.weights import Weights 18 | from federatedml.util import consts 19 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import Variable 20 | 21 | 22 | class Arbiter(object): 23 | 24 | # noinspection PyAttributeOutsideInit 25 | def register_model_broadcaster(self, model_transfer: Variable): 26 | self._models_broadcast = model_transfer 27 | return self 28 | 29 | def send_model(self, model: Weights, ciphers_dict=None, suffix=tuple()): 30 | if ciphers_dict: 31 | self._models_broadcast.remote(obj=model.for_remote(), 32 | role=consts.GUEST, 33 | idx=0, 34 | suffix=suffix) 35 | for i, cipher in ciphers_dict.items(): 36 | if cipher: 37 | self._models_broadcast.remote(obj=model.encrypted(cipher, inplace=False).for_remote(), 38 | role=consts.HOST, 39 | idx=i, 40 | suffix=suffix) 41 | else: 42 | self._models_broadcast.remote(obj=model.for_remote(), 43 | role=consts.HOST, 44 | idx=i, 45 | suffix=suffix) 46 | else: 47 | self._models_broadcast.remote(obj=model.for_remote(), 48 | role=None, 49 | idx=-1, 50 | suffix=suffix) 51 | 52 | 53 | class Client(object): 54 | # noinspection PyAttributeOutsideInit 55 | def register_model_broadcaster(self, model_transfer: Variable): 56 | self._models_broadcast = model_transfer 57 | return self 58 | 59 | def get_model(self, suffix=tuple()): 60 | return self._models_broadcast.get(idx=0, suffix=suffix) 61 | 62 | 63 | Guest = Client 64 | Host = Client 65 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/model_scatter_sync.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.weights import TransferableWeights 18 | from federatedml.framework.homo.util import scatter 19 | from federatedml.util import consts 20 | 21 | 22 | class Arbiter(object): 23 | 24 | # noinspection PyAttributeOutsideInit 25 | def register_model_scatter(self, host_model_transfer, guest_model_transfer): 26 | self._models_sync = scatter.Scatter(host_model_transfer, guest_model_transfer) 27 | return self 28 | 29 | def get_models(self, ciphers_dict=None, suffix=tuple()): 30 | 31 | # guest model 32 | models_iter = self._models_sync.get(suffix=suffix) 33 | guest_model = next(models_iter) 34 | yield (guest_model.weights, guest_model.get_degree() or 1.0) 35 | 36 | # host model 37 | index = 0 38 | for model in models_iter: 39 | weights = model.weights 40 | if ciphers_dict and ciphers_dict.get(index, None): 41 | weights = weights.decrypted(ciphers_dict[index]) 42 | yield (weights, model.get_degree() or 1.0) 43 | index += 1 44 | 45 | 46 | class _Client(object): 47 | # noinspection PyAttributeOutsideInit 48 | def register_model_scatter(self, model_transfer): 49 | self._models_sync = model_transfer 50 | return self 51 | 52 | def send_model(self, weights: TransferableWeights, suffix=tuple()): 53 | self._models_sync.remote(obj=weights, role=consts.ARBITER, idx=0, suffix=suffix) 54 | return weights 55 | 56 | 57 | Guest = _Client 58 | Host = _Client 59 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/sync/paillier_keygen_sync.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | from federatedml.secureprotol.encrypt import PaillierEncrypt 19 | from federatedml.secureprotol.fate_paillier import PaillierPublicKey 20 | from federatedml.util import consts 21 | 22 | 23 | class Arbiter(object): 24 | 25 | # noinspection PyAttributeOutsideInit 26 | def _register_paillier_keygen(self, use_encrypt_transfer, pubkey_transfer): 27 | self._use_encrypt_transfer = use_encrypt_transfer 28 | self._pubkey_transfer = pubkey_transfer 29 | return self 30 | 31 | def paillier_keygen(self, key_length, suffix=tuple()) -> dict: 32 | hosts_use_cipher = self._use_encrypt_transfer.get(suffix=suffix) 33 | host_ciphers = dict() 34 | for idx, use_encryption in enumerate(hosts_use_cipher): 35 | if not use_encryption: 36 | host_ciphers[idx] = None 37 | else: 38 | cipher = PaillierEncrypt() 39 | cipher.generate_key(key_length) 40 | pub_key = cipher.get_public_key() 41 | self._pubkey_transfer.remote(obj=pub_key, role=consts.HOST, idx=idx, suffix=suffix) 42 | host_ciphers[idx] = cipher 43 | return host_ciphers 44 | 45 | 46 | class Host(object): 47 | 48 | # noinspection PyAttributeOutsideInit 49 | def _register_paillier_keygen(self, use_encrypt_transfer, pubkey_transfer): 50 | self._use_encrypt_transfer = use_encrypt_transfer 51 | self._pubkey_transfer = pubkey_transfer 52 | return self 53 | 54 | def gen_paillier_pubkey(self, enable, suffix=tuple()) -> PaillierPublicKey: 55 | self._use_encrypt_transfer.remote(obj=enable, role=consts.ARBITER, idx=0, suffix=suffix) 56 | if enable: 57 | return self._pubkey_transfer.get(idx=0, suffix=suffix) 58 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoszh/BatchCrypt/e1067faeae54e8472ac8d7a987456b0e72b08b7a/FATE_plugin/federatedml/framework/homo/util/__init__.py -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/homo/util/scatter.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | class Scatter(object): 19 | 20 | def __init__(self, host_variable, guest_variable): 21 | """ 22 | scatter values from guest and hosts 23 | 24 | Args: 25 | host_variable: a variable represents `Host -> Arbiter` 26 | guest_variable: a variable represent `Guest -> Arbiter` 27 | 28 | Examples: 29 | 30 | >>> from federatedml.framework.homo.util import scatter 31 | >>> s = scatter.Scatter(host_variable, guest_variable) 32 | >>> for v in s.get(): 33 | print(v) 34 | 35 | 36 | """ 37 | self._host_variable = host_variable 38 | self._guest_variable = guest_variable 39 | 40 | def get(self, suffix=tuple(), host_ids=None): 41 | """ 42 | create a generator of values from guest and hosts. 43 | 44 | Args: 45 | suffix: tag suffix 46 | host_ids: ids of hosts to get value from. 47 | If None provided, get values from all hosts. 48 | If a list of int provided, get values from all hosts listed. 49 | 50 | Returns: 51 | a generator of scatted values 52 | 53 | Raises: 54 | if host_ids is neither None nor a list of int, ValueError raised 55 | """ 56 | yield self._guest_variable.get(idx=0, suffix=suffix) 57 | if host_ids is None: 58 | host_ids = -1 59 | for ret in self._host_variable.get(idx=host_ids, suffix=suffix): 60 | yield ret 61 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/aggregator_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.homo.procedure import aggregator 18 | from federatedml.framework.weights import OrderDictWeights 19 | from federatedml.util import consts 20 | from .homo_test_sync_base import TestSyncBase 21 | import numpy as np 22 | import random 23 | import copy 24 | 25 | 26 | class AggregatorTest(TestSyncBase): 27 | @classmethod 28 | def call(cls, role, transfer_variable, ind, *args): 29 | agg = aggregator.with_role(role, transfer_variable, enable_secure_aggregate=True) 30 | if role == consts.ARBITER: 31 | agg.aggregate_and_broadcast() 32 | print(agg.aggregate_loss()) 33 | else: 34 | # disorder dit 35 | order = list(range(5)) 36 | np.random.seed(random.SystemRandom().randint(1, 100)) 37 | np.random.shuffle(order) 38 | raw = {k: np.random.rand(10, 10) for k in order} 39 | 40 | w = OrderDictWeights(copy.deepcopy(raw)) 41 | d = random.random() 42 | aggregated = agg.aggregate_then_get(w, degree=d) 43 | 44 | agg.send_loss(2.0) 45 | return aggregated, raw, d 46 | 47 | def run_with_num_hosts(self, num_hosts): 48 | _, guest, *hosts = self.run_results(num_hosts) 49 | expert = OrderDictWeights(guest[1]) * guest[2] 50 | total_weights = guest[2] 51 | aggregated = [guest[0]] 52 | for host in hosts: 53 | expert += OrderDictWeights(host[1]) * host[2] 54 | total_weights += host[2] 55 | aggregated.append(host[0]) 56 | expert /= total_weights 57 | expert = expert.unboxed 58 | aggregated = [w.unboxed for w in aggregated] 59 | 60 | for k in expert: 61 | for w in aggregated: 62 | self.assertAlmostEqual(np.linalg.norm(expert[k] - w[k]), 0.0) 63 | 64 | def test_host_1(self): 65 | self.run_with_num_hosts(1) 66 | 67 | def test_host_10(self): 68 | self.run_with_num_hosts(10) 69 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/identify_uuid_sync_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from federatedml.framework.homo.sync import identify_uuid_sync 17 | from federatedml.util import consts 18 | from .homo_test_sync_base import TestSyncBase 19 | 20 | 21 | class IdentifyUUIDTest(TestSyncBase): 22 | 23 | @classmethod 24 | def call(cls, role, transfer_variable, ind, *args): 25 | if role == consts.ARBITER: 26 | sync = identify_uuid_sync.Arbiter() 27 | sync.register_identify_uuid(transfer_variable.guest_uuid, 28 | transfer_variable.host_uuid, 29 | transfer_variable.uuid_conflict_flag) 30 | return sync.validate_uuid() 31 | elif role == consts.HOST: 32 | sync = identify_uuid_sync.Host() 33 | return sync.register_identify_uuid(transfer_variable.host_uuid, 34 | conflict_flag_transfer_variable=transfer_variable.uuid_conflict_flag) \ 35 | .generate_uuid() 36 | else: 37 | sync = identify_uuid_sync.Guest() 38 | sync.register_identify_uuid(transfer_variable.guest_uuid, 39 | conflict_flag_transfer_variable=transfer_variable.uuid_conflict_flag) 40 | return sync.generate_uuid() 41 | 42 | def run_with_num_hosts(self, num_hosts): 43 | results = self.run_results(num_hosts=num_hosts) 44 | arbiter = results[0] 45 | guest = results[1] 46 | hosts = results[2:] 47 | 48 | self.assertEqual(len(results), num_hosts + 2) 49 | self.assertIn(guest, arbiter) 50 | for host in hosts: 51 | self.assertIn(host, arbiter) 52 | 53 | def test_host_1(self): 54 | self.run_with_num_hosts(1) 55 | 56 | def test_host_10(self): 57 | self.run_with_num_hosts(10) 58 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/is_converge_sync_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.homo.sync import is_converge_sync 18 | from federatedml.util import consts 19 | from .homo_test_sync_base import TestSyncBase 20 | 21 | 22 | class IsConvergeTest(TestSyncBase): 23 | 24 | @classmethod 25 | def call(cls, role, transfer_variable, ind, *args): 26 | epsilon = 0.001 27 | funcs = [lambda x: abs(x) < epsilon, lambda x, y: abs(x - y) < epsilon] 28 | args = [ 29 | [(0.00005,), (3.14159, 3.1415926)], 30 | [(0.0015,), (2.71, 3.14)] 31 | ] 32 | if role == consts.ARBITER: 33 | status = [] 34 | is_converge = is_converge_sync.Arbiter().register_is_converge(transfer_variable.is_converge) 35 | for i in range(2): 36 | for j in range(2): 37 | status.append(is_converge.check_converge_status(funcs[i], args[j][i], suffix=(i, j))) 38 | return status 39 | elif role == consts.HOST: 40 | status = [] 41 | is_converge = is_converge_sync.Host().register_is_converge(transfer_variable.is_converge) 42 | for i in range(2): 43 | for j in range(2): 44 | status.append(is_converge.get_converge_status(suffix=(i, j))) 45 | return status 46 | else: 47 | status = [] 48 | is_converge = is_converge_sync.Guest().register_is_converge(transfer_variable.is_converge) 49 | for i in range(2): 50 | for j in range(2): 51 | status.append(is_converge.get_converge_status(suffix=(i, j))) 52 | return status 53 | 54 | def run_with_num_hosts(self, num_hosts): 55 | arbiter, guest, *hosts = self.run_results(num_hosts=num_hosts) 56 | self.assertListEqual(arbiter, guest) 57 | for state in hosts: 58 | self.assertLessEqual(arbiter, state) 59 | 60 | def test_host_1(self): 61 | self.run_with_num_hosts(1) 62 | 63 | def test_host_10(self): 64 | self.run_with_num_hosts(10) 65 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/loss_transfer_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import random 18 | 19 | from federatedml.framework.homo.sync import loss_transfer_sync 20 | from federatedml.util import consts 21 | from .homo_test_sync_base import TestSyncBase 22 | 23 | 24 | class LossTransferTest(TestSyncBase): 25 | 26 | @classmethod 27 | def call(cls, role, transfer_variable, ind, *args): 28 | host_list = args[0] 29 | 30 | if role == consts.ARBITER: 31 | return list(loss_transfer_sync.Arbiter() 32 | .register_loss_transfer(transfer_variable.host_loss, 33 | transfer_variable.guest_loss) 34 | .get_losses(host_list)) 35 | elif role == consts.HOST: 36 | import random 37 | has_loss = ind in host_list 38 | loss = random.random() 39 | if has_loss: 40 | loss_transfer_sync.Host() \ 41 | .register_loss_transfer(transfer_variable.host_loss) \ 42 | .send_loss(loss) 43 | return has_loss, loss 44 | else: 45 | import random 46 | return loss_transfer_sync.Guest() \ 47 | .register_loss_transfer(transfer_variable.guest_loss) \ 48 | .send_loss(random.random()) 49 | 50 | def run_with_num_hosts(self, num_hosts): 51 | ratio = 0.3 52 | host_list = [i for i in range(num_hosts) if random.random() > ratio] 53 | arbiter, guest, *hosts = self.run_results(num_hosts, host_list) 54 | self.assertEqual(guest, arbiter[0]) 55 | for i in range(len(host_list)): 56 | host_id = host_list[i] 57 | self.assertTrue(hosts[host_id][0]) 58 | self.assertEqual(arbiter[i+1], hosts[host_id][1]) 59 | 60 | def test_host_1(self): 61 | self.run_with_num_hosts(1) 62 | 63 | def test_host_10(self): 64 | self.run_with_num_hosts(10) 65 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/paillier_keygen_sync_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.homo.sync import paillier_keygen_sync 18 | from federatedml.util import consts 19 | from .homo_test_sync_base import TestSyncBase 20 | 21 | 22 | class PaillierKeyGenTest(TestSyncBase): 23 | 24 | @classmethod 25 | def call(cls, role, transfer_variable, ind, *args): 26 | if role == consts.ARBITER: 27 | return paillier_keygen_sync.Arbiter() \ 28 | ._register_paillier_keygen(transfer_variable.use_encrypt, 29 | transfer_variable.paillier_pubkey) \ 30 | .paillier_keygen(1024) 31 | elif role == consts.HOST: 32 | import random 33 | enable = random.random() > 0.3 34 | return paillier_keygen_sync.Host() \ 35 | ._register_paillier_keygen(transfer_variable.use_encrypt, 36 | transfer_variable.paillier_pubkey)\ 37 | .gen_paillier_pubkey(enable=enable) 38 | else: 39 | pass 40 | 41 | def run_with_num_hosts(self, num_hosts): 42 | arbiter, guest, *hosts = self.run_results(num_hosts=num_hosts) 43 | enabled = [] 44 | for i in range(len(hosts)): 45 | if hosts[i]: 46 | enabled.append(i) 47 | self.assertEqual(len([x for x in arbiter.values() if x]), len(enabled)) 48 | for i in enabled: 49 | self.assertEqual(arbiter[i].get_public_key(), hosts[i]) 50 | 51 | def test_host_1(self): 52 | self.run_with_num_hosts(1) 53 | 54 | def test_host_10(self): 55 | self.run_with_num_hosts(10) 56 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/framework/test/homo/random_padding_cipher_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.framework.homo.procedure import random_padding_cipher 18 | from federatedml.util import consts 19 | from .homo_test_sync_base import TestSyncBase 20 | 21 | 22 | class RandomPaddingCipherTest(TestSyncBase): 23 | @classmethod 24 | def call(cls, role, transfer_variable, ind, *args): 25 | if role == consts.ARBITER: 26 | rp_cipher = random_padding_cipher.Arbiter() 27 | rp_cipher.register_random_padding_cipher(transfer_variable) 28 | rp_cipher.exchange_secret_keys() 29 | return 30 | 31 | elif role == consts.HOST: 32 | rp_cipher = random_padding_cipher.Host() 33 | rp_cipher.register_random_padding_cipher(transfer_variable) 34 | rp_cipher.create_cipher() 35 | return rp_cipher 36 | else: 37 | rp_cipher = random_padding_cipher.Guest() 38 | rp_cipher.register_random_padding_cipher(transfer_variable) 39 | rp_cipher.create_cipher() 40 | return rp_cipher 41 | 42 | def run_with_num_hosts(self, num_hosts): 43 | 44 | arbiter, guest, *hosts = self.run_results(num_hosts) 45 | import numpy as np 46 | raw = np.zeros((10, 10)) 47 | encrypted = np.zeros((10, 10)) 48 | 49 | guest_matrix = np.random.rand(10, 10) 50 | raw += guest_matrix 51 | encrypted += guest.encrypt(guest_matrix) 52 | 53 | for host in hosts: 54 | host_matrix = np.random.rand(10, 10) 55 | raw += host_matrix 56 | encrypted += host.encrypt(host_matrix) 57 | 58 | self.assertTrue(np.allclose(raw, encrypted)) 59 | 60 | def test_host_1(self): 61 | self.run_with_num_hosts(1) 62 | 63 | def test_host_10(self): 64 | self.run_with_num_hosts(10) 65 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/linear_model_weight.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import numpy as np 21 | 22 | from federatedml.framework.weights import ListWeights, TransferableWeights 23 | 24 | 25 | class LinearModelWeights(ListWeights): 26 | def __init__(self, l, fit_intercept): 27 | super().__init__(l) 28 | self.fit_intercept = fit_intercept 29 | 30 | def for_remote(self): 31 | return TransferableWeights(self._weights, self.__class__, self.fit_intercept) 32 | 33 | @property 34 | def coef_(self): 35 | if self.fit_intercept: 36 | return np.array(self._weights[:-1]) 37 | return np.array(self._weights) 38 | 39 | @property 40 | def intercept_(self): 41 | if self.fit_intercept: 42 | return self._weights[-1] 43 | return 0.0 44 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # from federatedml.logistic_regression import homo_logsitic_regression, hetero_logistic_regression 17 | # from federatedml.logistic_regression.logistic_regression import LogisticRegression 18 | # 19 | # __all__ = ['homo_logsitic_regression', 'hetero_logistic_regression', 'LogisticRegression'] -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/hetero_logistic_regression/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | __all__ = ["HeteroLRHost", "HeteroLRGuest", "HeteroLRArbiter"] -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_arbiter.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from arch.api.utils import log_utils 18 | from federatedml.linear_model.base_linear_model_arbiter import HeteroBaseArbiter 19 | from federatedml.linear_model.logistic_regression.hetero_logistic_regression.hetero_lr_base import HeteroLRBase 20 | from federatedml.optim.gradient import hetero_lr_gradient_and_loss 21 | from federatedml.param.logistic_regression_param import HeteroLogisticParam 22 | from federatedml.util import consts 23 | 24 | 25 | LOGGER = log_utils.getLogger() 26 | 27 | 28 | class HeteroLRArbiter(HeteroBaseArbiter, HeteroLRBase): 29 | def __init__(self): 30 | super(HeteroLRArbiter, self).__init__() 31 | self.gradient_loss_operator = hetero_lr_gradient_and_loss.Arbiter() 32 | self.model_param = HeteroLogisticParam() 33 | self.n_iter_ = 0 34 | self.header = None 35 | self.is_converged = False 36 | self.model_param_name = 'HeteroLogisticRegressionParam' 37 | self.model_meta_name = 'HeteroLogisticRegressionMeta' 38 | self.need_one_vs_rest = None 39 | self.in_one_vs_rest = False 40 | self.mode = consts.HETERO 41 | 42 | def fit(self, data_instances=None, validate_data=None): 43 | LOGGER.debug("Need one_vs_rest: {}".format(self.need_one_vs_rest)) 44 | classes = self.one_vs_rest_obj.get_data_classes(data_instances) 45 | if len(classes) > 2: 46 | self.need_one_vs_rest = True 47 | self.in_one_vs_rest = True 48 | self.one_vs_rest_fit(train_data=data_instances, validate_data=validate_data) 49 | else: 50 | self.need_one_vs_rest = False 51 | super().fit(data_instances, validate_data) 52 | 53 | def fit_binary(self, data_instances, validate_data): 54 | super().fit(data_instances, validate_data) 55 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_logsitic_regression/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # 18 | __all__ = ["HomoLRHost", 'HomoLRGuest', 'HomoLRArbiter'] -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # 18 | __all__ = ["HomoLRHost", 'HomoLRGuest', 'HomoLRArbiter'] -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist/fmnist_init.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoszh/BatchCrypt/e1067faeae54e8472ac8d7a987456b0e72b08b7a/FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist/fmnist_init.h5 -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist/fmnist_init.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "Flatten", "config": {"name": "flatten", "trainable": true, "batch_input_shape": [null, 28, 28], "dtype": "float32", "data_format": "channels_last"}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 128, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null, "dtype": "float32"}}, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 10, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null, "dtype": "float32"}}, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4-tf", "backend": "tensorflow"} -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist_batch/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # 18 | __all__ = ["HomoLRHost", 'HomoLRGuest', 'HomoLRArbiter'] -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist_batch/fmnist_init.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoszh/BatchCrypt/e1067faeae54e8472ac8d7a987456b0e72b08b7a/FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist_batch/fmnist_init.h5 -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/homo_zcl_fmnist_batch/fmnist_init.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "Flatten", "config": {"name": "flatten", "trainable": true, "batch_input_shape": [null, 28, 28], "dtype": "float32", "data_format": "channels_last"}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 128, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null, "dtype": "float32"}}, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "units": 10, "activation": "softmax", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null, "dtype": "float32"}}, "bias_initializer": {"class_name": "Zeros", "config": {"dtype": "float32"}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "keras_version": "2.2.4-tf", "backend": "tensorflow"} -------------------------------------------------------------------------------- /FATE_plugin/federatedml/linear_model/logistic_regression/images/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoszh/BatchCrypt/e1067faeae54e8472ac8d7a987456b0e72b08b7a/FATE_plugin/federatedml/linear_model/logistic_regression/images/__init__.py -------------------------------------------------------------------------------- /FATE_plugin/federatedml/loss/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.loss.cross_entropy import SigmoidBinaryCrossEntropyLoss 18 | from federatedml.loss.cross_entropy import SoftmaxCrossEntropyLoss 19 | from federatedml.loss.regression_loss import LeastSquaredErrorLoss 20 | from federatedml.loss.regression_loss import LeastAbsoluteErrorLoss 21 | from federatedml.loss.regression_loss import HuberLoss 22 | from federatedml.loss.regression_loss import FairLoss 23 | from federatedml.loss.regression_loss import LogCoshLoss 24 | from federatedml.loss.regression_loss import TweedieLoss 25 | 26 | __all__ = ["SigmoidBinaryCrossEntropyLoss", 27 | "SoftmaxCrossEntropyLoss", 28 | "LeastSquaredEroorLoss", 29 | "LeastAbsoluteErrorLoss", 30 | "HuberLoss", 31 | "FairLoss", 32 | "LogCoshLoss", 33 | "TweedieLoss"] 34 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/loss/test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/activation.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import numpy as np 18 | from arch.api.utils import log_utils 19 | 20 | LOGGER = log_utils.getLogger() 21 | 22 | 23 | def hard_sigmoid(x): 24 | y = 0.2 * x + 0.5 25 | return np.clip(y, 0, 1) 26 | 27 | 28 | def softmax(x, axis=-1): 29 | y = np.exp(x - np.max(x, axis, keepdims=True)) 30 | return y / np.sum(y, axis, keepdims=True) 31 | 32 | 33 | def sigmoid(x): 34 | if x <= 0: 35 | a = np.exp(x) 36 | a /= (1. + a) 37 | else: 38 | a = 1. / (1. + np.exp(-x)) 39 | return a 40 | 41 | 42 | def softplus(x): 43 | return np.log(1. + np.exp(x)) 44 | 45 | 46 | def softsign(x): 47 | return x / (1 + np.abs(x)) 48 | 49 | 50 | def tanh(x): 51 | return np.tanh(x) 52 | 53 | 54 | def log_logistic(x): 55 | if x <= 0: 56 | a = x - np.log(1 + np.exp(x)) 57 | else: 58 | a = - np.log(1 + np.exp(-x)) 59 | return a 60 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/gradient/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/gradient/homo_lr_gradient.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import numpy as np 18 | 19 | from arch.api.utils import log_utils 20 | from federatedml.util import fate_operator 21 | 22 | LOGGER = log_utils.getLogger() 23 | 24 | 25 | def load_data(data_instance): 26 | X = [] 27 | Y = [] 28 | for iter_key, instant in data_instance: 29 | weighted_feature = instant.weight * instant.features 30 | X.append(weighted_feature) 31 | if instant.label == 1: 32 | Y.append([1]) 33 | else: 34 | Y.append([-1]) 35 | X = np.array(X) 36 | Y = np.array(Y) 37 | return X, Y 38 | 39 | 40 | class LogisticGradient(object): 41 | 42 | @staticmethod 43 | def compute_loss(values, coef, intercept): 44 | X, Y = load_data(values) 45 | tot_loss = np.log(1 + np.exp(np.multiply(-Y.transpose(), X.dot(coef) + intercept))).sum() 46 | return tot_loss 47 | 48 | @staticmethod 49 | def compute_gradient(values, coef, intercept, fit_intercept): 50 | X, Y = load_data(values) 51 | batch_size = len(X) 52 | if batch_size == 0: 53 | LOGGER.warning("This partition got 0 data") 54 | return None 55 | 56 | d = (1.0 / (1 + np.exp(-np.multiply(Y.transpose(), X.dot(coef) + intercept))) - 1).transpose() * Y 57 | grad_batch = d * X 58 | if fit_intercept: 59 | grad_batch = np.c_[grad_batch, d] 60 | grad = sum(grad_batch) 61 | return grad 62 | 63 | 64 | class TaylorLogisticGradient(object): 65 | 66 | @staticmethod 67 | def compute_gradient(values, coef, intercept, fit_intercept): 68 | LOGGER.debug("Get in compute_gradient") 69 | X, Y = load_data(values) 70 | batch_size = len(X) 71 | if batch_size == 0: 72 | return None 73 | 74 | one_d_y = Y.reshape([-1, ]) 75 | d = (0.25 * np.array(fate_operator.dot(X, coef) + intercept).transpose() + 0.5 * one_d_y * -1) 76 | 77 | grad_batch = X.transpose() * d 78 | grad_batch = grad_batch.transpose() 79 | if fit_intercept: 80 | grad_batch = np.c_[grad_batch, d] 81 | grad = sum(grad_batch) 82 | LOGGER.debug("Finish compute_gradient") 83 | return grad 84 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/gradient/test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.optim.gradient.logistic_gradient import LogisticGradient, HeteroLogisticGradient, TaylorLogisticGradient 18 | 19 | __all__ = ["LogisticGradient", "HeteroLogisticGradient", "TaylorLogisticGradient"] -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/test/activation_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | import math 20 | import unittest 21 | import numpy as np 22 | from federatedml.optim import activation 23 | 24 | 25 | class TestConvergeFunction(unittest.TestCase): 26 | def test_numeric_stability(self): 27 | x_list = np.linspace(-709, 709, 10000) 28 | 29 | # Original function 30 | # a = 1. / (1. + np.exp(-x)) 31 | for x in x_list: 32 | a1 = 1. / (1. + np.exp(-x)) 33 | a2 = activation.sigmoid(x) 34 | self.assertTrue(np.abs(a1 - a2) < 1e-5) 35 | 36 | 37 | if __name__ == '__main__': 38 | unittest.main() 39 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/test/convergence_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import math 18 | import unittest 19 | 20 | from federatedml.optim.convergence import converge_func_factory 21 | 22 | 23 | class TestConvergeFunction(unittest.TestCase): 24 | def test_diff_converge(self): 25 | loss = 50 26 | eps = 0.00001 27 | # converge_func = DiffConverge(eps=eps) 28 | converge_func = converge_func_factory(early_stop='diff', tol=eps) 29 | iter_num = 0 30 | pre_loss = loss 31 | while iter_num < 500: 32 | loss *= 0.5 33 | converge_flag = converge_func.is_converge(loss) 34 | if converge_flag: 35 | break 36 | iter_num += 1 37 | pre_loss = loss 38 | self.assertTrue(math.fabs(pre_loss - loss) <= eps) 39 | 40 | def test_abs_converge(self): 41 | loss = 50 42 | eps = 0.00001 43 | # converge_func = AbsConverge(eps=eps) 44 | converge_func = converge_func_factory(early_stop='abs', tol=eps) 45 | 46 | iter_num = 0 47 | while iter_num < 500: 48 | loss *= 0.5 49 | converge_flag = converge_func.is_converge(loss) 50 | if converge_flag: 51 | break 52 | iter_num += 1 53 | self.assertTrue(math.fabs(loss) <= eps) 54 | 55 | 56 | if __name__ == '__main__': 57 | unittest.main() 58 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/optim/test/initialize_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | 19 | from federatedml.optim.initialize import Initializer 20 | from federatedml.param import InitParam 21 | from federatedml.util import consts 22 | import numpy as np 23 | 24 | 25 | class TestInitialize(unittest.TestCase): 26 | def test_initializer(self): 27 | initializer = Initializer() 28 | data_shape = 10 29 | init_param_obj = InitParam(init_method=consts.RANDOM_NORMAL, 30 | init_const=20, 31 | fit_intercept=False 32 | ) 33 | model = initializer.init_model(model_shape=data_shape, init_params=init_param_obj) 34 | model_shape = np.array(model.unboxed).shape 35 | self.assertTrue(model_shape == (10,)) 36 | 37 | 38 | if __name__ == '__main__': 39 | unittest.main() 40 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.param.dataio_param import DataIOParam 18 | from federatedml.param.encrypt_param import EncryptParam 19 | from federatedml.param.logistic_regression_param import InitParam 20 | from federatedml.param.logistic_regression_param import LogisticParam 21 | from federatedml.param.linear_regression_param import LinearParam 22 | from federatedml.param.encrypted_mode_calculation_param import EncryptedModeCalculatorParam 23 | from federatedml.param.boosting_tree_param import ObjectiveParam 24 | from federatedml.param.boosting_tree_param import DecisionTreeParam 25 | from federatedml.param.boosting_tree_param import BoostingTreeParam 26 | from federatedml.param.predict_param import PredictParam 27 | from federatedml.param.evaluation_param import EvaluateParam 28 | from federatedml.param.feature_binning_param import FeatureBinningParam 29 | 30 | 31 | __all__ = ["DataIOParam", "DecisionTreeParam", "InitParam", "LogisticParam", "ObjectiveParam", 32 | "EncryptParam", "BoostingTreeParam", "EvaluateParam", "PredictParam", 'FeatureBinningParam', 33 | "LinearParam"] 34 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/cross_validation_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | import copy 20 | 21 | from federatedml.param.base_param import BaseParam 22 | # from federatedml.param.evaluation_param import EvaluateParam 23 | from federatedml.util import consts 24 | 25 | 26 | class CrossValidationParam(BaseParam): 27 | """ 28 | Define cross validation params 29 | 30 | Parameters 31 | ---------- 32 | n_splits: int, default: 5 33 | Specify how many splits used in KFold 34 | 35 | mode: str, default: 'Hetero' 36 | Indicate what mode is current task 37 | 38 | role: str, default: 'Guest' 39 | Indicate what role is current party 40 | 41 | shuffle: bool, default: True 42 | Define whether do shuffle before KFold or not. 43 | 44 | random_seed: int, default: 1 45 | Specify the random seed for numpy shuffle 46 | 47 | need_run: bool, default True 48 | Indicate if this module needed to be run 49 | 50 | """ 51 | 52 | def __init__(self, n_splits=5, mode=consts.HETERO, role=consts.GUEST, shuffle=True, random_seed=1, 53 | need_cv=False): 54 | super(CrossValidationParam, self).__init__() 55 | self.n_splits = n_splits 56 | self.mode = mode 57 | self.role = role 58 | self.shuffle = shuffle 59 | self.random_seed = random_seed 60 | # self.evaluate_param = copy.deepcopy(evaluate_param) 61 | self.need_cv = need_cv 62 | 63 | def check(self): 64 | model_param_descr = "cross validation param's " 65 | self.check_positive_integer(self.n_splits, model_param_descr) 66 | self.check_valid_value(self.mode, model_param_descr, valid_values=[consts.HOMO, consts.HETERO]) 67 | self.check_valid_value(self.role, model_param_descr, valid_values=[consts.HOST, consts.GUEST, consts.ARBITER]) 68 | self.check_boolean(self.shuffle, model_param_descr) 69 | if self.random_seed is not None: 70 | self.check_positive_integer(self.random_seed, model_param_descr) 71 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/encrypted_mode_calculation_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from federatedml.param.base_param import BaseParam 20 | 21 | 22 | class EncryptedModeCalculatorParam(BaseParam): 23 | """ 24 | Define the encrypted_mode_calulator parameters. 25 | 26 | Parameters 27 | ---------- 28 | mode: str, support 'strict', 'fast', 'balance' only, default: strict 29 | 30 | re_encrypted_rate: float or int, numeric number in [0, 1], use when mode equals to 'balance, default: 1 31 | 32 | """ 33 | 34 | def __init__(self, mode="strict", re_encrypted_rate=1): 35 | self.mode = mode 36 | self.re_encrypted_rate = re_encrypted_rate 37 | 38 | def check(self): 39 | descr = "encrypted_mode_calculator param" 40 | self.mode = self.check_and_change_lower(self.mode, 41 | ["strict", "fast", "balance"], 42 | descr) 43 | 44 | if self.mode == "balance": 45 | if type(self.re_encrypted_rate).__name__ not in ["int", "long", "float"]: 46 | raise ValueError("re_encrypted_rate should be a numeric number") 47 | 48 | if not (self.re_encrypted_rate >= 0.0 and self.re_encrypted_rate <= 1): 49 | raise ValueError("r_encrypted_rate should in [0, 1]") 50 | 51 | return True 52 | 53 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/evaluation_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from arch.api.utils import log_utils 20 | from federatedml.util import consts 21 | from federatedml.param.base_param import BaseParam 22 | 23 | LOGGER = log_utils.getLogger() 24 | 25 | 26 | class EvaluateParam(BaseParam): 27 | """ 28 | Define the evaluation method of binary/multiple classification and regression 29 | 30 | Parameters 31 | ---------- 32 | eval_type: string, support 'binary' for HomoLR, HeteroLR and Secureboosting. support 'regression' for Secureboosting. 'multi' is not support these version 33 | 34 | pos_label: specify positive label type, can be int, float and str, this depend on the data's label, this parameter effective only for 'binary' 35 | 36 | need_run: bool, default True 37 | Indicate if this module needed to be run 38 | """ 39 | 40 | def __init__(self, eval_type="binary", pos_label=1, need_run=True): 41 | super().__init__() 42 | self.eval_type = eval_type 43 | self.pos_label = pos_label 44 | self.need_run = need_run 45 | 46 | def check(self): 47 | descr = "evaluate param's " 48 | self.eval_type = self.check_and_change_lower(self.eval_type, 49 | [consts.BINARY, consts.MULTY, consts.REGRESSION], 50 | descr) 51 | 52 | if type(self.pos_label).__name__ not in ["str", "float", "int"]: 53 | raise ValueError( 54 | "evaluate param's pos_label {} not supported, should be str or float or int type".format( 55 | self.pos_label)) 56 | 57 | if type(self.need_run).__name__ != "bool": 58 | raise ValueError( 59 | "evaluate param's need_run {} not supported, should be bool".format( 60 | self.need_run)) 61 | 62 | LOGGER.info("Finish evaluation parameter check!") 63 | return True 64 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/one_vs_rest_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from arch.api.utils import log_utils 20 | from federatedml.param.base_param import BaseParam 21 | 22 | LOGGER = log_utils.getLogger() 23 | 24 | 25 | class OneVsRestParam(BaseParam): 26 | """ 27 | Define the one_vs_rest parameters. 28 | 29 | Parameters 30 | ---------- 31 | has_arbiter: bool. For some algorithm, may not has arbiter, for instances, secureboost of FATE, 32 | for these algorithms, it should be set to false. 33 | default true 34 | """ 35 | 36 | def __init__(self, need_one_vs_rest=False, has_arbiter=True): 37 | super().__init__() 38 | self.need_one_vs_rest = need_one_vs_rest 39 | self.has_arbiter = has_arbiter 40 | 41 | def check(self): 42 | if type(self.has_arbiter).__name__ != "bool": 43 | raise ValueError( 44 | "one_vs_rest param's has_arbiter {} not supported, should be bool type".format( 45 | self.has_arbiter)) 46 | 47 | LOGGER.debug("Finish one_vs_rest parameter check!") 48 | return True 49 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/onehot_encoder_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from arch.api.utils import log_utils 20 | from federatedml.param.base_param import BaseParam 21 | 22 | LOGGER = log_utils.getLogger() 23 | 24 | 25 | class OneHotEncoderParam(BaseParam): 26 | """ 27 | 28 | Parameters 29 | ---------- 30 | 31 | cols: list or int, default: -1 32 | Specify which columns need to calculated. -1 represent for all columns. 33 | 34 | need_run: bool, default True 35 | Indicate if this module needed to be run 36 | """ 37 | 38 | def __init__(self, cols=-1, need_run=True): 39 | super(OneHotEncoderParam, self).__init__() 40 | self.cols = cols 41 | self.need_run = need_run 42 | 43 | def check(self): 44 | descr = "One-hot encoder param's" 45 | self.check_defined_type(self.cols, descr, ['list', 'int']) 46 | return True 47 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/predict_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | ################################################################################ 20 | 21 | from arch.api.utils import log_utils 22 | from federatedml.param.base_param import BaseParam 23 | 24 | LOGGER = log_utils.getLogger() 25 | 26 | 27 | class PredictParam(BaseParam): 28 | """ 29 | Define the predict method of HomoLR, HeteroLR, SecureBoosting 30 | 31 | Parameters 32 | ---------- 33 | 34 | threshold: float or int, The threshold use to separate positive and negative class. Normally, it should be (0,1) 35 | """ 36 | 37 | def __init__(self, threshold=0.5): 38 | self.threshold = threshold 39 | 40 | def check(self): 41 | if type(self.with_proba).__name__ != "bool": 42 | raise ValueError( 43 | "predict param's with_proba {} not supported, should be bool type".format(self.with_proba)) 44 | 45 | if type(self.threshold).__name__ not in ["float", "int"]: 46 | raise ValueError("predict param's predict_param {} not supported, should be float or int".format( 47 | self.threshold)) 48 | 49 | LOGGER.debug("Finish predict parameter check!") 50 | return True 51 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/rsa_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from federatedml.param.base_param import BaseParam 20 | 21 | 22 | class RsaParam(BaseParam): 23 | """ 24 | Define the sample method 25 | 26 | Parameters 27 | ---------- 28 | rsa_key_n: integer, RSA modulus, default: None 29 | rsa_key_e: integer, RSA public exponent, default: None 30 | rsa_key_d: integer, RSA private exponent, default: None 31 | save_out_table_namespace: str, namespace of dtable where stores the output data. default: None 32 | save_out_table_name: str, name of dtable where stores the output data. default: None 33 | """ 34 | 35 | def __init__(self, rsa_key_n=None, rsa_key_e=None, rsa_key_d=None, save_out_table_namespace=None, save_out_table_name=None): 36 | self.rsa_key_n = rsa_key_n 37 | self.rsa_key_e = rsa_key_e 38 | self.rsa_key_d = rsa_key_d 39 | self.save_out_table_namespace = save_out_table_namespace 40 | self.save_out_table_name = save_out_table_name 41 | 42 | def check(self): 43 | descr = "rsa param" 44 | self.check_positive_integer(self.rsa_key_n, descr) 45 | self.check_positive_integer(self.rsa_key_e, descr) 46 | self.check_positive_integer(self.rsa_key_d, descr) 47 | self.check_string(self.save_out_table_namespace, descr) 48 | self.check_string(self.save_out_table_name, descr) 49 | return True 50 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/sample_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from federatedml.param.base_param import BaseParam 20 | 21 | 22 | class SampleParam(BaseParam): 23 | """ 24 | Define the sample method 25 | 26 | Parameters 27 | ---------- 28 | mode: str, accepted 'random','stratified'' only in this version, specify sample to use, default: 'random' 29 | 30 | method: str, accepted 'downsample','upsample' only in this version. default: 'downsample' 31 | 32 | fractions: None or float or list, if mode equals to random, it should be a float number greater than 0, 33 | otherwise a list of float elements. default: None 34 | 35 | random_state: int, RandomState instance or None, default: None 36 | 37 | need_run: bool, default True 38 | Indicate if this module needed to be run 39 | """ 40 | 41 | def __init__(self, mode="random", method="downsample", fractions=None, random_state=None, task_type="hetero", need_run=True): 42 | self.mode = mode 43 | self.method = method 44 | self.fractions = fractions 45 | self.random_state = random_state 46 | self.task_type = task_type 47 | self.need_run = need_run 48 | 49 | def check(self): 50 | descr = "sample param" 51 | self.mode = self.check_and_change_lower(self.mode, 52 | ["random", "stratified"], 53 | descr) 54 | 55 | self.method = self.check_and_change_lower(self.method, 56 | ["upsample", "downsample"], 57 | descr) 58 | 59 | return True 60 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/secure_add_example_param.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | from federatedml.param.base_param import BaseParam 20 | 21 | 22 | class SecureAddExampleParam(BaseParam): 23 | def __init__(self, seed=None, partition=1, data_num=1000): 24 | self.seed = seed 25 | self.partition = partition 26 | self.data_num = data_num 27 | 28 | def check(self): 29 | if self.seed is not None and type(self.seed).__name__ != "int": 30 | raise ValueError("random seed should be None or integers") 31 | 32 | if type(self.partition).__name__ != "int" or self.partition < 1: 33 | raise ValueError("partition should be an integer large than 0") 34 | 35 | if type(self.data_num).__name__ != "int" or self.data_num < 1: 36 | raise ValueError("data_num should be an integer large than 0") 37 | 38 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/test/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/test/param.json: -------------------------------------------------------------------------------- 1 | { 2 | "BoostingTreeParam": { 3 | "task_type": "classification", 4 | "learning_rate": 0.1, 5 | "num_trees": 2, 6 | "subsample_feature_rate": 1, 7 | "n_iter_no_change": false, 8 | "tol": 0.0001, 9 | "quantile_method": "bin_by_sample_data", 10 | "bin_num": 50, 11 | "bin_gap": 0.000001, 12 | "bin_sample_num": 150000, 13 | "tree_param": { 14 | "criterion_method": "xgboost", 15 | "criterion_params": [0.1], 16 | "max_depth": 5, 17 | "min_sample_split": 2, 18 | "min_leaf_node": 1, 19 | "min_impurity_split": 0.001, 20 | "max_split_nodes": 1024 21 | }, 22 | "objective_param": { 23 | "objective": "cross_entropy", 24 | "params": [1.5] 25 | }, 26 | "encrypt_param": { 27 | "method": "paillier" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/test/param_feature_binning.json: -------------------------------------------------------------------------------- 1 | { 2 | "FeatureBinningParam": { 3 | "process_method": "fit", 4 | "method": "quantile", 5 | "compress_thres": 10000, 6 | "head_size": 10000, 7 | "error": 0.001, 8 | "adjustment_factor": 0.5, 9 | "bin_num": 10, 10 | "cols": -1, 11 | "local_only": false, 12 | "display_result": ["iv"], 13 | "transform_param": { 14 | "transform_cols": null, 15 | "transform_type": "bin_num" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/test/param_feature_selection.json: -------------------------------------------------------------------------------- 1 | { 2 | "FeatureSelectionParam": { 3 | "method": "fit", 4 | "filter_method": ["unique_value", "iv_value_thres", 5 | "coefficient_of_variation_value_thres", "outlier_cols"], 6 | "select_cols": -1, 7 | "local_only": false, 8 | "iv_value_param" :{ 9 | "value_threshold": 1.0 10 | }, 11 | "iv_percentile_param": { 12 | "percentile_threshold": 1.0 13 | }, 14 | "coe_param": { 15 | "value_threshold": 0.3 16 | }, 17 | "outlier_param": { 18 | "percentile": 1.0, 19 | "upper_threshold": 1000.0 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/test/param_json_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | import json 20 | import os 21 | import unittest 22 | 23 | from federatedml.param.feature_binning_param import FeatureBinningParam 24 | from federatedml.util.param_extract import ParamExtract 25 | 26 | home_dir = os.path.split(os.path.realpath(__file__))[0] 27 | 28 | 29 | class TestParamExtract(unittest.TestCase): 30 | def setUp(self): 31 | self.param = FeatureBinningParam() 32 | json_config_file = home_dir + '/param_feature_binning.json' 33 | self.config_path = json_config_file 34 | with open(json_config_file, 'r', encoding='utf-8') as load_f: 35 | role_config = json.load(load_f) 36 | self.config_json = role_config 37 | 38 | # def tearDown(self): 39 | # os.system("rm -r " + self.config_path) 40 | 41 | def test_directly_extract(self): 42 | param_obj = FeatureBinningParam() 43 | extractor = ParamExtract() 44 | param_obj = extractor.parse_param_from_config(param_obj, self.config_json) 45 | self.assertTrue(param_obj.method == "quantile") 46 | self.assertTrue(param_obj.transform_param.transform_type == 'bin_num') 47 | 48 | 49 | if __name__ == '__main__': 50 | unittest.main() 51 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/param/test/param_one_hot.json: -------------------------------------------------------------------------------- 1 | { 2 | "OneHotEncoderParam": { 3 | "cols": [0, 1] 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/protobuf/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/protobuf/generated/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from federatedml.secureprotol.encrypt import RsaEncrypt, PaillierEncrypt, FakeEncrypt, AffineEncrypt, \ 18 | IterativeAffineEncrypt 19 | from federatedml.secureprotol.encrypt_mode import EncryptModeCalculator 20 | from federatedml.secureprotol import aciq, batch_encryption 21 | 22 | __all__ = ['RsaEncrypt', 'PaillierEncrypt', 'FakeEncrypt', 'EncryptModeCalculator', 'AffineEncrypt', 23 | 'IterativeAffineEncrypt', 'aciq', 'batch_encryption'] 24 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/affine_encoder.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | class AffineEncoder(object): 19 | def __init__(self, mult=2 ** 100, trans=0): 20 | self.mult = mult 21 | self.trans = trans 22 | 23 | def encode(self, plaintext): 24 | return int(self.mult * (plaintext + self.trans)) 25 | 26 | def decode(self, ciphertext, multiplier=1): 27 | return ciphertext / self.mult - multiplier * self.trans 28 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/gmpy_math.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import random 19 | import gmpy2 20 | 21 | POWMOD_GMP_SIZE = pow(2, 64) 22 | 23 | def powmod(a, b, c): 24 | """ 25 | return int: (a ** b) % c 26 | """ 27 | 28 | if a == 1: 29 | return 1 30 | 31 | if max(a, b, c) < POWMOD_GMP_SIZE: 32 | return pow(a, b, c) 33 | 34 | else: 35 | return int(gmpy2.powmod(a, b, c)) 36 | 37 | 38 | def invert(a, b): 39 | """return int: x, where a * x == 1 mod b 40 | """ 41 | x = int(gmpy2.invert(a, b)) 42 | 43 | if x == 0: 44 | raise ZeroDivisionError('invert(a, b) no inverse exists') 45 | 46 | return x 47 | 48 | 49 | def getprimeover(n): 50 | """return a random n-bit prime number 51 | """ 52 | r = gmpy2.mpz(random.SystemRandom().getrandbits(n)) 53 | r = gmpy2.bit_set(r, n - 1) 54 | 55 | return int(gmpy2.next_prime(r)) 56 | 57 | 58 | def isqrt(n): 59 | """ return the integer square root of N """ 60 | 61 | return int(gmpy2.isqrt(n)) 62 | 63 | 64 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/random.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from numpy.random import RandomState 18 | 19 | 20 | class RandomPads(object): 21 | """random pads utils for secret homogeneous aggregation 22 | currently use numpy.random, which use bit generator MT19937 23 | other algorithms such as pcg and xoroshiro may be supported in the future 24 | """ 25 | 26 | def __init__(self, init_seed=None): 27 | self._rand = RandomState(init_seed) 28 | 29 | def rand(self, d0, *more, **kwargs): 30 | return self._rand.rand(d0, *more, **kwargs) 31 | 32 | def randn(self, d0, *more, **kwargs): 33 | return self._rand.randn(d0, *more, **kwargs) 34 | 35 | def add_randn_pads(self, a, w): 36 | """a + r * w, 37 | where r is random array with nominal distribution N(0,1) and r.shape == a.shape 38 | """ 39 | return a + self._rand.randn(*a.shape) * w 40 | 41 | def add_rand_pads(self, a, w): 42 | """a + r * w, 43 | where r is random array with uniform distribution U[0,1) and r.shape == a.shape 44 | """ 45 | return a + self._rand.rand(*a.shape) * w 46 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/test/affine_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import numpy as np 18 | import unittest 19 | 20 | from federatedml.secureprotol.affine import AffineCipher 21 | 22 | 23 | class TestAffine(unittest.TestCase): 24 | def setUp(self): 25 | self.key = AffineCipher.generate_keypair() 26 | 27 | def tearDown(self): 28 | unittest.TestCase.tearDown(self) 29 | 30 | def test_add(self): 31 | x_li = np.ones(100) * np.random.randint(100) 32 | y_li = np.ones(100) * np.random.randint(1000) 33 | z_li = np.ones(100) * np.random.rand() 34 | t_li = range(100) 35 | 36 | for i in range(x_li.shape[0]): 37 | x = x_li[i] 38 | y = y_li[i] 39 | z = z_li[i] 40 | t = t_li[i] 41 | en_x = self.key.encrypt(x) 42 | en_y = self.key.encrypt(y) 43 | en_z = self.key.encrypt(z) 44 | en_t = self.key.encrypt(t) 45 | 46 | en_res = en_x + en_y + en_z + en_t 47 | 48 | res = x + y + z + t 49 | 50 | de_en_res = self.key.decrypt(en_res) 51 | self.assertAlmostEqual(de_en_res, res) 52 | 53 | def test_mul(self): 54 | x_li = (np.ones(100) * np.random.randint(100)).tolist() 55 | y_li = (np.ones(100) * np.random.randint(1000) * -1).tolist() 56 | z_li = (np.ones(100) * np.random.rand()).tolist() 57 | t_li = range(100) 58 | 59 | for i in range(len(x_li)): 60 | x = x_li[i] 61 | y = int(y_li[i]) 62 | z = z_li[i] 63 | t = int(t_li[i]) 64 | en_x = self.key.encrypt(x) 65 | en_z = self.key.encrypt(z) 66 | 67 | en_res = (en_x * y + en_z) * t 68 | 69 | res = (x * y + z) * t 70 | 71 | de_en_res = self.key.decrypt(en_res) 72 | self.assertAlmostEqual(de_en_res, res) 73 | 74 | 75 | if __name__ == '__main__': 76 | unittest.main() 77 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/secureprotol/test/iterative_affine_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import numpy as np 18 | import unittest 19 | 20 | from federatedml.secureprotol.iterative_affine import IterativeAffineCipher 21 | 22 | 23 | class TestAffine(unittest.TestCase): 24 | def setUp(self): 25 | self.key = IterativeAffineCipher.generate_keypair() 26 | 27 | def tearDown(self): 28 | unittest.TestCase.tearDown(self) 29 | 30 | def test_add(self): 31 | x_li = np.ones(100) * np.random.randint(100) 32 | y_li = np.ones(100) * np.random.randint(1000) 33 | z_li = np.ones(100) * np.random.rand() 34 | t_li = range(100) 35 | 36 | for i in range(x_li.shape[0]): 37 | x = x_li[i] 38 | y = y_li[i] 39 | z = z_li[i] 40 | t = t_li[i] 41 | en_x = self.key.encrypt(x) 42 | en_y = self.key.encrypt(y) 43 | en_z = self.key.encrypt(z) 44 | en_t = self.key.encrypt(t) 45 | 46 | en_res = en_x + en_y + en_z + en_t 47 | 48 | res = x + y + z + t 49 | 50 | de_en_res = self.key.decrypt(en_res) 51 | self.assertAlmostEqual(de_en_res, res) 52 | 53 | 54 | if __name__ == '__main__': 55 | unittest.main() 56 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/cross_validation.json: -------------------------------------------------------------------------------- 1 | { 2 | "CrossValidationTransferVariable": { 3 | "train_sid": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | }, 9 | "test_sid": { 10 | "src": "guest", 11 | "dst": [ 12 | "host" 13 | ] 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_decision_tree.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroDecisionTreeTransferVariable": { 3 | "encrypted_grad_and_hess": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | }, 9 | "tree_node_queue": { 10 | "src": "guest", 11 | "dst": [ 12 | "host" 13 | ] 14 | }, 15 | "node_positions": { 16 | "src": "guest", 17 | "dst": [ 18 | "host" 19 | ] 20 | }, 21 | "encrypted_splitinfo_host": { 22 | "src": "host", 23 | "dst": [ 24 | "guest" 25 | ] 26 | }, 27 | "federated_best_splitinfo_host": { 28 | "src": "guest", 29 | "dst": [ 30 | "host" 31 | ] 32 | }, 33 | "final_splitinfo_host": { 34 | "src": "host", 35 | "dst": [ 36 | "guest" 37 | ] 38 | }, 39 | "dispatch_node_host": { 40 | "src": "guest", 41 | "dst": [ 42 | "host" 43 | ] 44 | }, 45 | "dispatch_node_host_result": { 46 | "src": "host", 47 | "dst": [ 48 | "guest" 49 | ] 50 | }, 51 | "tree": { 52 | "src": "guest", 53 | "dst": [ 54 | "host" 55 | ] 56 | }, 57 | "predict_data": { 58 | "src": "guest", 59 | "dst": [ 60 | "host" 61 | ] 62 | }, 63 | "predict_data_by_host": { 64 | "src": "host", 65 | "dst": [ 66 | "guest" 67 | ] 68 | }, 69 | "predict_finish_tag": { 70 | "src": "guest", 71 | "dst": [ 72 | "host" 73 | ] 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_dnn_lr.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroDNNLRTransferVariable": { 3 | "guest_dec_gradient": { 4 | "src": "arbiter", 5 | "dst": [ 6 | "guest" 7 | ] 8 | }, 9 | "guest_enc_gradient": { 10 | "src": "guest", 11 | "dst": [ 12 | "arbiter" 13 | ] 14 | }, 15 | "host_dec_gradient": { 16 | "src": "arbiter", 17 | "dst": [ 18 | "host" 19 | ] 20 | }, 21 | "host_enc_gradient": { 22 | "src": "host", 23 | "dst": [ 24 | "arbiter" 25 | ] 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_feature_binning.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroFeatureBinningTransferVariable": { 3 | "paillier_pubkey": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | }, 9 | "encrypted_label": { 10 | "src": "guest", 11 | "dst": [ 12 | "host" 13 | ] 14 | }, 15 | "encrypted_bin_sum": { 16 | "src": "host", 17 | "dst": [ 18 | "guest" 19 | ] 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_feature_selection.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroFeatureSelectionTransferVariable": { 3 | "result_left_cols": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | }, 9 | "host_select_cols": { 10 | "src": "host", 11 | "dst": [ 12 | "guest" 13 | ] 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_linr.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroLinRTransferVariable": { 3 | "paillier_pubkey": { 4 | "src": "arbiter", 5 | "dst": [ 6 | "host", 7 | "guest" 8 | ] 9 | }, 10 | "batch_data_index": { 11 | "src": "guest", 12 | "dst": [ 13 | "host" 14 | ] 15 | }, 16 | "host_forward": { 17 | "src": "host", 18 | "dst": [ 19 | "guest" 20 | ] 21 | }, 22 | "loss_intermediate": { 23 | "src": "host", 24 | "dst": [ 25 | "guest" 26 | ] 27 | }, 28 | "fore_gradient": { 29 | "src": "guest", 30 | "dst": [ 31 | "host" 32 | ] 33 | }, 34 | "guest_gradient": { 35 | "src": "guest", 36 | "dst": [ 37 | "arbiter" 38 | ] 39 | }, 40 | "guest_optim_gradient": { 41 | "src": "arbiter", 42 | "dst": [ 43 | "guest" 44 | ] 45 | }, 46 | "host_loss_regular": { 47 | "src": "host", 48 | "dst": [ 49 | "guest" 50 | ] 51 | }, 52 | "loss": { 53 | "src": "guest", 54 | "dst": [ 55 | "arbiter" 56 | ] 57 | }, 58 | "converge_flag": { 59 | "src": "arbiter", 60 | "dst": [ 61 | "host", 62 | "guest" 63 | ] 64 | }, 65 | "batch_info": { 66 | "src": "guest", 67 | "dst": [ 68 | "host", 69 | "arbiter" 70 | ] 71 | }, 72 | "host_optim_gradient": { 73 | "src": "arbiter", 74 | "dst": [ 75 | "host" 76 | ] 77 | }, 78 | "host_gradient": { 79 | "src": "host", 80 | "dst": [ 81 | "arbiter" 82 | ] 83 | }, 84 | "host_partial_prediction": { 85 | "src": "host", 86 | "dst": [ 87 | "guest" 88 | ] 89 | } 90 | } 91 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_lr.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroLRTransferVariable": { 3 | "paillier_pubkey": { 4 | "src": "arbiter", 5 | "dst": [ 6 | "host", 7 | "guest" 8 | ] 9 | }, 10 | "batch_data_index": { 11 | "src": "guest", 12 | "dst": [ 13 | "host" 14 | ] 15 | }, 16 | "host_forward_dict": { 17 | "src": "host", 18 | "dst": [ 19 | "guest" 20 | ] 21 | }, 22 | "fore_gradient": { 23 | "src": "guest", 24 | "dst": [ 25 | "host" 26 | ] 27 | }, 28 | "guest_gradient": { 29 | "src": "guest", 30 | "dst": [ 31 | "arbiter" 32 | ] 33 | }, 34 | "guest_optim_gradient": { 35 | "src": "arbiter", 36 | "dst": [ 37 | "guest" 38 | ] 39 | }, 40 | "host_loss_regular": { 41 | "src": "host", 42 | "dst": [ 43 | "guest" 44 | ] 45 | }, 46 | "loss": { 47 | "src": "guest", 48 | "dst": [ 49 | "arbiter" 50 | ] 51 | }, 52 | "loss_intermediate": { 53 | "src": "host", 54 | "dst": [ 55 | "guest" 56 | ] 57 | }, 58 | "converge_flag": { 59 | "src": "arbiter", 60 | "dst": [ 61 | "host", 62 | "guest" 63 | ] 64 | }, 65 | "batch_info": { 66 | "src": "guest", 67 | "dst": [ 68 | "host", 69 | "arbiter" 70 | ] 71 | }, 72 | "host_optim_gradient": { 73 | "src": "arbiter", 74 | "dst": [ 75 | "host" 76 | ] 77 | }, 78 | "host_gradient": { 79 | "src": "host", 80 | "dst": [ 81 | "arbiter" 82 | ] 83 | }, 84 | "host_prob": { 85 | "src": "host", 86 | "dst": [ 87 | "guest" 88 | ] 89 | } 90 | } 91 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_poisson.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroPoissonTransferVariable": { 3 | "paillier_pubkey": { 4 | "src": "arbiter", 5 | "dst": [ 6 | "host", 7 | "guest" 8 | ] 9 | }, 10 | "batch_data_index": { 11 | "src": "guest", 12 | "dst": [ 13 | "host" 14 | ] 15 | }, 16 | "host_forward": { 17 | "src": "host", 18 | "dst": [ 19 | "guest" 20 | ] 21 | }, 22 | "loss_intermediate": { 23 | "src": "host", 24 | "dst": [ 25 | "guest" 26 | ] 27 | }, 28 | "fore_gradient": { 29 | "src": "guest", 30 | "dst": [ 31 | "host" 32 | ] 33 | }, 34 | "guest_gradient": { 35 | "src": "guest", 36 | "dst": [ 37 | "arbiter" 38 | ] 39 | }, 40 | "guest_optim_gradient": { 41 | "src": "arbiter", 42 | "dst": [ 43 | "guest" 44 | ] 45 | }, 46 | "host_loss_regular": { 47 | "src": "host", 48 | "dst": [ 49 | "guest" 50 | ] 51 | }, 52 | "loss": { 53 | "src": "guest", 54 | "dst": [ 55 | "arbiter" 56 | ] 57 | }, 58 | "converge_flag": { 59 | "src": "arbiter", 60 | "dst": [ 61 | "host", 62 | "guest" 63 | ] 64 | }, 65 | "batch_info": { 66 | "src": "guest", 67 | "dst": [ 68 | "host", 69 | "arbiter" 70 | ] 71 | }, 72 | "host_optim_gradient": { 73 | "src": "arbiter", 74 | "dst": [ 75 | "host" 76 | ] 77 | }, 78 | "host_gradient": { 79 | "src": "host", 80 | "dst": [ 81 | "arbiter" 82 | ] 83 | }, 84 | "host_partial_prediction": { 85 | "src": "host", 86 | "dst": [ 87 | "guest" 88 | ] 89 | } 90 | } 91 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/hetero_secure_boost.json: -------------------------------------------------------------------------------- 1 | { 2 | "HeteroSecureBoostingTreeTransferVariable": { 3 | "tree_dim": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | }, 9 | "stop_flag": { 10 | "src": "guest", 11 | "dst": [ 12 | "host" 13 | ] 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/one_vs_rest.json: -------------------------------------------------------------------------------- 1 | { 2 | "OneVsRestTransferVariable": { 3 | "host_classes": { 4 | "src": "host", 5 | "dst": [ 6 | "guest" 7 | ] 8 | }, 9 | "aggregate_classes": { 10 | "src": "guest", 11 | "dst": [ 12 | "host", 13 | "arbiter" 14 | ] 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/raw_intersect.json: -------------------------------------------------------------------------------- 1 | { 2 | "RawIntersectTransferVariable": { 3 | "send_ids_host": { 4 | "src": "host", 5 | "dst": [ 6 | "guest" 7 | ] 8 | }, 9 | "send_ids_guest": { 10 | "src": "guest", 11 | "dst": [ 12 | "host" 13 | ] 14 | }, 15 | "intersect_ids_host": { 16 | "src": "host", 17 | "dst": [ 18 | "guest" 19 | ] 20 | }, 21 | "intersect_ids_guest": { 22 | "src": "guest", 23 | "dst": [ 24 | "host" 25 | ] 26 | }, 27 | "sync_intersect_ids_multi_hosts": { 28 | "src": "guest", 29 | "dst": [ 30 | "host" 31 | ] 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/rsa_intersect.json: -------------------------------------------------------------------------------- 1 | { 2 | "RsaIntersectTransferVariable": { 3 | "rsa_pubkey": { 4 | "src": "host", 5 | "dst": [ 6 | "guest" 7 | ] 8 | }, 9 | "intersect_guest_ids": { 10 | "src": "guest", 11 | "dst": [ 12 | "host" 13 | ] 14 | }, 15 | "intersect_host_ids_process": { 16 | "src": "host", 17 | "dst": [ 18 | "guest" 19 | ] 20 | }, 21 | "intersect_guest_ids_process": { 22 | "src": "host", 23 | "dst": [ 24 | "guest" 25 | ] 26 | }, 27 | "intersect_ids": { 28 | "src": "guest", 29 | "dst": [ 30 | "host" 31 | ] 32 | }, 33 | "cache_version_info":{ 34 | "src": "guest", 35 | "dst": [ 36 | "host" 37 | ] 38 | }, 39 | "cache_version_match_info":{ 40 | "src": "host", 41 | "dst": [ 42 | "guest" 43 | ] 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "SampleTransferVariable": { 3 | "sample_ids": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/definition/secure_add_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "SecureAddExampleTransferVariable": { 3 | "guest_share": { 4 | "src": "guest", 5 | "dst": [ 6 | "host" 7 | ] 8 | }, 9 | "host_share": { 10 | "src": "host", 11 | "dst": [ 12 | "guest" 13 | ] 14 | }, 15 | "host_sum": { 16 | "src": "host", 17 | "dst": [ 18 | "guest" 19 | ] 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/cross_validation_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class CrossValidationTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.train_sid = Variable(name='CrossValidationTransferVariable.train_sid', auth=dict(src='guest', dst=['host']), transfer_variable=self) 33 | self.test_sid = Variable(name='CrossValidationTransferVariable.test_sid', auth=dict(src='guest', dst=['host']), transfer_variable=self) 34 | pass 35 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/hetero_dnn_lr_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class HeteroDNNLRTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.guest_dec_gradient = Variable(name='HeteroDNNLRTransferVariable.guest_dec_gradient', auth=dict(src='arbiter', dst=['guest']), transfer_variable=self) 33 | self.guest_enc_gradient = Variable(name='HeteroDNNLRTransferVariable.guest_enc_gradient', auth=dict(src='guest', dst=['arbiter']), transfer_variable=self) 34 | self.host_dec_gradient = Variable(name='HeteroDNNLRTransferVariable.host_dec_gradient', auth=dict(src='arbiter', dst=['host']), transfer_variable=self) 35 | self.host_enc_gradient = Variable(name='HeteroDNNLRTransferVariable.host_enc_gradient', auth=dict(src='host', dst=['arbiter']), transfer_variable=self) 36 | pass 37 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/hetero_feature_binning_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class HeteroFeatureBinningTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.paillier_pubkey = Variable(name='HeteroFeatureBinningTransferVariable.paillier_pubkey', auth=dict(src='guest', dst=['host']), transfer_variable=self) 33 | self.encrypted_label = Variable(name='HeteroFeatureBinningTransferVariable.encrypted_label', auth=dict(src='guest', dst=['host']), transfer_variable=self) 34 | self.encrypted_bin_sum = Variable(name='HeteroFeatureBinningTransferVariable.encrypted_bin_sum', auth=dict(src='host', dst=['guest']), transfer_variable=self) 35 | pass 36 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/hetero_feature_selection_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class HeteroFeatureSelectionTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.result_left_cols = Variable(name='HeteroFeatureSelectionTransferVariable.result_left_cols', auth=dict(src='guest', dst=['host']), transfer_variable=self) 33 | self.host_select_cols = Variable(name='HeteroFeatureSelectionTransferVariable.host_select_cols', auth=dict(src='host', dst=['guest']), transfer_variable=self) 34 | pass 35 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/hetero_secure_boost_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class HeteroSecureBoostingTreeTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.tree_dim = Variable(name='HeteroSecureBoostingTreeTransferVariable.tree_dim', auth=dict(src='guest', dst=['host']), transfer_variable=self) 33 | self.stop_flag = Variable(name='HeteroSecureBoostingTreeTransferVariable.stop_flag', auth=dict(src='guest', dst=['host']), transfer_variable=self) 34 | pass 35 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/one_vs_rest_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class OneVsRestTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.host_classes = Variable(name='OneVsRestTransferVariable.host_classes', auth=dict(src='host', dst=['guest']), transfer_variable=self) 33 | self.aggregate_classes = Variable(name='OneVsRestTransferVariable.aggregate_classes', auth=dict(src='guest', dst=['host', 'arbiter']), transfer_variable=self) 34 | pass 35 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/raw_intersect_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class RawIntersectTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.send_ids_host = Variable(name='RawIntersectTransferVariable.send_ids_host', auth=dict(src='host', dst=['guest']), transfer_variable=self) 33 | self.send_ids_guest = Variable(name='RawIntersectTransferVariable.send_ids_guest', auth=dict(src='guest', dst=['host']), transfer_variable=self) 34 | self.intersect_ids_host = Variable(name='RawIntersectTransferVariable.intersect_ids_host', auth=dict(src='host', dst=['guest']), transfer_variable=self) 35 | self.intersect_ids_guest = Variable(name='RawIntersectTransferVariable.intersect_ids_guest', auth=dict(src='guest', dst=['host']), transfer_variable=self) 36 | self.sync_intersect_ids_multi_hosts = Variable(name='RawIntersectTransferVariable.sync_intersect_ids_multi_hosts', auth=dict(src='guest', dst=['host']), transfer_variable=self) 37 | pass 38 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/rsa_intersect_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class RsaIntersectTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.rsa_pubkey = Variable(name='RsaIntersectTransferVariable.rsa_pubkey', auth=dict(src='host', dst=['guest']), transfer_variable=self) 33 | self.intersect_guest_ids = Variable(name='RsaIntersectTransferVariable.intersect_guest_ids', auth=dict(src='guest', dst=['host']), transfer_variable=self) 34 | self.intersect_host_ids_process = Variable(name='RsaIntersectTransferVariable.intersect_host_ids_process', auth=dict(src='host', dst=['guest']), transfer_variable=self) 35 | self.intersect_guest_ids_process = Variable(name='RsaIntersectTransferVariable.intersect_guest_ids_process', auth=dict(src='host', dst=['guest']), transfer_variable=self) 36 | self.intersect_ids = Variable(name='RsaIntersectTransferVariable.intersect_ids', auth=dict(src='guest', dst=['host']), transfer_variable=self) 37 | self.cache_version_info = Variable(name='RsaIntersectTransferVariable.cache_version_info', auth=dict(src='guest', dst=['host']), transfer_variable=self) 38 | self.cache_version_match_info = Variable(name='RsaIntersectTransferVariable.cache_version_match_info', auth=dict(src='host', dst=['guest']), transfer_variable=self) 39 | pass 40 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/sample_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class SampleTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.sample_ids = Variable(name='SampleTransferVariable.sample_ids', auth=dict(src='guest', dst=['host']), transfer_variable=self) 33 | pass 34 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_class/secure_add_example_transfer_variable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | # noinspection PyAttributeOutsideInit 30 | class SecureAddExampleTransferVariable(BaseTransferVariable): 31 | def define_transfer_variable(self): 32 | self.guest_share = Variable(name='SecureAddExampleTransferVariable.guest_share', auth=dict(src='guest', dst=['host']), transfer_variable=self) 33 | self.host_share = Variable(name='SecureAddExampleTransferVariable.host_share', auth=dict(src='host', dst=['guest']), transfer_variable=self) 34 | self.host_sum = Variable(name='SecureAddExampleTransferVariable.host_sum', auth=dict(src='host', dst=['guest']), transfer_variable=self) 35 | pass 36 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/transfer_variable/transfer_variable_template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | ################################################################################ 21 | # 22 | # AUTO GENERATED TRANSFER VARIABLE CLASS. DO NOT MODIFY 23 | # 24 | ################################################################################ 25 | 26 | from federatedml.transfer_variable.transfer_class.base_transfer_variable import BaseTransferVariable, Variable 27 | 28 | 29 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/util/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/util/abnormal_detection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from federatedml.statistic import data_overview 20 | 21 | 22 | def empty_table_detection(data_instances): 23 | num_data = data_instances.count() 24 | if num_data == 0: 25 | raise ValueError("Count of data_instance is 0") 26 | 27 | 28 | def empty_feature_detection(data_instances): 29 | is_empty_feature = data_overview.is_empty_feature(data_instances) 30 | if is_empty_feature: 31 | raise ValueError("Number of features of DTable is 0.") 32 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/util/param_extract.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Copyright 2019 The FATE Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # ============================================================================= 21 | # Param Exact Class 22 | # ============================================================================= 23 | import builtins 24 | from federatedml.util import consts 25 | 26 | 27 | class ParamExtract(object): 28 | def __init__(self): 29 | self.builtin_types = dir(builtins) 30 | 31 | def parse_param_from_config(self, param, config_json): 32 | if config_json is None or type(config_json).__name__ != "dict": 33 | raise Exception("config file is not a valid dict type, please have a check!") 34 | 35 | default_section = type(param).__name__ 36 | if default_section not in config_json: 37 | return param 38 | 39 | param = self.recursive_parse_param_from_config(param, config_json.get(default_section), 40 | param_parse_depth=0) 41 | 42 | return param 43 | 44 | def recursive_parse_param_from_config(self, param, config_json, param_parse_depth): 45 | if param_parse_depth > consts.PARAM_MAXDEPTH: 46 | raise ValueError("Param define nesting too deep!!!, can not parse it") 47 | 48 | inst_variables = param.__dict__ 49 | 50 | for variable in inst_variables: 51 | attr = getattr(param, variable) 52 | 53 | if type(attr).__name__ in self.builtin_types or attr is None: 54 | if variable in config_json: 55 | option = config_json[variable] 56 | setattr(param, variable, option) 57 | elif variable in config_json: 58 | sub_params = self.recursive_parse_param_from_config(attr, config_json.get(variable), 59 | param_parse_depth + 1) 60 | setattr(param, variable, sub_params) 61 | 62 | return param 63 | 64 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/util/test/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/util/test/classify_label_checker_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | import random 19 | 20 | from arch.api import session 21 | from federatedml.feature.instance import Instance 22 | from federatedml.util.classify_label_checker import ClassifyLabelChecker, RegressionLabelChecker 23 | 24 | 25 | class TeskClassifyLabelChecker(unittest.TestCase): 26 | def setUp(self): 27 | session.init("test_label_checker") 28 | 29 | self.small_label_set = [Instance(label=i % 5) for i in range(100)] 30 | self.classify_inst = session.parallelize(self.small_label_set, include_key=False) 31 | self.regression_label = [Instance(label=random.random()) for i in range(100)] 32 | self.regression_inst = session.parallelize(self.regression_label) 33 | self.classify_checker = ClassifyLabelChecker() 34 | self.regression_checker = RegressionLabelChecker() 35 | 36 | def test_classify_label_checkert(self): 37 | num_class, classes = self.classify_checker.validate_label(self.classify_inst) 38 | self.assertTrue(num_class == 5) 39 | self.assertTrue(sorted(classes) == [0, 1, 2, 3, 4]) 40 | 41 | def test_regression_label_checker(self): 42 | self.regression_checker.validate_label(self.regression_inst) 43 | 44 | 45 | if __name__ == '__main__': 46 | unittest.main() 47 | -------------------------------------------------------------------------------- /FATE_plugin/federatedml/util/test/param_extract_test.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import unittest 18 | import os 19 | 20 | from federatedml.util.param_extract import ParamExtract 21 | from federatedml.param import InitParam 22 | from federatedml.param import BoostingTreeParam 23 | 24 | 25 | class TestParamExtract(unittest.TestCase): 26 | def setUp(self): 27 | self.init_param = InitParam() 28 | self.boosting_tree_param = BoostingTreeParam() 29 | self.config_dict = \ 30 | {"BoostingTreeParam": { 31 | "init_param": {"init_method": "test_init", "fit_intercept": False}, 32 | "tree_param": {"criterion_method": "test_decisiontree"}, 33 | "task_type": "test_boostingtree", 34 | "test_variable": "test"} 35 | } 36 | 37 | def test_directly_extract(self): 38 | boosting_tree_param = BoostingTreeParam() 39 | extractor = ParamExtract() 40 | boosting_tree_param = extractor.parse_param_from_config(boosting_tree_param, self.config_dict) 41 | self.assertTrue(boosting_tree_param.task_type == "test_boostingtree") 42 | 43 | def test_undefine_variable_extract(self): 44 | boosting_tree_param = BoostingTreeParam() 45 | extractor = ParamExtract() 46 | boosting_tree_param = extractor.parse_param_from_config(boosting_tree_param, self.config_dict) 47 | self.assertTrue(not hasattr(boosting_tree_param, "test_variable")) 48 | 49 | def test_param_embedding(self): 50 | boosting_tree_param = BoostingTreeParam() 51 | extractor = ParamExtract() 52 | boosting_tree_param = extractor.parse_param_from_config(boosting_tree_param, self.config_dict) 53 | print ("boosting_tree_param.tree_param.criterion_method {}".format(boosting_tree_param.tree_param.criterion_method)) 54 | self.assertTrue(boosting_tree_param.tree_param.criterion_method == "test_decisiontree") 55 | 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | -------------------------------------------------------------------------------- /FATE_plugin/profile/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoszh/BatchCrypt/e1067faeae54e8472ac8d7a987456b0e72b08b7a/FATE_plugin/profile/__init__.py -------------------------------------------------------------------------------- /FATE_plugin/profile/mem_stat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime 3 | import threading 4 | import random 5 | 6 | PATH = '/data/profile/network_logs/memory_usage.log' 7 | 8 | last_used = -100.0 9 | last_free = -100.0 10 | 11 | def run(sec): 12 | global last_used 13 | global last_free 14 | t = threading.Timer(sec, run, [sec]) 15 | tot_m, used_m, free_m = map(int, os.popen('free -t -m').readlines()[-1].split()[1:]) 16 | ratio_used = used_m * 100.0 / tot_m 17 | ratio_free = free_m * 100.0 / tot_m 18 | if abs(ratio_used - last_used) > 0.1 or abs(ratio_free - last_free) > 0.1: 19 | last_used = ratio_used 20 | last_free = ratio_free 21 | with open(PATH, 'a+') as f: 22 | f.write(f'[{str(datetime.datetime.now())}] Used: {ratio_used:.2f}% | Free: {ratio_free:.2f}%\n') 23 | t.start() 24 | 25 | if __name__ == '__main__': 26 | 27 | run(3.0) 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # batchcrypt 2 | 3 | This is the source repo for paper ATC'20 paper "BatchCrypt: Efficient Homomorphic Encryption for Cross-Silo Federated Learning" 4 | 5 | ## Structure 6 | 7 | FATE_plugin contains our patch to FATE 1.0.1 8 | accuracy_eval contains our scripts to validate and evaluate our quantization and batching scheme for Homomorphic Encryption 9 | -------------------------------------------------------------------------------- /accuracy_eval/augmentation.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def flip(x): 5 | """Flip augmentation 6 | 7 | Args: 8 | x: Image to flip 9 | 10 | Returns: 11 | Augmented image 12 | """ 13 | x = tf.image.random_flip_left_right(x) 14 | # x = tf.image.random_flip_up_down(x) 15 | 16 | return x 17 | 18 | 19 | def color(x): 20 | """Color augmentation 21 | 22 | Args: 23 | x: Image 24 | 25 | Returns: 26 | Augmented image 27 | """ 28 | x = tf.image.random_hue(x, 0.08) 29 | x = tf.image.random_saturation(x, 0.6, 1.6) 30 | x = tf.image.random_brightness(x, 0.05) 31 | x = tf.image.random_contrast(x, 0.7, 1.3) 32 | return x 33 | 34 | 35 | def zoom(x): 36 | """Zoom augmentation 37 | 38 | Args: 39 | x: Image 40 | 41 | Returns: 42 | Augmented image 43 | """ 44 | scale = np.random.uniform(0.8, 1.0) 45 | 46 | x1 = y1 = 0.5 - (0.5 * scale) 47 | x2 = y2 = 0.5 + (0.5 * scale) 48 | box = [x1, y1, x2, y2] 49 | 50 | def random_crop(img): 51 | # Create different crops for an image 52 | crops = tf.image.crop_and_resize([img], boxes=[box], box_indices=[0], crop_size=(32, 32)) 53 | # Return a random crop 54 | return crops[0] 55 | 56 | x = random_crop(x) 57 | 58 | return x 59 | 60 | 61 | def augment_img(x, y): 62 | if np.random.uniform(0.0, 1.0) > 0.5: 63 | x = flip(x) 64 | if np.random.uniform(0.0, 1.0) > 0.5: 65 | x = color(x) 66 | if np.random.uniform(0.0, 1.0) > 0.75: 67 | x = zoom(x) 68 | return x, y 69 | -------------------------------------------------------------------------------- /accuracy_eval/encryption/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marcoszh/BatchCrypt/e1067faeae54e8472ac8d7a987456b0e72b08b7a/accuracy_eval/encryption/__init__.py -------------------------------------------------------------------------------- /accuracy_eval/encryption/gmpy_math.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import random 19 | import gmpy2 20 | 21 | POWMOD_GMP_SIZE = pow(2, 64) 22 | 23 | 24 | def powmod(a, b, c): 25 | """ 26 | return int: (a ** b) % c 27 | """ 28 | 29 | if a == 1: 30 | return 1 31 | 32 | if max(a, b, c) < POWMOD_GMP_SIZE: 33 | return pow(a, b, c) 34 | 35 | else: 36 | return int(gmpy2.powmod(a, b, c)) 37 | 38 | 39 | def invert(a, b): 40 | """return int: x, where a * x == 1 mod b 41 | """ 42 | x = int(gmpy2.invert(a, b)) 43 | 44 | if x == 0: 45 | raise ZeroDivisionError('invert(a, b) no inverse exists') 46 | 47 | return x 48 | 49 | 50 | def getprimeover(n): 51 | """return a random n-bit prime number 52 | """ 53 | r = gmpy2.mpz(random.SystemRandom().getrandbits(n)) 54 | r = gmpy2.bit_set(r, n - 1) 55 | 56 | return int(gmpy2.next_prime(r)) 57 | 58 | 59 | def isqrt(n): 60 | """ return the integer square root of N """ 61 | 62 | return int(gmpy2.isqrt(n)) 63 | 64 | 65 | -------------------------------------------------------------------------------- /accuracy_eval/regularization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # We create a simple class, called EarlyStoppingCheckPoint, that combines simplified version of Early Stoping and ModelCheckPoint classes from Keras. 4 | # References: 5 | # https://keras.io/callbacks/ 6 | # https://github.com/keras-team/keras/blob/master/keras/callbacks.py#L458 7 | # https://github.com/keras-team/keras/blob/master/keras/callbacks.py#L358 8 | 9 | 10 | class EarlyStoppingCheckPoint(object): 11 | 12 | def __init__(self, monitor, patience, file_path=None): 13 | 14 | self.model = None 15 | self.monitor = monitor 16 | self.patience = patience 17 | self.file_path = file_path 18 | self.wait = 0 19 | self.stopped_epoch = 0 20 | 21 | def set_model(self, model): 22 | self.model = model 23 | 24 | def on_train_begin(self): 25 | self.wait = 0 26 | self.stopped_epoch = 0 27 | self.best = -np.Inf 28 | 29 | def on_iteration_end(self, epoch, batch, logs=None): 30 | 31 | current = logs.get(self.monitor) 32 | if current is None: 33 | print('monitor does not available in logs') 34 | return 35 | 36 | if current > self.best: 37 | self.best = current 38 | print("find best acc: ", self.best, "at epoch:", epoch, "batch:", batch) 39 | if self.file_path is not None: 40 | self.model.save_model(self.file_path) 41 | self.wait = 0 42 | else: 43 | self.wait += 1 44 | if self.wait >= self.patience: 45 | self.stopped_epoch = epoch 46 | self.model.stop_training = True 47 | -------------------------------------------------------------------------------- /accuracy_eval/test_10parties/utils.py: -------------------------------------------------------------------------------- 1 | def assert_matrix(X, Y): 2 | assert X.shape[0] == X.shape[0] 3 | assert Y.shape[1] == Y.shape[1] 4 | 5 | row_num = X.shape[0] 6 | col_num = Y.shape[1] 7 | for row in range(row_num): 8 | for col in range(col_num): 9 | assert X[row][col] == Y[row][col] 10 | --------------------------------------------------------------------------------