├── src ├── data │ ├── celeba.py │ ├── __init__.py │ ├── __pycache__ │ │ ├── data.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── partition.cpython-37.pyc │ │ ├── pmnist_dl.cpython-37.pyc │ │ ├── datasetzoo.cpython-37.pyc │ │ ├── pcifar10_dl.cpython-37.pyc │ │ ├── utils_data.cpython-37.pyc │ │ ├── fmnist_superclass.cpython-37.pyc │ │ ├── cifar100_superclass.cpython-37.pyc │ │ └── cifar10_superclass.cpython-37.pyc │ └── datasetzoo.py ├── pruning │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── structured.cpython-37.pyc │ │ └── unstructured.cpython-37.pyc │ └── unstructured.py ├── sub_fedavg │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── sub_fedavg_s.cpython-37.pyc │ │ └── sub_fedavg_u.cpython-37.pyc │ └── sub_fedavg_u.py ├── clustering │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── cluster_fl.cpython-37.pyc │ │ ├── utils_flis.cpython-37.pyc │ │ └── hierarchical_clustering.cpython-37.pyc │ └── hierarchical_clustering.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── model.cpython-37.pyc │ │ ├── vgg.cpython-37.pyc │ │ ├── lenet5.cpython-37.pyc │ │ ├── models.cpython-37.pyc │ │ ├── resnet.cpython-37.pyc │ │ ├── resnet9.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── structured.cpython-37.pyc │ │ ├── vggmodel.cpython-37.pyc │ │ ├── moon_models.cpython-37.pyc │ │ ├── resnetcifar.cpython-37.pyc │ │ ├── unstructured.cpython-37.pyc │ │ ├── models_structured.cpython-37.pyc │ │ └── models_unstructured.cpython-37.pyc │ ├── mlp.py │ ├── vgg.py │ ├── subfedavg_models.py │ └── resnet.py ├── utils │ ├── __pycache__ │ │ ├── utils.cpython-37.pyc │ │ ├── utils2.cpython-37.pyc │ │ ├── utils3.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── args_lg.cpython-37.pyc │ │ ├── datasets.cpython-37.pyc │ │ ├── args_fedavg.cpython-37.pyc │ │ ├── args_ifca.cpython-37.pyc │ │ ├── args_fednova.cpython-37.pyc │ │ ├── args_fedprox.cpython-37.pyc │ │ ├── args_scaffold.cpython-37.pyc │ │ ├── args_perfedavg.cpython-37.pyc │ │ └── options_cluster.cpython-37.pyc │ ├── __init__.py │ ├── args_fedavg.py │ ├── args_fednova.py │ ├── args_perfedavg.py │ ├── args_lg.py │ ├── args_fedprox.py │ ├── args_ifca.py │ ├── args_scaffold.py │ ├── args_fedrep.py │ └── options_cluster.py ├── client │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── client_cfl.cpython-37.pyc │ │ ├── client_mtl.cpython-37.pyc │ │ ├── client_s.cpython-37.pyc │ │ ├── client_u.cpython-37.pyc │ │ ├── client_apfl.cpython-37.pyc │ │ ├── client_ditto.cpython-37.pyc │ │ ├── client_feddf.cpython-37.pyc │ │ ├── client_fedem.cpython-37.pyc │ │ ├── client_moon.cpython-37.pyc │ │ ├── client_fedavg.cpython-37.pyc │ │ ├── client_fedavg2.cpython-37.pyc │ │ ├── client_fednova.cpython-37.pyc │ │ ├── client_fedprox.cpython-37.pyc │ │ ├── client_fedrep.cpython-37.pyc │ │ ├── client_pfedme.cpython-37.pyc │ │ ├── client_scaffold.cpython-37.pyc │ │ ├── client_scafold.cpython-37.pyc │ │ ├── client_cluster_fl.cpython-37.pyc │ │ ├── client_per_fedavg.cpython-37.pyc │ │ ├── client_subfedavg_s.cpython-37.pyc │ │ └── client_subfedavg_u.cpython-37.pyc │ ├── __init__.py │ ├── client_fedavg.py │ ├── client_cluster_fl.py │ ├── client_mtl.py │ ├── client_fedrep.py │ ├── client_ditto.py │ ├── client_fedprox.py │ ├── client_feddf.py │ ├── client_fednova.py │ ├── client_apfl.py │ ├── client_moon.py │ ├── client_per_fedavg.py │ ├── client_scaffold.py │ ├── client_feddyn.py │ ├── client_pfedme.py │ └── client_cfl.py └── benchmarks │ ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── main_cfl.cpython-37.pyc │ ├── main_lg.cpython-37.pyc │ ├── main_mtl.cpython-37.pyc │ ├── main_apfl.cpython-37.pyc │ ├── main_ditto.cpython-37.pyc │ ├── main_fedavg.cpython-37.pyc │ ├── main_fedbn.cpython-37.pyc │ ├── main_feddf.cpython-37.pyc │ ├── main_fedem.cpython-37.pyc │ ├── main_fedper.cpython-37.pyc │ ├── main_fedrep.cpython-37.pyc │ ├── main_flis.cpython-37.pyc │ ├── main_ifca.cpython-37.pyc │ ├── main_moon.cpython-37.pyc │ ├── main_pacfl.cpython-37.pyc │ ├── main_pfedme.cpython-37.pyc │ ├── main_solo.cpython-37.pyc │ ├── main_fedavg2.cpython-37.pyc │ ├── main_fednova.cpython-37.pyc │ ├── main_fedprox.cpython-37.pyc │ ├── main_scaffold.cpython-37.pyc │ ├── main_scafold.cpython-37.pyc │ ├── main_centralized.cpython-37.pyc │ ├── main_per_fedavg.cpython-37.pyc │ └── main_subfedavg_u.cpython-37.pyc │ └── __init__.py ├── requirements.txt ├── scripts_rci ├── run_all.sh ├── fedprox.sh ├── ifca.sh ├── scaffold.sh ├── test.sh ├── solo.sh ├── comp_acc_gfl.sh ├── comp_acc_pfl.sh ├── fedavg2.sh ├── master.sh ├── train_sample.sh ├── test_sample.sh ├── wtp.sh ├── comm.sh ├── lr_mom.sh ├── num_clients_cpu.sh ├── num_clients.sh ├── tsvsts.sh └── client_samp.sh ├── __pycache__ └── randaugment.cpython-37.pyc ├── setup.py ├── .gitignore ├── LICENSE ├── scripts_v2 ├── alg_gfl.sh ├── alg_feddf.sh └── alg_newcommers.sh └── main.py /src/data/celeba.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=2.2.0 2 | torchvision==0.14.0 3 | numpy==1.19.4 4 | -------------------------------------------------------------------------------- /src/pruning/__init__.py: -------------------------------------------------------------------------------- 1 | from .structured import * 2 | from .unstructured import * -------------------------------------------------------------------------------- /src/sub_fedavg/__init__.py: -------------------------------------------------------------------------------- 1 | from .sub_fedavg_u import * 2 | from .sub_fedavg_s import * -------------------------------------------------------------------------------- /src/clustering/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils_flis import * 2 | from .hierarchical_clustering import * -------------------------------------------------------------------------------- /src/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .datasetzoo import * 2 | from .partition import * 3 | from .utils_data import * -------------------------------------------------------------------------------- /scripts_rci/run_all.sh: -------------------------------------------------------------------------------- 1 | sh fedavg.sh && sh fedprox.sh && sh fednova.sh && sh lg.sh && sh per_fedavg.sh && sh ifca.sh -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .vgg import * 2 | from .resnet import * 3 | from .lenet5 import * 4 | from .moon_models import * -------------------------------------------------------------------------------- /__pycache__/randaugment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/__pycache__/randaugment.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/vgg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/vgg.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/utils2.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/utils2.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/utils3.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/utils3.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/partition.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/partition.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/pmnist_dl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/pmnist_dl.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/lenet5.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/lenet5.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/models.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnet9.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/resnet9.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_lg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_lg.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/datasets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/datasets.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_cfl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_cfl.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_mtl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_mtl.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_s.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_s.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_u.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_u.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/datasetzoo.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/datasetzoo.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/pcifar10_dl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/pcifar10_dl.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/utils_data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/utils_data.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/structured.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/structured.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/vggmodel.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/vggmodel.cpython-37.pyc -------------------------------------------------------------------------------- /src/pruning/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/pruning/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_fedavg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_fedavg.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_ifca.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_ifca.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_cfl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_cfl.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_lg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_lg.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_mtl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_mtl.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_apfl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_apfl.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_ditto.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_ditto.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_feddf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_feddf.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_fedem.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_fedem.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_moon.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_moon.cpython-37.pyc -------------------------------------------------------------------------------- /src/clustering/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/clustering/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/moon_models.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/moon_models.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/resnetcifar.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/resnetcifar.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/unstructured.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/unstructured.cpython-37.pyc -------------------------------------------------------------------------------- /src/pruning/__pycache__/structured.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/pruning/__pycache__/structured.cpython-37.pyc -------------------------------------------------------------------------------- /src/sub_fedavg/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/sub_fedavg/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_fednova.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_fednova.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_fedprox.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_fedprox.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_scaffold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_scaffold.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_apfl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_apfl.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_ditto.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_ditto.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedavg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedavg.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedbn.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedbn.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_feddf.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_feddf.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedem.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedem.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedper.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedper.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedrep.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedrep.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_flis.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_flis.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_ifca.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_ifca.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_moon.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_moon.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_pacfl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_pacfl.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_pfedme.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_pfedme.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_solo.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_solo.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_fedavg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_fedavg.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_fedavg2.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_fedavg2.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_fednova.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_fednova.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_fedprox.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_fedprox.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_fedrep.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_fedrep.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_pfedme.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_pfedme.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_scaffold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_scaffold.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_scafold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_scafold.cpython-37.pyc -------------------------------------------------------------------------------- /src/clustering/__pycache__/cluster_fl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/clustering/__pycache__/cluster_fl.cpython-37.pyc -------------------------------------------------------------------------------- /src/clustering/__pycache__/utils_flis.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/clustering/__pycache__/utils_flis.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/fmnist_superclass.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/fmnist_superclass.cpython-37.pyc -------------------------------------------------------------------------------- /src/pruning/__pycache__/unstructured.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/pruning/__pycache__/unstructured.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/args_perfedavg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/args_perfedavg.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/options_cluster.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/utils/__pycache__/options_cluster.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedavg2.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedavg2.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fednova.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fednova.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_fedprox.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_fedprox.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_scaffold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_scaffold.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_scafold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_scafold.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_cluster_fl.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_cluster_fl.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_per_fedavg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_per_fedavg.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/cifar100_superclass.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/cifar100_superclass.cpython-37.pyc -------------------------------------------------------------------------------- /src/data/__pycache__/cifar10_superclass.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/data/__pycache__/cifar10_superclass.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/models_structured.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/models_structured.cpython-37.pyc -------------------------------------------------------------------------------- /src/sub_fedavg/__pycache__/sub_fedavg_s.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/sub_fedavg/__pycache__/sub_fedavg_s.cpython-37.pyc -------------------------------------------------------------------------------- /src/sub_fedavg/__pycache__/sub_fedavg_u.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/sub_fedavg/__pycache__/sub_fedavg_u.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_centralized.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_centralized.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_per_fedavg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_per_fedavg.cpython-37.pyc -------------------------------------------------------------------------------- /src/benchmarks/__pycache__/main_subfedavg_u.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/benchmarks/__pycache__/main_subfedavg_u.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_subfedavg_s.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_subfedavg_s.cpython-37.pyc -------------------------------------------------------------------------------- /src/client/__pycache__/client_subfedavg_u.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/client/__pycache__/client_subfedavg_u.cpython-37.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/models_unstructured.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/models/__pycache__/models_unstructured.cpython-37.pyc -------------------------------------------------------------------------------- /src/clustering/__pycache__/hierarchical_clustering.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMorafah/FedZoo-Bench/HEAD/src/clustering/__pycache__/hierarchical_clustering.cpython-37.pyc -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .options_cluster import * 2 | from .utils import * 3 | # from .args_fedavg import * 4 | # from .args_fedprox import * 5 | # from .args_fednova import * 6 | # from .args_scaffold import * 7 | # from .args_ifca import * 8 | # from .args_perfedavg import * 9 | # from .args_lg import * 10 | -------------------------------------------------------------------------------- /src/models/mlp.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | # a simple perceptron model for generated 3D data 6 | class PerceptronModel(nn.Module): 7 | def __init__(self, input_dim=3, output_dim=2): 8 | super(PerceptronModel, self).__init__() 9 | 10 | self.fc1 = nn.Linear(input_dim, output_dim) 11 | 12 | def forward(self, x): 13 | 14 | x = self.fc1(x) 15 | return x -------------------------------------------------------------------------------- /src/client/__init__.py: -------------------------------------------------------------------------------- 1 | from .client_fedavg import * 2 | from .client_cluster_fl import * 3 | from .client_fedprox import * 4 | from .client_fednova import * 5 | from .client_scaffold import * 6 | from .client_per_fedavg import * 7 | from .client_pfedme import * 8 | from .client_feddf import * 9 | from .client_cfl import * 10 | from .client_fedrep import * 11 | from .client_fedem import * 12 | from .client_mtl import * 13 | from .client_ditto import * 14 | from .client_apfl import * 15 | from .client_moon import * 16 | from .client_subfedavg_u import * 17 | from .client_subfedavg_s import * -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import setuptools 3 | 4 | with open("README.md", "r", encoding="utf-8") as fh: 5 | long_description = fh.read() 6 | 7 | setuptools.setup( 8 | name='FedZoo-Bench', 9 | version='1.0.0', 10 | author='Mahdi Morafah, Weijia Wang', 11 | author_email='wweijia@eng.ucsd.edu', 12 | description='PyTorch implementation of the state-of-the-art federated learning benchmarks', 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | url='https://github.com/MMorafah/FedZoo-Bench', 16 | license='MIT', 17 | packages=['FedZoo-Bench'], 18 | install_requires=['torch', 'torchvision', 'numpy'], # TODO: specify versions 19 | ) 20 | -------------------------------------------------------------------------------- /src/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | from .main_solo import * 2 | from .main_fedavg import * 3 | from .main_fedprox import * 4 | from .main_fednova import * 5 | from .main_scaffold import * 6 | from .main_lg import * 7 | from .main_ifca import * 8 | from .main_per_fedavg import * 9 | from .main_pfedme import * 10 | from .main_cfl import * 11 | from .main_fedrep import * 12 | from .main_fedper import * 13 | from .main_fedem import * 14 | from .main_fedbn import * 15 | from .main_mtl import * 16 | from .main_feddf import * 17 | from .main_ditto import * 18 | from .main_apfl import * 19 | from .main_moon import * 20 | from .main_subfedavg_u import * 21 | from .main_pacfl import * 22 | from .main_flis import * 23 | from .main_centralized import * 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # http://www.gnu.org/software/automake 2 | 3 | Makefile.in 4 | /ar-lib 5 | /mdate-sh 6 | /py-compile 7 | /test-driver 8 | /ylwrap 9 | 10 | # http://www.gnu.org/software/autoconf 11 | 12 | autom4te.cache 13 | /autoscan.log 14 | /autoscan-*.log 15 | /aclocal.m4 16 | /compile 17 | /config.guess 18 | /config.h.in 19 | /config.log 20 | /config.status 21 | /config.sub 22 | /configure 23 | /configure.scan 24 | /depcomp 25 | /install-sh 26 | /missing 27 | /stamp-h1 28 | 29 | # https://www.gnu.org/software/libtool/ 30 | 31 | /ltmain.sh 32 | 33 | # http://www.gnu.org/software/texinfo 34 | 35 | /texinfo.tex 36 | 37 | # http://www.gnu.org/software/m4/ 38 | 39 | m4/libtool.m4 40 | m4/ltoptions.m4 41 | m4/ltsugar.m4 42 | m4/ltversion.m4 43 | m4/lt~obsolete.m4 44 | 45 | # Generated Makefile 46 | # (meta build system like autotools, 47 | # can automatically generate from config.status script 48 | # (which is called by configure script)) 49 | Makefile 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Mahdi Morafah 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts_rci/fedprox.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedprox_cifar10_superclass_1 11 | #SBATCH --err=results/fedprox_cifar10_superclass_1.err 12 | #SBATCH --out=results/fedprox_cifar10_superclass_1.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for trial in 3 25 | do 26 | python ../main.py \ 27 | --ntrials=3 \ 28 | --rounds=100 \ 29 | --num_users=100 \ 30 | --frac=0.1 \ 31 | --local_ep=10 \ 32 | --local_bs=10 \ 33 | --lr=0.01 \ 34 | --momentum=0.9 \ 35 | --model=resnet9 \ 36 | --dataset=cifar100 \ 37 | --partition='sc_niid_dir' \ 38 | --datadir='../../data/' \ 39 | --logdir='../save_results/' \ 40 | --log_filename=$trial \ 41 | --alg='fedprox' \ 42 | --beta=0.5 \ 43 | --local_view \ 44 | --noise=0 \ 45 | --gpu=0 \ 46 | --print_freq=10 47 | done 48 | -------------------------------------------------------------------------------- /scripts_v2/alg_gfl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=40G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=alg_gfl_cifar10 11 | #SBATCH --err=results/alg_gfl_cifar10.err 12 | #SBATCH --out=results/alg_gfl_cifar10.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in fedavg 25 | do 26 | python ../main.py \ 27 | --ntrials=3 \ 28 | --rounds=100 \ 29 | --num_users=100 \ 30 | --frac=0.1 \ 31 | --local_ep=10 \ 32 | --local_bs=10 \ 33 | --lr=0.01 \ 34 | --momentum=0.9 \ 35 | --model=lenet5 \ 36 | --dataset=cifar10 \ 37 | --partition='niid-labeldir' \ 38 | --datadir='../../data/' \ 39 | --logdir='../results/' \ 40 | --log_filename='lenet5_E10_C0.1_sgd_0.01' \ 41 | --alg=$alg \ 42 | --iid_beta=0.5 \ 43 | --niid_beta=0.5 \ 44 | --local_view \ 45 | --noise=0 \ 46 | --gpu=0 \ 47 | --print_freq=10 48 | done 49 | -------------------------------------------------------------------------------- /scripts_rci/ifca.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=ifca_cifar10_superclass_1 11 | #SBATCH --err=results/ifca_cifar10_superclass_1.err 12 | #SBATCH --out=results/ifca_cifar10_superclass_1.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for trial in 1 25 | do 26 | python ../main.py \ 27 | --ntrials=3 \ 28 | --nclusters=2 \ 29 | --rounds=50 \ 30 | --num_users=25 \ 31 | --frac=0.3 \ 32 | --local_ep=10 \ 33 | --local_bs=10 \ 34 | --lr=0.01 \ 35 | --momentum=0.5 \ 36 | --model=resnet9 \ 37 | --dataset=stl10 \ 38 | --partition='sc_old_niid2' \ 39 | --datadir='../../data/' \ 40 | --logdir='../save_results/' \ 41 | --log_filename=$trial \ 42 | --alg='ifca' \ 43 | --beta=0.5 \ 44 | --local_view \ 45 | --noise=0 \ 46 | --gpu=0 \ 47 | --print_freq=10 48 | done 49 | -------------------------------------------------------------------------------- /scripts_rci/scaffold.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=scafold_cifar10_superclass_1 11 | #SBATCH --err=results/scafold_cifar10_superclass_1.err 12 | #SBATCH --out=results/scafold_cifar10_superclass_1.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for trial in 2 25 | do 26 | python ../main.py \ 27 | --ntrials=3 \ 28 | --rounds=100 \ 29 | --num_users=100 \ 30 | --frac=0.1 \ 31 | --local_ep=10 \ 32 | --local_bs=10 \ 33 | --lr=0.01 \ 34 | --momentum=0.9 \ 35 | --model=resnet9 \ 36 | --dataset=cifar100 \ 37 | --partition='sc_old_niid2' \ 38 | --datadir='../../data/' \ 39 | --logdir='../save_results/' \ 40 | --log_filename=$trial \ 41 | --alg='scaffold' \ 42 | --beta=0.5 \ 43 | --local_view \ 44 | --noise=0 \ 45 | --gpu=0 \ 46 | --print_freq=10 47 | done 48 | -------------------------------------------------------------------------------- /scripts_rci/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=test 11 | #SBATCH --err=results/test.err 12 | #SBATCH --out=results/test.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in pacfl 25 | do 26 | dir='../save_results/'$alg'/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=1 \ 33 | --rounds=20 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=10 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --partition='niid-labelskew' \ 43 | --datadir='../../data/' \ 44 | --logdir='../save_results/' \ 45 | --log_filename='test' \ 46 | --alg=$alg \ 47 | --iid_beta=0.5 \ 48 | --niid_beta=2 \ 49 | --pacfl_beta=14 \ 50 | --noise=0 \ 51 | --gpu=1 \ 52 | --print_freq=10 53 | done 54 | -------------------------------------------------------------------------------- /scripts_v2/alg_feddf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=40G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=alg_feddf_cifar100 11 | #SBATCH --err=results/alg_feddf_cifar100.err 12 | #SBATCH --out=results/alg_feddf_cifar100.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in feddf 25 | do 26 | python ../main.py \ 27 | --ntrials=3 \ 28 | --rounds=100 \ 29 | --num_users=100 \ 30 | --frac=0.1 \ 31 | --local_ep=5 \ 32 | --local_bs=10 \ 33 | --lr=0.01 \ 34 | --momentum=0.9 \ 35 | --model=lenet5 \ 36 | --dataset=cifar10 \ 37 | --partition='niid-labelskew' \ 38 | --datadir='../../data/' \ 39 | --logdir='../results_performance_comp/' \ 40 | --log_filename='lenet5_E10_C0.1_sgd_0.01_5eKL_T3' \ 41 | --alg=$alg \ 42 | --iid_beta=0.5 \ 43 | --niid_beta=8 \ 44 | --distill_T=3 \ 45 | --distill_E=5 \ 46 | --distill_lr=0.00001 \ 47 | --public_dataset='cifar100'\ 48 | --gpu=0 \ 49 | --print_freq=10 50 | done 51 | -------------------------------------------------------------------------------- /scripts_rci/solo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=solo_cifar10_superclass_1 11 | #SBATCH --err=results/solo_cifar10_superclass_1.err 12 | #SBATCH --out=results/solo_cifar10_superclass_1.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for trial in 1 25 | do 26 | # dir='../save_results/solo/cifar10' 27 | # if [ ! -e $dir ]; then 28 | # mkdir -p $dir 29 | # fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=10 \ 34 | --num_users=100 \ 35 | --frac=1.0 \ 36 | --local_ep=10 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.5 \ 40 | --model=simple-cnn \ 41 | --dataset=cifar10 \ 42 | --partition='sc_old_niid2' \ 43 | --datadir='../../data/' \ 44 | --logdir='../save_results/' \ 45 | --log_filename=$trial \ 46 | --alg='solo' \ 47 | --beta=0.5 \ 48 | --local_view \ 49 | --noise=0 \ 50 | --gpu=0 \ 51 | --print_freq=10 52 | done 53 | -------------------------------------------------------------------------------- /scripts_rci/comp_acc_gfl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=comp_acc_gfl 11 | #SBATCH --err=results/comp_acc_gfl.err 12 | #SBATCH --out=results/comp_acc_gfl.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in 'fedavg' 'fedprox' 'fednova' 'scaffold' 25 | do 26 | dir='../save_results/fedavg/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=100 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=5 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --partition='niid-labeldir' \ 43 | --datadir='../../data/' \ 44 | --logdir='../results_acc/gfl/' \ 45 | --log_filename=$alg'_epoch5' \ 46 | --alg=$alg \ 47 | --iid_beta=0.5 \ 48 | --niid_beta=0.5 \ 49 | --local_view \ 50 | --noise=0 \ 51 | --gpu=0 \ 52 | --print_freq=10 53 | done -------------------------------------------------------------------------------- /scripts_rci/comp_acc_pfl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=comp_acc_pfl 11 | #SBATCH --err=results/comp_acc_pfl.err 12 | #SBATCH --out=results/comp_acc_pfl.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in 'fedavg' 'lg' 'perfedavg' 'ifca' 'pfedme' 25 | do 26 | dir='../save_results/fedavg/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=100 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=10 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --partition='niid-labeldir' \ 43 | --datadir='../../data/' \ 44 | --logdir='../results_acc/pfl/' \ 45 | --log_filename=$alg'_epoch10' \ 46 | --alg=$alg \ 47 | --iid_beta=0.5 \ 48 | --niid_beta=0.1 \ 49 | --local_view \ 50 | --noise=0 \ 51 | --gpu=0 \ 52 | --print_freq=10 53 | done -------------------------------------------------------------------------------- /scripts_rci/fedavg2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_superclass_1 11 | #SBATCH --err=results/fedavg_cifar10_superclass_1.err 12 | #SBATCH --out=results/fedavg_cifar10_superclass_1.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for trial in 3 25 | do 26 | dir='../save_results/fedavg/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=20 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=10 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --partition='niid-labeldir' \ 43 | --datadir='../../data/' \ 44 | --logdir='../save_results/' \ 45 | --log_filename=$trial \ 46 | --alg='fedavg' \ 47 | --iid_beta=0.5 \ 48 | --niid_beta=0.5 \ 49 | --local_view \ 50 | --noise=0 \ 51 | --gpu=0 \ 52 | --print_freq=10 53 | done 54 | -------------------------------------------------------------------------------- /scripts_v2/alg_newcommers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=40G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=alg_newcomers_cifar10 11 | #SBATCH --err=results/alg_newcomers_cifar10.err 12 | #SBATCH --out=results/alg_newcomers_cifar10.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in apfl 25 | do 26 | python ../main.py \ 27 | --ntrials=3 \ 28 | --rounds=100 \ 29 | --num_users=20 \ 30 | --frac=0.2 \ 31 | --local_ep=10 \ 32 | --local_bs=10 \ 33 | --lr=0.01 \ 34 | --momentum=0.9 \ 35 | --model=resnet9 \ 36 | --dataset=cifar100 \ 37 | --partition='niid-labeldir' \ 38 | --datadir='../../data/' \ 39 | --logdir='../results_newcomers_comp/' \ 40 | --log_filename='resnet9_E10_C0.2_sgd_0.01' \ 41 | --alg=$alg \ 42 | --iid_beta=0.5 \ 43 | --niid_beta=0.1 \ 44 | --new_comer \ 45 | --ft_epoch=15 \ 46 | --pruning_percent_subfedavg=5 \ 47 | --pruning_target_subfedavg=50 \ 48 | --dist_thresh_subfedavg=0.0001 \ 49 | --acc_thresh_subfedavg=55 \ 50 | --gpu=0 \ 51 | --print_freq=10 52 | done 53 | -------------------------------------------------------------------------------- /scripts_rci/master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=solo_cifar10_wtp_labelskew 11 | #SBATCH --err=results/solo_cifar10_wtp_labelskew.err 12 | #SBATCH --out=results/solo_cifar10_wtp_labelskew.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for alg in fedavg fedprox fednova scaffold ifca lg per_fedavg fedrep fedper fedem fedbn solo 25 | do 26 | dir='../save_results/fedavg/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=100 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=10 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --partition='niid-labelskew' \ 43 | --datadir='../../data/' \ 44 | --logdir='../results_master/' \ 45 | --log_filename='niid_labelskew_2' \ 46 | --alg=$alg \ 47 | --iid_beta=0.5 \ 48 | --niid_beta=2 \ 49 | --local_view \ 50 | --noise=0 \ 51 | --gpu=0 \ 52 | --print_freq=10 53 | done -------------------------------------------------------------------------------- /scripts_rci/train_sample.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_test_sample_labeldir 11 | #SBATCH --err=results/fedavg_cifar10_test_sample_labeldir.err 12 | #SBATCH --out=results/fedavg_cifar10_test_sample_labeldir.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for test in 1.0 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2 0.1 0.05 25 | do 26 | dir='../save_results/fedavg/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=100 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=10 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --p_train=1.0 \ 43 | --p_test=$test \ 44 | --partition='niid-labeldir' \ 45 | --datadir='../../data/' \ 46 | --logdir='../results_test_sample/' \ 47 | --log_filename='ptest_'$test \ 48 | --alg='fedavg' \ 49 | --iid_beta=0.5 \ 50 | --niid_beta=0.1 \ 51 | --local_view \ 52 | --noise=0 \ 53 | --gpu=0 \ 54 | --print_freq=10 55 | done -------------------------------------------------------------------------------- /scripts_rci/test_sample.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_test_sample_labeldir 11 | #SBATCH --err=results/fedavg_cifar10_test_sample_labeldir.err 12 | #SBATCH --out=results/fedavg_cifar10_test_sample_labeldir.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for test in 1.0 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2 0.1 0.05 25 | do 26 | dir='../save_results/fedavg/cifar10' 27 | if [ ! -e $dir ]; then 28 | mkdir -p $dir 29 | fi 30 | 31 | python ../main.py \ 32 | --ntrials=3 \ 33 | --rounds=100 \ 34 | --num_users=100 \ 35 | --frac=0.1 \ 36 | --local_ep=5 \ 37 | --local_bs=10 \ 38 | --lr=0.01 \ 39 | --momentum=0.9 \ 40 | --model=lenet5 \ 41 | --dataset=cifar10 \ 42 | --p_train=1.0 \ 43 | --p_test=$test \ 44 | --partition='niid-labeldir' \ 45 | --datadir='../../data/' \ 46 | --logdir='../results_test_sample/' \ 47 | --log_filename='epoch_5_ptest_'$test \ 48 | --alg='fedavg' \ 49 | --iid_beta=0.5 \ 50 | --niid_beta=0.1 \ 51 | --local_view \ 52 | --noise=0 \ 53 | --gpu=0 \ 54 | --print_freq=10 55 | done 56 | -------------------------------------------------------------------------------- /scripts_rci/wtp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_wtp_niid_labelskew 11 | #SBATCH --err=results/fedavg_cifar10_wtp_niid_labelskew.err 12 | #SBATCH --out=results/fedavg_cifar10_wtp_niid_labelskew.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for epoch in 20 25 | do 26 | for skew in 5 6 7 8 9 27 | do 28 | dir='../save_results/fedavg/cifar10' 29 | if [ ! -e $dir ]; then 30 | mkdir -p $dir 31 | fi 32 | 33 | python ../main.py \ 34 | --ntrials=3 \ 35 | --rounds=100 \ 36 | --num_users=100 \ 37 | --frac=0.1 \ 38 | --local_ep=$epoch \ 39 | --local_bs=10 \ 40 | --lr=0.01 \ 41 | --momentum=0.9 \ 42 | --model=lenet5 \ 43 | --dataset=cifar10 \ 44 | --partition='niid-labelskew' \ 45 | --datadir='../../data/' \ 46 | --logdir='../results_wtp/' \ 47 | --log_filename='epoch_'$epoch \ 48 | --alg='fedavg' \ 49 | --iid_beta=0.5 \ 50 | --niid_beta=$skew \ 51 | --local_view \ 52 | --noise=0 \ 53 | --gpu=0 \ 54 | --print_freq=10 55 | done 56 | done 57 | -------------------------------------------------------------------------------- /scripts_rci/comm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_comm_niid_labeldir 11 | #SBATCH --err=results/fedavg_cifar10_comm_niid_labeldir.err 12 | #SBATCH --out=results/fedavg_cifar10_comm_niid_labeldir.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for epoch in 20 25 | do 26 | for round in 500 250 27 | do 28 | dir='../save_results/fedavg/cifar10' 29 | if [ ! -e $dir ]; then 30 | mkdir -p $dir 31 | fi 32 | 33 | python ../main.py \ 34 | --ntrials=3 \ 35 | --rounds=$round \ 36 | --num_users=100 \ 37 | --frac=0.1 \ 38 | --local_ep=$epoch \ 39 | --local_bs=10 \ 40 | --lr=0.01 \ 41 | --momentum=0.9 \ 42 | --model=lenet5 \ 43 | --dataset=cifar10 \ 44 | --partition='niid-labelskew' \ 45 | --datadir='../../data/' \ 46 | --logdir='../results_comm/' \ 47 | --log_filename='epoch_'$epoch'_comm_'$round \ 48 | --alg='fedavg' \ 49 | --iid_beta=0.5 \ 50 | --niid_beta=3 \ 51 | --local_view \ 52 | --noise=0 \ 53 | --gpu=0 \ 54 | --print_freq=10 55 | done 56 | done 57 | -------------------------------------------------------------------------------- /scripts_rci/lr_mom.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_lr_mom_niid_labeldir 11 | #SBATCH --err=results/fedavg_cifar10_lr_mom_niid_labeldir.err 12 | #SBATCH --out=results/fedavg_cifar10_lr_mom_niid_labeldir.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for epoch in 1 5 25 | do 26 | for lr in 0.1 0.05 0.005 0.001 27 | do 28 | dir='../save_results/fedavg/cifar10' 29 | if [ ! -e $dir ]; then 30 | mkdir -p $dir 31 | fi 32 | 33 | python ../main.py \ 34 | --ntrials=3 \ 35 | --rounds=100 \ 36 | --num_users=100 \ 37 | --frac=0.1 \ 38 | --local_ep=$epoch \ 39 | --local_bs=10 \ 40 | --lr=$lr \ 41 | --momentum=0.5 \ 42 | --model=lenet5 \ 43 | --dataset=cifar10 \ 44 | --partition='niid-labeldir' \ 45 | --datadir='../../data/' \ 46 | --logdir='../results_lr/m0.5/' \ 47 | --log_filename='epoch_'$epoch'_lr_'$lr'_comm_100' \ 48 | --alg='fedavg' \ 49 | --iid_beta=0.5 \ 50 | --niid_beta=0.1 \ 51 | --local_view \ 52 | --noise=0 \ 53 | --gpu=0 \ 54 | --print_freq=10 55 | done 56 | done 57 | -------------------------------------------------------------------------------- /scripts_rci/num_clients_cpu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=40G 8 | #SBATCH --partition=cpulong 9 | #SBATCH --job-name=fedavg_cifar10_nclients_niid_labeldir 10 | #SBATCH --err=results/fedavg_cifar10_nclients_niid_labeldir.err 11 | #SBATCH --out=results/fedavg_cifar10_nclients_niid_labeldir.out 12 | 13 | #DIR = "save_async/cifar" 14 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 15 | 16 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 17 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 18 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 19 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 20 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 21 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 22 | 23 | for epoch in 10 24 | do 25 | for clients in 50 26 | do 27 | dir='../save_results/fedavg/cifar10' 28 | if [ ! -e $dir ]; then 29 | mkdir -p $dir 30 | fi 31 | 32 | python ../main.py \ 33 | --ntrials=1 \ 34 | --rounds=100 \ 35 | --num_users=$clients \ 36 | --frac=0.25 \ 37 | --local_ep=$epoch \ 38 | --local_bs=10 \ 39 | --lr=0.01 \ 40 | --momentum=0.9 \ 41 | --model=lenet5 \ 42 | --dataset=cifar10 \ 43 | --partition='niid-labelskew' \ 44 | --datadir='../../data/' \ 45 | --logdir='../results_num_clients/adjusted_frac/' \ 46 | --log_filename='epoch_'$epoch'_clients_'$clients \ 47 | --alg='fedavg' \ 48 | --iid_beta=0.5 \ 49 | --niid_beta=3 \ 50 | --local_view \ 51 | --noise=0 \ 52 | --gpu=-1 \ 53 | --print_freq=10 54 | done 55 | done 56 | -------------------------------------------------------------------------------- /scripts_rci/num_clients.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=40G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_nclients_niid_labeldir 11 | #SBATCH --err=results/fedavg_cifar10_nclients_niid_labeldir.err 12 | #SBATCH --out=results/fedavg_cifar10_nclients_niid_labeldir.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for epoch in 1 25 | do 26 | for clients in 500 27 | do 28 | dir='../save_results/fedavg/cifar10' 29 | if [ ! -e $dir ]; then 30 | mkdir -p $dir 31 | fi 32 | 33 | python ../main.py \ 34 | --ntrials=1 \ 35 | --rounds=100 \ 36 | --num_users=$clients \ 37 | --frac=0.03 \ 38 | --local_ep=$epoch \ 39 | --local_bs=10 \ 40 | --lr=0.01 \ 41 | --momentum=0.9 \ 42 | --model=lenet5 \ 43 | --dataset=cifar10 \ 44 | --partition='niid-labeldir' \ 45 | --datadir='../../data/' \ 46 | --logdir='../results_num_clients/adjusted_frac/' \ 47 | --log_filename='epoch_'$epoch'_clients_'$clients \ 48 | --alg='fedavg' \ 49 | --iid_beta=0.5 \ 50 | --niid_beta=0.15 \ 51 | --local_view \ 52 | --noise=0 \ 53 | --gpu=0 \ 54 | --print_freq=10 55 | done 56 | done 57 | -------------------------------------------------------------------------------- /scripts_rci/tsvsts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=1 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_tsvsts_labeldir 11 | #SBATCH --err=results/fedavg_cifar10_tsvsts_labeldir.err 12 | #SBATCH --out=results/fedavg_cifar10_tsvsts_labeldir.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | 24 | for test in 1.0 0.8 0.5 0.3 0.1 0.05 25 | do 26 | for train in 0.3 0.1 0.05 27 | do 28 | dir='../save_results/fedavg/cifar10' 29 | if [ ! -e $dir ]; then 30 | mkdir -p $dir 31 | fi 32 | 33 | python ../main.py \ 34 | --ntrials=3 \ 35 | --rounds=100 \ 36 | --num_users=100 \ 37 | --frac=0.1 \ 38 | --local_ep=10 \ 39 | --local_bs=10 \ 40 | --lr=0.01 \ 41 | --momentum=0.9 \ 42 | --model=lenet5 \ 43 | --dataset=cifar10 \ 44 | --p_train=$train \ 45 | --p_test=$test \ 46 | --partition='niid-labeldir' \ 47 | --datadir='../../data/' \ 48 | --logdir='../results_train_test_samples/' \ 49 | --log_filename='epoch_10_ptrain'$train'_ptest_'$test \ 50 | --alg='fedavg' \ 51 | --iid_beta=0.5 \ 52 | --niid_beta=0.1 \ 53 | --local_view \ 54 | --noise=0 \ 55 | --gpu=0 \ 56 | --print_freq=10 57 | done 58 | done 59 | -------------------------------------------------------------------------------- /scripts_rci/client_samp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #SBATCH --time=70:00:00 3 | 4 | #SBATCH --nodes=1 5 | #SBATCH --ntasks-per-node=4 6 | #SBATCH --cpus-per-task=1 7 | #SBATCH --mem=50G 8 | #SBATCH --partition=gpulong 9 | #SBATCH --gres=gpu:1 10 | #SBATCH --job-name=fedavg_cifar10_client_samp_niid_labelskew 11 | #SBATCH --err=results/fedavg_cifar10_client_samp_niid_labelskew.err 12 | #SBATCH --out=results/fedavg_cifar10_client_samp_niid_labelskew.out 13 | 14 | #DIR = "save_async/cifar" 15 | #[ ! -d "$DIR" ] && mkdir -p "$DIR" 16 | 17 | ml TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4 18 | ml matplotlib/3.1.1-fosscuda-2019b-Python-3.7.4 19 | ml SciPy-bundle/2019.10-fosscuda-2019b-Python-3.7.4 20 | ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 21 | ml torchvision/0.9.1-fosscuda-2019b-PyTorch-1.8.0 22 | ml scikit-learn/0.21.3-fosscuda-2019b-Python-3.7.4 23 | #ml matplotlib/3.4.2-foss-2021a 24 | #ml SciPy-bundle/2021.10-foss-2021b 25 | #ml PyTorch/1.8.0-fosscuda-2019b-Python-3.7.4 26 | #ml torchvision/0.10.0-fosscuda-2020b-PyTorch-1.9.0 27 | #ml scikit-learn/0.24.2-foss-2021a 28 | 29 | for epoch in 20 30 | do 31 | for frac in 1.0 32 | do 33 | dir='../save_results/fedavg/cifar10' 34 | if [ ! -e $dir ]; then 35 | mkdir -p $dir 36 | fi 37 | 38 | python ../main.py \ 39 | --ntrials=1 \ 40 | --rounds=100 \ 41 | --num_users=100 \ 42 | --frac=$frac \ 43 | --local_ep=$epoch \ 44 | --local_bs=10 \ 45 | --lr=0.001 \ 46 | --momentum=0.9 \ 47 | --model=lenet5 \ 48 | --dataset=cifar10 \ 49 | --partition='niid-labeldir' \ 50 | --datadir='../../data/' \ 51 | --logdir='../results_client_samp/' \ 52 | --log_filename='niid_labeldir_epoch_'$epoch'sample_rate_'$frac \ 53 | --alg='fedavg' \ 54 | --iid_beta=0.5 \ 55 | --niid_beta=0.1 \ 56 | --local_view \ 57 | --noise=0 \ 58 | --gpu=0 \ 59 | --print_freq=10 60 | done 61 | done 62 | -------------------------------------------------------------------------------- /src/utils/args_fedavg.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_fedavg(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 15 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 16 | parser.add_argument('--dataset', type=str, default='cifar10', 17 | help="name of dataset: mnist, cifar10, cifar100") 18 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 19 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 20 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 21 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 22 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 23 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 24 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 25 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 26 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 27 | parser.add_argument('--is_print', action='store_true', help='verbose print') 28 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 29 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 30 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 31 | 32 | args = parser.parse_args() 33 | return args 34 | -------------------------------------------------------------------------------- /src/utils/args_fednova.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_fednova(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 15 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 16 | parser.add_argument('--dataset', type=str, default='cifar10', 17 | help="name of dataset: mnist, cifar10, cifar100") 18 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 19 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 20 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 21 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 22 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 23 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 24 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 25 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 26 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 27 | parser.add_argument('--is_print', action='store_true', help='verbose print') 28 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 29 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 30 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 31 | 32 | args = parser.parse_args() 33 | return args 34 | -------------------------------------------------------------------------------- /src/utils/args_perfedavg.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_perfedavg(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 15 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 16 | parser.add_argument('--dataset', type=str, default='cifar10', 17 | help="name of dataset: mnist, cifar10, cifar100") 18 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 19 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 20 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 21 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 22 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 23 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 24 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 25 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 26 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 27 | parser.add_argument('--is_print', action='store_true', help='verbose print') 28 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 29 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 30 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 31 | 32 | args = parser.parse_args() 33 | return args 34 | -------------------------------------------------------------------------------- /src/utils/args_lg.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_lg(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--mu', type=float, default=0.001, help="FedProx Regularizer") 15 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 16 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 17 | parser.add_argument('--dataset', type=str, default='cifar10', 18 | help="name of dataset: mnist, cifar10, cifar100") 19 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 20 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 21 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 22 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 23 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 24 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 25 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 26 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 27 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 28 | parser.add_argument('--is_print', action='store_true', help='verbose print') 29 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 30 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 31 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 32 | 33 | args = parser.parse_args() 34 | return args 35 | -------------------------------------------------------------------------------- /src/utils/args_fedprox.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_fedprox(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--mu', type=float, default=0.001, help="FedProx Regularizer") 15 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 16 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 17 | parser.add_argument('--dataset', type=str, default='cifar10', 18 | help="name of dataset: mnist, cifar10, cifar100") 19 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 20 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 21 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 22 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 23 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 24 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 25 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 26 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 27 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 28 | parser.add_argument('--is_print', action='store_true', help='verbose print') 29 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 30 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 31 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 32 | 33 | args = parser.parse_args() 34 | return args 35 | -------------------------------------------------------------------------------- /src/utils/args_ifca.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_ifca(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--nclusters', type=int, default=3, help="Number of Clusters for IFCA") 15 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 16 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 17 | parser.add_argument('--dataset', type=str, default='cifar10', 18 | help="name of dataset: mnist, cifar10, cifar100") 19 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 20 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 21 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 22 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 23 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 24 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 25 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 26 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 27 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 28 | parser.add_argument('--is_print', action='store_true', help='verbose print') 29 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 30 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 31 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 32 | 33 | args = parser.parse_args() 34 | return args 35 | -------------------------------------------------------------------------------- /src/utils/args_scaffold.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_scaffold(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 10 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 11 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 12 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 13 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 14 | parser.add_argument('--mu', type=float, default=0.001, help="FedProx Regularizer") 15 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 16 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 17 | parser.add_argument('--dataset', type=str, default='cifar10', 18 | help="name of dataset: mnist, cifar10, cifar100") 19 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 20 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 21 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 22 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 23 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 24 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 25 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 26 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 27 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 28 | parser.add_argument('--is_print', action='store_true', help='verbose print') 29 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 30 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 31 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 32 | 33 | args = parser.parse_args() 34 | return args 35 | -------------------------------------------------------------------------------- /src/utils/args_fedrep.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | def args_parser_fedavg(): 4 | parser = argparse.ArgumentParser() 5 | # federated arguments 6 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 7 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 8 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 9 | parser.add_argument('--local_ep', type=int, default=10, help="the number of total local epochs: E") 10 | parser.add_argument('--head_ep', type=int, default=5, help="the number of local updates for the head") 11 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 12 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 13 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 14 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 15 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 16 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 17 | parser.add_argument('--dataset', type=str, default='cifar10', 18 | help="name of dataset: mnist, cifar10, cifar100") 19 | parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 20 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 21 | parser.add_argument('--partition', type=str, default='noniid-#label2', help='method of partitioning') 22 | parser.add_argument('--alg', type=str, default='fedavg', help='Algorithm') 23 | parser.add_argument('--noise', type=float, default=0, help='how much noise we add to some party') 24 | parser.add_argument('--noise_type', type=str, default='level', help='Different level of noise or different space of noise') 25 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 26 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 27 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 28 | parser.add_argument('--is_print', action='store_true', help='verbose print') 29 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 30 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 31 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 32 | 33 | args = parser.parse_args() 34 | return args 35 | -------------------------------------------------------------------------------- /src/sub_fedavg/sub_fedavg_u.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | 5 | def Sub_FedAVG_U(w_server, w_clients, masks, weight_avg=None): 6 | ''' 7 | This function performs Sub-FedAvg-U (for unstructured pruning) as stated in the paper. 8 | This function updates the server model based on Sub-FedAvg. It is called at the end of each round. 9 | 10 | :param w_server: server model's state_dict 11 | :param w_clients: list of clients' model state_dict to be averaged 12 | :param masks: list of clients' pruning masks to be averaged 13 | 14 | :return w_server: updated server model's state_dict 15 | ''' 16 | 17 | if weight_avg == None: 18 | weight_avg = [1/len(w_clients) for i in range(len(w_clients))] 19 | 20 | step = 0 21 | for name in w_server.keys(): 22 | 23 | if 'weight' in name: 24 | 25 | weight_dev = w_server[name].device 26 | 27 | indices = [] 28 | count = np.zeros_like(masks[0][step].reshape([-1])) 29 | avg = np.zeros_like(w_server[name].data.cpu().numpy().reshape([-1])) 30 | for i in range(len(masks)): 31 | count += masks[i][step].reshape([-1]) 32 | avg += w_clients[i][name].data.cpu().numpy().reshape([-1])*weight_avg[i] 33 | 34 | #final_avg = np.divide(avg, count) 35 | final_avg = avg 36 | ind = np.isfinite(final_avg) 37 | 38 | temp_server = w_server[name].data.cpu().numpy().reshape([-1]) 39 | temp_server[ind] = final_avg[ind] 40 | 41 | #print(f'Name: {name}, NAN: {np.mean(np.isnan(temp_server))}, INF: {np.mean(np.isinf(temp_server))}') 42 | 43 | shape = w_server[name].data.cpu().numpy().shape 44 | w_server[name].data = torch.from_numpy(temp_server.reshape(shape)).to(weight_dev) 45 | 46 | step += 1 47 | else: 48 | 49 | avg = np.zeros_like(w_server[name].data.cpu().numpy().reshape([-1])) 50 | for i in range(len(masks)): 51 | arr = w_clients[i][name].data.cpu().numpy().reshape([-1])*weight_avg[i] 52 | avg = np.add(avg, arr, out=avg, casting="unsafe") 53 | #avg += w_clients[i][name].data.cpu().numpy().reshape([-1])*weight_avg[i] 54 | #avg /= len(masks) 55 | 56 | #print(f'Name: {name}, NAN: {np.mean(np.isnan(avg))}, INF: {np.mean(np.isinf(avg))}') 57 | weight_dev = w_server[name].device 58 | shape = w_server[name].data.cpu().numpy().shape 59 | w_server[name].data = torch.from_numpy(avg.reshape(shape)).to(weight_dev) 60 | 61 | return w_server 62 | 63 | 64 | def Sub_FedAvg_U_initial(mask, model, w_server): 65 | ''' 66 | This function initializes each client's subnetwork by the server's model at the begining of each round. 67 | It is called at the begining of each round 68 | 69 | :param mask: pruning mask of the client receiving the initial from the server 70 | :param model: client model 71 | :param w_server: server model's state_dict 72 | 73 | :return updated client model's state_dict: 74 | ''' 75 | model.load_state_dict(w_server) 76 | step = 0 77 | for name, param in model.named_parameters(): 78 | if "weight" in name: 79 | weight_dev = param.device 80 | param.data = torch.from_numpy(mask[step] * w_server[name].cpu().numpy()).to(weight_dev) 81 | step = step + 1 82 | if "bias" in name: 83 | param.data = w_server[name] 84 | return model.state_dict() 85 | -------------------------------------------------------------------------------- /src/models/vgg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch.nn as nn 4 | import torch.nn.init as init 5 | 6 | __all__ = [ 7 | 'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', 8 | 'vgg19_bn', 'vgg19', 9 | ] 10 | 11 | 12 | class VGG(nn.Module): 13 | ''' 14 | VGG model 15 | ''' 16 | def __init__(self, features, num_classes=10): 17 | super(VGG, self).__init__() 18 | self.features = features 19 | self.classifier = nn.Sequential( 20 | nn.Dropout(), 21 | nn.Linear(512, 512), 22 | nn.ReLU(True), 23 | nn.Dropout(), 24 | nn.Linear(512, 512), 25 | nn.ReLU(True), 26 | nn.Linear(512, num_classes), 27 | ) 28 | # Initialize weights 29 | for m in self.modules(): 30 | if isinstance(m, nn.Conv2d): 31 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 32 | m.weight.data.normal_(0, np.sqrt(2. / n)) 33 | m.bias.data.zero_() 34 | 35 | 36 | def forward(self, x): 37 | x = self.features(x) 38 | x = x.view(x.size(0), -1) 39 | x = self.classifier(x) 40 | return x 41 | 42 | 43 | def make_layers(cfg, batch_norm=False): 44 | layers = [] 45 | in_channels = 3 46 | for v in cfg: 47 | if v == 'M': 48 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)] 49 | else: 50 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) 51 | if batch_norm: 52 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] 53 | else: 54 | layers += [conv2d, nn.ReLU(inplace=True)] 55 | in_channels = v 56 | return nn.Sequential(*layers) 57 | 58 | 59 | cfg = { 60 | 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 61 | 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 62 | 'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 63 | 'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 64 | 512, 512, 512, 512, 'M'], 65 | 'F': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M'], 66 | } 67 | 68 | def vgg9(num_classes=10): 69 | """VGG 9-layer model (configuration "A")""" 70 | return VGG(make_layers(cfg['F']), num_classes) 71 | 72 | def vgg11(num_classes=10): 73 | """VGG 11-layer model (configuration "A")""" 74 | return VGG(make_layers(cfg['A']), num_classes) 75 | 76 | 77 | def vgg11_bn(num_classes=10): 78 | """VGG 11-layer model (configuration "A") with batch normalization""" 79 | return VGG(make_layers(cfg['A'], batch_norm=True), num_classes) 80 | 81 | 82 | def vgg13(num_classes=10): 83 | """VGG 13-layer model (configuration "B")""" 84 | return VGG(make_layers(cfg['B']), num_classes) 85 | 86 | 87 | def vgg13_bn(num_classes=10): 88 | """VGG 13-layer model (configuration "B") with batch normalization""" 89 | return VGG(make_layers(cfg['B'], batch_norm=True), num_classes) 90 | 91 | 92 | def vgg16(num_classes=10): 93 | """VGG 16-layer model (configuration "D")""" 94 | return VGG(make_layers(cfg['D']), num_classes) 95 | 96 | 97 | def vgg16_bn(num_classes=10): 98 | """VGG 16-layer model (configuration "D") with batch normalization""" 99 | return VGG(make_layers(cfg['D'], batch_norm=True), num_classes) 100 | 101 | 102 | def vgg19(num_classes=10): 103 | """VGG 19-layer model (configuration "E")""" 104 | return VGG(make_layers(cfg['E']), num_classes) 105 | 106 | 107 | def vgg19_bn(num_classes=10): 108 | """VGG 19-layer model (configuration 'E') with batch normalization""" 109 | return VGG(make_layers(cfg['E'], batch_norm=True), num_classes) 110 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import copy 4 | import os 5 | import gc 6 | import pickle 7 | import time 8 | import sys 9 | import datetime 10 | 11 | import torch 12 | from torch import nn 13 | import torch.nn.functional as F 14 | from torch.utils.data import DataLoader, Dataset 15 | from torchvision import datasets, transforms 16 | 17 | from src.data import * 18 | from src.models import * 19 | from src.client import * 20 | from src.clustering import * 21 | from src.utils import * 22 | from src.benchmarks import * 23 | 24 | if __name__ == '__main__': 25 | print('-'*40) 26 | 27 | args = args_parser() 28 | if args.gpu == -1: 29 | args.device = torch.device('cpu') 30 | else: 31 | args.device = torch.device('cuda:{}'.format(args.gpu) if torch.cuda.is_available() else 'cpu') 32 | torch.cuda.set_device(args.gpu) ## Setting cuda on GPU 33 | 34 | args.path = args.logdir + args.alg +'/' + args.dataset + '/' + args.partition + '/' 35 | if args.partition != 'iid': 36 | if args.partition == 'iid_qskew': 37 | args.path = args.path + str(args.iid_beta) + '/' 38 | else: 39 | if args.niid_beta.is_integer(): 40 | args.path = args.path + str(int(args.niid_beta)) + '/' 41 | else: 42 | args.path = args.path + str(args.niid_beta) + '/' 43 | 44 | mkdirs(args.path) 45 | 46 | if args.log_filename is None: 47 | filename='logs_%s.txt' % datetime.datetime.now().strftime("%Y_%m_%d_%H_%M") 48 | else: 49 | filename='logs_'+args.log_filename+'.txt' 50 | 51 | sys.stdout = Logger(fname=args.path+filename) 52 | 53 | fname=args.path+filename 54 | fname=fname[0:-4] 55 | if args.alg == 'solo': 56 | alg_name = 'SOLO' 57 | run_solo(args, fname=fname) 58 | elif args.alg == 'fedavg': 59 | alg_name = 'FedAvg' 60 | run_fedavg(args, fname=fname) 61 | elif args.alg == 'fedprox': 62 | alg_name = 'FedProx' 63 | run_fedprox(args, fname=fname) 64 | elif args.alg == 'fednova': 65 | alg_name = 'FedNova' 66 | run_fednova(args, fname=fname) 67 | elif args.alg == 'scaffold': 68 | alg_name = 'Scaffold' 69 | run_scaffold(args, fname=fname) 70 | elif args.alg == 'lg': 71 | alg_name = 'LG' 72 | run_lg(args, fname=fname) 73 | elif args.alg == 'perfedavg': 74 | alg_name = 'Per-FedAvg' 75 | run_per_fedavg(args, fname=fname) 76 | elif args.alg == 'pfedme': 77 | alg_name = 'pFedMe' 78 | run_pfedme(args, fname=fname) 79 | elif args.alg == 'ifca': 80 | alg_name = 'IFCA' 81 | run_ifca(args, fname=fname) 82 | elif args.alg == 'cfl': 83 | alg_name = 'CFL' 84 | run_cfl(args, fname=fname) 85 | elif args.alg == 'fedrep': 86 | alg_name = 'FedRep' 87 | run_fedrep(args, fname=fname) 88 | elif args.alg == 'fedper': 89 | alg_name = 'FedPer' 90 | run_fedper(args, fname=fname) 91 | elif args.alg == 'feddf': 92 | alg_name = 'FedDF' 93 | run_feddf(args, fname=fname) 94 | elif args.alg == 'ditto': 95 | alg_name = 'Ditto' 96 | run_ditto(args, fname=fname) 97 | elif args.alg == 'fedem': 98 | alg_name = 'FedEM' 99 | run_fedem(args, fname=fname) 100 | elif args.alg == 'fedbn': 101 | alg_name = 'FedBN' 102 | run_fedbn(args, fname=fname) 103 | elif args.alg == 'mtl': 104 | alg_name = 'MTL' 105 | run_mtl(args, fname=fname) 106 | elif args.alg == 'apfl': 107 | alg_name = 'APFL' 108 | run_apfl(args, fname=fname) 109 | elif args.alg == 'moon': 110 | alg_name = 'MOON' 111 | run_moon(args, fname=fname) 112 | elif args.alg == 'subfedavg_u': 113 | alg_name = 'SubFedAvg_U' 114 | run_subfedavg_u(args, fname=fname) 115 | elif args.alg == 'pacfl': 116 | alg_name = 'PACFL' 117 | run_pacfl(args, fname=fname) 118 | elif args.alg == 'flis': 119 | alg_name = 'FLIS' 120 | run_flis(args, fname=fname) 121 | elif args.alg == 'centralized': 122 | alg_name = 'Centralized' 123 | run_centralized(args, fname=fname) 124 | else: 125 | print('Algorithm Does Not Exist') 126 | sys.exit() 127 | -------------------------------------------------------------------------------- /src/clustering/hierarchical_clustering.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from typing import Iterable 3 | import copy 4 | 5 | def flatten(items): 6 | """Yield items from any nested iterable; see Reference.""" 7 | for x in items: 8 | if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): 9 | for sub_x in flatten(x): 10 | yield sub_x 11 | else: 12 | yield x 13 | 14 | 15 | def calculating_adjacency(clients_idxs, U): 16 | 17 | nclients = len(clients_idxs) 18 | 19 | sim_mat = np.zeros([nclients, nclients]) 20 | for idx1 in range(nclients): 21 | for idx2 in range(nclients): 22 | #print(idx1) 23 | #print(U) 24 | #print(idx1) 25 | U1 = copy.deepcopy(U[clients_idxs[idx1]]) 26 | U2 = copy.deepcopy(U[clients_idxs[idx2]]) 27 | 28 | #sim_mat[idx1,idx2] = np.where(np.abs(U1.T@U2) > 1e-2)[0].shape[0] 29 | #sim_mat[idx1,idx2] = 10*np.linalg.norm(U1.T@U2 - np.eye(15), ord='fro') 30 | #sim_mat[idx1,idx2] = 100/np.pi*(np.sort(np.arccos(U1.T@U2).reshape(-1))[0:4]).sum() 31 | mul = np.clip(U1.T@U2 ,a_min =-1.0, a_max=1.0) 32 | sim_mat[idx1,idx2] = np.min(np.arccos(mul))*180/np.pi 33 | 34 | return sim_mat 35 | 36 | def hierarchical_clustering(A, thresh=1.5, linkage='maximum'): 37 | ''' 38 | Hierarchical Clustering Algorithm. It is based on single linkage, finds the minimum element and merges 39 | rows and columns replacing the minimum elements. It is working on adjacency matrix. 40 | 41 | :param: A (adjacency matrix), thresh (stopping threshold) 42 | :type: A (np.array), thresh (int) 43 | 44 | :return: clusters 45 | ''' 46 | label_assg = {i: i for i in range(A.shape[0])} 47 | 48 | step = 0 49 | while A.shape[0] > 1: 50 | np.fill_diagonal(A,-np.NINF) 51 | #print(f'step {step} \n {A}') 52 | step+=1 53 | ind=np.unravel_index(np.argmin(A, axis=None), A.shape) 54 | 55 | if A[ind[0],ind[1]]>thresh: 56 | print('Breaking HC') 57 | break 58 | else: 59 | np.fill_diagonal(A,0) 60 | if linkage == 'maximum': 61 | Z=np.maximum(A[:,ind[0]], A[:,ind[1]]) 62 | elif linkage == 'minimum': 63 | Z=np.minimum(A[:,ind[0]], A[:,ind[1]]) 64 | elif linkage == 'average': 65 | Z= (A[:,ind[0]] + A[:,ind[1]])/2 66 | 67 | A[:,ind[0]]=Z 68 | A[:,ind[1]]=Z 69 | A[ind[0],:]=Z 70 | A[ind[1],:]=Z 71 | A = np.delete(A, (ind[1]), axis=0) 72 | A = np.delete(A, (ind[1]), axis=1) 73 | 74 | if type(label_assg[ind[0]]) == list: 75 | label_assg[ind[0]].append(label_assg[ind[1]]) 76 | else: 77 | label_assg[ind[0]] = [label_assg[ind[0]], label_assg[ind[1]]] 78 | 79 | label_assg.pop(ind[1], None) 80 | 81 | temp = [] 82 | for k,v in label_assg.items(): 83 | if k > ind[1]: 84 | kk = k-1 85 | vv = v 86 | else: 87 | kk = k 88 | vv = v 89 | temp.append((kk,vv)) 90 | 91 | label_assg = dict(temp) 92 | 93 | clusters = [] 94 | for k in label_assg.keys(): 95 | if type(label_assg[k]) == list: 96 | clusters.append(list(flatten(label_assg[k]))) 97 | elif type(label_assg[k]) == int: 98 | clusters.append([label_assg[k]]) 99 | 100 | return clusters 101 | 102 | def error_gen(actual, rounded): 103 | divisor = np.sqrt(1.0 if actual < 1.0 else actual) 104 | return abs(rounded - actual) ** 2 / divisor 105 | 106 | def round_to(percents, budget=100): 107 | if not np.isclose(sum(percents), budget): 108 | raise ValueError 109 | n = len(percents) 110 | rounded = [int(x) for x in percents] 111 | up_count = budget - sum(rounded) 112 | errors = [(error_gen(percents[i], rounded[i] + 1) - error_gen(percents[i], rounded[i]), i) for i in range(n)] 113 | rank = sorted(errors) 114 | for i in range(up_count): 115 | rounded[rank[i][1]] += 1 116 | return rounded -------------------------------------------------------------------------------- /src/pruning/unstructured.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy import linalg 3 | from scipy.spatial import distance 4 | 5 | import torch 6 | import torch.nn as nn 7 | 8 | def make_init_mask(model, is_print=False): 9 | ''' 10 | Makes the initial pruning mask for the given model. For example, for LeNet-5 architecture it return a list of 11 | 5 arrays, each array is the same size of each layer's weights and with all 1 entries. We do not prune bias 12 | 13 | :param model: a pytorch model 14 | :return mask: a list of pruning masks 15 | ''' 16 | step = 0 17 | for name, param in model.named_parameters(): 18 | if 'weight' in name: 19 | step = step + 1 20 | mask = [None]* step 21 | 22 | step = 0 23 | for name, param in model.named_parameters(): 24 | if 'weight' in name: 25 | tensor = param.data.cpu().numpy() 26 | mask[step] = np.ones_like(tensor) 27 | if is_print: 28 | print(f'step {step}, shape: {mask[step].shape}') 29 | step = step + 1 30 | 31 | return mask 32 | 33 | def fake_prune(percent, model, mask): 34 | ''' 35 | This function derives the new pruning mask, it put 0 for the weights under the given percentile 36 | 37 | :param percent: pruning percent 38 | :param model: a pytorch model 39 | :param mask: the pruning mask 40 | 41 | :return mask: updated pruning mask 42 | ''' 43 | # Calculate percentile value 44 | step = 0 45 | for name, param in model.named_parameters(): 46 | 47 | # We do not prune bias term 48 | if 'weight' in name: 49 | weight_dev = param.device 50 | 51 | tensor = param.data.cpu().numpy() 52 | alive = tensor[np.nonzero(tensor * mask[step])] # flattened array of nonzero values 53 | percentile_value = np.percentile(abs(alive), percent) 54 | 55 | new_mask = np.where(abs(tensor) < percentile_value, 0, mask[step]) 56 | 57 | # Apply mask 58 | mask[step] = new_mask 59 | step += 1 60 | 61 | return mask 62 | 63 | def real_prune(model, mask): 64 | ''' 65 | This function applies the derived mask. It zeros the weights needed to be pruned based on the updated mask 66 | 67 | :param model: a pytorch model 68 | :param mask: pruning mask 69 | 70 | :return state_dict: updated (pruned) model state_dict 71 | ''' 72 | # Calculate percentile value 73 | step = 0 74 | for name, param in model.named_parameters(): 75 | 76 | # We do not prune bias term 77 | if 'weight' in name: 78 | tensor = param.data.cpu().numpy() 79 | weight_dev = param.device 80 | 81 | # Apply new weight and mask 82 | param.data = torch.from_numpy(tensor * mask[step]).to(weight_dev) 83 | step += 1 84 | 85 | return model.state_dict() 86 | 87 | def print_pruning(model, is_print = False): 88 | ''' 89 | This function prints the pruning percentage and status of a given model 90 | 91 | :param model: a pytorch model 92 | 93 | :return pruning percentage, number of remaining weights: 94 | ''' 95 | nonzero = 0 96 | total = 0 97 | for name, p in model.named_parameters(): 98 | tensor = p.data.cpu().numpy() 99 | nz_count = np.count_nonzero(tensor) 100 | total_params = np.prod(tensor.shape) 101 | nonzero += nz_count 102 | total += total_params 103 | if is_print: 104 | print(f'alive: {nonzero}, pruned : {total - nonzero}, total: {total},' 105 | f'Compression rate : {total/nonzero:10.2f}x ({100 * (total-nonzero) / total:3.2f}% pruned)') 106 | return 100 * (total-nonzero) / total, nonzero 107 | 108 | def dist_masks(m1, m2): 109 | ''' 110 | Calculates hamming distance of two pruning masks. It averages the hamming distance of all layers and returns it 111 | 112 | :param m1: pruning mask 1 113 | :param m2: pruning mask 2 114 | 115 | :return average hamming distance of two pruning masks: 116 | ''' 117 | temp_dist = [] 118 | for step in range(len(m1)): 119 | #1 - float(m1[step].reshape([-1]) == m2[step].reshape([-1])) / len(m2[step].reshape([-1])) 120 | temp_dist.append(distance.hamming(m1[step].reshape([-1]), m2[step].reshape([-1]))) 121 | dist = np.mean(temp_dist) 122 | return dist -------------------------------------------------------------------------------- /src/client/client_fedavg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_FedAvg(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | def train(self, is_print = False): 27 | self.net.to(self.device) 28 | self.net.train() 29 | 30 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 31 | 32 | epoch_loss = [] 33 | for iteration in range(self.local_ep): 34 | batch_loss = [] 35 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 36 | images, labels = images.to(self.device), labels.to(self.device) 37 | labels = labels.type(torch.LongTensor).to(self.device) 38 | 39 | self.net.zero_grad() 40 | #optimizer.zero_grad() 41 | log_probs = self.net(images) 42 | loss = self.loss_func(log_probs, labels) 43 | loss.backward() 44 | 45 | optimizer.step() 46 | batch_loss.append(loss.item()) 47 | 48 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 49 | 50 | # if self.save_best: 51 | # _, acc = self.eval_test() 52 | # if acc > self.acc_best: 53 | # self.acc_best = acc 54 | 55 | return sum(epoch_loss) / len(epoch_loss) 56 | 57 | def get_state_dict(self): 58 | return self.net.state_dict() 59 | def get_best_acc(self): 60 | return self.acc_best 61 | def get_count(self): 62 | return self.count 63 | def get_net(self): 64 | return self.net 65 | def set_state_dict(self, state_dict): 66 | self.net.load_state_dict(state_dict) 67 | 68 | def eval_test(self): 69 | self.net.to(self.device) 70 | self.net.eval() 71 | test_loss = 0 72 | correct = 0 73 | with torch.no_grad(): 74 | for data, target in self.ldr_test: 75 | data, target = data.to(self.device), target.to(self.device) 76 | target = target.type(torch.LongTensor).to(self.device) 77 | 78 | output = self.net(data) 79 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 80 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 81 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 82 | test_loss /= len(self.ldr_test.dataset) 83 | accuracy = 100. * correct / len(self.ldr_test.dataset) 84 | return test_loss, accuracy 85 | 86 | def eval_test_glob(self, glob_dl): 87 | self.net.to(self.device) 88 | self.net.eval() 89 | test_loss = 0 90 | correct = 0 91 | with torch.no_grad(): 92 | for data, target in glob_dl: 93 | data, target = data.to(self.device), target.to(self.device) 94 | target = target.type(torch.LongTensor).to(self.device) 95 | 96 | output = self.net(data) 97 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 98 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 99 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 100 | test_loss /= len(glob_dl.dataset) 101 | accuracy = 100. * correct / len(glob_dl.dataset) 102 | return test_loss, accuracy 103 | 104 | def eval_train(self): 105 | self.net.to(self.device) 106 | self.net.eval() 107 | train_loss = 0 108 | correct = 0 109 | with torch.no_grad(): 110 | for data, target in self.ldr_train: 111 | data, target = data.to(self.device), target.to(self.device) 112 | target = target.type(torch.LongTensor).to(self.device) 113 | 114 | output = self.net(data) 115 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 116 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 117 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 118 | train_loss /= len(self.ldr_train.dataset) 119 | accuracy = 100. * correct / len(self.ldr_train.dataset) 120 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/client/client_cluster_fl.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_ClusterFL(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | def train(self, is_print = False): 27 | self.net.to(self.device) 28 | self.net.train() 29 | 30 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 31 | 32 | epoch_loss = [] 33 | for iteration in range(self.local_ep): 34 | batch_loss = [] 35 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 36 | images, labels = images.to(self.device), labels.to(self.device) 37 | labels = labels.type(torch.LongTensor).to(self.device) 38 | 39 | self.net.zero_grad() 40 | #optimizer.zero_grad() 41 | log_probs = self.net(images) 42 | loss = self.loss_func(log_probs, labels) 43 | loss.backward() 44 | 45 | optimizer.step() 46 | batch_loss.append(loss.item()) 47 | 48 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 49 | 50 | # if self.save_best: 51 | # _, acc = self.eval_test() 52 | # if acc > self.acc_best: 53 | # self.acc_best = acc 54 | 55 | return sum(epoch_loss) / len(epoch_loss) 56 | 57 | def get_state_dict(self): 58 | return self.net.state_dict() 59 | def get_best_acc(self): 60 | return self.acc_best 61 | def get_count(self): 62 | return self.count 63 | def get_net(self): 64 | return self.net 65 | def set_state_dict(self, state_dict): 66 | self.net.load_state_dict(state_dict) 67 | 68 | def eval_test(self): 69 | self.net.to(self.device) 70 | self.net.eval() 71 | test_loss = 0 72 | correct = 0 73 | with torch.no_grad(): 74 | for data, target in self.ldr_test: 75 | data, target = data.to(self.device), target.to(self.device) 76 | target = target.type(torch.LongTensor).to(self.device) 77 | 78 | output = self.net(data) 79 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 80 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 81 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 82 | test_loss /= len(self.ldr_test.dataset) 83 | accuracy = 100. * correct / len(self.ldr_test.dataset) 84 | return test_loss, accuracy 85 | 86 | def eval_test_glob(self, glob_dl): 87 | self.net.to(self.device) 88 | self.net.eval() 89 | test_loss = 0 90 | correct = 0 91 | with torch.no_grad(): 92 | for data, target in glob_dl: 93 | data, target = data.to(self.device), target.to(self.device) 94 | target = target.type(torch.LongTensor).to(self.device) 95 | 96 | output = self.net(data) 97 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 98 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 99 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 100 | test_loss /= len(glob_dl.dataset) 101 | accuracy = 100. * correct / len(glob_dl.dataset) 102 | return test_loss, accuracy 103 | 104 | def eval_train(self): 105 | self.net.to(self.device) 106 | self.net.eval() 107 | train_loss = 0 108 | correct = 0 109 | with torch.no_grad(): 110 | for data, target in self.ldr_train: 111 | data, target = data.to(self.device), target.to(self.device) 112 | target = target.type(torch.LongTensor).to(self.device) 113 | 114 | output = self.net(data) 115 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 116 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 117 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 118 | train_loss /= len(self.ldr_train.dataset) 119 | accuracy = 100. * correct / len(self.ldr_train.dataset) 120 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/data/datasetzoo.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from torchvision import datasets 3 | from torch.utils.data import DataLoader, Dataset 4 | from PIL import Image 5 | import os 6 | from .custom_data import * 7 | 8 | class DatasetZoo(Dataset): 9 | def __init__(self, root, dataset='cifar10', dataidxs=None, train=True, transform=None, target_transform=None, 10 | download=False, p_data=1.0): 11 | 12 | self.root = root 13 | self.dataset = dataset 14 | self.dataidxs = dataidxs 15 | self.train = train 16 | self.transform = transform 17 | self.target_transform = target_transform 18 | self.download = download 19 | self.p_data = p_data 20 | 21 | self.data, self.target, self.dataobj, self.mode = self.__init_dataset__() 22 | 23 | def __init_dataset__(self): 24 | 25 | if self.dataset == 'mnist': 26 | dataobj = datasets.MNIST(self.root, self.train, self.transform, self.target_transform, self.download) 27 | mode = 'L' 28 | elif self.dataset == 'usps': 29 | dataobj = datasets.USPS(self.root, self.train, self.transform, self.target_transform, self.download) 30 | mode = 'L' 31 | elif self.dataset == 'fmnist': 32 | dataobj = datasets.FashionMNIST(self.root, self.train, self.transform, self.target_transform, self.download) 33 | mode = 'L' 34 | elif self.dataset == 'cifar10': 35 | dataobj = datasets.CIFAR10(self.root, self.train, self.transform, self.target_transform, self.download) 36 | mode = 'RGB' 37 | elif self.dataset == 'cifar100': 38 | dataobj = datasets.CIFAR100(self.root, self.train, self.transform, self.target_transform, self.download) 39 | mode = 'RGB' 40 | elif self.dataset == 'svhn': 41 | if self.train: 42 | dataobj = datasets.SVHN(self.root, 'train', self.transform, self.target_transform, self.download) 43 | else: 44 | dataobj = datasets.SVHN(self.root, 'test', self.transform, self.target_transform, self.download) 45 | mode = 'RGB' 46 | elif self.dataset == 'stl10': 47 | if self.train: 48 | dataobj = datasets.STL10(self.root, 'train', self.transform, self.target_transform, self.download) 49 | else: 50 | dataobj = datasets.STL10(self.root, 'test', self.transform, self.target_transform, self.download) 51 | mode = 'RGB' 52 | elif self.dataset == 'celeba': 53 | X_train, y_train, X_test, y_test = load_celeba_data(datadir) 54 | mode = 'RGB' 55 | elif self.dataset == 'tinyimagenet': 56 | dataobj = load_tinyimagenet_data(self.root, self.train) 57 | mode = 'RGB' 58 | elif self.dataset == 'femnist': 59 | dataobj = datasets.FEMNIST(self.root, self.train, self.transform, self.target_transform, self.download) 60 | mode = 'L' 61 | 62 | data = np.array(dataobj.data) 63 | try: 64 | target = np.array(dataobj.targets) 65 | except: 66 | target = np.array(dataobj.labels) 67 | 68 | if len(data.shape) > 2 and data.shape[2]==data.shape[3]: 69 | data = data.transpose(0,2,3,1) ## STL-10 70 | 71 | if self.dataidxs is not None: 72 | data = data[self.dataidxs] 73 | target = target[self.dataidxs] 74 | 75 | if self.dataidxs is None: 76 | idxs_data = np.arange(len(data)) 77 | idxs_target = np.arange(len(target)) 78 | 79 | perm_data = idxs_data #np.random.permutation(idxs_data) 80 | 81 | p_data1 = int(len(idxs_data)*self.p_data) 82 | perm_data = perm_data[0:p_data1] 83 | 84 | data = data[perm_data] 85 | target = target[perm_data] 86 | 87 | return data, target, dataobj, mode 88 | 89 | def __getitem__(self, index): 90 | """ 91 | Args: 92 | index (int): Index 93 | 94 | Returns: 95 | tuple: (image, target) where target is index of the target class. 96 | """ 97 | img, target = self.data[index], self.target[index] 98 | if len(img.shape) == 1: ## tinyimagenet 99 | img = self.dataobj.loader(img[0]) 100 | elif img.shape[1]==img.shape[2]: ## SVHN 101 | img = Image.fromarray(np.transpose(img, (1, 2, 0))) 102 | else: 103 | img = Image.fromarray(img, mode=self.mode) 104 | 105 | if self.transform is not None: 106 | img = self.transform(img) 107 | 108 | if self.target_transform is not None: 109 | target = self.target_transform(target) 110 | target.type(torch.LongTensor) 111 | 112 | return img, target 113 | 114 | def __len__(self): 115 | return len(self.data) 116 | 117 | def load_tinyimagenet_data(datadir, train=True): 118 | transform = transforms.Compose([transforms.ToTensor()]) 119 | if train: 120 | ds = ImageFolder(datadir+'tiny-imagenet-200/train/', transform=transform) 121 | else: 122 | ds = ImageFolder(datadir+'tiny-imagenet-200/val/', transform=transform) 123 | ds.data = ds.imgs 124 | ds.target = ds.targets 125 | return ds -------------------------------------------------------------------------------- /src/client/client_mtl.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | import math 4 | 5 | import torch 6 | from torch import nn, optim 7 | import torch.nn.functional as F 8 | 9 | class Client_MTL(object): 10 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 11 | train_dl_local = None, test_dl_local = None): 12 | 13 | self.name = name 14 | self.net = model 15 | self.local_bs = local_bs 16 | self.local_ep = local_ep 17 | self.lr = lr 18 | self.momentum = momentum 19 | self.device = device 20 | self.loss_func = nn.CrossEntropyLoss() 21 | self.ldr_train = train_dl_local 22 | self.ldr_test = test_dl_local 23 | self.acc_best = 0 24 | self.count = 0 25 | self.save_best = True 26 | 27 | def train(self, idx, W_glob, omega, is_print = False): 28 | self.net.to(self.device) 29 | self.net.train() 30 | 31 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 32 | 33 | epoch_loss = [] 34 | for iteration in range(self.local_ep): 35 | batch_loss = [] 36 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 37 | images, labels = images.to(self.device), labels.to(self.device) 38 | self.net.zero_grad() 39 | #optimizer.zero_grad() 40 | log_probs = self.net(images) 41 | loss = self.loss_func(log_probs, labels) 42 | 43 | W = W_glob.clone() 44 | 45 | W_local = [v.flatten() for v in self.get_state_dict(keep_vars=True).values()] 46 | W_local = torch.cat(W_local) 47 | W[:, idx] = W_local 48 | 49 | loss_regularizer = 0 50 | loss_regularizer += W.norm() ** 2 51 | 52 | k = 4000 53 | for i in range(W.shape[0] // k): 54 | x = W[i * k:(i+1) * k, :] 55 | loss_regularizer += x.mm(omega).mm(x.T).trace() 56 | f = (int)(math.log10(W.shape[0])+1) + 1 57 | loss_regularizer *= 10 ** (-f) 58 | 59 | loss = loss + loss_regularizer 60 | loss.backward() 61 | 62 | optimizer.step() 63 | batch_loss.append(loss.item()) 64 | 65 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 66 | 67 | # if self.save_best: 68 | # _, acc = self.eval_test() 69 | # if acc > self.acc_best: 70 | # self.acc_best = acc 71 | 72 | return sum(epoch_loss) / len(epoch_loss) 73 | 74 | def get_state_dict(self, keep_vars=False): 75 | return self.net.state_dict(keep_vars=keep_vars) 76 | def get_best_acc(self): 77 | return self.acc_best 78 | def get_count(self): 79 | return self.count 80 | def get_net(self): 81 | return self.net 82 | def set_state_dict(self, state_dict): 83 | self.net.load_state_dict(state_dict) 84 | 85 | def eval_test(self): 86 | self.net.to(self.device) 87 | self.net.eval() 88 | test_loss = 0 89 | correct = 0 90 | with torch.no_grad(): 91 | for data, target in self.ldr_test: 92 | data, target = data.to(self.device), target.to(self.device) 93 | output = self.net(data) 94 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 95 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 96 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 97 | test_loss /= len(self.ldr_test.dataset) 98 | accuracy = 100. * correct / len(self.ldr_test.dataset) 99 | return test_loss, accuracy 100 | 101 | def eval_train(self): 102 | self.net.to(self.device) 103 | self.net.eval() 104 | train_loss = 0 105 | correct = 0 106 | with torch.no_grad(): 107 | for data, target in self.ldr_train: 108 | data, target = data.to(self.device), target.to(self.device) 109 | output = self.net(data) 110 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 111 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 112 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 113 | train_loss /= len(self.ldr_train.dataset) 114 | accuracy = 100. * correct / len(self.ldr_train.dataset) 115 | return train_loss, accuracy 116 | 117 | def eval_test(net, args, ldr_test): 118 | net.to(args.device) 119 | net.eval() 120 | test_loss = 0 121 | correct = 0 122 | with torch.no_grad(): 123 | for data, target in ldr_test: 124 | data, target = data.to(args.device), target.to(args.device) 125 | output = net(data) 126 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 127 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 128 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 129 | test_loss /= len(ldr_test.dataset) 130 | accuracy = 100. * correct / len(ldr_test.dataset) 131 | return test_loss, accuracy 132 | -------------------------------------------------------------------------------- /src/client/client_fedrep.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_FedRep(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | def train(self, global_keys, local_rep_ep, is_print = False): 27 | self.net.to(self.device) 28 | self.net.train() 29 | 30 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 31 | head_ep = self.local_ep - local_rep_ep 32 | 33 | epoch_loss = [] 34 | for iteration in range(self.local_ep): 35 | batch_loss = [] 36 | 37 | if iteration < head_ep: 38 | for name, param in self.net.named_parameters(): 39 | if name in global_keys: 40 | param.requires_grad = False 41 | else: 42 | param.requires_grad = True 43 | else: 44 | for name, param in self.net.named_parameters(): 45 | if name in global_keys: 46 | param.requires_grad = True 47 | else: 48 | param.requires_grad = False 49 | 50 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 51 | images, labels = images.to(self.device), labels.to(self.device) 52 | labels = labels.type(torch.LongTensor).to(self.device) 53 | 54 | self.net.zero_grad() 55 | #optimizer.zero_grad() 56 | log_probs = self.net(images) 57 | loss = self.loss_func(log_probs, labels) 58 | loss.backward() 59 | 60 | optimizer.step() 61 | batch_loss.append(loss.item()) 62 | 63 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 64 | 65 | return sum(epoch_loss) / len(epoch_loss) 66 | 67 | def get_state_dict(self): 68 | return self.net.state_dict() 69 | def get_best_acc(self): 70 | return self.acc_best 71 | def get_count(self): 72 | return self.count 73 | def get_net(self): 74 | return self.net 75 | def set_state_dict(self, state_dict): 76 | self.net.load_state_dict(state_dict) 77 | 78 | def eval_test(self): 79 | self.net.to(self.device) 80 | self.net.eval() 81 | test_loss = 0 82 | correct = 0 83 | with torch.no_grad(): 84 | for data, target in self.ldr_test: 85 | data, target = data.to(self.device), target.to(self.device) 86 | target = target.type(torch.LongTensor).to(self.device) 87 | 88 | output = self.net(data) 89 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 90 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 91 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 92 | test_loss /= len(self.ldr_test.dataset) 93 | accuracy = 100. * correct / len(self.ldr_test.dataset) 94 | return test_loss, accuracy 95 | 96 | def eval_test_glob(self, glob_dl): 97 | self.net.to(self.device) 98 | self.net.eval() 99 | test_loss = 0 100 | correct = 0 101 | with torch.no_grad(): 102 | for data, target in glob_dl: 103 | data, target = data.to(self.device), target.to(self.device) 104 | target = target.type(torch.LongTensor).to(self.device) 105 | 106 | output = self.net(data) 107 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 108 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 109 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 110 | test_loss /= len(glob_dl.dataset) 111 | accuracy = 100. * correct / len(glob_dl.dataset) 112 | return test_loss, accuracy 113 | 114 | def eval_train(self): 115 | self.net.to(self.device) 116 | self.net.eval() 117 | train_loss = 0 118 | correct = 0 119 | with torch.no_grad(): 120 | for data, target in self.ldr_train: 121 | data, target = data.to(self.device), target.to(self.device) 122 | target = target.type(torch.LongTensor).to(self.device) 123 | 124 | output = self.net(data) 125 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 126 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 127 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 128 | train_loss /= len(self.ldr_train.dataset) 129 | accuracy = 100. * correct / len(self.ldr_train.dataset) 130 | return train_loss, accuracy 131 | -------------------------------------------------------------------------------- /src/client/client_ditto.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_Ditto(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None, lam=1.0): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | self.lam = lam 26 | 27 | def train(self, w_ditto=None, is_print = False): 28 | self.net.to(self.device) 29 | self.net.train() 30 | 31 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 32 | 33 | epoch_loss = [] 34 | for iteration in range(self.local_ep): 35 | batch_loss = [] 36 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 37 | images, labels = images.to(self.device), labels.to(self.device) 38 | labels = labels.type(torch.LongTensor).to(self.device) 39 | 40 | # personalized model 41 | w_0 = self.net.state_dict() 42 | self.net.zero_grad() 43 | optimizer.zero_grad() 44 | 45 | log_probs = self.net(images) 46 | loss = self.loss_func(log_probs, labels) 47 | loss.backward() 48 | optimizer.step() 49 | 50 | if w_ditto is not None: 51 | w_net = copy.deepcopy(self.net.state_dict()) 52 | for key in w_net.keys(): 53 | w_net[key] = w_net[key].to(self.device) - (self.lr*self.lam*( 54 | w_0[key].to(self.device) - w_ditto[key].to(self.device))).to(self.device) 55 | 56 | self.net.load_state_dict(w_net) 57 | self.net.zero_grad() 58 | optimizer.zero_grad() 59 | 60 | batch_loss.append(loss.item()) 61 | 62 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 63 | 64 | return sum(epoch_loss) / len(epoch_loss) 65 | 66 | def get_state_dict(self): 67 | return self.net.state_dict() 68 | def get_best_acc(self): 69 | return self.acc_best 70 | def get_count(self): 71 | return self.count 72 | def get_net(self): 73 | return self.net 74 | def set_state_dict(self, state_dict): 75 | self.net.load_state_dict(state_dict) 76 | 77 | def eval_test(self): 78 | self.net.to(self.device) 79 | self.net.eval() 80 | test_loss = 0 81 | correct = 0 82 | with torch.no_grad(): 83 | for data, target in self.ldr_test: 84 | data, target = data.to(self.device), target.to(self.device) 85 | target = target.type(torch.LongTensor).to(self.device) 86 | 87 | output = self.net(data) 88 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 89 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 90 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 91 | test_loss /= len(self.ldr_test.dataset) 92 | accuracy = 100. * correct / len(self.ldr_test.dataset) 93 | return test_loss, accuracy 94 | 95 | def eval_test_glob(self, glob_dl): 96 | self.net.to(self.device) 97 | self.net.eval() 98 | test_loss = 0 99 | correct = 0 100 | with torch.no_grad(): 101 | for data, target in glob_dl: 102 | data, target = data.to(self.device), target.to(self.device) 103 | target = target.type(torch.LongTensor).to(self.device) 104 | 105 | output = self.net(data) 106 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 107 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 108 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 109 | test_loss /= len(glob_dl.dataset) 110 | accuracy = 100. * correct / len(glob_dl.dataset) 111 | return test_loss, accuracy 112 | 113 | def eval_train(self): 114 | self.net.to(self.device) 115 | self.net.eval() 116 | train_loss = 0 117 | correct = 0 118 | with torch.no_grad(): 119 | for data, target in self.ldr_train: 120 | data, target = data.to(self.device), target.to(self.device) 121 | target = target.type(torch.LongTensor).to(self.device) 122 | 123 | output = self.net(data) 124 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 125 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 126 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 127 | train_loss /= len(self.ldr_train.dataset) 128 | accuracy = 100. * correct / len(self.ldr_train.dataset) 129 | return train_loss, accuracy 130 | -------------------------------------------------------------------------------- /src/client/client_fedprox.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_FedProx(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None, mu=0.001): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | self.mu = mu 26 | 27 | def train(self, is_print = False): 28 | self.net.to(self.device) 29 | self.net.train() 30 | 31 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 32 | global_weight_collector = list(self.net.parameters()) 33 | 34 | epoch_loss = [] 35 | for iteration in range(self.local_ep): 36 | batch_loss = [] 37 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 38 | images, labels = images.to(self.device), labels.to(self.device) 39 | labels = labels.type(torch.LongTensor).to(self.device) 40 | 41 | self.net.zero_grad() 42 | #optimizer.zero_grad() 43 | log_probs = self.net(images) 44 | loss = self.loss_func(log_probs, labels) 45 | 46 | fed_prox_reg = 0.0 47 | for param_index, param in enumerate(self.net.parameters()): 48 | fed_prox_reg += ((self.mu / 2) * torch.norm((param - global_weight_collector[param_index]))**2) 49 | 50 | loss += fed_prox_reg 51 | 52 | loss.backward() 53 | 54 | optimizer.step() 55 | batch_loss.append(loss.item()) 56 | 57 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 58 | 59 | # if self.save_best: 60 | # _, acc = self.eval_test() 61 | # if acc > self.acc_best: 62 | # self.acc_best = acc 63 | 64 | return sum(epoch_loss) / len(epoch_loss) 65 | 66 | def get_state_dict(self): 67 | return self.net.state_dict() 68 | def get_best_acc(self): 69 | return self.acc_best 70 | def get_count(self): 71 | return self.count 72 | def get_net(self): 73 | return self.net 74 | def set_state_dict(self, state_dict): 75 | self.net.load_state_dict(state_dict) 76 | 77 | def eval_test(self): 78 | self.net.to(self.device) 79 | self.net.eval() 80 | test_loss = 0 81 | correct = 0 82 | with torch.no_grad(): 83 | for data, target in self.ldr_test: 84 | data, target = data.to(self.device), target.to(self.device) 85 | target = target.type(torch.LongTensor).to(self.device) 86 | 87 | output = self.net(data) 88 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 89 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 90 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 91 | test_loss /= len(self.ldr_test.dataset) 92 | accuracy = 100. * correct / len(self.ldr_test.dataset) 93 | return test_loss, accuracy 94 | 95 | def eval_test_glob(self, glob_dl): 96 | self.net.to(self.device) 97 | self.net.eval() 98 | test_loss = 0 99 | correct = 0 100 | with torch.no_grad(): 101 | for data, target in glob_dl: 102 | data, target = data.to(self.device), target.to(self.device) 103 | target = target.type(torch.LongTensor).to(self.device) 104 | 105 | output = self.net(data) 106 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 107 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 108 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 109 | test_loss /= len(glob_dl.dataset) 110 | accuracy = 100. * correct / len(glob_dl.dataset) 111 | return test_loss, accuracy 112 | 113 | def eval_train(self): 114 | self.net.to(self.device) 115 | self.net.eval() 116 | train_loss = 0 117 | correct = 0 118 | with torch.no_grad(): 119 | for data, target in self.ldr_train: 120 | data, target = data.to(self.device), target.to(self.device) 121 | target = target.type(torch.LongTensor).to(self.device) 122 | 123 | output = self.net(data) 124 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 125 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 126 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 127 | train_loss /= len(self.ldr_train.dataset) 128 | accuracy = 100. * correct / len(self.ldr_train.dataset) 129 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/client/client_feddf.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_FedDF(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | def train(self, is_print = False): 27 | self.net.to(self.device) 28 | self.net.train() 29 | 30 | #optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 31 | optimizer = torch.optim.Adam(self.net.parameters(), lr=self.lr) 32 | 33 | epoch_loss = [] 34 | for iteration in range(self.local_ep): 35 | batch_loss = [] 36 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 37 | images, labels = images.to(self.device), labels.to(self.device) 38 | labels = labels.type(torch.LongTensor).to(self.device) 39 | 40 | self.net.zero_grad() 41 | #optimizer.zero_grad() 42 | log_probs = self.net(images) 43 | loss = self.loss_func(log_probs, labels) 44 | loss.backward() 45 | 46 | optimizer.step() 47 | batch_loss.append(loss.item()) 48 | 49 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 50 | 51 | # if self.save_best: 52 | # _, acc = self.eval_test() 53 | # if acc > self.acc_best: 54 | # self.acc_best = acc 55 | 56 | return sum(epoch_loss) / len(epoch_loss) 57 | 58 | def get_state_dict(self): 59 | return self.net.state_dict() 60 | def get_best_acc(self): 61 | return self.acc_best 62 | def get_count(self): 63 | return self.count 64 | def get_net(self): 65 | return self.net 66 | def set_state_dict(self, state_dict): 67 | self.net.load_state_dict(state_dict) 68 | 69 | def inference(self, public_ds): 70 | public_dl = torch.utils.data.DataLoader(public_ds, batch_size=self.local_bs, shuffle=False, drop_last=False) 71 | 72 | outs = [] 73 | for data, _,_ in public_dl: 74 | data = data.to(self.device) 75 | out = self.net(data) 76 | outs.append(out.detach().cpu()) 77 | 78 | outputs = torch.cat(outs).numpy() 79 | return outputs 80 | 81 | def eval_test(self): 82 | self.net.to(self.device) 83 | self.net.eval() 84 | test_loss = 0 85 | correct = 0 86 | with torch.no_grad(): 87 | for data, target in self.ldr_test: 88 | data, target = data.to(self.device), target.to(self.device) 89 | target = target.type(torch.LongTensor).to(self.device) 90 | 91 | output = self.net(data) 92 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 93 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 94 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 95 | test_loss /= len(self.ldr_test.dataset) 96 | accuracy = 100. * correct / len(self.ldr_test.dataset) 97 | return test_loss, accuracy 98 | 99 | def eval_test_glob(self, glob_dl): 100 | self.net.to(self.device) 101 | self.net.eval() 102 | test_loss = 0 103 | correct = 0 104 | with torch.no_grad(): 105 | for data, target in glob_dl: 106 | data, target = data.to(self.device), target.to(self.device) 107 | target = target.type(torch.LongTensor).to(self.device) 108 | 109 | output = self.net(data) 110 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 111 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 112 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 113 | test_loss /= len(glob_dl.dataset) 114 | accuracy = 100. * correct / len(glob_dl.dataset) 115 | return test_loss, accuracy 116 | 117 | def eval_train(self): 118 | self.net.to(self.device) 119 | self.net.eval() 120 | train_loss = 0 121 | correct = 0 122 | with torch.no_grad(): 123 | for data, target in self.ldr_train: 124 | data, target = data.to(self.device), target.to(self.device) 125 | target = target.type(torch.LongTensor).to(self.device) 126 | 127 | output = self.net(data) 128 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 129 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 130 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 131 | train_loss /= len(self.ldr_train.dataset) 132 | accuracy = 100. * correct / len(self.ldr_train.dataset) 133 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/client/client_fednova.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_FedNova(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | def train(self, w_glob, is_print = False): 27 | self.net.to(self.device) 28 | self.net.train() 29 | 30 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 31 | 32 | tau = 0 33 | epoch_loss = [] 34 | for iteration in range(self.local_ep): 35 | batch_loss = [] 36 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 37 | images, labels = images.to(self.device), labels.to(self.device) 38 | labels = labels.type(torch.LongTensor).to(self.device) 39 | 40 | self.net.zero_grad() 41 | #optimizer.zero_grad() 42 | log_probs = self.net(images) 43 | loss = self.loss_func(log_probs, labels) 44 | loss.backward() 45 | 46 | optimizer.step() 47 | batch_loss.append(loss.item()) 48 | 49 | tau = tau + 1 50 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 51 | 52 | a_i = (tau - self.momentum * (1 - np.power(self.momentum, tau)) / (1 - self.momentum)) / (1 - self.momentum) 53 | 54 | net_para = copy.deepcopy(self.net.state_dict()) 55 | norm_grad = copy.deepcopy(w_glob) 56 | for key in norm_grad: 57 | #norm_grad[key] = (global_model_para[key] - net_para[key]) / a_i 58 | norm_grad[key] = torch.true_divide(w_glob[key].to(self.device)-net_para[key].to(self.device), a_i) 59 | 60 | # if self.save_best: 61 | # _, acc = self.eval_test() 62 | # if acc > self.acc_best: 63 | # self.acc_best = acc 64 | 65 | return sum(epoch_loss)/len(epoch_loss), a_i, norm_grad 66 | 67 | def get_state_dict(self): 68 | return self.net.state_dict() 69 | def get_best_acc(self): 70 | return self.acc_best 71 | def get_count(self): 72 | return self.count 73 | def get_net(self): 74 | return self.net 75 | def set_state_dict(self, state_dict): 76 | self.net.load_state_dict(state_dict) 77 | 78 | def eval_test(self): 79 | self.net.to(self.device) 80 | self.net.eval() 81 | test_loss = 0 82 | correct = 0 83 | with torch.no_grad(): 84 | for data, target in self.ldr_test: 85 | data, target = data.to(self.device), target.to(self.device) 86 | target = target.type(torch.LongTensor).to(self.device) 87 | 88 | output = self.net(data) 89 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 90 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 91 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 92 | test_loss /= len(self.ldr_test.dataset) 93 | accuracy = 100. * correct / len(self.ldr_test.dataset) 94 | return test_loss, accuracy 95 | 96 | def eval_test_glob(self, glob_dl): 97 | self.net.to(self.device) 98 | self.net.eval() 99 | test_loss = 0 100 | correct = 0 101 | with torch.no_grad(): 102 | for data, target in glob_dl: 103 | data, target = data.to(self.device), target.to(self.device) 104 | target = target.type(torch.LongTensor).to(self.device) 105 | 106 | output = self.net(data) 107 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 108 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 109 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 110 | test_loss /= len(glob_dl.dataset) 111 | accuracy = 100. * correct / len(glob_dl.dataset) 112 | return test_loss, accuracy 113 | 114 | def eval_train(self): 115 | self.net.to(self.device) 116 | self.net.eval() 117 | train_loss = 0 118 | correct = 0 119 | with torch.no_grad(): 120 | for data, target in self.ldr_train: 121 | data, target = data.to(self.device), target.to(self.device) 122 | target = target.type(torch.LongTensor).to(self.device) 123 | 124 | output = self.net(data) 125 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 126 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 127 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 128 | train_loss /= len(self.ldr_train.dataset) 129 | accuracy = 100. * correct / len(self.ldr_train.dataset) 130 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/models/subfedavg_models.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from torch import nn 4 | import torch.nn.functional as F 5 | import torch.nn.init as init 6 | 7 | ## Defining MODELS For Structured Pruning 8 | 9 | class LeNetBN5Mnist(nn.Module): 10 | def __init__(self, cfg=None, ks=5): 11 | super(LeNetBN5Mnist, self).__init__() 12 | if cfg == None: 13 | self.cfg = [10, 'M', 20, 'M'] 14 | else: 15 | self.cfg = cfg 16 | 17 | self.ks = ks 18 | self.main = nn.Sequential() 19 | self.make_layers(self.cfg, True) 20 | 21 | #self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 22 | #self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 23 | 24 | self.fc1 = nn.Linear(320, 50) 25 | self.fc2 = nn.Linear(50, 10) 26 | 27 | self._initialize_weights() 28 | 29 | def make_layers(self, cfg, batch_norm=False): 30 | layers = [] 31 | in_channels = 1 32 | idx_maxpool = 1 33 | idx_bn = 1 34 | idx_conv = 1 35 | idx_relu = 1 36 | for v in self.cfg: 37 | if v == 'M': 38 | layers += [('maxpool{}'.format(idx_maxpool), nn.MaxPool2d(kernel_size=2, stride=2))] 39 | idx_maxpool += 1 40 | else: 41 | conv2d = nn.Conv2d(in_channels, v, kernel_size=self.ks) 42 | if batch_norm: 43 | layers += [('conv{}'.format(idx_conv), conv2d), ('bn{}'.format(idx_bn), nn.BatchNorm2d(v)), 44 | ('relu{}'.format(idx_relu), nn.ReLU(inplace=True))] 45 | idx_bn += 1 46 | else: 47 | layers += [conv2d, nn.ReLU(inplace=True)] 48 | idx_conv += 1 49 | idx_relu += 1 50 | in_channels = v 51 | 52 | [self.main.add_module(n, l) for n, l in layers] 53 | 54 | def _initialize_weights(self): 55 | for m in self.modules(): 56 | if isinstance(m, nn.Conv2d): 57 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 58 | m.weight.data.normal_(0, np.sqrt(2. / n)) 59 | if m.bias is not None: 60 | m.bias.data.zero_() 61 | elif isinstance(m, nn.BatchNorm2d): 62 | m.weight.data.fill_(0.5) 63 | m.bias.data.zero_() 64 | elif isinstance(m, nn.Linear): 65 | m.weight.data.normal_(0, 0.01) 66 | m.bias.data.zero_() 67 | 68 | def forward(self, x): 69 | x = self.main(x) 70 | #print(x.shape) 71 | #x = x.view(-1, self.cfg[-2] * self.ks * self.ks) 72 | x = x.view(-1, x.shape[1]*x.shape[2]*x.shape[3]) 73 | x = F.relu(self.fc1(x)) 74 | x = self.fc2(x) 75 | return x 76 | 77 | class LeNetBN5Cifar(nn.Module): 78 | def __init__(self, nclasses = 10, cfg=None, ks=5): 79 | super(LeNetBN5Cifar, self).__init__() 80 | if cfg == None: 81 | self.cfg = [6, 'M', 16, 'M'] 82 | else: 83 | self.cfg = cfg 84 | 85 | self.ks = ks 86 | fc_cfg = [120, 84, 100] 87 | 88 | self.main = nn.Sequential() 89 | self.make_layers(self.cfg, True) 90 | 91 | self.fc1 = nn.Linear(self.cfg[-2] * 5 * 5, 120) 92 | self.fc2 = nn.Linear(120, 84) 93 | self.fc3 = nn.Linear(84, nclasses) 94 | 95 | self._initialize_weights() 96 | 97 | def make_layers(self, cfg, batch_norm=False): 98 | layers = [] 99 | in_channels = 3 100 | idx_maxpool = 1 101 | idx_bn = 1 102 | idx_conv = 1 103 | idx_relu = 1 104 | for v in self.cfg: 105 | if v == 'M': 106 | layers += [('maxpool{}'.format(idx_maxpool), nn.MaxPool2d(kernel_size=2, stride=2))] 107 | idx_maxpool += 1 108 | else: 109 | conv2d = nn.Conv2d(in_channels, v, kernel_size=self.ks) 110 | if batch_norm: 111 | layers += [('conv{}'.format(idx_conv), conv2d), ('bn{}'.format(idx_bn), nn.BatchNorm2d(v)), 112 | ('relu{}'.format(idx_relu), nn.ReLU(inplace=True))] 113 | idx_bn += 1 114 | else: 115 | layers += [conv2d, nn.ReLU(inplace=True)] 116 | idx_conv += 1 117 | idx_relu += 1 118 | in_channels = v 119 | 120 | [self.main.add_module(n, l) for n, l in layers] 121 | 122 | def forward(self, x): 123 | #x = self.main.conv1(x) 124 | x = self.main(x) 125 | 126 | #print(x.shape) 127 | #print(self.cfg[2]) 128 | x = x.view(-1, x.shape[1]*x.shape[2]*x.shape[3]) 129 | x = F.relu(self.fc1(x)) 130 | x = F.relu(self.fc2(x)) 131 | x = self.fc3(x) 132 | 133 | return x 134 | 135 | def _initialize_weights(self): 136 | for m in self.modules(): 137 | if isinstance(m, nn.Conv2d): 138 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 139 | m.weight.data.normal_(0, np.sqrt(2. / n)) 140 | if m.bias is not None: 141 | m.bias.data.zero_() 142 | elif isinstance(m, nn.BatchNorm2d): 143 | m.weight.data.fill_(0.5) 144 | m.bias.data.zero_() 145 | elif isinstance(m, nn.Linear): 146 | m.weight.data.normal_(0, 0.01) 147 | m.bias.data.zero_() 148 | return 149 | 150 | def updateBN(mymodel, args): 151 | for m in mymodel.modules(): 152 | if isinstance(m, nn.BatchNorm2d): 153 | m.weight.grad.data.add_(args.s*torch.sign(m.weight.data)) # L1 154 | return -------------------------------------------------------------------------------- /src/models/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | '''ResNet in PyTorch. 6 | For Pre-activation ResNet, see 'preact_resnet.py'. 7 | Reference: 8 | [1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun 9 | Deep Residual Learning for Image Recognition. arXiv:1512.03385 10 | ''' 11 | 12 | def conv_bn_relu_pool(in_channels, out_channels, pool=False): 13 | layers = [ 14 | nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, padding=1), 15 | nn.BatchNorm2d(out_channels), 16 | #nn.GroupNorm(32,out_channels), 17 | nn.ReLU(inplace=True) 18 | ] 19 | if pool: 20 | layers.append(nn.MaxPool2d(2)) 21 | return nn.Sequential(*layers) 22 | 23 | class ResNet9(nn.Module): 24 | def __init__(self, in_channels, num_classes, dim=512): 25 | super().__init__() 26 | self.prep = conv_bn_relu_pool(in_channels, 64) 27 | self.layer1_head = conv_bn_relu_pool(64, 128, pool=True) 28 | self.layer1_residual = nn.Sequential(conv_bn_relu_pool(128, 128), conv_bn_relu_pool(128, 128)) 29 | self.layer2 = conv_bn_relu_pool(128, 256, pool=True) 30 | self.layer3_head = conv_bn_relu_pool(256, 512, pool=True) 31 | self.layer3_residual = nn.Sequential(conv_bn_relu_pool(512, 512), conv_bn_relu_pool(512, 512)) 32 | self.MaxPool2d = nn.Sequential( 33 | nn.MaxPool2d(4)) 34 | self.linear = nn.Linear(dim, num_classes) 35 | # self.classifier = nn.Sequential( 36 | # nn.MaxPool2d(4), 37 | # nn.Flatten(), 38 | # nn.Linear(512, num_classes)) 39 | 40 | 41 | def forward(self, x): 42 | x = self.prep(x) 43 | x = self.layer1_head(x) 44 | x = self.layer1_residual(x) + x 45 | x = self.layer2(x) 46 | x = self.layer3_head(x) 47 | x = self.layer3_residual(x) + x 48 | x = self.MaxPool2d(x) 49 | x = x.view(x.size(0), -1) 50 | #print(x.shape) 51 | x = self.linear(x) 52 | return x 53 | 54 | class BasicBlock(nn.Module): 55 | expansion = 1 56 | 57 | def __init__(self, in_planes, planes, stride=1): 58 | super(BasicBlock, self).__init__() 59 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 60 | self.bn1 = nn.BatchNorm2d(planes) 61 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) 62 | self.bn2 = nn.BatchNorm2d(planes) 63 | 64 | self.shortcut = nn.Sequential() 65 | if stride != 1 or in_planes != self.expansion*planes: 66 | self.shortcut = nn.Sequential( 67 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 68 | nn.BatchNorm2d(self.expansion*planes) 69 | ) 70 | 71 | def forward(self, x): 72 | out = F.relu(self.bn1(self.conv1(x))) 73 | out = self.bn2(self.conv2(out)) 74 | out += self.shortcut(x) 75 | out = F.relu(out) 76 | return out 77 | 78 | 79 | class Bottleneck(nn.Module): 80 | expansion = 4 81 | 82 | def __init__(self, in_planes, planes, stride=1): 83 | super(Bottleneck, self).__init__() 84 | self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) 85 | self.bn1 = nn.BatchNorm2d(planes) 86 | self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 87 | self.bn2 = nn.BatchNorm2d(planes) 88 | self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) 89 | self.bn3 = nn.BatchNorm2d(self.expansion*planes) 90 | 91 | self.shortcut = nn.Sequential() 92 | if stride != 1 or in_planes != self.expansion*planes: 93 | self.shortcut = nn.Sequential( 94 | nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), 95 | nn.BatchNorm2d(self.expansion*planes) 96 | ) 97 | 98 | def forward(self, x): 99 | out = F.relu(self.bn1(self.conv1(x))) 100 | out = F.relu(self.bn2(self.conv2(out))) 101 | out = self.bn3(self.conv3(out)) 102 | out += self.shortcut(x) 103 | out = F.relu(out) 104 | return out 105 | 106 | 107 | class ResNet(nn.Module): 108 | def __init__(self, block, num_blocks, num_classes=10): 109 | super(ResNet, self).__init__() 110 | self.in_planes = 64 111 | 112 | self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 113 | self.bn1 = nn.BatchNorm2d(64) 114 | self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) 115 | self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) 116 | self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) 117 | self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) 118 | self.linear = nn.Linear(512*block.expansion, num_classes) 119 | 120 | def _make_layer(self, block, planes, num_blocks, stride): 121 | strides = [stride] + [1]*(num_blocks-1) 122 | layers = [] 123 | for stride in strides: 124 | layers.append(block(self.in_planes, planes, stride)) 125 | self.in_planes = planes * block.expansion 126 | return nn.Sequential(*layers) 127 | 128 | def forward(self, x): 129 | out = F.relu(self.bn1(self.conv1(x))) 130 | out = self.layer1(out) 131 | out = self.layer2(out) 132 | out = self.layer3(out) 133 | out = self.layer4(out) 134 | out = F.avg_pool2d(out, 4) 135 | out = out.view(out.size(0), -1) 136 | out = self.linear(out) 137 | return out 138 | 139 | 140 | def ResNet18(): 141 | return ResNet(BasicBlock, [2,2,2,2]) 142 | 143 | def ResNet34(): 144 | return ResNet(BasicBlock, [3,4,6,3]) 145 | 146 | def ResNet50(): 147 | return ResNet(Bottleneck, [3,4,6,3]) 148 | 149 | def ResNet101(): 150 | return ResNet(Bottleneck, [3,4,23,3]) 151 | 152 | def ResNet152(): 153 | return ResNet(Bottleneck, [3,8,36,3]) 154 | 155 | 156 | def test(): 157 | net = ResNet18() 158 | y = net(torch.randn(1,3,32,32)) 159 | print(y.size()) 160 | 161 | # test() -------------------------------------------------------------------------------- /src/client/client_apfl.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | # final update is not implemented 9 | class Client_APFL(object): 10 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, global_model, 11 | alpha, train_dl_local = None, test_dl_local = None, alpha_apfl=0.8): 12 | 13 | self.name = name 14 | self.net = model # local model 15 | self.local_bs = local_bs 16 | self.local_ep = local_ep 17 | self.lr = lr 18 | self.momentum = momentum 19 | self.device = device 20 | self.loss_func = nn.CrossEntropyLoss() 21 | self.ldr_train = train_dl_local 22 | self.ldr_test = test_dl_local 23 | self.acc_best = 0 24 | self.count = 0 25 | self.save_best = True 26 | self.alpha_apfl = alpha_apfl 27 | 28 | def train(self, w_local=None, is_print = False): 29 | self.net.to(self.device) 30 | self.net.train() 31 | 32 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 33 | 34 | w_glob = copy.deepcopy(self.net.state_dict()) 35 | epoch_loss = [] 36 | for iteration in range(self.local_ep): 37 | batch_loss = [] 38 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 39 | images, labels = images.to(self.device), labels.to(self.device) 40 | labels = labels.type(torch.LongTensor).to(self.device) 41 | 42 | # for k in self.net.state_dict().keys(): 43 | # w_loc_new[k] = self.alpha_apfl*w_local[k] + (1-self.alpha_apfl)*w_glob[k] 44 | 45 | self.net.load_state_dict(w_glob) 46 | self.net.to(self.device) 47 | optimizer.zero_grad() 48 | self.net.zero_grad() 49 | log_probs = self.net(images) 50 | loss = self.loss_func(log_probs, labels) 51 | loss.backward() 52 | optimizer.step() 53 | batch_loss.append(loss.item()) 54 | 55 | wt = copy.deepcopy(self.net.state_dict()) 56 | 57 | self.net.load_state_dict(w_local) 58 | self.net.to(self.device) 59 | optimizer.zero_grad() 60 | self.net.zero_grad() 61 | log_probs = self.net(images) 62 | loss = self.loss_func(log_probs, labels) 63 | loss.backward() 64 | optimizer.step() 65 | 66 | vt = copy.deepcopy(self.net.state_dict()) 67 | for k in self.net.state_dict().keys(): 68 | w_local[k] = (self.alpha_apfl*vt[k]).to(self.device) + ((1-self.alpha_apfl)*wt[k]).to(self.device) 69 | w_local[k] = w_local[k].to(self.device) 70 | 71 | w_glob = copy.deepcopy(wt) 72 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 73 | 74 | return sum(epoch_loss)/len(epoch_loss), w_local, w_glob 75 | 76 | def get_state_dict(self): 77 | return self.global_model.state_dict() 78 | def get_best_acc(self): 79 | return self.acc_best 80 | def get_count(self): 81 | return self.count 82 | def get_net(self): 83 | return self.net 84 | def set_state_dict(self, state_dict): 85 | self.net.load_state_dict(state_dict) 86 | 87 | def eval_test(self): 88 | self.net.to(self.device) 89 | self.net.eval() 90 | test_loss = 0 91 | correct = 0 92 | with torch.no_grad(): 93 | for data, target in self.ldr_test: 94 | data, target = data.to(self.device), target.to(self.device) 95 | target = target.type(torch.LongTensor).to(self.device) 96 | 97 | output = self.net(data) 98 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 99 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 100 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 101 | test_loss /= len(self.ldr_test.dataset) 102 | accuracy = 100. * correct / len(self.ldr_test.dataset) 103 | return test_loss, accuracy 104 | 105 | def eval_test_glob(self, glob_dl): 106 | self.net.to(self.device) 107 | self.net.eval() 108 | test_loss = 0 109 | correct = 0 110 | with torch.no_grad(): 111 | for data, target in glob_dl: 112 | data, target = data.to(self.device), target.to(self.device) 113 | target = target.type(torch.LongTensor).to(self.device) 114 | 115 | output = self.net(data) 116 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 117 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 118 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 119 | test_loss /= len(glob_dl.dataset) 120 | accuracy = 100. * correct / len(glob_dl.dataset) 121 | return test_loss, accuracy 122 | 123 | def eval_train(self): 124 | self.net.to(self.device) 125 | self.net.eval() 126 | train_loss = 0 127 | correct = 0 128 | with torch.no_grad(): 129 | for data, target in self.ldr_train: 130 | data, target = data.to(self.device), target.to(self.device) 131 | target = target.type(torch.LongTensor).to(self.device) 132 | 133 | output = self.net(data) 134 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 135 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 136 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 137 | train_loss /= len(self.ldr_train.dataset) 138 | accuracy = 100. * correct / len(self.ldr_train.dataset) 139 | return train_loss, accuracy 140 | -------------------------------------------------------------------------------- /src/client/client_moon.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_Moon(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None, mu=0.001, temperature=0.5): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | self.mu = mu 26 | self.temperature = temperature 27 | self.prevnet = copy.deepcopy(self.net) 28 | 29 | def train(self, is_print = False): 30 | self.net.to(self.device) 31 | self.net.train() 32 | 33 | self.prevnet.to(self.device) 34 | self.prevnet.eval() 35 | 36 | global_net = copy.deepcopy(self.net).to(self.device).eval() 37 | 38 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 39 | cos = torch.nn.CosineSimilarity(dim=-1) 40 | 41 | epoch_loss = [] 42 | for iteration in range(self.local_ep): 43 | batch_loss = [] 44 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 45 | images, labels = images.to(self.device), labels.to(self.device) 46 | labels = labels.type(torch.LongTensor).to(self.device) 47 | #labels = labels.view(images.size(0), -1) 48 | 49 | self.net.zero_grad() 50 | optimizer.zero_grad() 51 | ## Calculating loss1 52 | _, pro1, out = self.net(images) 53 | out = out.view(images.size(0), -1) 54 | #print(f'out: {out.shape}') 55 | #print(f'labels: {labels.shape}') 56 | loss1 = self.loss_func(out, labels) 57 | 58 | ## Calculating loss2 59 | _, pro2, _ = global_net(images) 60 | logits = cos(pro1, pro2).reshape(-1,1) 61 | 62 | self.prevnet.to(self.device) 63 | _, pro3, _ = self.prevnet(images) 64 | nega = cos(pro1, pro3) 65 | logits = torch.cat((logits, nega.reshape(-1,1)), dim=1) 66 | 67 | logits /= self.temperature 68 | fake_labels = torch.zeros(images.size(0)).to(self.device).long() 69 | loss2 = self.mu * self.loss_func(logits, fake_labels) 70 | 71 | loss = loss1 + loss2 72 | loss.backward() 73 | 74 | optimizer.step() 75 | batch_loss.append(loss.item()) 76 | 77 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 78 | 79 | self.prevnet = copy.deepcopy(self.net) 80 | self.prevnet.to('cpu') 81 | self.prevnet.eval() 82 | del global_net 83 | return sum(epoch_loss) / len(epoch_loss) 84 | 85 | def get_state_dict(self): 86 | return self.net.state_dict() 87 | def get_best_acc(self): 88 | return self.acc_best 89 | def get_count(self): 90 | return self.count 91 | def get_net(self): 92 | return self.net 93 | def set_state_dict(self, state_dict): 94 | self.net.load_state_dict(state_dict) 95 | 96 | def eval_test(self): 97 | self.net.to(self.device) 98 | self.net.eval() 99 | test_loss = 0 100 | correct = 0 101 | with torch.no_grad(): 102 | for data, target in self.ldr_test: 103 | data, target = data.to(self.device), target.to(self.device) 104 | target = target.type(torch.LongTensor).to(self.device) 105 | 106 | _,_,output = self.net(data) 107 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 108 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 109 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 110 | test_loss /= len(self.ldr_test.dataset) 111 | accuracy = 100. * correct / len(self.ldr_test.dataset) 112 | return test_loss, accuracy 113 | 114 | def eval_test_glob(self, glob_dl): 115 | self.net.to(self.device) 116 | self.net.eval() 117 | test_loss = 0 118 | correct = 0 119 | with torch.no_grad(): 120 | for data, target in glob_dl: 121 | data, target = data.to(self.device), target.to(self.device) 122 | target = target.type(torch.LongTensor).to(self.device) 123 | 124 | _,_,output = self.net(data) 125 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 126 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 127 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 128 | test_loss /= len(glob_dl.dataset) 129 | accuracy = 100. * correct / len(glob_dl.dataset) 130 | return test_loss, accuracy 131 | 132 | def eval_train(self): 133 | self.net.to(self.device) 134 | self.net.eval() 135 | train_loss = 0 136 | correct = 0 137 | with torch.no_grad(): 138 | for data, target in self.ldr_train: 139 | data, target = data.to(self.device), target.to(self.device) 140 | target = target.type(torch.LongTensor).to(self.device) 141 | 142 | _,_,output = self.net(data) 143 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 144 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 145 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 146 | train_loss /= len(self.ldr_train.dataset) 147 | accuracy = 100. * correct / len(self.ldr_train.dataset) 148 | return train_loss, accuracy 149 | -------------------------------------------------------------------------------- /src/client/client_per_fedavg.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_Per_FedAvg(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | def train(self, is_print = False): 27 | self.net.to(self.device) 28 | self.net.train() 29 | 30 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 31 | beta = 0.001 32 | 33 | epoch_loss = [] 34 | for iteration in range(self.local_ep): 35 | dataloader_iterator = iter(self.ldr_train) 36 | batch_loss = [] 37 | for batch_idx in range(len(self.ldr_train) // 2): 38 | temp_w = copy.deepcopy(self.get_state_dict()) 39 | 40 | images, labels = next(dataloader_iterator) 41 | images, labels = images.to(self.device), labels.to(self.device) 42 | labels = labels.type(torch.LongTensor).to(self.device) 43 | 44 | self.net.zero_grad() 45 | #optimizer.zero_grad() 46 | log_probs = self.net(images) 47 | loss = self.loss_func(log_probs, labels) 48 | loss.backward() 49 | adjust_learning_rate(optimizer, self.lr) 50 | optimizer.step() 51 | 52 | batch_loss.append(loss.item()) 53 | 54 | images, labels = next(dataloader_iterator) 55 | images, labels = images.to(self.device), labels.to(self.device) 56 | labels = labels.type(torch.LongTensor).to(self.device) 57 | 58 | self.net.zero_grad() 59 | #optimizer.zero_grad() 60 | log_probs = self.net(images) 61 | loss = self.loss_func(log_probs, labels) 62 | loss.backward() 63 | 64 | self.set_state_dict(temp_w) 65 | adjust_learning_rate(optimizer, beta) 66 | optimizer.step() 67 | 68 | if batch_loss != []: 69 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 70 | 71 | # if self.save_best: 72 | # _, acc = self.eval_test() 73 | # if acc > self.acc_best: 74 | # self.acc_best = acc 75 | # return sum(epoch_loss) / len(epoch_loss) 76 | 77 | if epoch_loss == []: 78 | return 0 79 | else: 80 | return sum(epoch_loss) / len(epoch_loss) 81 | 82 | def get_state_dict(self, keep_vars=False): 83 | return self.net.state_dict(keep_vars=keep_vars) 84 | def get_best_acc(self): 85 | return self.acc_best 86 | def get_count(self): 87 | return self.count 88 | def get_net(self): 89 | return self.net 90 | def set_state_dict(self, state_dict): 91 | self.net.load_state_dict(state_dict) 92 | 93 | def eval_test(self): 94 | self.net.to(self.device) 95 | self.net.eval() 96 | test_loss = 0 97 | correct = 0 98 | with torch.no_grad(): 99 | for data, target in self.ldr_test: 100 | data, target = data.to(self.device), target.to(self.device) 101 | target = target.type(torch.LongTensor).to(self.device) 102 | 103 | output = self.net(data) 104 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 105 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 106 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 107 | test_loss /= len(self.ldr_test.dataset) 108 | accuracy = 100. * correct / len(self.ldr_test.dataset) 109 | return test_loss, accuracy 110 | 111 | def eval_train(self): 112 | self.net.to(self.device) 113 | self.net.eval() 114 | train_loss = 0 115 | correct = 0 116 | with torch.no_grad(): 117 | for data, target in self.ldr_train: 118 | data, target = data.to(self.device), target.to(self.device) 119 | target = target.type(torch.LongTensor).to(self.device) 120 | 121 | output = self.net(data) 122 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 123 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 124 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 125 | train_loss /= len(self.ldr_train.dataset) 126 | accuracy = 100. * correct / len(self.ldr_train.dataset) 127 | return train_loss, accuracy 128 | 129 | def adjust_learning_rate(optimizer, lr): 130 | for param_group in optimizer.param_groups: 131 | param_group['lr'] = lr 132 | 133 | def eval_test(net, args, ldr_test): 134 | net.to(args.device) 135 | net.eval() 136 | test_loss = 0 137 | correct = 0 138 | with torch.no_grad(): 139 | for data, target in ldr_test: 140 | data, target = data.to(args.device), target.to(args.device) 141 | target = target.type(torch.LongTensor).to(self.device) 142 | 143 | output = net(data) 144 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 145 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 146 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 147 | test_loss /= len(ldr_test.dataset) 148 | accuracy = 100. * correct / len(ldr_test.dataset) 149 | return test_loss, accuracy -------------------------------------------------------------------------------- /src/client/client_scaffold.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_Scaffold(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local=None, test_dl_local=None, c_local=None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | 26 | #self.c_global = c_global 27 | self.c_local = c_local 28 | 29 | for key in self.c_local.keys(): 30 | self.c_local[key] = self.c_local[key].to(self.device) 31 | 32 | def train(self, w_glob, c_global, is_print = False): 33 | self.net.to(self.device) 34 | self.net.train() 35 | 36 | for key in c_global.keys(): 37 | c_global[key] = c_global[key].to(self.device) 38 | 39 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 40 | 41 | cnt=0 42 | epoch_loss = [] 43 | for iteration in range(self.local_ep): 44 | batch_loss = [] 45 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 46 | images, labels = images.to(self.device), labels.to(self.device) 47 | labels = labels.type(torch.LongTensor).to(self.device) 48 | 49 | self.net.zero_grad() 50 | #optimizer.zero_grad() 51 | log_probs = self.net(images) 52 | loss = self.loss_func(log_probs, labels) 53 | 54 | loss.backward() 55 | optimizer.step() 56 | 57 | net_para = self.net.state_dict() 58 | for key in net_para: 59 | net_para[key] = net_para[key].to(self.device) - self.lr * (c_global[key] - self.c_local[key]) 60 | self.net.load_state_dict(net_para) 61 | 62 | cnt+=1 63 | batch_loss.append(loss.item()) 64 | 65 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 66 | 67 | c_new = copy.deepcopy(self.c_local) 68 | c_delta = copy.deepcopy(self.c_local) 69 | net_para = self.net.state_dict() 70 | for key in net_para: 71 | c_new[key] = c_new[key].to(self.device) - c_global[key].to(self.device) + (w_glob[key].to(self.device) - net_para[key].to(self.device))/(cnt* self.lr) 72 | c_delta[key] = c_new[key].to(self.device) - self.c_local[key].to(self.device) 73 | 74 | self.c_local = copy.deepcopy(c_new) 75 | 76 | # if self.save_best: 77 | # _, acc = self.eval_test() 78 | # if acc > self.acc_best: 79 | # self.acc_best = acc 80 | 81 | return sum(epoch_loss)/len(epoch_loss), c_delta 82 | 83 | def get_state_dict(self): 84 | return self.net.state_dict() 85 | def get_best_acc(self): 86 | return self.acc_best 87 | def get_count(self): 88 | return self.count 89 | def get_net(self): 90 | return self.net 91 | def set_state_dict(self, state_dict): 92 | self.net.load_state_dict(state_dict) 93 | 94 | def eval_test(self): 95 | self.net.to(self.device) 96 | self.net.eval() 97 | test_loss = 0 98 | correct = 0 99 | with torch.no_grad(): 100 | for data, target in self.ldr_test: 101 | data, target = data.to(self.device), target.to(self.device) 102 | target = target.type(torch.LongTensor).to(self.device) 103 | 104 | output = self.net(data) 105 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 106 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 107 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 108 | test_loss /= len(self.ldr_test.dataset) 109 | accuracy = 100. * correct / len(self.ldr_test.dataset) 110 | return test_loss, accuracy 111 | 112 | def eval_test_glob(self, glob_dl): 113 | self.net.to(self.device) 114 | self.net.eval() 115 | test_loss = 0 116 | correct = 0 117 | with torch.no_grad(): 118 | for data, target in glob_dl: 119 | data, target = data.to(self.device), target.to(self.device) 120 | target = target.type(torch.LongTensor).to(self.device) 121 | 122 | output = self.net(data) 123 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 124 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 125 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 126 | test_loss /= len(glob_dl.dataset) 127 | accuracy = 100. * correct / len(glob_dl.dataset) 128 | return test_loss, accuracy 129 | 130 | def eval_train(self): 131 | self.net.to(self.device) 132 | self.net.eval() 133 | train_loss = 0 134 | correct = 0 135 | with torch.no_grad(): 136 | for data, target in self.ldr_train: 137 | data, target = data.to(self.device), target.to(self.device) 138 | target = target.type(torch.LongTensor).to(self.device) 139 | 140 | output = self.net(data) 141 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 142 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 143 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 144 | train_loss /= len(self.ldr_train.dataset) 145 | accuracy = 100. * correct / len(self.ldr_train.dataset) 146 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/client/client_feddyn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | 4 | import torch 5 | from torch import nn, optim 6 | import torch.nn.functional as F 7 | 8 | class Client_FedDyn(object): 9 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 10 | train_dl_local = None, test_dl_local = None): 11 | 12 | self.name = name 13 | self.net = model 14 | self.local_bs = local_bs 15 | self.local_ep = local_ep 16 | self.lr = lr 17 | self.momentum = momentum 18 | self.device = device 19 | self.loss_func = nn.CrossEntropyLoss() 20 | self.ldr_train = train_dl_local 21 | self.ldr_test = test_dl_local 22 | self.acc_best = 0 23 | self.count = 0 24 | self.save_best = True 25 | self.prev_grads = None 26 | for param in self.net.parameters(): 27 | if not isinstance(self.prev_grads, torch.Tensor): 28 | self.prev_grads = torch.zeros_like(param.view(-1)) 29 | else: 30 | self.prev_grads = torch.cat((self.prev_grads, torch.zeros_like(param.view(-1))), dim=0) 31 | 32 | def train(self, w_glob, is_print = False): 33 | self.net.to(self.device) 34 | self.net.train() 35 | 36 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 37 | 38 | epoch_loss = [] 39 | for iteration in range(self.local_ep): 40 | batch_loss = [] 41 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 42 | images, labels = images.to(self.device), labels.to(self.device) 43 | labels = labels.type(torch.LongTensor).to(self.device) 44 | 45 | self.net.zero_grad() 46 | #optimizer.zero_grad() 47 | log_probs = self.net(images) 48 | loss = self.loss_func(log_probs, labels) 49 | 50 | #=== Dynamic regularization === # 51 | # Linear penalty 52 | lin_penalty = 0.0 53 | curr_params = None 54 | for name, param in self.net.named_parameters(): 55 | if not isinstance(curr_params, torch.Tensor): 56 | curr_params = param.view(-1) 57 | else: 58 | curr_params = torch.cat((curr_params, param.view(-1)), dim=0) 59 | 60 | lin_penalty = torch.sum(curr_params * self.prev_grads) 61 | loss -= lin_penalty 62 | epoch_loss['Lin Penalty'] = lin_penalty.item() 63 | 64 | # Quadratic Penalty 65 | quad_penalty = 0.0 66 | for name, param in self.net.named_parameters(): 67 | quad_penalty += F.mse_loss(param, w_glob[name], reduction='sum') 68 | 69 | loss += self.alpha/2.0 * quad_penalty 70 | loss.backward() 71 | 72 | self.prev_grads = None 73 | for param in self.net.parameters(): 74 | if not isinstance(self.prev_grads, torch.Tensor): 75 | self.prev_grads = param.grad.view(-1).clone() 76 | else: 77 | self.prev_grads = torch.cat((self.prev_grads, param.grad.view(-1).clone()), dim=0) 78 | 79 | optimizer.step() 80 | 81 | batch_loss.append(loss.item()) 82 | 83 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 84 | 85 | return sum(epoch_loss) / len(epoch_loss) 86 | 87 | def get_state_dict(self): 88 | return self.net.state_dict() 89 | def get_best_acc(self): 90 | return self.acc_best 91 | def get_count(self): 92 | return self.count 93 | def get_net(self): 94 | return self.net 95 | def set_state_dict(self, state_dict): 96 | self.net.load_state_dict(state_dict) 97 | 98 | def eval_test(self): 99 | self.net.to(self.device) 100 | self.net.eval() 101 | test_loss = 0 102 | correct = 0 103 | with torch.no_grad(): 104 | for data, target in self.ldr_test: 105 | data, target = data.to(self.device), target.to(self.device) 106 | target = target.type(torch.LongTensor).to(self.device) 107 | 108 | output = self.net(data) 109 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 110 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 111 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 112 | test_loss /= len(self.ldr_test.dataset) 113 | accuracy = 100. * correct / len(self.ldr_test.dataset) 114 | return test_loss, accuracy 115 | 116 | def eval_test_glob(self, glob_dl): 117 | self.net.to(self.device) 118 | self.net.eval() 119 | test_loss = 0 120 | correct = 0 121 | with torch.no_grad(): 122 | for data, target in glob_dl: 123 | data, target = data.to(self.device), target.to(self.device) 124 | target = target.type(torch.LongTensor).to(self.device) 125 | 126 | output = self.net(data) 127 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 128 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 129 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 130 | test_loss /= len(glob_dl.dataset) 131 | accuracy = 100. * correct / len(glob_dl.dataset) 132 | return test_loss, accuracy 133 | 134 | def eval_train(self): 135 | self.net.to(self.device) 136 | self.net.eval() 137 | train_loss = 0 138 | correct = 0 139 | with torch.no_grad(): 140 | for data, target in self.ldr_train: 141 | data, target = data.to(self.device), target.to(self.device) 142 | target = target.type(torch.LongTensor).to(self.device) 143 | 144 | output = self.net(data) 145 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 146 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 147 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 148 | train_loss /= len(self.ldr_train.dataset) 149 | accuracy = 100. * correct / len(self.ldr_train.dataset) 150 | return train_loss, accuracy 151 | -------------------------------------------------------------------------------- /src/client/client_pfedme.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import copy 3 | import math 4 | 5 | import torch 6 | from torch import nn, optim 7 | import torch.nn.functional as F 8 | 9 | class Client_pFedMe(object): 10 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 11 | train_dl_local = None, test_dl_local = None): 12 | 13 | self.name = name 14 | self.net = model 15 | self.local_bs = local_bs 16 | self.local_ep = local_ep 17 | self.lr = lr 18 | self.momentum = momentum 19 | self.device = device 20 | self.loss_func = nn.CrossEntropyLoss() 21 | self.ldr_train = train_dl_local 22 | self.ldr_test = test_dl_local 23 | self.acc_best = 0 24 | self.count = 0 25 | self.save_best = True 26 | self.K = 5 27 | self.lam = 15 28 | self.personal_lr = 0.09 29 | # self.K = 5 30 | # self.lam = 15 31 | # self.personal_lr = 0.01 32 | 33 | def train(self, is_print = False): 34 | self.net.to(self.device) 35 | self.net.train() 36 | 37 | optimizer = pFedMeOptimizer(self.net.parameters(), lr=self.personal_lr, lam=self.lam) 38 | 39 | epoch_loss = [] 40 | w_local = copy.deepcopy(list(self.net.parameters())) 41 | for iteration in range(self.local_ep): 42 | batch_loss = [] 43 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 44 | images, labels = images.to(self.device), labels.to(self.device) 45 | labels = labels.type(torch.LongTensor).to(self.device) 46 | 47 | for _ in range(self.K): 48 | optimizer.zero_grad() 49 | log_probs = self.net(images) 50 | loss = self.loss_func(log_probs, labels) 51 | loss.backward() 52 | personalized_model_bar, _ = optimizer.step(w_local) 53 | 54 | for new_param, localweight, w in zip(personalized_model_bar, w_local, self.net.parameters()): 55 | w.data = localweight.data - self.lam * self.lr * (localweight.data - new_param.data) 56 | 57 | batch_loss.append(loss.item()) 58 | 59 | if batch_loss != []: 60 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 61 | 62 | # if self.save_best: 63 | # _, acc = self.eval_test() 64 | # if acc > self.acc_best: 65 | # self.acc_best = acc 66 | # return sum(epoch_loss) / len(epoch_loss) 67 | 68 | if epoch_loss == []: 69 | return 0 70 | else: 71 | return sum(epoch_loss) / len(epoch_loss) 72 | 73 | def get_state_dict(self, keep_vars=False): 74 | return self.net.state_dict(keep_vars=keep_vars) 75 | def get_best_acc(self): 76 | return self.acc_best 77 | def get_count(self): 78 | return self.count 79 | def get_net(self): 80 | return self.net 81 | def set_state_dict(self, state_dict): 82 | self.net.load_state_dict(state_dict) 83 | 84 | def eval_test(self): 85 | self.net.to(self.device) 86 | self.net.eval() 87 | test_loss = 0 88 | correct = 0 89 | with torch.no_grad(): 90 | for data, target in self.ldr_test: 91 | data, target = data.to(self.device), target.to(self.device) 92 | target = target.type(torch.LongTensor).to(self.device) 93 | 94 | output = self.net(data) 95 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 96 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 97 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 98 | test_loss /= len(self.ldr_test.dataset) 99 | accuracy = 100. * correct / len(self.ldr_test.dataset) 100 | return test_loss, accuracy 101 | 102 | def eval_train(self): 103 | self.net.to(self.device) 104 | self.net.eval() 105 | train_loss = 0 106 | correct = 0 107 | with torch.no_grad(): 108 | for data, target in self.ldr_train: 109 | data, target = data.to(self.device), target.to(self.device) 110 | target = target.type(torch.LongTensor).to(self.device) 111 | 112 | output = self.net(data) 113 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 114 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 115 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 116 | train_loss /= len(self.ldr_train.dataset) 117 | accuracy = 100. * correct / len(self.ldr_train.dataset) 118 | return train_loss, accuracy 119 | 120 | class pFedMeOptimizer(optim.Optimizer): 121 | def __init__(self, params, lr=0.01, lam=15 , mu=0.001): 122 | #self.local_weight_updated = local_weight # w_i,K 123 | if lr < 0.0: 124 | raise ValueError("Invalid learning rate: {}".format(lr)) 125 | defaults = dict(lr=lr, lam=lam, mu=mu) 126 | super(pFedMeOptimizer, self).__init__(params, defaults) 127 | 128 | def step(self, local_weight_updated, closure=None): 129 | loss = None 130 | if closure is not None: 131 | loss = closure 132 | weight_update = local_weight_updated.copy() 133 | for group in self.param_groups: 134 | for p, localweight in zip( group['params'], weight_update): 135 | p.data = p.data - group['lr'] * (p.grad.data + group['lam'] * (p.data - localweight.data) + group['mu']*p.data) 136 | return group['params'], loss 137 | 138 | # def eval_test(net, args, ldr_test): 139 | # net.to(args.device) 140 | # net.eval() 141 | # test_loss = 0 142 | # correct = 0 143 | # with torch.no_grad(): 144 | # for data, target in ldr_test: 145 | # data, target = data.to(args.device), target.to(args.device) 146 | # target = target.type(torch.LongTensor).to(self.device) 147 | 148 | # output = net(data) 149 | # test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 150 | # pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 151 | # correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 152 | # test_loss /= len(ldr_test.dataset) 153 | # accuracy = 100. * correct / len(ldr_test.dataset) 154 | # return test_loss, accuracy 155 | -------------------------------------------------------------------------------- /src/client/client_cfl.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import torch 4 | from torch import nn, optim 5 | import torch.nn.functional as F 6 | from sklearn.cluster import AgglomerativeClustering 7 | 8 | def mycopy(target, source): 9 | for name in target: 10 | target[name].data = source[name].data.clone() 11 | 12 | def subtract_(target, minuend, subtrahend): 13 | for name in target: 14 | target[name].data = minuend[name].data.clone()-subtrahend[name].data.clone() 15 | 16 | def flatten(source): 17 | return torch.cat([value.flatten() for value in source.values()]) 18 | 19 | def reduce_add_average(targets, sources): 20 | for target in targets: 21 | for name in target: 22 | tmp = torch.mean(torch.stack([source[name].data for source in sources]), dim=0).clone() 23 | target[name].data += tmp 24 | 25 | def pairwise_angles(sources): 26 | angles = torch.zeros([len(sources), len(sources)]) 27 | for i, source1 in enumerate(sources): 28 | for j, source2 in enumerate(sources): 29 | s1 = flatten(source1) 30 | s2 = flatten(source2) 31 | angles[i,j] = torch.sum(s1*s2)/(torch.norm(s1)*torch.norm(s2)+1e-12) 32 | 33 | return angles.numpy() 34 | 35 | def compute_pairwise_similarities(clients): 36 | return pairwise_angles([client.dw for client in clients]) 37 | 38 | def cluster_clients(S): 39 | clustering = AgglomerativeClustering(affinity="precomputed", linkage="complete").fit(-S) 40 | 41 | c1 = np.argwhere(clustering.labels_ == 0).flatten() 42 | c2 = np.argwhere(clustering.labels_ == 1).flatten() 43 | return c1, c2 44 | 45 | def aggregate_clusterwise(client_clusters): 46 | for cluster in client_clusters: 47 | reduce_add_average(targets=[client.w for client in cluster], 48 | sources=[client.dw for client in cluster]) 49 | 50 | def compute_max_update_norm(cluster): 51 | return np.max([torch.norm(flatten(client.dw)).item() for client in cluster]) 52 | 53 | def compute_mean_update_norm(cluster): 54 | return torch.norm(torch.mean(torch.stack([flatten(client.dw) for client in cluster]), 55 | dim=0)).item() 56 | 57 | class Client_CFL(object): 58 | def __init__(self, name, model, local_bs, local_ep, lr, momentum, device, 59 | train_dl_local = None, test_dl_local = None): 60 | 61 | self.name = name 62 | self.net = model 63 | self.local_bs = local_bs 64 | self.local_ep = local_ep 65 | self.lr = lr 66 | self.momentum = momentum 67 | self.device = device 68 | self.net.to(self.device) 69 | self.loss_func = nn.CrossEntropyLoss() 70 | self.ldr_train = train_dl_local 71 | self.ldr_test = test_dl_local 72 | self.acc_best = 0 73 | self.count = 0 74 | self.save_best = True 75 | self.w = {key : value for key, value in self.net.named_parameters()} 76 | self.w_old = {key : torch.zeros_like(value) for key, value in self.net.named_parameters()} 77 | self.dw = {key : torch.zeros_like(value) for key, value in self.net.named_parameters()} 78 | 79 | def train(self, is_print = False): 80 | self.net.to(self.device) 81 | self.net.train() 82 | mycopy(target=self.w_old, source=self.w) 83 | 84 | optimizer = torch.optim.SGD(self.net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=0) 85 | 86 | epoch_loss = [] 87 | for iteration in range(self.local_ep): 88 | batch_loss = [] 89 | for batch_idx, (images, labels) in enumerate(self.ldr_train): 90 | images, labels = images.to(self.device), labels.to(self.device) 91 | labels = labels.type(torch.LongTensor).to(self.device) 92 | 93 | self.net.zero_grad() 94 | #optimizer.zero_grad() 95 | log_probs = self.net(images) 96 | loss = self.loss_func(log_probs, labels) 97 | loss.backward() 98 | 99 | optimizer.step() 100 | batch_loss.append(loss.item()) 101 | 102 | epoch_loss.append(sum(batch_loss)/len(batch_loss)) 103 | 104 | # if self.save_best: 105 | # _, acc = self.eval_test() 106 | # if acc > self.acc_best: 107 | # self.acc_best = acc 108 | subtract_(target=self.dw, minuend=self.w, subtrahend=self.w_old) 109 | mycopy(target=self.w, source=self.w_old) 110 | return sum(epoch_loss) / len(epoch_loss) 111 | 112 | def get_state_dict(self): 113 | return self.net.state_dict() 114 | def get_best_acc(self): 115 | return self.acc_best 116 | def get_count(self): 117 | return self.count 118 | def get_net(self): 119 | return self.net 120 | def set_state_dict(self, state_dict): 121 | self.net.load_state_dict(state_dict) 122 | 123 | def eval_test(self): 124 | self.net.to(self.device) 125 | self.net.eval() 126 | test_loss = 0 127 | correct = 0 128 | with torch.no_grad(): 129 | for data, target in self.ldr_test: 130 | data, target = data.to(self.device), target.to(self.device) 131 | target = target.type(torch.LongTensor).to(self.device) 132 | 133 | output = self.net(data) 134 | test_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 135 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 136 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 137 | test_loss /= len(self.ldr_test.dataset) 138 | accuracy = 100. * correct / len(self.ldr_test.dataset) 139 | return test_loss, accuracy 140 | 141 | def eval_train(self): 142 | self.net.to(self.device) 143 | self.net.eval() 144 | train_loss = 0 145 | correct = 0 146 | with torch.no_grad(): 147 | for data, target in self.ldr_train: 148 | data, target = data.to(self.device), target.to(self.device) 149 | target = target.type(torch.LongTensor).to(self.device) 150 | 151 | output = self.net(data) 152 | train_loss += F.cross_entropy(output, target, reduction='sum').item() # sum up batch loss 153 | pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability 154 | correct += pred.eq(target.data.view_as(pred)).long().cpu().sum() 155 | train_loss /= len(self.ldr_train.dataset) 156 | accuracy = 100. * correct / len(self.ldr_train.dataset) 157 | return train_loss, accuracy -------------------------------------------------------------------------------- /src/utils/options_cluster.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | ## CIFAR-10 has 50000 training images (5000 per class), 10 classes, 10000 test images (1000 per class) 4 | ## CIFAR-100 has 50000 training images (500 per class), 100 classes, 10000 test images (100 per class) 5 | ## MNIST has 60000 training images (min: 5421, max: 6742 per class), 10000 test images (min: 892, max: 1135 6 | ## per class) --> in the code we fixed 5000 training image per class, and 900 test image per class to be 7 | ## consistent with CIFAR-10 8 | 9 | ## CIFAR-10 Non-IID 250 samples per label for 2 class non-iid is the benchmark (500 samples for each client) 10 | 11 | def args_parser(): 12 | parser = argparse.ArgumentParser() 13 | # general federated arguments 14 | parser.add_argument('--rounds', type=int, default=500, help="rounds of training") 15 | parser.add_argument('--num_users', type=int, default=100, help="number of users: K") 16 | parser.add_argument('--model', type=str, default='lenet5', help='model name') 17 | parser.add_argument('--dataset', type=str, default='cifar10', help="name of dataset: mnist, cifar10, cifar100") 18 | parser.add_argument('--partition', type=str, default='iid', help='method of partitioning') 19 | parser.add_argument('--niid_beta', type=float, default=0.5, help='The parameter for non-iid data partitioning') 20 | parser.add_argument('--iid_beta', type=float, default=0.5, help='The parameter for iid data partitioning') 21 | parser.add_argument('--frac', type=float, default=0.1, help="the fraction of clients: C") 22 | parser.add_argument('--local_ep', type=int, default=10, help="the number of local epochs: E") 23 | parser.add_argument('--local_bs', type=int, default=10, help="local batch size: B") 24 | parser.add_argument('--lr', type=float, default=0.01, help="learning rate") 25 | parser.add_argument('--momentum', type=float, default=0.5, help="SGD momentum (default: 0.5)") 26 | parser.add_argument('--ntrials', type=int, default=1, help="the number of trials") 27 | parser.add_argument('--log_filename', type=str, default=None, help='The log file name') 28 | parser.add_argument('--p_train', type=float, default=1.0, help="Percentage of Train Data") 29 | parser.add_argument('--p_test', type=float, default=1.0, help="Percentage of Test Data") 30 | parser.add_argument('--datadir', type=str, default='../data/', help='data directory') 31 | parser.add_argument('--alg', type=str, default='cluster_fl', help='Algorithm') 32 | parser.add_argument('--logdir', type=str, default='../logs/', help='logs directory') 33 | parser.add_argument('--warmup_epoch', type=int, default=0, help="the number of pretrain local ep") 34 | 35 | ## newcomers 36 | parser.add_argument('--new_comer', action='store_true', help='if true, 80% clients are involved for FL training and 20% are evaluated afterwards') 37 | parser.add_argument('--ft_epoch', type=int, default=20, help="finetune epochs for new_comer") 38 | 39 | # clustering arguments for pacfl 40 | parser.add_argument('--pacfl_beta', type=float, default=4.6, help="PACFL clustering threshold") 41 | parser.add_argument('--pacfl_n_basis', type=int, default=3, help="PACFL number of basis per label") 42 | parser.add_argument('--pacfl_linkage', type=str, default='average', help="PACFL Type of Linkage for HC") 43 | 44 | ## FLIS 45 | parser.add_argument('--nclasses', type=int, default=10, help="number of classes") 46 | parser.add_argument('--flis_cluster_alpha', type=float, default=0.5, help="FLIS clustering threshold") 47 | 48 | ## IFCA 49 | parser.add_argument('--nclusters', type=int, default=2, help="Number of Clusters for IFCA") 50 | #parser.add_argument('--num_incluster_layers', type=int, default=2, help="Number of Clusters for IFCA") 51 | 52 | # pruning arguments for subfedavg 53 | parser.add_argument('--pruning_percent_subfedavg', type=float, default=5, 54 | help="Pruning percent for layers (0-100)") 55 | parser.add_argument('--pruning_target_subfedavg', type=float, default=35, 56 | help="Total Pruning target percentage (0-100)") 57 | parser.add_argument('--dist_thresh_subfedavg', type=float, default=0.0001, 58 | help="threshold for fcs masks difference ") 59 | parser.add_argument('--acc_thresh_subfedavg', type=float, default=55, 60 | help="accuracy threshold to apply the derived pruning mask") 61 | parser.add_argument('--ks', type=int, default=5, help='kernel size to use for convolutions') 62 | parser.add_argument('--in_ch', type=int, default=3, help='input channels of the first conv layer') 63 | # parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float, 64 | # metavar='W', help='weight decay (default: 1e-4)') 65 | 66 | ## FedDF 67 | parser.add_argument('--distill_lr', type=float, default=0.01, help="Distillation learning rate") 68 | parser.add_argument('--distill_T', type=float, default=1.0, help="Distillation Temprature") 69 | parser.add_argument('--distill_E', type=int, default=10, help="Distillation Epoch") 70 | parser.add_argument('--public_dataset', type=str, default='cifar100', help="Public Distillation Dataset cifar100") 71 | 72 | # algorithm specific arguments 73 | parser.add_argument('--pfedme_beta', type=float, default=1.0, help="pFedMe beta for global model update") 74 | parser.add_argument('--local_rep_ep', type=int, default=1, help="the number of local rep layers updates for FedRep") 75 | parser.add_argument('--n_models', type=int, default=3, help="number of mixture distributions M for FedEM") 76 | parser.add_argument('--mu', type=float, default=0.001, help="FedProx Regularizer") 77 | parser.add_argument('--alpha', type=float, default=0.25, help="alpha for APFL and FedDyn") 78 | parser.add_argument('--glob_layers', type=int, default=4, help='number of global or personal layers') 79 | parser.add_argument('--lam_ditto', type=float, default=0.8, help='Ditto parameter lambda') 80 | parser.add_argument('--alpha_apfl', type=float, default=0.75, help='APFL parameter Alpha') 81 | parser.add_argument('--use_project_head', action='store_true', help='whether use projection head or not') 82 | parser.add_argument('--temperature_moon', type=float, default=0.5, help='Moon parameter Temperature') 83 | parser.add_argument('--mu_moon', type=float, default=1.0, help='Moon parameter Mu') 84 | 85 | # other arguments 86 | parser.add_argument('--gpu', type=int, default=0, help="GPU ID, -1 for CPU") 87 | parser.add_argument('--is_print', action='store_true', help='verbose print') 88 | parser.add_argument('--print_freq', type=int, default=100, help="printing frequency during training rounds") 89 | parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') 90 | parser.add_argument('--load_initial', type=str, default='', help='define initial model path') 91 | #parser.add_argument('--savedir', type=str, default='../save/', help='save directory') 92 | #parser.add_argument('--results_save', type=str, default='/', help='define fed results save folder') 93 | #parser.add_argument('--start_saving', type=int, default=0, help='when to start saving models') 94 | 95 | args = parser.parse_args() 96 | return args 97 | --------------------------------------------------------------------------------