├── .flake8 ├── .github └── workflows │ ├── lint.yml │ └── mkdocs-gh-pages.yml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── LICENSE ├── README.md ├── conf └── extensions.yaml ├── data ├── coco │ └── download_coco.py ├── fddb │ └── download.sh ├── fruit │ └── download_fruit.py ├── mnist │ ├── checksum │ └── download.sh ├── roadsign_voc │ ├── download_roadsign_voc.py │ └── label_list.txt ├── voc │ ├── create_list.py │ ├── download_voc.py │ ├── generic_det_label_list.txt │ ├── generic_det_label_list_zh.txt │ └── label_list.txt └── wider_face │ └── download.sh ├── deploy └── deploy_tookit │ ├── .gitignore │ ├── MANIFEST.in │ ├── README.md │ ├── fedvision_deploy_toolkit │ ├── __init__.py │ ├── _build.py │ ├── _deploy.py │ ├── _generate_template.py │ ├── _service.py │ ├── cli.py │ ├── data │ │ └── __init__.py │ └── template │ │ ├── __init__.py │ │ ├── standalone_template.yaml │ │ └── template.yaml │ ├── pyproject.toml │ ├── setup.cfg │ └── setup.py ├── deps ├── PaddleFL │ ├── AUTHORS.md │ ├── LICENSE │ └── python │ │ └── paddle_fl │ │ ├── __init__.py │ │ └── paddle_fl │ │ ├── __init__.py │ │ └── core │ │ ├── __init__.py │ │ ├── master │ │ ├── __init__.py │ │ ├── fl_job.py │ │ └── job_generator.py │ │ └── strategy │ │ ├── __init__.py │ │ ├── details │ │ ├── __init__.py │ │ ├── checkport.py │ │ ├── program_utils.py │ │ ├── ps_dispatcher.py │ │ ├── ufind.py │ │ └── vars_distributed.py │ │ ├── fl_distribute_transpiler.py │ │ └── fl_strategy_base.py └── README.md ├── docs ├── apis │ ├── cluster.md │ ├── coordinator.md │ └── master.md ├── css │ ├── custom.css │ └── termynal.css ├── deploy │ └── cli.md ├── develop │ └── codestyle.md ├── framework │ ├── overview.md │ ├── paddledetection.md │ └── paddlefl.md ├── img │ ├── Architecture.png │ ├── federated_detection.png │ ├── fedvision.png │ └── job.png ├── index.md ├── js │ ├── custom.js │ └── termynal.js ├── quickstart │ └── quickstart.md ├── release │ ├── release.md │ └── v0.1.md └── requirements.txt ├── examples ├── paddle_detection │ ├── README.md │ ├── config.yaml │ ├── run.sh │ └── yolov3_mobilenet_v1_fruit.yml └── paddle_mnist │ ├── README.md │ ├── cnn.yml │ ├── config.yaml │ └── run.sh ├── fedvision ├── __init__.py ├── cli.py ├── framework │ ├── __init__.py │ ├── abc │ │ ├── __init__.py │ │ ├── executor.py │ │ ├── job.py │ │ └── task.py │ ├── cli │ │ ├── __init__.py │ │ ├── cluster_manager.py │ │ ├── cluster_worker.py │ │ ├── coordinator.py │ │ ├── master.py │ │ └── submitter.py │ ├── cluster │ │ ├── __init__.py │ │ ├── executor.py │ │ ├── manager.py │ │ └── worker.py │ ├── coordinator │ │ ├── __init__.py │ │ └── coordinator.py │ ├── extensions │ │ ├── __init__.py │ │ └── _extensions.py │ ├── master │ │ ├── __init__.py │ │ └── master.py │ ├── protobuf │ │ └── __init__.py │ └── utils │ │ ├── __init__.py │ │ ├── exception.py │ │ ├── logger.py │ │ └── logger.pyi ├── ml │ ├── __init__.py │ └── paddle │ │ ├── __init__.py │ │ ├── paddle_detection │ │ ├── README.md │ │ ├── __init__.py │ │ ├── _empty_optimizer.py │ │ ├── fl_master.py │ │ └── fl_trainer.py │ │ └── paddle_mnist │ │ ├── README.md │ │ ├── __init__.py │ │ ├── fl_master.py │ │ └── fl_trainer.py └── paddle_fl │ ├── __init__.py │ ├── job.py │ ├── protobuf │ └── __init__.py │ └── tasks │ ├── __init__.py │ ├── cli │ ├── __init__.py │ ├── fl_scheduler.py │ └── fl_server.py │ ├── task │ ├── __init__.py │ ├── aggregator.py │ └── trainer.py │ └── utils │ ├── __init__.py │ └── _trainer.py ├── mkdocs.yml ├── proto ├── build_config.yaml └── fedvision │ ├── framework │ └── protobuf │ │ ├── cluster.proto │ │ ├── coordinator.proto │ │ └── job.proto │ └── paddle_fl │ └── protobuf │ ├── fl_job.proto │ └── scheduler.proto ├── requirements.txt ├── requirements_dev.txt ├── sbin ├── cluster_manager.sh ├── cluster_worker.sh ├── coordinator.sh ├── env.sh ├── master.sh └── service.sh ├── schema └── paddle_fl.json ├── setup.py └── tools └── protobuf.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 119 3 | ignore = D203, E501, W503, C901, F541 4 | exclude =.git,__pycache__,.venv,deps 5 | max-complexity = 10 6 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: push 4 | 5 | jobs: 6 | run-linters: 7 | name: Run linters 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - name: Check out Git repository 12 | uses: actions/checkout@v2 13 | 14 | - name: Set up Python 15 | uses: actions/setup-python@v1 16 | with: 17 | python-version: 3.8 18 | 19 | - name: Install Python dependencies 20 | run: pip install black flake8 21 | 22 | - name: Run linters 23 | uses: wearerequired/lint-action@v1 24 | with: 25 | github_token: ${{ secrets.GITHUB_TOKEN }} 26 | # Enable linters 27 | black: true 28 | black_args: "--exclude deps/" 29 | flake8: true 30 | flake8_args: "--exclude deps/" -------------------------------------------------------------------------------- /.github/workflows/mkdocs-gh-pages.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Pages 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | deploy: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | with: 12 | submodules: "recursive" 13 | fetch-depth: 0 14 | - name: Setup Python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: '3.7' 18 | architecture: 'x64' 19 | - name: Install dependencies 20 | run: | 21 | python3 -m pip install --upgrade pip # install pip 22 | python3 -m pip install -r docs/requirements.txt # install doc dependency 23 | python3 -m pip install -r requirements.txt # install dependency 24 | python3 -m pip install -r requirements_dev.txt 25 | - name: Generate Proto Buffer 26 | run: | 27 | python3 tools/protobuf.py build 28 | python3 tools/protobuf.py doc 29 | - name: Build site 30 | run: mkdocs build 31 | - name: Deploy 32 | uses: peaceiris/actions-gh-pages@v3 33 | with: 34 | github_token: ${{ secrets.GITHUB_TOKEN }} 35 | publish_dir: ./site -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv/ 2 | .idea/ 3 | */__pycache__ 4 | *.egg-info 5 | dist/ 6 | build/ 7 | fedvision/framework/protobuf/* 8 | fedvision/paddle_fl/protobuf/* 9 | !fedvision/framework/protobuf/__init__.py 10 | !fedvision/paddle_fl/protobuf/__init__.py 11 | docs/apis/proto.md 12 | *.log 13 | logs/ 14 | FedVision.egg-info/ 15 | temp_data/ 16 | *.org 17 | **/__pycache__ 18 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/PaddleDetection"] 2 | path = deps/PaddleDetection 3 | url = https://github.com/PaddlePaddle/PaddleDetection.git 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/ambv/black 3 | rev: stable 4 | hooks: 5 | - id: black 6 | language_version: python3.7 7 | args: [--exclude=deps/] 8 | - repo: https://gitlab.com/pycqa/flake8 9 | rev: 3.7.9 10 | hooks: 11 | - id: flake8 12 | args: [--exclude=deps/] 13 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | mkdocs: 4 | configuration: mkdocs.yml 5 | fail_on_warning: false 6 | 7 | formats: all 8 | 9 | python: 10 | version: 3.7 11 | install: 12 | - requirements: docs/requirements.txt 13 | - requirements: requirements.txt 14 | - requirements: requirements_dev.txt 15 | - method: setuptools 16 | path: "." -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FedVision 2 | 3 | [![Documentation Read the Docs](https://img.shields.io/readthedocs/fedvision?label=Read%20the%20Docs)](https://fedvision.readthedocs.io/en/latest/?badge=latest) 4 | [![Documentation Github Page](https://github.com/FederatedAI/FedVision/workflows/GitHub%20Pages/badge.svg)](http://federatedai.github.io/FedVision) 5 | [![Support Python Versions](https://img.shields.io/pypi/pyversions/fedvision)](https://img.shields.io/pypi/pyversions/fedvision) 6 | [![PyPI Version](https://img.shields.io/pypi/v/FedVision)](https://pypi.org/project/fedvision/) 7 | 8 | 9 | FedVision is a Visual Object Detection Platform Powered by Federated Learning 10 | ![FederatedDetection](docs/img/federated_detection.png) 11 | 12 | ## Quick start 13 | 14 | ### Prerequisites 15 | 16 | Too run FedVision, following dependency or tools required: 17 | 18 | - machine to install fedvision-deploy-toolkit: 19 | 20 | - python virtualenv with Python>=3 21 | 22 | - setup SSH password less login to machine(s) for deploy fedvision framework. 23 | 24 | - machine(s) to deploy fedvision framework: 25 | 26 | - Python>=3.7(with pip) 27 | 28 | - an isolated directory (each directory will be deployed with a copy of code) 29 | 30 | ### Deploy 31 | 32 | 1. install fedvision deploy toolkit 33 | 34 | ``` bash 35 | # ceate a python virtual envirement (recommanded) or use an exist one. 36 | source /bin/activate 37 | python -m pip install -U pip && python -m pip install fedvision_deploy_toolkit 38 | ``` 39 | 40 | 2. generate deploy template 41 | 42 | ```bash 43 | fedvision-deploy template standalone 44 | ``` 45 | 46 | 3. read comments in generated template `standalone_template.yaml` and modify as you want. 47 | 48 | 4. run deploy cmd 49 | 50 | ```bash 51 | fedvision-deploy deploy deploy standalone_template.yaml 52 | ``` 53 | 54 | ### Services start and stop 55 | 56 | Services could be start/stop with scripts in `Fedvision/sbin` or, use fedvision deploy toolkits: 57 | 58 | ```bash 59 | fedvision-deploy services all start standalone_template.yaml 60 | ``` 61 | 62 | ### Run examples 63 | 64 | Jobs could be submitted at each deployed machine with master service started. 65 | 66 | ```bash 67 | cd /data/projects/fedvision 68 | source venv/bin/activate 69 | export PYTHONPATH=$PYTHONPATH:/data/projects/fedvision/FedVision 70 | sh FedVision/examples/paddle_mnist/run.sh 127.0.0.1:10002 71 | ``` 72 | 73 | ## Architecture and Runtime Framework 74 | 75 | We utilize `PaddleFL` to makes `PaddlePaddle` programs federated and utilize `PaddleDetection` to generate object detection program. 76 | This project may be extended to utilize `pytorch's Ecology` in future versions as well. 77 | 78 | ![architecture](docs/img/Architecture.png) 79 | 80 | At runtime, each `Party` connects with coordinator and proposal jobs to or subscribe jobs from coordinator to participate in federated learning training cycle. 81 | 82 | ![framework](docs/img/fedvision.png) 83 | 84 | 85 | ## Documentation 86 | 87 | Documentation can be generated using [Mkdocs](https://www.mkdocs.org/). 88 | Users can build the documentation using 89 | 90 | ``` 91 | source venv/bin/activate # venv to build docs 92 | pip install -r requirement.txt 93 | pip install -r docs/requirements.txt 94 | mkdocs serve 95 | ``` 96 | 97 | Open up http://127.0.0.1:8000/ in your browser. 98 | 99 | The latest version of online documentation can be found at 100 | 101 | - [![Documentation Read the Docs](https://img.shields.io/readthedocs/fedvision?label=Read%20the%20Docs)](https://fedvision.readthedocs.io/en/latest/?badge=latest) 102 | 103 | - [![Documentation Github Page](https://github.com/FederatedAI/FedVision/workflows/GitHub%20Pages/badge.svg)](http://federatedai.github.io/FedVision) 104 | 105 | ## License 106 | 107 | FedVision is released under Apache License 2.0. 108 | Please note that third-party libraries may not have the same license as FedVision. 109 | 110 | ## Contributing 111 | 112 | Any contributions you make are greatly appreciated! 113 | 114 | - Please report bugs by submitting a GitHub issue. 115 | 116 | - Please submit contributions using pull requests. -------------------------------------------------------------------------------- /conf/extensions.yaml: -------------------------------------------------------------------------------- 1 | PaddleFL: 2 | jobs: 3 | - name: paddle_fl 4 | schema: ../schema/paddle_fl.json 5 | loader: fedvision.paddle_fl.job:PaddleFLJob 6 | tasks: 7 | - name: fl_trainer 8 | loader: fedvision.paddle_fl.tasks.task.trainer:FLTrainer 9 | - name: fl_aggregator 10 | loader: fedvision.paddle_fl.tasks.task.aggregator:FLAggregator 11 | -------------------------------------------------------------------------------- /data/coco/download_coco.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import os.path as osp 17 | import logging 18 | 19 | # add python path of PadleDetection to sys.path 20 | parent_path = osp.abspath( 21 | osp.join(__file__, osp.pardir, osp.pardir, osp.pardir, "deps", "PaddleDetection") 22 | ) 23 | if parent_path not in sys.path: 24 | sys.path.append(parent_path) 25 | 26 | from ppdet.utils.download import download_dataset # noqa: E402 27 | 28 | logging.basicConfig(level=logging.INFO) 29 | 30 | download_path = osp.split(osp.realpath(sys.argv[0]))[0] 31 | download_dataset(download_path, "coco") 32 | -------------------------------------------------------------------------------- /data/fddb/download.sh: -------------------------------------------------------------------------------- 1 | # All rights `PaddleDetection` reserved 2 | # References: 3 | # @TechReport{fddbTech, 4 | # author = {Vidit Jain and Erik Learned-Miller}, 5 | # title = {FDDB: A Benchmark for Face Detection in Unconstrained Settings}, 6 | # institution = {University of Massachusetts, Amherst}, 7 | # year = {2010}, 8 | # number = {UM-CS-2010-009} 9 | # } 10 | 11 | DIR="$( cd "$(dirname "$0")" ; pwd -P )" 12 | cd "$DIR" 13 | 14 | # Download the data. 15 | echo "Downloading..." 16 | # external link to the Faces in the Wild data set and annotations file 17 | wget http://tamaraberg.com/faceDataset/originalPics.tar.gz 18 | wget http://vis-www.cs.umass.edu/fddb/FDDB-folds.tgz 19 | wget http://vis-www.cs.umass.edu/fddb/evaluation.tgz 20 | 21 | # Extract the data. 22 | echo "Extracting..." 23 | tar -zxf originalPics.tar.gz 24 | tar -zxf FDDB-folds.tgz 25 | tar -zxf evaluation.tgz 26 | 27 | # Generate full image path list and groundtruth in FDDB-folds: 28 | cd FDDB-folds 29 | cat `ls|grep -v"ellipse"` > filePath.txt && cat *ellipse* > fddb_annotFile.txt 30 | cd .. 31 | echo "------------- All done! --------------" 32 | -------------------------------------------------------------------------------- /data/fruit/download_fruit.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import os.path as osp 17 | import logging 18 | 19 | # add python path of PadleDetection to sys.path 20 | parent_path = osp.abspath( 21 | osp.join(__file__, osp.pardir, osp.pardir, osp.pardir, "deps", "PaddleDetection") 22 | ) 23 | if parent_path not in sys.path: 24 | sys.path.append(parent_path) 25 | 26 | from ppdet.utils.download import download_dataset # noqa: E402 27 | 28 | logging.basicConfig(level=logging.INFO) 29 | 30 | download_path = osp.split(osp.realpath(sys.argv[0]))[0] 31 | download_dataset(download_path, "fruit") 32 | -------------------------------------------------------------------------------- /data/mnist/checksum: -------------------------------------------------------------------------------- 1 | 9fb629c4189551a2d022fa330f9573f3 t10k-images-idx3-ubyte.gz 2 | ec29112dd5afa0611ce80d1b7f02629c t10k-labels-idx1-ubyte.gz 3 | f68b3c2dcbeaaa9fbdd348bbdeb94873 train-images-idx3-ubyte.gz 4 | d53e105ee54ea40749a09fcbcd1e9432 train-labels-idx1-ubyte.gz 5 | -------------------------------------------------------------------------------- /data/mnist/download.sh: -------------------------------------------------------------------------------- 1 | DIR="$( 2 | cd "$(dirname "$0")" 3 | pwd -P 4 | )" 5 | cd "$DIR" 6 | 7 | URL_PREFIX="https://dataset.bj.bcebos.com/mnist/" 8 | TEST_IMAGE="t10k-images-idx3-ubyte.gz" 9 | TEST_LABEL="t10k-labels-idx1-ubyte.gz" 10 | TRAIN_IMAGE="train-images-idx3-ubyte.gz" 11 | TRAIN_LABEL="train-labels-idx1-ubyte.gz" 12 | 13 | echo "Downloading..." 14 | wget "${URL_PREFIX}${TEST_IMAGE}" 15 | wget "${URL_PREFIX}${TEST_LABEL}" 16 | wget "${URL_PREFIX}${TRAIN_IMAGE}" 17 | wget "${URL_PREFIX}${TRAIN_LABEL}" 18 | 19 | echo "checking" 20 | md5sum --check checksum 21 | -------------------------------------------------------------------------------- /data/roadsign_voc/download_roadsign_voc.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import os.path as osp 17 | import logging 18 | 19 | # add python path of PadleDetection to sys.path 20 | parent_path = osp.abspath( 21 | osp.join(__file__, osp.pardir, osp.pardir, osp.pardir, "deps", "PaddleDetection") 22 | ) 23 | if parent_path not in sys.path: 24 | sys.path.append(parent_path) 25 | 26 | from ppdet.utils.download import download_dataset # noqa: E402 27 | 28 | logging.basicConfig(level=logging.INFO) 29 | 30 | download_path = osp.split(osp.realpath(sys.argv[0]))[0] 31 | download_dataset(download_path, "roadsign_voc") 32 | -------------------------------------------------------------------------------- /data/roadsign_voc/label_list.txt: -------------------------------------------------------------------------------- 1 | speedlimit 2 | crosswalk 3 | trafficlight 4 | stop -------------------------------------------------------------------------------- /data/voc/create_list.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import os.path as osp 17 | import logging 18 | import argparse 19 | 20 | # add python path of PadleDetection to sys.path 21 | parent_path = osp.abspath( 22 | osp.join(__file__, osp.pardir, osp.pardir, osp.pardir, "deps", "PaddleDetection") 23 | ) 24 | if parent_path not in sys.path: 25 | sys.path.append(parent_path) 26 | 27 | from ppdet.utils.download import create_voc_list # noqa: E402 28 | 29 | logging.basicConfig(level=logging.INFO) 30 | 31 | 32 | def main(config): 33 | voc_path = config.dataset_dir 34 | create_voc_list(voc_path) 35 | 36 | 37 | if __name__ == "__main__": 38 | parser = argparse.ArgumentParser() 39 | default_voc_path = osp.split(osp.realpath(sys.argv[0]))[0] 40 | parser.add_argument( 41 | "-d", 42 | "--dataset_dir", 43 | default=default_voc_path, 44 | type=str, 45 | help="VOC dataset directory, default is current directory.", 46 | ) 47 | config = parser.parse_args() 48 | 49 | main(config) 50 | -------------------------------------------------------------------------------- /data/voc/download_voc.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import os.path as osp 17 | import logging 18 | 19 | # add python path of PadleDetection to sys.path 20 | parent_path = osp.abspath( 21 | osp.join(__file__, osp.pardir, osp.pardir, osp.pardir, "deps", "PaddleDetection") 22 | ) 23 | if parent_path not in sys.path: 24 | sys.path.append(parent_path) 25 | 26 | from ppdet.utils.download import download_dataset, create_voc_list # noqa: E402 27 | 28 | logging.basicConfig(level=logging.INFO) 29 | 30 | download_path = osp.split(osp.realpath(sys.argv[0]))[0] 31 | download_dataset(download_path, "voc") 32 | create_voc_list(download_path) 33 | -------------------------------------------------------------------------------- /data/voc/generic_det_label_list_zh.txt: -------------------------------------------------------------------------------- 1 | 婴儿床 2 | 玫瑰 3 | 旗 4 | 手电筒 5 | 海龟 6 | 照相机 7 | 动物 8 | 手套 9 | 鳄鱼 10 | 牛 11 | 房子 12 | 鳄梨酱 13 | 企鹅 14 | 车辆牌照 15 | 凳子 16 | 瓢虫 17 | 人鼻 18 | 西瓜 19 | 长笛 20 | 蝴蝶 21 | 洗衣机 22 | 浣熊 23 | 赛格威 24 | 墨西哥玉米薄饼卷 25 | 海蜇 26 | 蛋糕 27 | 笔 28 | 加农炮 29 | 面包 30 | 树 31 | 贝类 32 | 床 33 | 仓鼠 34 | 帽子 35 | 烤面包机 36 | 帽帽 37 | 冠状头饰 38 | 碗 39 | 蜻蜓 40 | 飞蛾和蝴蝶 41 | 羚羊 42 | 蔬菜 43 | 火炬 44 | 建筑物 45 | 电源插头和插座 46 | 搅拌机 47 | 台球桌 48 | 切割板 49 | 青铜雕塑 50 | 乌龟 51 | 西兰花 52 | 老虎 53 | 镜子 54 | 熊 55 | 西葫芦 56 | 礼服 57 | 排球 58 | 吉他 59 | 爬行动物 60 | 高尔夫球车 61 | 蛋挞 62 | 费多拉 63 | 食肉动物 64 | 小型车 65 | 灯塔 66 | 咖啡壶 67 | 食品加工厂 68 | 卡车 69 | 书柜 70 | 冲浪板 71 | 鞋类 72 | 凳子 73 | 项链 74 | 花 75 | 萝卜 76 | 海洋哺乳动物 77 | 煎锅 78 | 水龙头 79 | 桃 80 | 刀 81 | 手提包 82 | 笔记本电脑 83 | 帐篷 84 | 救护车 85 | 圣诞树 86 | 鹰 87 | 豪华轿车 88 | 厨房和餐桌 89 | 北极熊 90 | 塔楼 91 | 足球 92 | 柳树 93 | 人头 94 | 停车标志 95 | 香蕉 96 | 搅拌机 97 | 双筒望远镜 98 | 甜点 99 | 蜜蜂 100 | 椅子 101 | 烧柴炉 102 | 花盆 103 | 烧杯 104 | 牡蛎 105 | 啄木鸟 106 | 竖琴 107 | 浴缸 108 | 挂钟 109 | 运动服 110 | 犀牛 111 | 蜂箱 112 | 橱柜 113 | 鸡 114 | 人 115 | 冠蓝鸦 116 | 黄瓜 117 | 气球 118 | 风筝 119 | 壁炉 120 | 灯笼 121 | 导弹 122 | 书 123 | 勺子 124 | 葡萄柚 125 | 松鼠 126 | 橙色 127 | 外套 128 | 打孔袋 129 | 斑马 130 | 广告牌 131 | 自行车 132 | 门把手 133 | 机械风扇 134 | 环形粘结剂 135 | 桌子 136 | 鹦鹉 137 | 袜子 138 | 花瓶 139 | 武器 140 | 猎枪 141 | 玻璃杯 142 | 海马 143 | 腰带 144 | 船舶 145 | 窗口 146 | 长颈鹿 147 | 狮子 148 | 轮胎 149 | 车辆 150 | 独木舟 151 | 领带 152 | 架子 153 | 相框 154 | 打印机 155 | 人腿 156 | 小船 157 | 慢炖锅 158 | 牛角包 159 | 蜡烛 160 | 煎饼 161 | 枕头 162 | 硬币 163 | 担架 164 | 凉鞋 165 | 女人 166 | 楼梯 167 | 拨弦键琴 168 | 凳子 169 | 公共汽车 170 | 手提箱 171 | 人口学 172 | 果汁 173 | 颅骨 174 | 门 175 | 小提琴 176 | 筷子 177 | 数字时钟 178 | 向日葵 179 | 豹 180 | 甜椒 181 | 海港海豹 182 | 蛇 183 | 缝纫机 184 | 鹅 185 | 直升机 186 | 座椅安全带 187 | 咖啡杯 188 | 微波炉 189 | 热狗 190 | 台面 191 | 服务托盘 192 | 狗床 193 | 啤酒 194 | 太阳镜 195 | 高尔夫球 196 | 华夫饼干 197 | 棕榈树 198 | 小号 199 | 尺子 200 | 头盔 201 | 梯子 202 | 办公楼 203 | 平板电脑 204 | 厕纸 205 | 石榴 206 | 裙子 207 | 煤气炉 208 | 曲奇饼干 209 | 大车 210 | 掠夺 211 | 鸡蛋 212 | 墨西哥煎饼 213 | 山羊 214 | 菜刀 215 | 滑板 216 | 盐和胡椒瓶 217 | 猞猁 218 | 靴子 219 | 大浅盘 220 | 滑雪板 221 | 泳装 222 | 游泳池 223 | 吸管 224 | 扳手 225 | 鼓 226 | 蚂蚁 227 | 人耳 228 | 耳机 229 | 喷泉 230 | 鸟 231 | 牛仔裤 232 | 电视机 233 | 蟹 234 | 话筒 235 | 家用电器 236 | 除雪机 237 | 甲虫 238 | 朝鲜蓟 239 | 喷气式滑雪板 240 | 固定自行车 241 | 人发 242 | 棕熊 243 | 海星 244 | 叉子 245 | 龙虾 246 | 有线电话 247 | 饮料 248 | 碟 249 | 胡萝卜 250 | 昆虫 251 | 时钟 252 | 城堡 253 | 网球拍 254 | 吊扇 255 | 芦笋 256 | 美洲虎 257 | 乐器 258 | 火车 259 | 猫 260 | 来复枪 261 | 哑铃 262 | 手机 263 | 出租车 264 | 淋浴 265 | 投掷者 266 | 柠檬 267 | 无脊椎动物 268 | 火鸡 269 | 高跟鞋 270 | 打破 271 | 大象 272 | 围巾 273 | 枪管 274 | 长号 275 | 南瓜 276 | 盒子 277 | 番茄 278 | 蛙 279 | 坐浴盆 280 | 人脸 281 | 室内植物 282 | 厢式货车 283 | 鲨鱼 284 | 冰淇淋 285 | 游泳帽 286 | 隼 287 | 鸵鸟 288 | 手枪 289 | 白板 290 | 蜥蜴 291 | 面食 292 | 雪车 293 | 灯泡 294 | 窗盲 295 | 松饼 296 | 椒盐脆饼 297 | 计算机显示器 298 | 喇叭 299 | 家具 300 | 三明治 301 | 福克斯 302 | 便利店 303 | 鱼 304 | 水果 305 | 耳环 306 | 帷幕 307 | 葡萄 308 | 沙发床 309 | 马 310 | 行李和行李 311 | 书桌 312 | 拐杖 313 | 自行车头盔 314 | 滴答声 315 | 飞机 316 | 金丝雀 317 | 铲 318 | 手表 319 | 莉莉 320 | 厨房用具 321 | 文件柜 322 | 飞机 323 | 蛋糕架 324 | 糖果 325 | 水槽 326 | 鼠标 327 | 葡萄酒 328 | 轮椅 329 | 金鱼 330 | 冰箱 331 | 炸薯条 332 | 抽屉 333 | 单调的工作 334 | 野餐篮子 335 | 骰子 336 | 甘蓝 337 | 足球头盔 338 | 猪 339 | 人 340 | 短裤 341 | 贡多拉 342 | 蜂巢 343 | 炸圈饼 344 | 抽屉柜 345 | 陆地车辆 346 | 蝙蝠 347 | 猴子 348 | 匕首 349 | 餐具 350 | 人足 351 | 马克杯 352 | 闹钟 353 | 高压锅 354 | 人手 355 | 乌龟 356 | 棒球手套 357 | 剑 358 | 梨 359 | 迷你裙 360 | 交通标志 361 | 女孩 362 | 旱冰鞋 363 | 恐龙 364 | 门廊 365 | 胡须 366 | 潜艇三明治 367 | 螺丝起子 368 | 草莓 369 | 酒杯 370 | 海鲜 371 | 球拍 372 | 车轮 373 | 海狮 374 | 玩具 375 | 茶叶 376 | 网球 377 | 废物容器 378 | 骡子 379 | 板球 380 | 菠萝 381 | 椰子 382 | 娃娃 383 | 咖啡桌 384 | 雪人 385 | 薰衣草 386 | 小虾 387 | 枫树 388 | 牛仔帽 389 | 护目镜 390 | 橄榄球 391 | 毛虫 392 | 海报 393 | 火箭 394 | 器官 395 | 萨克斯 396 | 交通灯 397 | 鸡尾酒 398 | 塑料袋 399 | 壁球 400 | 蘑菇 401 | 汉堡包 402 | 电灯开关 403 | 降落伞 404 | 泰迪熊 405 | 冬瓜 406 | 鹿 407 | 音乐键盘 408 | 卫生器具 409 | 记分牌 410 | 棒球棒 411 | 包络线 412 | 胶带 413 | 公文包 414 | 桨 415 | 弓箭 416 | 电话 417 | 羊 418 | 夹克 419 | 男孩 420 | 披萨 421 | 水獭 422 | 办公用品 423 | 沙发 424 | 大提琴 425 | 公牛 426 | 骆驼 427 | 球 428 | 鸭子 429 | 鲸鱼 430 | 衬衫 431 | 坦克 432 | 摩托车 433 | 手风琴 434 | 猫头鹰 435 | 豪猪 436 | 太阳帽 437 | 钉子 438 | 剪刀 439 | 天鹅 440 | 灯 441 | 皇冠 442 | 钢琴 443 | 雕塑 444 | 猎豹 445 | 双簧管 446 | 罐头罐 447 | 芒果 448 | 三脚架 449 | 烤箱 450 | 鼠标 451 | 驳船 452 | 咖啡 453 | 滑雪板 454 | 普通无花果 455 | 沙拉 456 | 无脊椎动物 457 | 雨伞 458 | 袋鼠 459 | 人手臂 460 | 量杯 461 | 蜗牛 462 | 相思 463 | 西服 464 | 茶壶 465 | 瓶 466 | 羊驼 467 | 水壶 468 | 裤子 469 | 爆米花 470 | 蜈蚣 471 | 蜘蛛 472 | 麻雀 473 | 盘子 474 | 百吉饼 475 | 个人护理 476 | 苹果 477 | 胸罩 478 | 浴室柜 479 | 演播室沙发 480 | 电脑键盘 481 | 乒乓球拍 482 | 寿司 483 | 橱柜 484 | 路灯 485 | 毛巾 486 | 床头柜 487 | 兔 488 | 海豚 489 | 狗 490 | 大罐 491 | 炒锅 492 | 消火栓 493 | 人眼 494 | 摩天大楼 495 | 背包 496 | 马铃薯 497 | 纸巾 498 | 小精灵 499 | 自行车车轮 500 | 卫生间 501 | 大号 502 | 地毯 503 | 手推车 504 | 电视 505 | 风扇 506 | 美洲驼 507 | 订书机 508 | 三轮车 509 | 耳机 510 | 空调器 511 | 饼干 512 | 毛巾/餐巾 513 | 靴子 514 | 香肠 515 | 运动型多用途汽车 516 | 肥皂 517 | 棒球 518 | 行李 519 | 扑克牌 520 | 铲子 521 | 标记笔 522 | 耳机 523 | 投影机 524 | 铅笔盒 525 | 法国圆号 526 | 橘子 527 | 路由器 528 | 文件夹 529 | 甜甜圈 530 | 榴莲 531 | 帆船 532 | 坚果 533 | 咖啡机 534 | 肉丸 535 | 篮子 536 | 插线板 537 | 青豆 538 | 鳄梨 539 | 英式足球 540 | 蛋挞 541 | 离合器 542 | 滑梯 543 | 鱼竿 544 | 衣架 545 | 面包 546 | 监控摄像头 547 | 地球仪 548 | 黑板/白板 549 | 救生员 550 | 鸽子 551 | 红卷心菜 552 | 铜钹 553 | 水龙头 554 | 牛排 555 | 秋千 556 | 山竹 557 | 奶酪 558 | 小便池 559 | 生菜 560 | 跨栏 561 | 戒指 562 | 篮球 563 | 盆栽植物 564 | 人力车 565 | 目标 566 | 赛车 567 | 蝴蝶结 568 | 熨斗 569 | 化妆品 570 | 驴 571 | 锯 572 | 铁锤 573 | 台球 574 | 切割/砧板 575 | 电源插座 576 | 吹风机 577 | 包子 578 | 奖章/奖牌 579 | 液体肥皂 580 | 野鸟 581 | 皮鞋 582 | 餐桌 583 | 游戏板 584 | 杠铃 585 | 收音机 586 | 路灯 587 | 磁带 588 | 曲棍球 589 | 春卷 590 | 大米 591 | 高尔夫俱乐部 592 | 打火机 593 | 炸薯条 594 | 显微镜 595 | 手机 596 | 消防车 597 | 面条 598 | 橱柜/架子 599 | 电磁炉和煤气炉 600 | 钥匙 601 | 梳子 602 | 垃圾箱/罐 603 | 牙刷 604 | 枣子 605 | 电钻 606 | 奶牛 607 | 茄子 608 | 扫帚 609 | 抽油烟机 610 | 钳子 611 | 大葱 612 | 扇贝 613 | 洁面乳 614 | 牙膏 615 | 哈密瓜 616 | 橡皮擦 617 | 洗发水/沐浴露 618 | 光盘 619 | 溜冰鞋和滑雪鞋 620 | 美式足球 621 | 拖鞋 622 | 火龙果 623 | 锅/平底锅 624 | 计算器 625 | 纸巾 626 | 乒乓球拍 627 | 板擦 628 | 扬声器 629 | 木瓜 630 | 雪茄 631 | 信纸 632 | 大蒜 633 | 电饭锅 634 | 罐装的 635 | 停车计时器 636 | 手电筒 637 | 画笔 638 | 杯子 639 | 球杆 640 | 人行横道标志 641 | 奇异果/猕猴桃 642 | 散热器 643 | 拖把 644 | 电锯 645 | 凉鞋拖鞋 646 | 储物箱 647 | 洋葱 648 | 手镯 649 | 灭火器 650 | 秤 651 | 秋葵 652 | 微波炉 653 | 运动鞋 654 | 胡椒 655 | 玉米 656 | 柚子 657 | 主机 658 | 钳子 659 | 奖杯 660 | 李子/梅子 661 | 刷子/画笔 662 | 机械车辆 663 | 牦牛 664 | 起重机 665 | 转换器 666 | 面膜 667 | 马车 668 | 皮卡车 669 | 交通锥 670 | 馅饼 671 | 钢笔/铅笔 672 | 跑车 673 | 飞盘 674 | 清洁用品/洗涤剂/洗衣液 675 | 遥控器 676 | 婴儿车/手推车 677 | -------------------------------------------------------------------------------- /data/voc/label_list.txt: -------------------------------------------------------------------------------- 1 | aeroplane 2 | bicycle 3 | bird 4 | boat 5 | bottle 6 | bus 7 | car 8 | cat 9 | chair 10 | cow 11 | diningtable 12 | dog 13 | horse 14 | motorbike 15 | person 16 | pottedplant 17 | sheep 18 | sofa 19 | train 20 | tvmonitor 21 | -------------------------------------------------------------------------------- /data/wider_face/download.sh: -------------------------------------------------------------------------------- 1 | # All rights `PaddleDetection` reserved 2 | # References: 3 | # @inproceedings{yang2016wider, 4 | # Author = {Yang, Shuo and Luo, Ping and Loy, Chen Change and Tang, Xiaoou}, 5 | # Booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 6 | # Title = {WIDER FACE: A Face Detection Benchmark}, 7 | # Year = {2016}} 8 | 9 | DIR="$( cd "$(dirname "$0")" ; pwd -P )" 10 | cd "$DIR" 11 | 12 | # Download the data. 13 | echo "Downloading..." 14 | wget https://dataset.bj.bcebos.com/wider_face/WIDER_train.zip 15 | wget https://dataset.bj.bcebos.com/wider_face/WIDER_val.zip 16 | wget https://dataset.bj.bcebos.com/wider_face/wider_face_split.zip 17 | # Extract the data. 18 | echo "Extracting..." 19 | unzip -q WIDER_train.zip 20 | unzip -q WIDER_val.zip 21 | unzip -q wider_face_split.zip 22 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/.gitignore: -------------------------------------------------------------------------------- 1 | fedvision_deploy_toolkit/data/fedvision.tar.gz 2 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include fedvision_deploy_toolkit/data/fedvision.tar.gz 2 | include fedvision_deploy_toolkit/template/* 3 | 4 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/README.md: -------------------------------------------------------------------------------- 1 | ## FedVision Deploy Toolkit 2 | 3 | ### Build package 4 | 5 | ```bash 6 | git clone https://github.com/FederatedAI/FedVision 7 | python3 -m venv venv 8 | source venv/bin/activate 9 | pip install -r FedVision/requirements.txt 10 | pip install -r FedVision/requirements_dev.txt 11 | python FedVision/tools/protobuf.py build 12 | cd FedVision/deploy/deploy_tookit 13 | export PYTHONPATH=$(pwd) 14 | python fedvision_deploy_toolkit/_build.py 15 | 16 | # build 17 | pip install pep517 18 | python -m pep517.build . 19 | ``` 20 | 21 | ### Usage 22 | 23 | See `Deploy` section in online documents: 24 | 25 | - [![Documentation Read the Docs](https://img.shields.io/readthedocs/fedvision?label=Read%20the%20Docs)](https://fedvision.readthedocs.io/en/latest/?badge=latest) 26 | 27 | - [![Documentation Github Page](https://github.com/FederatedAI/FedVision/workflows/GitHub%20Pages/badge.svg)](http://federatedai.github.io/FedVision) 28 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from pathlib import Path 17 | 18 | __base_dir__ = Path(__file__).parent 19 | __fedvision_tarball__ = __base_dir__.joinpath("data", "fedvision.tar.gz").absolute() 20 | __template__ = __base_dir__.joinpath("template").absolute() 21 | __BASE_NAME__ = "FedVision" 22 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/_build.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | import tarfile 16 | 17 | from fedvision_deploy_toolkit import __base_dir__, __fedvision_tarball__, __BASE_NAME__ 18 | 19 | _project_base = __base_dir__.parent.parent.parent 20 | 21 | 22 | def _build(): 23 | __fedvision_tarball__.parent.mkdir(exist_ok=True, parents=True) 24 | with tarfile.open(__fedvision_tarball__, "w:gz") as tar: 25 | for path in [ 26 | _project_base.joinpath("fedvision"), 27 | _project_base.joinpath("deps", "PaddleDetection", "ppdet"), 28 | _project_base.joinpath("deps", "PaddleFL", "python", "paddle_fl"), 29 | _project_base.joinpath("data"), 30 | _project_base.joinpath("conf"), 31 | _project_base.joinpath("examples"), 32 | _project_base.joinpath("schema"), 33 | _project_base.joinpath("sbin"), 34 | _project_base.joinpath("requirements.txt"), 35 | ]: 36 | tar.add(path, f"{__BASE_NAME__}/{os.path.basename(path)}") 37 | 38 | 39 | def _clean(): 40 | if __fedvision_tarball__.exists(): 41 | os.remove(__fedvision_tarball__) 42 | 43 | 44 | if __name__ == "__main__": 45 | _clean() 46 | _build() 47 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/_deploy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | from pathlib import Path 16 | 17 | import fabric 18 | import typer 19 | import yaml 20 | from fedvision_deploy_toolkit import __fedvision_tarball__, __BASE_NAME__ 21 | 22 | app = typer.Typer(help="deploy tools") 23 | 24 | 25 | @app.command() 26 | def deploy( 27 | config: Path = typer.Option( 28 | ..., 29 | file_okay=True, 30 | exists=True, 31 | dir_okay=False, 32 | readable=True, 33 | resolve_path=True, 34 | ) 35 | ): 36 | with config.open() as f: 37 | config_dict = yaml.safe_load(f) 38 | machines = config_dict.get("machines", []) 39 | typer.echo( 40 | f"deploying {len(machines)} machines: {[machine['name'] for machine in machines]}" 41 | ) 42 | with typer.progressbar(machines, length=len(machines)) as bar: 43 | for machine in bar: 44 | _maybe_create_python_venv(machine) 45 | _upload_code(machine) 46 | _install_deps(machine) 47 | typer.echo(f"deploy done") 48 | 49 | 50 | def _upload_code(machine): 51 | tarfile = os.path.abspath(__fedvision_tarball__) 52 | base_dir = Path(machine["base_dir"]) 53 | with fabric.Connection(machine["ssh_string"]) as c: 54 | c.run(f"mkdir -p {base_dir}") 55 | with c.cd(str(base_dir)): 56 | c.put(tarfile, f"{base_dir}") 57 | c.run(f"tar -xf {tarfile} -C {base_dir}") 58 | c.run(f"rm {os.path.join(base_dir, os.path.basename(tarfile))}") 59 | 60 | 61 | def _maybe_create_python_venv(machine: dict): 62 | with fabric.Connection(machine["ssh_string"]) as c: 63 | version = c.run( 64 | f"{machine['python_for_venv_create']} " 65 | f"-c 'import sys; assert sys.version_info.major >= 3 and sys.version_info.minor >= 7'", 66 | warn=True, 67 | ) 68 | if version.failed: 69 | raise RuntimeError(f"python executable {machine['python']} not valid") 70 | 71 | base_dir = Path(machine["base_dir"]) 72 | c.run(f"mkdir -p {base_dir}") 73 | with c.cd(str(base_dir)): 74 | if c.run( 75 | f"test -f {base_dir.joinpath('venv/bin/python')}", warn=True 76 | ).failed: 77 | c.run(f"{machine['python_for_venv_create']} -m venv venv") 78 | 79 | 80 | def _install_deps(machine): 81 | with fabric.Connection(machine["ssh_string"]) as c: 82 | base_dir = Path(machine["base_dir"]) 83 | with c.cd(str(base_dir)): 84 | c.run(f"venv/bin/python -m pip install -U pip --quiet") 85 | c.run( 86 | f"venv/bin/python -m pip install -r {__BASE_NAME__}/requirements.txt --log deps_install.log --quiet" 87 | ) 88 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/_generate_template.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | import shutil 16 | 17 | import typer 18 | from fedvision_deploy_toolkit import __template__ 19 | 20 | app = typer.Typer(help="template tools") 21 | 22 | 23 | @app.command() 24 | def generate(): 25 | """ 26 | generate template 27 | """ 28 | shutil.copy(os.path.join(__template__, "template.yaml"), os.getcwd()) 29 | 30 | 31 | @app.command(name="standalone") 32 | def standalone_template(): 33 | """ 34 | generate template for standalone deploy 35 | """ 36 | shutil.copy(os.path.join(__template__, "standalone_template.yaml"), os.getcwd()) 37 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/cli.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import typer 15 | from fedvision_deploy_toolkit import _deploy, _service, _generate_template 16 | 17 | app = typer.Typer() 18 | 19 | app.add_typer(_deploy.app, name="deploy") 20 | app.add_typer(_service.app, name="services") 21 | app.add_typer(_generate_template.app, name="template") 22 | 23 | if __name__ == "__main__": 24 | app() 25 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/template/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/template/standalone_template.yaml: -------------------------------------------------------------------------------- 1 | # standalone deploy config demo 2 | # deploy machine: single 3 | # coordinator: name=coordinator1, port=10000 4 | # clusters: single, name=cluster1, port=10001, single worker 5 | # masters: 4, share cluster `cluster1`, connect to same coordinator `coordinator1` 6 | # deploy structure: 7 | # 8 | #+----------------------------------------------------------------------------------------+ 9 | #| | 10 | #| +--------------------------------------------------------------------+ | 11 | #| | | | 12 | #| | coordinator1 | | 13 | #| | | | 14 | #| +-----+-----------------+------------------+-----------------+-------+ | 15 | #| | | | | | 16 | #| | | | | | 17 | #| +-----+------+ +-----+------+ +-----+------+ +-----+------+ | 18 | #| | master1 | | master2 | | master3 | | master4 | | 19 | #| | | | | | | | | | 20 | #| +-----+------+ +-----+------+ +-----+------+ +-----+------+ | 21 | #| | | | | | 22 | #| v v v v | 23 | #| +-----+-----------------+------------------+-----------------+-------+ | 24 | #| | | | 25 | #| | +-----------+ | | 26 | #| | |worker1 | | | 27 | #| | | | | | 28 | #| | +-----------+ cluster1 | | 29 | #| | | | 30 | #| +--------------------------------------------------------------------+ | 31 | #| | 32 | #| machine1 | 33 | #| | 34 | #+----------------------------------------------------------------------------------------+ 35 | 36 | 37 | machines: 38 | - name: machine1 39 | ip: 127.0.0.1 40 | ssh_string: 127.0.0.1:22 41 | base_dir: /data/projects/fedvision 42 | python_for_venv_create: python3 # use to create venv, python3.7+ required 43 | 44 | # coordinator start/stop only if machine provided 45 | coordinator: 46 | name: coordinator1 47 | machine: machine1 48 | port: 10000 49 | 50 | 51 | clusters: 52 | - name: cluster1 53 | manager: 54 | machine: machine1 55 | port: 10001 56 | workers: 57 | - name: worker1 58 | machine: machine1 59 | ports: 12000-12099 60 | max_tasks: 10 61 | 62 | masters: 63 | - name: master1 64 | machine: machine1 65 | submit_port: 10002 66 | coordinator: coordinator1 67 | cluster: cluster1 68 | 69 | - name: master2 70 | machine: machine1 71 | submit_port: 10003 72 | coordinator: coordinator1 73 | cluster: cluster1 74 | 75 | - name: master3 76 | machine: machine1 77 | submit_port: 10004 78 | coordinator: coordinator1 79 | cluster: cluster1 80 | 81 | - name: master4 82 | machine: machine1 83 | submit_port: 10005 84 | coordinator: coordinator1 85 | cluster: cluster1 86 | 87 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/fedvision_deploy_toolkit/template/template.yaml: -------------------------------------------------------------------------------- 1 | # cluster deploy config demo 2 | # deploy machine: machine1, machine2, machine3 3 | # coordinator: name=coordinator1, port=10000, at machine3 4 | # clusters: 5 | # - cluster1, name=cluster1, port=10001, workers: worker1, worker2 at machine1 6 | # - cluster2, name=cluster2, port=10001, workers: worker1, worker2 at machine2 7 | # masters: 2 8 | # - master1, use cluster `cluster1`, connect to coordinator `coordinator1` at machine1 9 | # - master2, use cluster `cluster2`, connect to coordinator `coordinator1` at machine1 10 | # deploy structure: 11 | # +---------------------------------+ 12 | # | | 13 | # | +------------------------+ | 14 | # | | coordinator1 | | 15 | # | +----------^-------------+ | 16 | # | | machine3 | 17 | # +---------------------------------+ 18 | # | 19 | # | 20 | #+---------------------------------------------+ | +----------------------------------------------+ 21 | #| | | | | 22 | #| +-------------+-----------+------+-------------+ | 23 | #| | | | | | | | 24 | #| | master1 | | | | master2 | | 25 | #| +-------------+ | | +-------------+ | 26 | #| +---------------------------------------+ | | +----------------------------------------+ | 27 | #| | | | | | | | 28 | #| | +-------------+ +-------------+ | | | | +-------------+ +-------------+ | | 29 | #| | | worker1 | | worker2 | | | | | | worker1 | | worker2 | | | 30 | #| | +-------------+ +-------------+ | | | | +-------------+ +-------------+ | | 31 | #| | cluster1| | | | cluster2 | | 32 | #| +---------------------------------------+ | | +----------------------------------------+ | 33 | #| | | | 34 | #| machine1 | | machine2 | 35 | #+---------------------------------------------+ +----------------------------------------------+ 36 | 37 | 38 | 39 | machines: 40 | - name: machine1 41 | ip: 42 | ssh_string: # [user@]ip:port 43 | base_dir: /data/projects/fedvision 44 | python_for_venv_create: python3 # use to create venv 45 | 46 | - name: machine2 47 | ip: 48 | ssh_string: 49 | base_dir: /data/projects/fedvision 50 | python_for_venv_create: python3 51 | 52 | - name: machine3 53 | ip: 54 | ssh_string: 55 | base_dir: /data/projects/fedvision 56 | python_for_venv_create: python3 57 | 58 | # coordinator start/stop only if machine provided 59 | coordinator: 60 | name: coordinator1 61 | machine: machine3 62 | port: 10000 63 | 64 | 65 | clusters: 66 | - name: cluster1 67 | manager: 68 | machine: machine1 69 | port: 10001 70 | workers: 71 | - name: worker1 72 | machine: machine1 73 | ports: 12000-12999 74 | max_tasks: 10 75 | - name: worker2 76 | machine: machine2 77 | ports: 13000-13999 78 | max_tasks: 10 79 | 80 | - name: cluster2 81 | manager: 82 | machine: machine2 83 | port: 10001 84 | workers: 85 | - name: worker1 86 | machine: machine1 87 | ports: 12000-12999 88 | max_tasks: 10 89 | - name: worker2 90 | machine: machine2 91 | ports: 13000-13999 92 | max_tasks: 10 93 | 94 | masters: 95 | - name: master1 96 | machine: machine1 97 | submit_port: 10002 98 | coordinator: coordinator1 99 | cluster: cluster1 100 | 101 | - name: master2 102 | machine: machine2 103 | submit_port: 10002 104 | coordinator: coordinator1 105 | cluster: cluster2 106 | 107 | -------------------------------------------------------------------------------- /deploy/deploy_tookit/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /deploy/deploy_tookit/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = fedvision_deploy_toolkit 3 | version = 0.1 4 | 5 | [options] 6 | packages = find: 7 | include_package_data = True 8 | install_requires = 9 | fabric 10 | typer 11 | pyyaml 12 | 13 | [options.entry_points] 14 | console_scripts = 15 | fedvision-deploy = fedvision_deploy_toolkit.cli:app -------------------------------------------------------------------------------- /deploy/deploy_tookit/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup() 4 | -------------------------------------------------------------------------------- /deps/PaddleFL/AUTHORS.md: -------------------------------------------------------------------------------- 1 | | Github account | name | 2 | |---|---| 3 | | guru4elephant | Daxiang Dong | 4 | | frankwhzhang | Wenhui Zhang | 5 | | qjing666 | Qinghe Jing | 6 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License" 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License" 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/master/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/details/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import print_function 16 | 17 | from .program_utils import * 18 | from .ufind import * 19 | from .checkport import * 20 | from .vars_distributed import * 21 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/details/checkport.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | import time 17 | import socket 18 | from contextlib import closing 19 | from six import string_types 20 | 21 | 22 | def wait_server_ready(endpoints): 23 | """ 24 | Wait until parameter servers are ready, use connext_ex to detect 25 | port readiness. 26 | 27 | Args: 28 | endpoints (list): endpoints string list, like: 29 | ["127.0.0.1:8080", "127.0.0.1:8081"] 30 | 31 | Examples: 32 | .. code-block:: python 33 | 34 | wait_server_ready(["127.0.0.1:8080", "127.0.0.1:8081"]) 35 | """ 36 | assert not isinstance(endpoints, string_types) 37 | while True: 38 | all_ok = True 39 | not_ready_endpoints = [] 40 | for ep in endpoints: 41 | ip_port = ep.split(":") 42 | with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: 43 | sock.settimeout(2) 44 | result = sock.connect_ex((ip_port[0], int(ip_port[1]))) 45 | if result != 0: 46 | all_ok = False 47 | not_ready_endpoints.append(ep) 48 | if not all_ok: 49 | sys.stderr.write("server not ready, wait 3 sec to retry...\n") 50 | sys.stderr.write("not ready endpoints:" + str(not_ready_endpoints) + "\n") 51 | sys.stderr.flush() 52 | time.sleep(3) 53 | else: 54 | break 55 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/details/program_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import print_function 16 | 17 | import six 18 | 19 | from paddle.fluid import core 20 | import paddle 21 | 22 | 23 | def delete_ops(block, ops): 24 | for op in ops: 25 | try: 26 | idx = list(block.ops).index(op) 27 | block._remove_op(idx) 28 | except Exception as e: 29 | print(e) 30 | 31 | 32 | def find_op_by_input_arg(block, arg_name): 33 | for index, op in enumerate(block.ops): 34 | if arg_name in op.input_arg_names: 35 | return index 36 | return -1 37 | 38 | 39 | def find_op_by_output_arg(block, arg_name, reverse=False): 40 | if reverse: 41 | pos = len(block.ops) - 1 42 | while pos >= 0: 43 | op = block.ops[pos] 44 | if arg_name in op.output_arg_names: 45 | return pos 46 | pos -= 1 47 | else: 48 | for index, op in enumerate(block.ops): 49 | if arg_name in op.output_arg_names: 50 | return index 51 | return -1 52 | 53 | 54 | def get_indent_space(indent, space_num=4): 55 | ret = "" 56 | for i in range(0, indent * space_num): 57 | ret += " " 58 | 59 | return ret 60 | 61 | 62 | def variable_to_code(var): 63 | """ 64 | Get readable codes of fluid variable. 65 | 66 | Args: 67 | var: A fluid operator. 68 | 69 | Returns: 70 | string: The formatted string. 71 | """ 72 | if ( 73 | var.type == core.VarDesc.VarType.SELECTED_ROWS 74 | or var.type == core.VarDesc.VarType.LOD_TENSOR 75 | ): 76 | var_str = "{name} : fluid.{type}.shape{shape}.astype({dtype})".format( 77 | i="{", e="}", name=var.name, type=var.type, shape=var.shape, dtype=var.dtype 78 | ) 79 | else: 80 | var_str = "{name} : fluid.{type})".format( 81 | i="{", e="}", name=var.name, type=var.type 82 | ) 83 | 84 | if type(var) == paddle.fluid.framework.Parameter: 85 | if var.trainable: 86 | var_str = "trainable parameter " + var_str 87 | else: 88 | var_str = "parameter " + var_str 89 | else: 90 | var_str = "var " + var_str 91 | 92 | if var.persistable: 93 | var_str = "persist " + var_str 94 | 95 | return var_str 96 | 97 | 98 | def op_to_code(op, skip_op_callstack=True): 99 | """ 100 | Get readable codes of fluid operator. 101 | 102 | Args: 103 | op: A fluid operator. 104 | 105 | Returns: 106 | string: The foramtted string. 107 | """ 108 | 109 | outputs_str = "{" 110 | for i in range(0, len(op.output_names)): 111 | outputs_str += "{name}=".format(name=op.output_names[i]) 112 | o = op.output(op.output_names[i]) 113 | outputs_str += "{value}".format(value=o) 114 | if i != len(op.output_names) - 1: 115 | outputs_str += ", " 116 | outputs_str += "}" 117 | 118 | inputs_str = "{" 119 | for i in range(0, len(op.input_names)): 120 | inputs_str += "{name}=".format(name=op.input_names[i]) 121 | o = op.input(op.input_names[i]) 122 | inputs_str += "{value}".format(value=o) 123 | 124 | if i != len(op.input_names) - 1: 125 | inputs_str += ", " 126 | inputs_str += "}" 127 | 128 | attr_names = sorted(op.attr_names) 129 | attrs_str = "" 130 | for i in range(0, len(attr_names)): 131 | name = attr_names[i] 132 | if skip_op_callstack and name == "op_callstack": 133 | continue 134 | 135 | attr_type = op.desc.attr_type(name) 136 | if attr_type == core.AttrType.BLOCK: 137 | a = "{name} = block[{value}]".format( 138 | name=name, type=attr_type, value=op._block_attr_id(name) 139 | ) 140 | attrs_str += a 141 | if i != len(attr_names) - 1: 142 | attrs_str += ", " 143 | continue 144 | 145 | if attr_type == core.AttrType.BLOCKS: 146 | a = "{name} = blocks{value}".format( 147 | name=name, type=attr_type, value=op._blocks_attr_ids(name) 148 | ) 149 | attrs_str += a 150 | if i != len(attr_names) - 1: 151 | attrs_str += ", " 152 | continue 153 | 154 | a = "{name} = {value}".format( 155 | name=name, type=attr_type, value=op.desc.attr(name) 156 | ) 157 | attrs_str += a 158 | if i != len(attr_names) - 1: 159 | attrs_str += ", " 160 | 161 | if outputs_str != "{}": 162 | op_str = "{outputs} = {op_type}(inputs={inputs}, {attrs})".format( 163 | outputs=outputs_str, op_type=op.type, inputs=inputs_str, attrs=attrs_str 164 | ) 165 | else: 166 | op_str = "{op_type}(inputs={inputs}, {attrs})".format( 167 | op_type=op.type, inputs=inputs_str, attrs=attrs_str 168 | ) 169 | return op_str 170 | 171 | 172 | def block_to_code(block, block_idx, fout=None, skip_op_callstack=False): 173 | indent = 0 174 | 175 | print( 176 | "{0}{1} // block {2}".format(get_indent_space(indent), "{", block_idx), 177 | file=fout, 178 | ) 179 | 180 | indent += 1 181 | # sort all vars 182 | all_vars = sorted(six.iteritems(block.vars), key=lambda x: x[0]) 183 | for var in all_vars: 184 | print( 185 | "{}{}".format(get_indent_space(indent), variable_to_code(var[1])), file=fout 186 | ) 187 | 188 | if len(all_vars) > 0: 189 | print("", file=fout) 190 | 191 | for op in block.ops: 192 | print( 193 | "{}{}".format(get_indent_space(indent), op_to_code(op, skip_op_callstack)), 194 | file=fout, 195 | ) 196 | indent -= 1 197 | 198 | print("{0}{1}".format(get_indent_space(indent), "}"), file=fout) 199 | 200 | 201 | def program_to_code(prog, fout=None, skip_op_callstack=True): 202 | """ 203 | Print readable codes of fluid program. 204 | 205 | Args: 206 | prog : A fluid program. 207 | 208 | An example result like bellow: 209 | https://github.com/PaddlePaddle/Paddle/pull/12673 210 | """ 211 | block_idx = 0 212 | for block in prog.blocks: 213 | block_to_code(block, block_idx, fout, skip_op_callstack) 214 | block_idx += 1 215 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/details/ps_dispatcher.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import print_function 16 | 17 | 18 | class PSDispatcher(object): 19 | """ 20 | PSDispatcher is the base class for dispatching vars 21 | into different pserver instance. 22 | You need to implement the `dispatch` inferface. 23 | """ 24 | 25 | def __init__(self, pserver_endpoints): 26 | self._eps = pserver_endpoints 27 | self._step = 0 28 | 29 | @property 30 | def eps(self): 31 | return self._eps 32 | 33 | def reset(self): 34 | self._step = 0 35 | 36 | def dispatch(self, varlist): 37 | """ 38 | Args: 39 | varlist(list): a list of Variables 40 | Returns: 41 | a map of pserver endpoint -> varname 42 | """ 43 | AssertionError("Interface has not been implemented.") 44 | 45 | 46 | class HashName(PSDispatcher): 47 | """ 48 | Hash variable names to several endpoints using python 49 | "hash()" function. 50 | 51 | Args: 52 | pserver_endpoints (list): list of endpoint(ip:port). 53 | 54 | Examples: 55 | .. code-block:: python 56 | 57 | pserver_endpoints = ["127.0.0.1:6007", "127.0.0.1:6008"] 58 | vars = ["var1","var2","var3","var4","var5"] 59 | 60 | rr = RoundRobin(pserver_endpoints) 61 | rr.dispatch(vars) 62 | 63 | """ 64 | 65 | def __init__(self, pserver_endpoints): 66 | super(self.__class__, self).__init__(pserver_endpoints) 67 | 68 | def _hash_block(self, block_str, total): 69 | return hash(block_str) % total 70 | 71 | def dispatch(self, varlist): 72 | eplist = [] 73 | for var in varlist: 74 | server_id = self._hash_block(var.name(), len(self._eps)) 75 | server_for_param = self._eps[server_id] 76 | eplist.append(server_for_param) 77 | return eplist 78 | 79 | 80 | class RoundRobin(PSDispatcher): 81 | """ 82 | Distribute variables to serveral endpoints using 83 | RondRobin method. 84 | 85 | Args: 86 | pserver_endpoints (list): list of endpoint(ip:port). 87 | 88 | Examples: 89 | .. code-block:: python 90 | 91 | pserver_endpoints = ["127.0.0.1:6007", "127.0.0.1:6008"] 92 | vars = ["var1","var2","var3","var4","var5"] 93 | 94 | rr = RoundRobin(pserver_endpoints) 95 | rr.dispatch(vars) 96 | 97 | """ 98 | 99 | def __init__(self, pserver_endpoints): 100 | super(self.__class__, self).__init__(pserver_endpoints) 101 | 102 | def dispatch(self, varlist): 103 | eplist = [] 104 | for var in varlist: 105 | server_for_param = self._eps[self._step] 106 | eplist.append(server_for_param) 107 | self._step += 1 108 | if self._step >= len(self._eps): 109 | self._step = 0 110 | return eplist 111 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/details/ufind.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import print_function 16 | 17 | 18 | class UnionFind(object): 19 | """Union-find data structure. 20 | 21 | Union-find is a data structure that keeps track of a set of elements partitioned 22 | into a number of disjoint (non-overlapping) subsets. 23 | 24 | Reference: 25 | https://en.wikipedia.org/wiki/Disjoint-set_data_structure 26 | 27 | Args: 28 | elements(list): The initialize element list. 29 | """ 30 | 31 | def __init__(self, elementes=None): 32 | self._parents = [] # index -> parent index 33 | self._index = {} # element -> index 34 | self._curr_idx = 0 35 | if not elementes: 36 | elementes = [] 37 | for ele in elementes: 38 | self._parents.append(self._curr_idx) 39 | self._index.update({ele: self._curr_idx}) 40 | self._curr_idx += 1 41 | 42 | def find(self, x): 43 | # Find the root index of given element x, 44 | # execute the path compress while findind the root index 45 | if not x in self._index: 46 | return -1 47 | idx = self._index[x] 48 | while idx != self._parents[idx]: 49 | t = self._parents[idx] 50 | self._parents[idx] = self._parents[t] 51 | idx = t 52 | return idx 53 | 54 | def union(self, x, y): 55 | # Union two given element 56 | x_root = self.find(x) 57 | y_root = self.find(y) 58 | 59 | if x_root == y_root: 60 | return 61 | self._parents[x_root] = y_root 62 | 63 | def is_connected(self, x, y): 64 | # If two given elements have the same root index, 65 | # then they are connected. 66 | return self.find(x) == self.find(y) 67 | -------------------------------------------------------------------------------- /deps/PaddleFL/python/paddle_fl/paddle_fl/core/strategy/fl_strategy_base.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .fl_distribute_transpiler import FLDistributeTranspiler 16 | 17 | 18 | class FLStrategyBase(object): 19 | """ 20 | FLStrategyBase is federated learning algorithm container 21 | """ 22 | 23 | def __init__(self): 24 | self._fed_avg = False 25 | self._dpsgd = False 26 | self._inner_step = 1 27 | pass 28 | 29 | def minimize(self, optimizer=None, losses=[]): 30 | """ 31 | minmize can do minimization as paddle.fluid.Optimizer.minimize does 32 | this function can be overloaded so that for some FLStrategy, the 33 | program should be transpiled before minimize 34 | 35 | Args: 36 | optimizer(paddle.fluid.optimizer): the user defined optimizer 37 | losses(List(Variable)): list of loss variables in paddle.fluid 38 | """ 39 | for loss in losses: 40 | optimizer.minimize(loss) 41 | 42 | def _build_trainer_program_for_job( 43 | self, 44 | trainer_id=0, 45 | program=None, 46 | ps_endpoints=[], 47 | trainers=0, 48 | sync_mode=True, 49 | startup_program=None, 50 | job=None, 51 | ): 52 | pass 53 | 54 | def _build_server_programs_for_job( 55 | self, 56 | program=None, 57 | ps_endpoints=[], 58 | trainers=0, 59 | sync_mode=True, 60 | startup_program=None, 61 | job=None, 62 | ): 63 | pass 64 | 65 | 66 | class FedAvgStrategy(FLStrategyBase): 67 | """ 68 | FedAvgStrategy: this is model averaging optimization proposed in 69 | H. Brendan McMahan, Eider Moore, Daniel Ramage, Blaise Aguera y Arcas. Federated Learning of Deep Networks using Model Averaging. 2017 70 | """ 71 | 72 | def __init__(self): 73 | super(FedAvgStrategy, self).__init__() 74 | 75 | def minimize(self, optimizer=None, losses=[]): 76 | """ 77 | minimize the first loss as in paddle.fluid 78 | """ 79 | optimizer.minimize(losses[0]) 80 | 81 | def _build_trainer_program_for_job( 82 | self, 83 | trainer_id=0, 84 | program=None, 85 | ps_endpoints=[], 86 | trainers=0, 87 | sync_mode=True, 88 | startup_program=None, 89 | job=None, 90 | ): 91 | transpiler = FLDistributeTranspiler() 92 | transpiler.transpile( 93 | trainer_id, 94 | program=program, 95 | pservers=",".join(ps_endpoints), 96 | trainers=trainers, 97 | sync_mode=sync_mode, 98 | startup_program=startup_program, 99 | ) 100 | recv, main, send = transpiler.get_trainer_program() 101 | job._trainer_startup_programs.append(startup_program) 102 | job._trainer_main_programs.append(main) 103 | job._trainer_send_programs.append(send) 104 | job._trainer_recv_programs.append(recv) 105 | 106 | def _build_server_programs_for_job( 107 | self, 108 | program=None, 109 | ps_endpoints=[], 110 | trainers=0, 111 | sync_mode=True, 112 | startup_program=None, 113 | job=None, 114 | ): 115 | transpiler = FLDistributeTranspiler() 116 | trainer_id = 0 117 | transpiler.transpile( 118 | trainer_id, 119 | program=program, 120 | pservers=",".join(ps_endpoints), 121 | trainers=trainers, 122 | sync_mode=sync_mode, 123 | startup_program=startup_program, 124 | ) 125 | job.set_server_endpoints(ps_endpoints) 126 | for endpoint in ps_endpoints: 127 | main_prog = transpiler.get_pserver_program(endpoint) 128 | startup_prog = transpiler.get_startup_program(endpoint, main_prog) 129 | job._server_startup_programs.append(startup_prog) 130 | job._server_main_programs.append(main_prog) 131 | -------------------------------------------------------------------------------- /deps/README.md: -------------------------------------------------------------------------------- 1 | ## 2 | 3 | This project is heavily depends on the Project [PaddleFL](https://github.com/PaddlePaddle/PaddleFL). 4 | 5 | While we meet some problem here. There are many packages we don't need right now and trying to install their requirements could be really painful. 6 | So, we have to simply copy and modify [PaddleFL's](https://github.com/PaddlePaddle/PaddleFL/tree/f1a6f8951ad78feb594064d165db951df1a0e0bd) codes here. 7 | 8 | We would report issues directly to the upstream and, these files would be delete once issues resolved. 9 | -------------------------------------------------------------------------------- /docs/apis/cluster.md: -------------------------------------------------------------------------------- 1 | ## manager 2 | 3 | ::: fedvision.framework.cluster.manager 4 | rendering: 5 | show_source: true 6 | show_root_heading: true 7 | 8 | ## worker 9 | 10 | ::: fedvision.framework.cluster.worker 11 | rendering: 12 | show_source: true 13 | show_root_heading: true 14 | selection: 15 | filters: [] 16 | -------------------------------------------------------------------------------- /docs/apis/coordinator.md: -------------------------------------------------------------------------------- 1 | ## Coordinator 2 | 3 | ::: fedvision.framework.coordinator 4 | rendering: 5 | show_source: true 6 | -------------------------------------------------------------------------------- /docs/apis/master.md: -------------------------------------------------------------------------------- 1 | ## Coordinator 2 | 3 | ::: fedvision.framework.master.master 4 | rendering: 5 | show_source: true 6 | -------------------------------------------------------------------------------- /docs/css/custom.css: -------------------------------------------------------------------------------- 1 | /* Indentation. */ 2 | div.doc-contents:not(.first) { 3 | padding-left: 25px; 4 | border-left: 4px solid rgba(230, 230, 230); 5 | margin-bottom: 80px; 6 | } 7 | 8 | /* Don't capitalize names. */ 9 | h5.doc-heading { 10 | text-transform: none !important; 11 | } 12 | 13 | /* Don't use vertical space on hidden ToC entries. */ 14 | h6.hidden-toc { 15 | margin: 0 !important; 16 | position: relative; 17 | top: -70px; 18 | } 19 | 20 | h6.hidden-toc::before { 21 | margin-top: 0 !important; 22 | padding-top: 0 !important; 23 | } 24 | 25 | /* Don't show permalink of hidden ToC entries. */ 26 | h6.hidden-toc a.headerlink { 27 | display: none; 28 | } 29 | 30 | /* Avoid breaking parameters name, etc. in table cells. */ 31 | td code { 32 | word-break: normal !important; 33 | } 34 | 35 | /* For pieces of Markdown rendered in table cells. */ 36 | td p { 37 | margin-top: 0 !important; 38 | margin-bottom: 0 !important; 39 | } -------------------------------------------------------------------------------- /docs/css/termynal.css: -------------------------------------------------------------------------------- 1 | /** 2 | * termynal.js 3 | * 4 | * @author Ines Montani 5 | * @version 0.0.1 6 | * @license MIT 7 | */ 8 | 9 | :root { 10 | --color-bg: #eee8d5; 11 | --color-text: #073642; 12 | --color-text-subtle: #cb4b16; 13 | background: linear-gradient(to right, #3a1c71, #d76d77, #ffaf7b); 14 | } 15 | 16 | [data-termynal] { 17 | width: 750px; 18 | max-width: 100%; 19 | background: var(--color-bg); 20 | color: var(--color-text); 21 | font-size: 18px; 22 | /* font-family: 'Fira Mono', Consolas, Menlo, Monaco, 'Courier New', Courier, monospace; */ 23 | font-family: 'Roboto Mono', 'Fira Mono', Consolas, Menlo, Monaco, 'Courier New', Courier, monospace; 24 | border-radius: 4px; 25 | padding: 75px 45px 35px; 26 | position: relative; 27 | -webkit-box-sizing: border-box; 28 | box-sizing: border-box; 29 | } 30 | 31 | [data-termynal]:before { 32 | content: ''; 33 | position: absolute; 34 | top: 15px; 35 | left: 15px; 36 | display: inline-block; 37 | width: 15px; 38 | height: 15px; 39 | border-radius: 50%; 40 | /* A little hack to display the window buttons in one pseudo element. */ 41 | background: #d9515d; 42 | -webkit-box-shadow: 25px 0 0 #f4c025, 50px 0 0 #3ec930; 43 | box-shadow: 25px 0 0 #f4c025, 50px 0 0 #3ec930; 44 | } 45 | 46 | [data-termynal]:after { 47 | content: 'bash'; 48 | position: absolute; 49 | color: var(--color-text-subtle); 50 | top: 5px; 51 | left: 0; 52 | width: 100%; 53 | text-align: center; 54 | } 55 | 56 | a[data-terminal-control] { 57 | text-align: right; 58 | display: block; 59 | color: #aebbff; 60 | } 61 | 62 | [data-ty] { 63 | display: block; 64 | line-height: 2; 65 | } 66 | 67 | [data-ty]:before { 68 | /* Set up defaults and ensure empty lines are displayed. */ 69 | content: ''; 70 | display: inline-block; 71 | vertical-align: middle; 72 | } 73 | 74 | [data-ty="input"]:before, 75 | [data-ty-prompt]:before { 76 | margin-right: 0.75em; 77 | color: var(--color-text-subtle); 78 | } 79 | 80 | [data-ty="input"]:before { 81 | content: '$'; 82 | } 83 | 84 | [data-ty][data-ty-prompt]:before { 85 | content: attr(data-ty-prompt); 86 | } 87 | 88 | [data-ty-cursor]:after { 89 | content: attr(data-ty-cursor); 90 | font-family: monospace; 91 | margin-left: 0.5em; 92 | -webkit-animation: blink 1s infinite; 93 | animation: blink 1s infinite; 94 | } 95 | 96 | 97 | /* Cursor animation */ 98 | 99 | @-webkit-keyframes blink { 100 | 50% { 101 | opacity: 0; 102 | } 103 | } 104 | 105 | @keyframes blink { 106 | 50% { 107 | opacity: 0; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /docs/develop/codestyle.md: -------------------------------------------------------------------------------- 1 | We use [black](https://github.com/psf/black) and [flake8](https://github.com/PyCQA/flake8) 2 | to format codes in `fedvision`. For developer, it's quiet easy to install pre-commit hooks: 3 | ```bash 4 | pip install pre-commit 5 | pre-commit install 6 | ``` 7 | 8 | This will check code style of changed files before you push codes. -------------------------------------------------------------------------------- /docs/framework/overview.md: -------------------------------------------------------------------------------- 1 | 2 | ### Overview 3 | 4 | ![framework](../img/fedvision.png) 5 | 6 | There are two `role` associated in `FedVision`: 7 | 8 | 1. coordinator 9 | 10 | 2. Party 11 | 12 | 13 | `Coordinator` is an independent role responsible for handling job publish and 14 | distribute subtasks to proper parties that have subscribed to the `coordinator`. 15 | Usually, the `coordinator` must be started before any party can subscribe to it. 16 | The `party` will post the job of the specified `job_type` to the coordinator and wait for `proposal_waiting_time` seconds. 17 | While waiting, all `party` subscribed to this `job_type` will received a message, 18 | and then decide whether to participate or not. After the waiting time is over, the `coordinator` selects a group of `party` 19 | as a participants in this job. 20 | 21 | `Party` is an independent role that publishes or subscribe jobs. usually, 22 | It has a `Cluster` to process assigned tasks, and a `submit service` to process work requests from `Users`, 23 | a `coordinator clients publish jobs to `Coordinator` or subscribe jobs from "Coordinator" and, a `master` to mixes them up. 24 | 25 | ### Job's Life Cycle 26 | 27 | ![job](../img/job.png) 28 | 29 | The FedVision framework is an extensible framework. 30 | When the master receives the job submit request, 31 | it will use different job loader according to the `job_type` parameter. 32 | This makes it possible to extent `FedVision` framework with various machine learning frameworks. 33 | 34 | Currently, only `PaddlePaddle` supported with extension configuration file: 35 | 36 | ```yaml 37 | PaddleFL: 38 | jobs: 39 | - name: paddle_fl 40 | schema: ../schema/paddle_fl.json 41 | loader: fedvision.paddle_fl.job:PaddleFLJob 42 | tasks: 43 | - name: fl_trainer 44 | loader: fedvision.paddle_fl.tasks.task.trainer:FLTrainer 45 | - name: fl_aggregator 46 | loader: fedvision.paddle_fl.tasks.task.aggregator:FLAggregator 47 | ``` 48 | 49 | To implement an new extension, one need to 50 | 51 | 1. add configuration to [extensions.yaml](https://github.com/FederatedAI/FedVision/blob/main/conf/extensions.yaml) 52 | 53 | 2. implement [abstract job class](https://github.com/FederatedAI/FedVision/blob/main/fedvision/framework/abc/job.py) 54 | 55 | 3. implement several [abstract task class](https://github.com/FederatedAI/FedVision/blob/main/fedvision/framework/abc/task.py) used by job. 56 | 57 | -------------------------------------------------------------------------------- /docs/framework/paddledetection.md: -------------------------------------------------------------------------------- 1 | Federated Job using `PaddleDetection` and `PaddleFL` are supported out of box. To submit a PaddleDetection jobs, 2 | one need a `yaml` config file to describe what algorithm and parameters to used. 3 | This is almost same as these 4 | [paddle detection configs](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.0-beta/configs) 5 | except that reference other config file are not supported. 6 | 7 | One can find an example using yolo mobilenet to detect `fruit` data in [examples](https://github.com/FederatedAI/FedVision/tree/main/examples/paddle_detection). 8 | -------------------------------------------------------------------------------- /docs/framework/paddlefl.md: -------------------------------------------------------------------------------- 1 | We provide a mnist demo to describes how to implement a job directly from `PaddleFL`. 2 | 3 | There are two `py` file related: 4 | 5 | 1. [fl_master](https://github.com/FederatedAI/FedVision/blob/main/fedvision/ml/paddle/paddle_mnist/fl_master.py): define 6 | CNN network to learning mnist dataset and compiled with `PaddleFL` 7 | 8 | 2. [fl_trainer](https://github.com/FederatedAI/FedVision/blob/main/fedvision/ml/paddle/paddle_mnist/fl_trainer.py): read dataset and training 9 | with compiled program. 10 | 11 | Just implement your demo by mimic this demo as you wish. -------------------------------------------------------------------------------- /docs/img/Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/docs/img/Architecture.png -------------------------------------------------------------------------------- /docs/img/federated_detection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/docs/img/federated_detection.png -------------------------------------------------------------------------------- /docs/img/fedvision.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/docs/img/fedvision.png -------------------------------------------------------------------------------- /docs/img/job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/docs/img/job.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | FedVision is a Visual Object Detection Platform Powered by Federated Learning 2 | 3 | ![FederatedDetection](img/federated_detection.png) 4 | 5 | 6 | At this version, FedVision utilize 7 | [PaddleFL](https://github.com/PaddlePaddle/PaddleFL) 8 | and [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) 9 | to achieve `Gradient Average` strategy. We aim to implement many 10 | awesome Features such as 11 | 12 | - Google's Secure Aggregation protocol 13 | 14 | - Gradient compression 15 | 16 | - pytorch support 17 | 18 | [Contribution is welcome!](https://github.com/FederatedAI/FedVision/issues/new) 19 | -------------------------------------------------------------------------------- /docs/js/custom.js: -------------------------------------------------------------------------------- 1 | document.querySelectorAll(".use-termynal").forEach(node => { 2 | node.style.display = "block"; 3 | new Termynal(node, { 4 | lineDelay: 500 5 | }); 6 | }); 7 | const progressLiteralStart = "---> 100%"; 8 | const promptLiteralStart = "$ "; 9 | const customPromptLiteralStart = "# "; 10 | const termynalActivateClass = "termy"; 11 | let termynals = []; 12 | 13 | function createTermynals() { 14 | document 15 | .querySelectorAll(`.${termynalActivateClass} .highlight`) 16 | .forEach(node => { 17 | const text = node.textContent; 18 | const lines = text.split("\n"); 19 | const useLines = []; 20 | let buffer = []; 21 | function saveBuffer() { 22 | if (buffer.length) { 23 | let isBlankSpace = true; 24 | buffer.forEach(line => { 25 | if (line) { 26 | isBlankSpace = false; 27 | } 28 | }); 29 | dataValue = {}; 30 | if (isBlankSpace) { 31 | dataValue["delay"] = 0; 32 | } 33 | if (buffer[buffer.length - 1] === "") { 34 | // A last single
won't have effect 35 | // so put an additional one 36 | buffer.push(""); 37 | } 38 | const bufferValue = buffer.join("
"); 39 | dataValue["value"] = bufferValue; 40 | useLines.push(dataValue); 41 | buffer = []; 42 | } 43 | } 44 | for (let line of lines) { 45 | if (line === progressLiteralStart) { 46 | saveBuffer(); 47 | useLines.push({ 48 | type: "progress" 49 | }); 50 | } else if (line.startsWith(promptLiteralStart)) { 51 | saveBuffer(); 52 | const value = line.replace(promptLiteralStart, "").trimEnd(); 53 | useLines.push({ 54 | type: "input", 55 | value: value 56 | }); 57 | } else if (line.startsWith("// ")) { 58 | saveBuffer(); 59 | const value = "💬 " + line.replace("// ", "").trimEnd(); 60 | useLines.push({ 61 | value: value, 62 | class: "termynal-comment", 63 | delay: 0 64 | }); 65 | } else if (line.startsWith(customPromptLiteralStart)) { 66 | saveBuffer(); 67 | const promptStart = line.indexOf(promptLiteralStart); 68 | if (promptStart === -1) { 69 | console.error("Custom prompt found but no end delimiter", line) 70 | } 71 | const prompt = line.slice(0, promptStart).replace(customPromptLiteralStart, "") 72 | let value = line.slice(promptStart + promptLiteralStart.length); 73 | useLines.push({ 74 | type: "input", 75 | value: value, 76 | prompt: prompt 77 | }); 78 | } else { 79 | buffer.push(line); 80 | } 81 | } 82 | saveBuffer(); 83 | const div = document.createElement("div"); 84 | node.replaceWith(div); 85 | const termynal = new Termynal(div, { 86 | lineData: useLines, 87 | noInit: true, 88 | lineDelay: 500 89 | }); 90 | termynals.push(termynal); 91 | }); 92 | } 93 | 94 | function loadVisibleTermynals() { 95 | termynals = termynals.filter(termynal => { 96 | if (termynal.container.getBoundingClientRect().top - innerHeight <= 0) { 97 | termynal.init(); 98 | return false; 99 | } 100 | return true; 101 | }); 102 | } 103 | window.addEventListener("scroll", loadVisibleTermynals); 104 | createTermynals(); 105 | loadVisibleTermynals(); 106 | -------------------------------------------------------------------------------- /docs/quickstart/quickstart.md: -------------------------------------------------------------------------------- 1 | This section describes how to quick deploy and run examples in standalone version of `FedVision` 2 | 3 | ### Prerequisites 4 | 5 | Too run Fedvision, following dependency or tools required: 6 | 7 | - machine to install fedvision-deploy-toolkit: 8 | 9 | - python virtualenv with Python>=3 10 | 11 | - setup SSH password less login to machine(s) for deploy fedvision framework. 12 | 13 | - machine(s) to deploy fedvision framework: 14 | 15 | - Python>=3.7(with pip) 16 | 17 | - an isolated directory (each directory will be deployed with a copy of code) 18 | 19 | ### Deploy 20 | 21 | 22 |
23 | 24 | ```console 25 | // create and activate python virtual environment 26 | $ python3 -V 27 | Python 3.9.0 28 | $ python3 -m venv venv && source venv/bin/activate 29 | 30 | // install fedvision_deploy_toolkit 31 | $ (venv) python -m pip install -U pip && python -m pip install fedvision_deploy_toolkit 32 | ---> 100% 33 | Successfully installed fedvision_deploy_toolkit 34 | 35 | // generate deploy template 36 | $ (venv) fedvision-deploy template standalone 37 | 38 | // read comments in generated template standalone_template.yaml` and modify as you want. 39 | 40 | 41 | // deploy now 42 | $ (venv) fedvision-deploy deploy deploy standalone_template.yaml 43 | deploying 2 machines: ['machine1'] 44 | ---> 100% 45 | deploy done 46 | 47 | ``` 48 | 49 |
50 | 51 | !!! note 52 | Deploying Cluster version is almost same except that you should generate template using 53 | ```bash 54 | fedvision-deploy template template 55 | ``` 56 | and modify generated template file according to comments. 57 | 58 | ### Services start 59 | 60 | Services could be start/stop with scripts in `Fedvision/sbin` or, use fedvision deploy toolkits: 61 | 62 |
63 | 64 | ```console 65 | // start services 66 | $ fedvision-deploy services all start standalone_template.yaml 67 | 68 | staring coordinator coordinator1 69 | coordinator service start successfully. pid: 92869 70 | 127.0.0.1:22:/data/projects/fedvision started coordinator: port=10000 71 | start coordinator coordinator1 done 72 | 73 | starting cluster cluster1 74 | clustermanager service start successfully. pid: 92907 75 | 127.0.0.1:22:/data/projects/fedvision started cluster manager: port=10001 76 | start cluster cluster1 done, success: True 77 | 78 | starting cluster workers for cluster cluster1 79 | starting worker worker1 80 | cluster worker service start successfully. pid: 92944 81 | 127.0.0.1:22:/data/projects/fedvision started cluster worker: name=worker1 82 | start worker worker1 done, success: True 83 | 84 | starting master master1 85 | master service start successfully. pid: 92975 86 | 127.0.0.1:22:/data/projects/fedvision started master: port=10002 87 | start master master1 done, success: True 88 | 89 | starting master master2 90 | master service start successfully. pid: 93022 91 | 127.0.0.1:22:/data/projects/fedvision started master: port=10003 92 | start master master2 done, success: True 93 | 94 | starting master master3 95 | master service start successfully. pid: 93067 96 | 127.0.0.1:22:/data/projects/fedvision started master: port=10004 97 | start master master3 done, success: True 98 | 99 | starting master master4 100 | master service start successfully. pid: 93112 101 | 127.0.0.1:22:/data/projects/fedvision started master: port=10005 102 | start master master4 done, success: True 103 | 104 | ``` 105 | 106 |
107 | 108 | ### Run examples 109 | 110 | Jobs could be submitted at each deployed machine with master service started. 111 | 112 | 113 |
114 | 115 | ```console 116 | $ cd /data/projects/fedvision 117 | $ source venv/bin/activate 118 | $ export PYTHONPATH=$PYTHONPATH:/data/projects/fedvision/FedVision 119 | 120 | // submit jobs to master1 121 | $ sh FedVision/examples/paddle_mnist/run.sh 127.0.0.1:10002 122 | { 123 | "job_id": "master1-20201218202835-1" 124 | } 125 | ``` 126 | 127 |
128 | 129 | **Note**: find logs in /data/projects/fedvision/FedVision/logs -------------------------------------------------------------------------------- /docs/release/release.md: -------------------------------------------------------------------------------- 1 | # Release Notes 2 | 3 | - [v0.1](v0.1.md) `[2020.12.31]` -------------------------------------------------------------------------------- /docs/release/v0.1.md: -------------------------------------------------------------------------------- 1 | # release v0.1 2 | 3 | Initial release of FedVision. 4 | 5 | ## Major Features 6 | 7 | * Support publish/subscribe job distribute workflow 8 | * Support PaddleFL extension 9 | * Support PaddleDetection job type 10 | * Support Cluster for distribute tasks to multi-machine environment 11 | * Support deploy-utils for deploy codes and start/stop services 12 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs 2 | python-markdown-math 3 | mkdocs-material 4 | mkdocstrings==0.13.6 5 | mkdocs-jupyter 6 | jupyter -------------------------------------------------------------------------------- /examples/paddle_detection/README.md: -------------------------------------------------------------------------------- 1 | ```sh 2 | sh run.sh 3 | ``` -------------------------------------------------------------------------------- /examples/paddle_detection/config.yaml: -------------------------------------------------------------------------------- 1 | job_type: paddle_fl 2 | job_config: 3 | program: paddle_detection 4 | proposal_wait_time: 5 5 | worker_num: 2 6 | max_iter: 10 7 | inner_step: 1 8 | device: cpu 9 | use_vdl: true 10 | algorithm_config: ./yolov3_mobilenet_v1_fruit.yml 11 | -------------------------------------------------------------------------------- /examples/paddle_detection/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # export Environment Variables 18 | # PYTHONPATH Python default search path for module files, add Fedvision, PaddleFL, PaddleDetection 19 | # FEDVISION_PYTHON_EXECUTABLE python executable path, such as python | python3 | venv/bin/python 20 | 21 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 22 | 23 | usage="Usage: run.sh " 24 | 25 | if [ $# -le 0 ]; then 26 | echo "$usage" 27 | exit 1 28 | fi 29 | 30 | python -m fedvision.framework.cli.submitter submit --config "${DIR}/config.yaml" --endpoint "$1" 31 | -------------------------------------------------------------------------------- /examples/paddle_detection/yolov3_mobilenet_v1_fruit.yml: -------------------------------------------------------------------------------- 1 | architecture: YOLOv3 2 | use_gpu: true 3 | max_iters: 20000 4 | log_iter: 20 5 | save_dir: output 6 | snapshot_iter: 200 7 | metric: VOC 8 | map_type: 11point 9 | pretrain_weights: https://paddlemodels.bj.bcebos.com/object_detection/yolov3_mobilenet_v1.tar 10 | weights: output/yolov3_mobilenet_v1_fruit/best_model 11 | num_classes: 3 12 | finetune_exclude_pretrained_params: ["yolo_output"] 13 | use_fine_grained_loss: false 14 | 15 | YOLOv3: 16 | backbone: MobileNet 17 | yolo_head: YOLOv3Head 18 | 19 | MobileNet: 20 | norm_type: sync_bn 21 | norm_decay: 0. 22 | conv_group_scale: 1 23 | with_extra_blocks: false 24 | 25 | YOLOv3Head: 26 | anchor_masks: [[6, 7, 8], [3, 4, 5], [0, 1, 2]] 27 | anchors: 28 | [ 29 | [10, 13], 30 | [16, 30], 31 | [33, 23], 32 | [30, 61], 33 | [62, 45], 34 | [59, 119], 35 | [116, 90], 36 | [156, 198], 37 | [373, 326], 38 | ] 39 | norm_decay: 0. 40 | yolo_loss: YOLOv3Loss 41 | nms: 42 | background_label: -1 43 | keep_top_k: 100 44 | nms_threshold: 0.45 45 | nms_top_k: 1000 46 | normalized: false 47 | score_threshold: 0.01 48 | 49 | YOLOv3Loss: 50 | ignore_thresh: 0.7 51 | label_smooth: true 52 | 53 | LearningRate: 54 | base_lr: 0.00001 55 | schedulers: 56 | - !PiecewiseDecay 57 | gamma: 0.1 58 | milestones: 59 | - 15000 60 | - 18000 61 | - !LinearWarmup 62 | start_factor: 0. 63 | steps: 100 64 | 65 | OptimizerBuilder: 66 | optimizer: 67 | momentum: 0.9 68 | type: Momentum 69 | regularizer: 70 | factor: 0.0005 71 | type: L2 72 | 73 | #_READER_: 'yolov3_reader.yml' 74 | # will merge TrainReader into yolov3_reader.yml 75 | TrainReader: 76 | inputs_def: 77 | image_shape: [3, 608, 608] 78 | fields: ["image", "gt_bbox", "gt_class", "gt_score"] 79 | num_max_boxes: 50 80 | use_dataloader: false 81 | dataset: !VOCDataSet 82 | dataset_dir: dataset/fruit 83 | anno_path: train.txt 84 | with_background: false 85 | use_default_label: false 86 | sample_transforms: 87 | - !DecodeImage 88 | to_rgb: true 89 | with_mixup: false 90 | - !NormalizeBox {} 91 | - !ExpandImage 92 | max_ratio: 4.0 93 | mean: [123.675, 116.28, 103.53] 94 | prob: 0.5 95 | - !RandomInterpImage 96 | max_size: 0 97 | target_size: 608 98 | - !RandomFlipImage 99 | is_normalized: true 100 | prob: 0.5 101 | - !NormalizeImage 102 | mean: [0.485, 0.456, 0.406] 103 | std: [0.229, 0.224, 0.225] 104 | is_scale: true 105 | is_channel_first: false 106 | - !PadBox 107 | num_max_boxes: 50 108 | - !BboxXYXY2XYWH {} 109 | batch_transforms: 110 | - !RandomShape 111 | sizes: [608] 112 | - !Permute 113 | channel_first: true 114 | to_bgr: false 115 | batch_size: 1 116 | shuffle: true 117 | mixup_epoch: -1 118 | 119 | EvalReader: 120 | batch_size: 1 121 | inputs_def: 122 | image_shape: [3, 608, 608] 123 | fields: ["image", "im_size", "im_id", "gt_bbox", "gt_class", "is_difficult"] 124 | num_max_boxes: 50 125 | dataset: !VOCDataSet 126 | dataset_dir: data/fruit 127 | anno_path: val.txt 128 | use_default_label: false 129 | with_background: false 130 | 131 | TestReader: 132 | batch_size: 1 133 | dataset: !ImageFolder 134 | anno_path: data/fruit/label_list.txt 135 | use_default_label: false 136 | with_background: false 137 | -------------------------------------------------------------------------------- /examples/paddle_mnist/README.md: -------------------------------------------------------------------------------- 1 | ```sh 2 | sh run.sh 3 | ``` -------------------------------------------------------------------------------- /examples/paddle_mnist/cnn.yml: -------------------------------------------------------------------------------- 1 | batch_size: 128 2 | need_shuffle: True -------------------------------------------------------------------------------- /examples/paddle_mnist/config.yaml: -------------------------------------------------------------------------------- 1 | job_type: paddle_fl 2 | job_config: 3 | program: paddle_mnist 4 | proposal_wait_time: 5 5 | worker_num: 2 6 | max_iter: 10 7 | inner_step: 10 8 | device: cpu 9 | use_vdl: true 10 | algorithm_config: ./cnn.yml 11 | -------------------------------------------------------------------------------- /examples/paddle_mnist/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # export Environment Variables 18 | # PYTHONPATH Python default search path for module files, add Fedvision, PaddleFL, PaddleDetection 19 | # FEDVISION_PYTHON_EXECUTABLE python executable path, such as python | python3 | venv/bin/python 20 | 21 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 22 | 23 | usage="Usage: run.sh " 24 | 25 | if [ $# -le 0 ]; then 26 | echo "$usage" 27 | exit 1 28 | fi 29 | 30 | python -m fedvision.framework.cli.submitter submit --config "${DIR}/config.yaml" --endpoint "$1" 31 | -------------------------------------------------------------------------------- /fedvision/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | __version__ = "0.1" 18 | __basedir__ = os.path.dirname(os.path.abspath(__file__)) 19 | __logs_dir__ = os.path.abspath(os.path.join(__basedir__, os.path.pardir, "logs")) 20 | __data_dir__ = os.path.abspath(os.path.join(__basedir__, os.path.pardir, "data")) 21 | 22 | FEDVISION_DATA_BASE_ENV = "FEDVISION_DATA_BASE" 23 | 24 | 25 | def get_data_dir(): 26 | if FEDVISION_DATA_BASE_ENV in os.environ and os.path.exists( 27 | os.environ.get(FEDVISION_DATA_BASE_ENV) 28 | ): 29 | return os.path.abspath(os.environ.get(FEDVISION_DATA_BASE_ENV)) 30 | else: 31 | return __data_dir__ 32 | -------------------------------------------------------------------------------- /fedvision/cli.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | import click 18 | 19 | from fedvision import __version__, __basedir__ 20 | 21 | CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) 22 | 23 | 24 | @click.group(context_settings=CONTEXT_SETTINGS) 25 | @click.version_option( 26 | f"{__version__} from {__basedir__} (Python {sys.version[:3]})", "-V", "--version" 27 | ) 28 | @click.pass_context 29 | def cli(ctx): 30 | ... 31 | -------------------------------------------------------------------------------- /fedvision/framework/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/abc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/abc/executor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import abc 16 | from pathlib import Path 17 | 18 | 19 | class Executor(object): 20 | def __init__(self, working_dir: Path): 21 | self._working_dir = working_dir 22 | self._working_dir.mkdir(parents=True, exist_ok=True) 23 | 24 | @abc.abstractmethod 25 | async def execute(self, cmd) -> int: 26 | ... 27 | 28 | @property 29 | def stderr(self): 30 | return "stderr" 31 | 32 | @property 33 | def stdout(self): 34 | return "stdout" 35 | 36 | @property 37 | def working_dir(self): 38 | return self._working_dir 39 | -------------------------------------------------------------------------------- /fedvision/framework/abc/job.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import abc 16 | from typing import List 17 | 18 | from fedvision.framework.protobuf import job_pb2, coordinator_pb2 19 | 20 | 21 | class Job(metaclass=abc.ABCMeta): 22 | 23 | job_type: str 24 | 25 | def __init__(self, job_id: str): 26 | self.job_id = job_id 27 | 28 | @property 29 | def resource_required(self): 30 | return None 31 | 32 | def set_required_resource(self, response): 33 | ... 34 | 35 | async def compile(self): 36 | ... 37 | 38 | @abc.abstractmethod 39 | def generate_proposal_request(self) -> coordinator_pb2.Proposal.REQ: 40 | ... 41 | 42 | @abc.abstractmethod 43 | def generate_local_tasks(self) -> List[job_pb2.Task]: 44 | ... 45 | 46 | @classmethod 47 | @abc.abstractmethod 48 | def load(cls, job_id: str, config, algorithm_config) -> "Job": 49 | ... 50 | 51 | def generate_task_id(self, task_name): 52 | return f"{self.job_id}-task_{task_name}" 53 | -------------------------------------------------------------------------------- /fedvision/framework/abc/task.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import abc 16 | 17 | from fedvision.framework.abc.executor import Executor 18 | from fedvision.framework.protobuf import job_pb2 19 | 20 | 21 | class Task(metaclass=abc.ABCMeta): 22 | """ 23 | A abstract Task class. 24 | """ 25 | 26 | task_type: str 27 | 28 | def __init__(self, job_id, task_id): 29 | self.job_id = job_id 30 | self.task_id = task_id 31 | 32 | @abc.abstractmethod 33 | async def exec(self, executor: Executor) -> int: 34 | ... 35 | 36 | def __str__(self): 37 | return f"{self.__class__.__name__}[{self.__dict__}]" 38 | 39 | @classmethod 40 | @abc.abstractmethod 41 | def deserialize(cls, pb: job_pb2.Task) -> "Task": 42 | ... 43 | -------------------------------------------------------------------------------- /fedvision/framework/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/cli/cluster_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | 17 | import click 18 | 19 | 20 | @click.command(name="start-manager") 21 | @click.option("--port", type=int, required=True, help="cluster manager address") 22 | def start_manager(port): 23 | """ 24 | start manager 25 | """ 26 | from fedvision.framework.utils import logger 27 | 28 | logger.set_logger("manager") 29 | from fedvision.framework.cluster.manager import ClusterManager 30 | 31 | loop = asyncio.get_event_loop() 32 | manager = ClusterManager( 33 | port=port, 34 | ) 35 | try: 36 | loop.run_until_complete(manager.start()) 37 | click.echo(f"cluster manager start") 38 | loop.run_forever() 39 | except KeyboardInterrupt: 40 | click.echo("keyboard interrupted") 41 | 42 | finally: 43 | loop.run_until_complete(manager.stop()) 44 | click.echo(f"cluster manager server stop") 45 | loop.close() 46 | 47 | 48 | if __name__ == "__main__": 49 | start_manager() 50 | -------------------------------------------------------------------------------- /fedvision/framework/cli/cluster_worker.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | 17 | import click 18 | 19 | 20 | @click.command(name="start-worker") 21 | @click.option("--name", type=str, required=True, help="worker name") 22 | @click.option("--worker-ip", type=str, required=True, help="worker ip") 23 | @click.option("--max-tasks", type=int, required=True, help="max tasks") 24 | @click.option("--port-start", type=int, required=True, help="port start") 25 | @click.option("--port-end", type=int, required=True, help="port end") 26 | @click.option( 27 | "--manager-address", type=str, required=True, help="cluster manager address" 28 | ) 29 | @click.option( 30 | "--data-base-dir", 31 | type=click.Path(exists=True, file_okay=False, dir_okay=True), 32 | required=False, 33 | help="data base dir", 34 | ) 35 | def start_worker( 36 | name, 37 | worker_ip, 38 | max_tasks, 39 | manager_address, 40 | port_start, 41 | port_end, 42 | data_base_dir, 43 | ): 44 | """ 45 | start worker 46 | """ 47 | from fedvision.framework.utils import logger 48 | 49 | logger.set_logger(f"worker-{worker_ip}") 50 | from fedvision.framework.cluster.worker import ClusterWorker 51 | 52 | loop = asyncio.get_event_loop() 53 | worker = ClusterWorker( 54 | worker_id=name, 55 | worker_ip=worker_ip, 56 | max_tasks=max_tasks, 57 | manager_address=manager_address, 58 | port_start=port_start, 59 | port_end=port_end, 60 | data_dir=data_base_dir, 61 | ) 62 | try: 63 | loop.run_until_complete(worker.start()) 64 | click.echo(f"worker {name} start") 65 | loop.run_until_complete(worker.wait_for_termination()) 66 | except KeyboardInterrupt: 67 | click.echo("keyboard interrupted") 68 | finally: 69 | loop.run_until_complete(worker.stop()) 70 | loop.run_until_complete(asyncio.sleep(1)) 71 | loop.close() 72 | click.echo(f"worker {name} stop") 73 | 74 | 75 | if __name__ == "__main__": 76 | start_worker() 77 | -------------------------------------------------------------------------------- /fedvision/framework/cli/coordinator.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | 17 | import click 18 | 19 | 20 | @click.command() 21 | @click.option("-p", "--port", type=int, required=True, help="port") 22 | def start_coordinator(port): 23 | """ 24 | start coordinator 25 | """ 26 | from fedvision.framework.utils import logger 27 | 28 | logger.set_logger("coordinator") 29 | from fedvision.framework.coordinator.coordinator import Coordinator 30 | 31 | loop = asyncio.get_event_loop() 32 | coordinator = Coordinator(port) 33 | try: 34 | loop.run_until_complete(coordinator.start()) 35 | click.echo(f"coordinator server start at port:{port}") 36 | loop.run_forever() 37 | except KeyboardInterrupt: 38 | click.echo("keyboard interrupted") 39 | finally: 40 | loop.run_until_complete(coordinator.stop()) 41 | click.echo(f"coordinator server stop") 42 | loop.close() 43 | 44 | 45 | if __name__ == "__main__": 46 | start_coordinator() 47 | -------------------------------------------------------------------------------- /fedvision/framework/cli/master.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | 17 | import click 18 | 19 | 20 | @click.command() 21 | @click.option("--party-id", type=str, required=True, help="party id") 22 | @click.option("--submitter-port", type=int, required=True, help="submitter port") 23 | @click.option( 24 | "--cluster-address", type=str, required=True, help="cluster manager address" 25 | ) 26 | @click.option( 27 | "--coordinator-address", type=str, required=True, help="coordinator address" 28 | ) 29 | def start_master(party_id, submitter_port, cluster_address, coordinator_address): 30 | """ 31 | start master 32 | """ 33 | from fedvision.framework.utils import logger 34 | 35 | logger.set_logger(f"master-{party_id}") 36 | from fedvision.framework.master.master import Master 37 | 38 | loop = asyncio.get_event_loop() 39 | master = Master( 40 | party_id=party_id, 41 | coordinator_address=coordinator_address, 42 | cluster_address=cluster_address, 43 | rest_port=submitter_port, 44 | ) 45 | try: 46 | loop.run_until_complete(master.start()) 47 | click.echo(f"master started") 48 | loop.run_forever() 49 | except KeyboardInterrupt: 50 | click.echo("keyboard interrupted") 51 | finally: 52 | loop.run_until_complete(master.stop()) 53 | click.echo(f"master stop") 54 | loop.close() 55 | 56 | 57 | if __name__ == "__main__": 58 | start_master() 59 | -------------------------------------------------------------------------------- /fedvision/framework/cli/submitter.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import urllib.parse 16 | from pathlib import Path 17 | 18 | import click 19 | import aiohttp 20 | import asyncio 21 | import json 22 | 23 | import yaml 24 | 25 | from fedvision.framework import extensions 26 | 27 | 28 | @click.group() 29 | def cli(): 30 | ... 31 | 32 | 33 | def post(endpoint, path, json_data): 34 | async def post_co(): 35 | async with aiohttp.ClientSession() as session: 36 | async with session.post( 37 | url=urllib.parse.urljoin(f"http://{endpoint}", path), json=json_data 38 | ) as resp: 39 | if not resp.ok: 40 | print(resp.status) 41 | resp.raise_for_status() 42 | else: 43 | print(json.dumps(await resp.json(), indent=2)) 44 | 45 | loop = asyncio.get_event_loop() 46 | loop.run_until_complete(post_co()) 47 | 48 | 49 | @cli.command() 50 | @click.option( 51 | "--config", 52 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 53 | required=True, 54 | ) 55 | @click.option( 56 | "--endpoint", 57 | type=str, 58 | required=True, 59 | ) 60 | def submit(endpoint, config): 61 | 62 | base = Path(config) 63 | with base.open("r") as f: 64 | config_json = yaml.load(f, yaml.Loader) 65 | job_type = config_json.get("job_type") 66 | job_config = config_json.get("job_config") 67 | algorithm_config_path = base.parent.joinpath( 68 | config_json.get("algorithm_config") 69 | ).absolute() 70 | with algorithm_config_path.open("r") as f: 71 | algorithm_config_string = f.read() 72 | 73 | extensions.get_job_schema_validator(job_type).validate(job_config) 74 | post( 75 | endpoint, 76 | "submit", 77 | dict( 78 | job_type=job_type, 79 | job_config=job_config, 80 | algorithm_config=algorithm_config_string, 81 | ), 82 | ) 83 | 84 | 85 | if __name__ == "__main__": 86 | cli() 87 | -------------------------------------------------------------------------------- /fedvision/framework/cluster/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/cluster/executor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | import os 17 | from pathlib import Path 18 | from typing import Optional 19 | 20 | from fedvision.framework.abc.executor import Executor 21 | from fedvision.framework.utils.logger import Logger 22 | from fedvision import FEDVISION_DATA_BASE_ENV 23 | 24 | 25 | class ProcessExecutor(Executor, Logger): 26 | def __init__(self, work_dir: Path, data_dir=None): 27 | super().__init__(work_dir) 28 | self._data_dir = data_dir 29 | 30 | async def execute(self, cmd) -> Optional[int]: 31 | self.info(f"execute cmd {cmd} at {self.working_dir}") 32 | try: 33 | env = os.environ.copy() 34 | if self._data_dir is not None: 35 | env[FEDVISION_DATA_BASE_ENV] = self._data_dir 36 | sub = await asyncio.subprocess.create_subprocess_shell( 37 | cmd, shell=True, cwd=self.working_dir, env=env 38 | ) 39 | await sub.communicate() 40 | return sub.returncode 41 | except Exception as e: 42 | self.error(e) 43 | -------------------------------------------------------------------------------- /fedvision/framework/coordinator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from fedvision.framework.extensions._extensions import ( 16 | get_task_class, 17 | get_job_class, 18 | get_job_schema_validator, 19 | ) 20 | 21 | __all__ = ["get_job_class", "get_task_class", "get_job_schema_validator"] 22 | -------------------------------------------------------------------------------- /fedvision/framework/extensions/_extensions.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import importlib 16 | import json 17 | from pathlib import Path 18 | from typing import Optional, MutableMapping, Type 19 | 20 | import jsonschema 21 | import yaml 22 | 23 | from fedvision.framework.abc.job import Job 24 | from fedvision.framework.abc.task import Task 25 | from fedvision.framework.utils.exception import FedvisionExtensionException 26 | from fedvision.framework.utils.logger import Logger 27 | 28 | 29 | class _ExtensionLoader(Logger): 30 | _job_classes: Optional[MutableMapping[str, Type[Job]]] = None 31 | _job_schema_validator: Optional[MutableMapping] = None 32 | _task_classes: Optional[MutableMapping[str, Type[Task]]] = None 33 | 34 | @classmethod 35 | def _load(cls): 36 | if cls._job_classes is not None: 37 | return cls 38 | 39 | cls._job_classes = {} 40 | cls._task_classes = {} 41 | cls._job_schema_validator = {} 42 | path = Path(__file__).parent.parent.parent.parent.joinpath( 43 | "conf/extensions.yaml" 44 | ) 45 | cls.trace(f"load extension configuration from {path}") 46 | with open(path) as f: 47 | try: 48 | extensions = yaml.safe_load(f) 49 | except yaml.YAMLError as e: 50 | raise FedvisionExtensionException("load extension failed") from e 51 | finally: 52 | cls.trace_lazy( 53 | "extension configuration:\n{yaml_config}", 54 | yaml_config=lambda: yaml.safe_dump(extensions, indent=2), 55 | ) 56 | 57 | for extension_name, configs in extensions.items(): 58 | cls.trace(f"loading extension: {extension_name}") 59 | 60 | cls.trace(f"loading job classes from extension {extension_name}") 61 | for extension_job in configs.get("jobs", []): 62 | job_module_name, job_cls_name = extension_job["loader"].split(":") 63 | module = importlib.import_module(job_module_name) 64 | job_cls = getattr(module, job_cls_name) 65 | if not issubclass(job_cls, Job): 66 | raise FedvisionExtensionException( 67 | f"JobLoader expected, {job_cls} found" 68 | ) 69 | cls._job_classes[extension_job["name"]] = job_cls 70 | 71 | if "schema" in extension_job: 72 | with path.parent.joinpath(extension_job["schema"]).open() as g: 73 | schema = json.load(g) 74 | print(schema) 75 | cls._job_schema_validator[ 76 | extension_job["name"] 77 | ] = jsonschema.Draft7Validator(schema) 78 | else: 79 | cls._job_schema_validator[ 80 | extension_job["name"] 81 | ] = jsonschema.Draft7Validator({}) 82 | 83 | cls.trace(f"loading task classes from extension {extension_name}") 84 | for extension_task in configs.get("tasks", []): 85 | loader_module, loader_cls = extension_task["loader"].split(":") 86 | module = importlib.import_module(loader_module) 87 | loader = getattr(module, loader_cls) 88 | if not issubclass(loader, Task): 89 | raise FedvisionExtensionException( 90 | f"JobLoader expected, {loader} found" 91 | ) 92 | cls._task_classes[extension_task["name"]] = loader 93 | cls.trace_lazy( 94 | "loading extensions done. job classes: {job_classes}, task classes: {task_classes}", 95 | job_classes=lambda: cls._job_classes, 96 | task_classes=lambda: cls._task_classes, 97 | ) 98 | return cls 99 | 100 | @classmethod 101 | def _load_schema(cls): 102 | if cls._job_schema_validator is not None: 103 | return cls 104 | 105 | cls._job_schema_validator = {} 106 | path = Path(__file__).parent.parent.parent.parent.joinpath( 107 | "conf/extensions.yaml" 108 | ) 109 | cls.trace(f"load extension configuration from {path}") 110 | with open(path) as f: 111 | try: 112 | extensions = yaml.safe_load(f) 113 | except yaml.YAMLError as e: 114 | raise FedvisionExtensionException("load extension failed") from e 115 | finally: 116 | cls.trace_lazy( 117 | "extension configuration:\n{yaml_config}", 118 | yaml_config=lambda: yaml.safe_dump(extensions, indent=2), 119 | ) 120 | 121 | for extension_name, configs in extensions.items(): 122 | cls.trace(f"loading extension: {extension_name}") 123 | 124 | for extension_job in configs.get("jobs", []): 125 | if "schema" in extension_job: 126 | with path.parent.joinpath(extension_job["schema"]).open() as g: 127 | schema = json.load(g) 128 | cls._job_schema_validator[ 129 | extension_job["name"] 130 | ] = jsonschema.Draft7Validator(schema) 131 | else: 132 | cls._job_schema_validator[ 133 | extension_job["name"] 134 | ] = jsonschema.Draft7Validator({}) 135 | return cls 136 | 137 | @classmethod 138 | def get_job_class(cls, name): 139 | return cls._load()._job_classes.get(name) 140 | 141 | @classmethod 142 | def get_task_class(cls, name): 143 | return cls._load()._task_classes.get(name) 144 | 145 | @classmethod 146 | def get_job_schema_validator(cls, name): 147 | return cls._load_schema()._job_schema_validator.get(name) 148 | 149 | 150 | def get_job_class(name) -> Type[Job]: 151 | return _ExtensionLoader.get_job_class(name) 152 | 153 | 154 | def get_task_class(name) -> Type[Task]: 155 | return _ExtensionLoader.get_task_class(name) 156 | 157 | 158 | def get_job_schema_validator(name): 159 | return _ExtensionLoader.get_job_schema_validator(name) 160 | -------------------------------------------------------------------------------- /fedvision/framework/master/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/protobuf/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # files in this folder are Generated by the protocol buffer compiler. DO NOT EDIT! 16 | -------------------------------------------------------------------------------- /fedvision/framework/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/framework/utils/exception.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class FedvisionBaseException(BaseException): 17 | ... 18 | 19 | 20 | class FedvisionException(FedvisionBaseException, Exception): 21 | ... 22 | 23 | 24 | class FedvisionExtensionException(FedvisionException): 25 | ... 26 | 27 | 28 | class FedvisionWorkerException(FedvisionException): 29 | ... 30 | 31 | 32 | class FedvisionJobCompileException(FedvisionException): 33 | ... 34 | 35 | 36 | class FedvisionDataNotFoundException(FedvisionException): 37 | ... 38 | -------------------------------------------------------------------------------- /fedvision/framework/utils/logger.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | from typing import Any, Union 17 | 18 | from google.protobuf import text_format 19 | from loguru import logger 20 | 21 | from fedvision import __logs_dir__ 22 | 23 | __BASE_LOGGER = None 24 | 25 | 26 | def set_logger(filename="unnamed"): 27 | log_dir = Path(__logs_dir__) 28 | if not log_dir.exists(): 29 | log_dir.mkdir(exist_ok=True) 30 | 31 | log_format = ( 32 | "[{extra[base]}]" 33 | "{time:YYYY-MM-DD HH:mm:ss.SSS} | " 34 | "{level: <8} | " 35 | "{name}:{function}:{line}:{message}" 36 | ) 37 | config = { 38 | "handlers": [ 39 | # dict(sink=sys.stdout, format=log_format, level="DEBUG"), 40 | dict( 41 | sink=f"{log_dir.joinpath(filename)}.log", 42 | format=log_format, 43 | level="DEBUG", 44 | ), 45 | ], 46 | "extra": {"base": "unknown"}, 47 | } 48 | logger.configure(**config) 49 | global __BASE_LOGGER 50 | __BASE_LOGGER = logger 51 | 52 | 53 | set_logger() 54 | 55 | 56 | class Logger(object): 57 | 58 | _logger = None 59 | 60 | @classmethod 61 | def get_logger(cls, lazy=False): 62 | if cls._logger is None: 63 | cls._logger = logger.bind(base=cls.__name__).opt(depth=1) 64 | if lazy: 65 | return cls._logger.opt(lazy=True, depth=1) 66 | return cls._logger 67 | 68 | @classmethod 69 | def log( 70 | cls, 71 | __level: Union[int, str], 72 | __message: str, 73 | *args: Any, 74 | lazy=False, 75 | **kwargs: Any, 76 | ): 77 | cls.get_logger(lazy=lazy).log(__level, __message, *args, **kwargs) 78 | 79 | @classmethod 80 | def trace(cls, __message: str, *args: Any, **kwargs: Any): 81 | cls.get_logger(lazy=False).trace(__message, *args, **kwargs) 82 | 83 | @classmethod 84 | def trace_lazy(cls, __message: str, *args: Any, **kwargs: Any): 85 | cls.get_logger(lazy=True).trace(__message, *args, **kwargs) 86 | 87 | @classmethod 88 | def debug(cls, __message: str, *args: Any, **kwargs: Any): 89 | cls.get_logger().debug(__message, *args, **kwargs) 90 | 91 | @classmethod 92 | def debug_lazy(cls, __message: str, *args: Any, **kwargs: Any): 93 | cls.get_logger(lazy=True).debug(__message, *args, **kwargs) 94 | 95 | @classmethod 96 | def info(cls, __message: str, *args: Any, **kwargs: Any): 97 | cls.get_logger().info(__message, *args, **kwargs) 98 | 99 | @classmethod 100 | def info_lazy(cls, __message: str, *args: Any, **kwargs: Any): 101 | cls.get_logger(lazy=True).info(__message, *args, **kwargs) 102 | 103 | @classmethod 104 | def warning(cls, __message: str, *args: Any, **kwargs: Any): 105 | cls.get_logger().warning(__message, *args, **kwargs) 106 | 107 | @classmethod 108 | def error(cls, __message: str, *args: Any, **kwargs: Any): 109 | cls.get_logger().error(__message, *args, **kwargs) 110 | 111 | @classmethod 112 | def critical(cls, __message: str, *args: Any, **kwargs: Any): 113 | cls.get_logger().critical(__message, *args, **kwargs) 114 | 115 | @classmethod 116 | def exception(cls, __message: str, *args: Any, **kwargs: Any): 117 | cls.get_logger().exception(__message, *args, **kwargs) 118 | 119 | 120 | def pretty_pb(pb): 121 | return text_format.MessageToString(pb, as_one_line=True) 122 | -------------------------------------------------------------------------------- /fedvision/framework/utils/logger.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any, overload, Union 2 | 3 | import loguru 4 | 5 | class Logger: 6 | @classmethod 7 | def get_logger(cls, lazy=False) -> loguru.Logger: ... 8 | @classmethod 9 | @overload 10 | def log( 11 | cls, 12 | __level: Union[int, str], 13 | __message: str, 14 | *args: Any, 15 | lazy=False, 16 | **kwargs: Any 17 | ): ... 18 | @classmethod 19 | @overload 20 | def log(cls, __level: Union[int, str], __message: Any, lazy=False) -> None: ... 21 | @classmethod 22 | @overload 23 | def trace(cls, __message: str, *args: Any, **kwargs: Any): ... 24 | @classmethod 25 | @overload 26 | def trace(cls, __message: Any): ... 27 | @classmethod 28 | @overload 29 | def trace_lazy(cls, __message: str, *args: Any, **kwargs: Any): ... 30 | @classmethod 31 | @overload 32 | def trace_lazy(cls, __message: Any): ... 33 | @classmethod 34 | @overload 35 | def debug(cls, __message: str, *args: Any, **kwargs: Any): ... 36 | @classmethod 37 | @overload 38 | def debug(cls, __message: Any): ... 39 | @classmethod 40 | @overload 41 | def debug_lazy(cls, __message: str, *args: Any, **kwargs: Any): ... 42 | @classmethod 43 | @overload 44 | def debug_lazy(cls, __message: Any): ... 45 | @classmethod 46 | @overload 47 | def info(cls, __message: str, *args: Any, **kwargs: Any): ... 48 | @classmethod 49 | @overload 50 | def info(cls, __message: Any): ... 51 | @classmethod 52 | @overload 53 | def info_lazy(cls, __message: str, *args: Any, **kwargs: Any): ... 54 | @classmethod 55 | @overload 56 | def info_lazy(cls, __message: Any): ... 57 | @classmethod 58 | @overload 59 | def warning(cls, __message: str, *args: Any, **kwargs: Any) -> None: ... 60 | @classmethod 61 | @overload 62 | def warning(cls, __message: Any) -> None: ... 63 | @classmethod 64 | @overload 65 | def error(cls, __message: str, *args: Any, **kwargs: Any) -> None: ... 66 | @classmethod 67 | @overload 68 | def error(cls, __message: Any) -> None: ... 69 | @classmethod 70 | @overload 71 | def critical(cls, __message: str, *args: Any, **kwargs: Any) -> None: ... 72 | @classmethod 73 | @overload 74 | def critical(cls, __message: Any) -> None: ... 75 | @classmethod 76 | @overload 77 | def exception(cls, __message: str, *args: Any, **kwargs: Any) -> None: ... 78 | @classmethod 79 | @overload 80 | def exception(cls, __message: Any) -> None: ... 81 | 82 | def pretty_pb(pb) -> str: ... 83 | def set_logger(filename="fedvision.log"): ... 84 | -------------------------------------------------------------------------------- /fedvision/ml/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/fedvision/ml/__init__.py -------------------------------------------------------------------------------- /fedvision/ml/paddle/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/fedvision/ml/paddle/__init__.py -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_detection/README.md: -------------------------------------------------------------------------------- 1 | ## Note 2 | 3 | This folder implement detection algorithms based on [PaddleDetection project](https://github.com/PaddlePaddle/PaddleDetection). 4 | 5 | The original detection programs are compiled into single server program and multiple trainer program by paddle_fl. 6 | Thus, all algorithms supported by [PaddleDetection project](https://github.com/PaddlePaddle/PaddleDetection) has their `federated counterpart`. 7 | 8 | While, there are plenty test should be task before we could claim it. 9 | 10 | ## How to use 11 | 12 | See [example](../../../../examples/paddle_detection) 13 | -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_detection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/fedvision/ml/paddle/paddle_detection/__init__.py -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_detection/_empty_optimizer.py: -------------------------------------------------------------------------------- 1 | class EmptyOptimizer(object): 2 | def minimize(self, loss): 3 | ... 4 | -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_detection/fl_master.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import json 20 | import logging 21 | 22 | import click 23 | from paddle import fluid 24 | 25 | from fedvision.ml.paddle.paddle_detection._empty_optimizer import ( 26 | EmptyOptimizer, 27 | ) 28 | from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator 29 | from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import ( 30 | FedAvgStrategy, 31 | ) 32 | 33 | 34 | class Model(object): 35 | def __init__(self): 36 | self.feeds = None 37 | self.startup_program = None 38 | self.loss = None 39 | 40 | def build_program(self, config): 41 | from ppdet.core.workspace import load_config, create 42 | from ppdet.utils.check import check_version, check_config 43 | 44 | cfg = load_config(config) 45 | check_config(cfg) 46 | check_version() 47 | 48 | lr_builder = create("LearningRate") 49 | optimizer_builder = create("OptimizerBuilder") 50 | 51 | # build program 52 | self.startup_program = fluid.Program() 53 | train_program = fluid.Program() 54 | with fluid.program_guard(train_program, self.startup_program): 55 | with fluid.unique_name.guard(): 56 | model = create(cfg.architecture) 57 | 58 | inputs_def = cfg["TrainReader"]["inputs_def"] 59 | # can't compile with dataloader now. 60 | inputs_def["use_dataloader"] = False 61 | feed_vars, _ = model.build_inputs(**inputs_def) 62 | 63 | train_fetches = model.train(feed_vars) 64 | loss = train_fetches["loss"] 65 | lr = lr_builder() 66 | optimizer = optimizer_builder(lr) 67 | optimizer.minimize(loss) 68 | 69 | self.loss = loss 70 | self.feeds = feed_vars 71 | 72 | 73 | @click.command() 74 | @click.option("--ps-endpoint", type=str, required=True) 75 | @click.option( 76 | "-c", 77 | "--config", 78 | type=click.Path(file_okay=True, dir_okay=False, exists=True), 79 | required=True, 80 | ) 81 | @click.option( 82 | "--algorithm-config", type=click.Path(exists=True, file_okay=True, dir_okay=False) 83 | ) 84 | def fl_master(algorithm_config, ps_endpoint, config): 85 | logging.basicConfig( 86 | level=logging.DEBUG, format="%(asctime)s-%(levelname)s: %(message)s" 87 | ) 88 | logger = logging.getLogger(__name__) # noqa: F841 89 | with open(config) as f: 90 | config_json = json.load(f) 91 | worker_num = config_json["worker_num"] 92 | 93 | model = Model() 94 | model.build_program(algorithm_config) 95 | 96 | job_generator = JobGenerator() 97 | job_generator.set_losses([model.loss]) 98 | job_generator.set_optimizer(EmptyOptimizer()) # optimizer defined in Model 99 | job_generator.set_startup_program(model.startup_program) 100 | job_generator.set_infer_feed_and_target_names( 101 | [name for name in model.feeds], [model.loss.name] 102 | ) 103 | job_generator.set_feeds(model.feeds.values()) 104 | 105 | strategy = FedAvgStrategy() 106 | strategy.fed_avg = True 107 | strategy.inner_step = 1 108 | 109 | endpoints = [ps_endpoint] 110 | output = "compile" 111 | job_generator.generate_fl_job( 112 | strategy, server_endpoints=endpoints, worker_num=worker_num, output=output 113 | ) 114 | 115 | 116 | if __name__ == "__main__": 117 | fl_master() 118 | -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_detection/fl_trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 16 | # 17 | # Licensed under the Apache License, Version 2.0 (the "License"); 18 | # you may not use this file except in compliance with the License. 19 | # You may obtain a copy of the License at 20 | # 21 | # http://www.apache.org/licenses/LICENSE-2.0 22 | # 23 | # Unless required by applicable law or agreed to in writing, software 24 | # distributed under the License is distributed on an "AS IS" BASIS, 25 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 26 | # See the License for the specific language governing permissions and 27 | # limitations under the License. 28 | 29 | import json 30 | import logging 31 | import os 32 | 33 | import click 34 | 35 | from fedvision.paddle_fl.tasks.utils import FedAvgTrainer 36 | from fedvision import get_data_dir 37 | 38 | 39 | @click.command() 40 | @click.option("--scheduler-ep", type=str, required=True) 41 | @click.option("--trainer-id", type=int, required=True) 42 | @click.option("--trainer-ep", type=str, required=True) 43 | @click.option( 44 | "--main-program", 45 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 46 | required=True, 47 | ) 48 | @click.option( 49 | "--startup-program", 50 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 51 | required=True, 52 | ) 53 | @click.option( 54 | "--send-program", 55 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 56 | required=True, 57 | ) 58 | @click.option( 59 | "--recv-program", 60 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 61 | required=True, 62 | ) 63 | @click.option( 64 | "--feed-names", 65 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 66 | required=True, 67 | ) 68 | @click.option( 69 | "--target-names", 70 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 71 | required=True, 72 | ) 73 | @click.option( 74 | "--strategy", 75 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 76 | required=True, 77 | ) 78 | @click.option( 79 | "--feeds", 80 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 81 | required=True, 82 | ) 83 | @click.option( 84 | "--config", 85 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 86 | required=True, 87 | ) 88 | @click.option( 89 | "--algorithm-config", 90 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 91 | required=True, 92 | ) 93 | def fl_trainer( 94 | trainer_id: int, 95 | trainer_ep: str, 96 | scheduler_ep: str, 97 | main_program, 98 | startup_program, 99 | send_program, 100 | recv_program, 101 | feed_names, 102 | target_names, 103 | strategy, 104 | feeds, 105 | config, 106 | algorithm_config, 107 | ): 108 | import numpy as np 109 | import paddle.fluid as fluid 110 | 111 | from ppdet.core.workspace import load_config 112 | from ppdet.data import create_reader 113 | from ppdet.utils import checkpoint 114 | from ppdet.utils.check import check_config, check_version 115 | 116 | logging.basicConfig( 117 | filename="trainer.log", 118 | filemode="w", 119 | format="%(asctime)s %(name)s:%(levelname)s:%(message)s", 120 | datefmt="%d-%M-%Y %H:%M:%S", 121 | level=logging.DEBUG, 122 | ) 123 | 124 | with open(config) as f: 125 | config_json = json.load(f) 126 | max_iter = config_json["max_iter"] 127 | device = config_json.get("device", "cpu") 128 | use_vdl = config_json.get("use_vdl", False) 129 | 130 | logging.debug(f"training program begin") 131 | trainer = FedAvgTrainer(scheduler_ep=scheduler_ep, trainer_ep=trainer_ep) 132 | logging.debug(f"job program loading") 133 | trainer.load_job( 134 | main_program=main_program, 135 | startup_program=startup_program, 136 | send_program=send_program, 137 | recv_program=recv_program, 138 | feed_names=feed_names, 139 | target_names=target_names, 140 | strategy=strategy, 141 | ) 142 | logging.debug(f"job program loaded") 143 | place = fluid.CPUPlace() if device != "cuda" else fluid.CUDAPlace(0) 144 | 145 | logging.debug(f"trainer starting with place {place}") 146 | trainer.start(place) 147 | logging.debug(f"trainer stared") 148 | 149 | cfg = load_config(algorithm_config) 150 | check_config(cfg) 151 | check_version() 152 | 153 | logging.debug(f"loading data") 154 | feed_list = trainer.load_feed_list(feeds) 155 | feeder = fluid.DataFeeder(feed_list=feed_list, place=place) 156 | logging.debug(f"data loader ready") 157 | 158 | epoch_id = -1 159 | step = 0 160 | 161 | # redirect dataset path to Fedvision/data 162 | cfg.TrainReader["dataset"].dataset_dir = os.path.join( 163 | get_data_dir(), cfg.TrainReader["dataset"].dataset_dir 164 | ) 165 | 166 | data_loader = create_reader( 167 | cfg.TrainReader, max_iter, cfg, devices_num=1, num_trainers=1 168 | ) 169 | logging.error(f"{cfg.TrainReader['dataset']}") 170 | 171 | if use_vdl: 172 | from visualdl import LogWriter 173 | 174 | vdl_writer = LogWriter("vdl_log") 175 | 176 | while epoch_id < max_iter: 177 | epoch_id += 1 178 | if not trainer.scheduler_agent.join(epoch_id): 179 | logging.debug(f"not join, waiting next round") 180 | continue 181 | 182 | logging.debug(f"epoch {epoch_id} start train") 183 | 184 | for step_id, data in enumerate(data_loader()): 185 | outs = trainer.run(feeder.feed(data), fetch=trainer._target_names) 186 | if use_vdl: 187 | stats = { 188 | k: np.array(v).mean() for k, v in zip(trainer._target_names, outs) 189 | } 190 | for loss_name, loss_value in stats.items(): 191 | vdl_writer.add_scalar(loss_name, loss_value, step) 192 | step += 1 193 | logging.debug(f"step: {step}, outs: {outs}") 194 | 195 | # save model 196 | logging.debug(f"saving model at {epoch_id}-th epoch") 197 | trainer.save_model(f"model/{epoch_id}") 198 | 199 | # info scheduler 200 | trainer.scheduler_agent.finish() 201 | checkpoint.save(trainer.exe, trainer._main_program, f"checkpoint/{epoch_id}") 202 | 203 | logging.debug(f"reach max iter, finish training") 204 | 205 | 206 | if __name__ == "__main__": 207 | fl_trainer() 208 | -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_mnist/README.md: -------------------------------------------------------------------------------- 1 | ## Note 2 | 3 | This folder implement cnn algorithms directly by paddle_fl. 4 | 5 | ## How to use 6 | 7 | See [example](../../../../examples/paddle_mnist) 8 | -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_mnist/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FedVision/87c3fa5aee2a45c7bfd694cba18f398ff135228f/fedvision/ml/paddle/paddle_mnist/__init__.py -------------------------------------------------------------------------------- /fedvision/ml/paddle/paddle_mnist/fl_master.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import json 15 | import logging 16 | 17 | import click 18 | from paddle import fluid 19 | 20 | from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator 21 | from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import ( 22 | FedAvgStrategy, 23 | ) 24 | 25 | 26 | class CNN(object): 27 | def __init__(self): 28 | inputs = fluid.layers.data(name="img", shape=[1, 28, 28], dtype="float64") 29 | label = fluid.layers.data(name="label", shape=[1], dtype="int64") 30 | conv_pool_1 = fluid.nets.simple_img_conv_pool( 31 | input=inputs, 32 | num_filters=20, 33 | filter_size=5, 34 | pool_size=2, 35 | pool_stride=2, 36 | act="relu", 37 | ) 38 | conv_pool_2 = fluid.nets.simple_img_conv_pool( 39 | input=conv_pool_1, 40 | num_filters=50, 41 | filter_size=5, 42 | pool_size=2, 43 | pool_stride=2, 44 | act="relu", 45 | ) 46 | predict = self.predict = fluid.layers.fc( 47 | input=conv_pool_2, size=62, act="softmax" 48 | ) 49 | cost = fluid.layers.cross_entropy(input=predict, label=label) 50 | accuracy = fluid.layers.accuracy(input=predict, label=label) 51 | self.loss = fluid.layers.mean(cost) 52 | self.startup_program = fluid.default_startup_program() 53 | 54 | self.feeds = [inputs, label] 55 | self.targets = [self.loss, accuracy] 56 | 57 | 58 | @click.command() 59 | @click.option("--ps-endpoint", type=str, required=True) 60 | @click.option( 61 | "-c", 62 | "--config", 63 | type=click.Path(file_okay=True, dir_okay=False, exists=True), 64 | required=True, 65 | ) 66 | @click.option( 67 | "--algorithm-config", type=click.Path(exists=True, file_okay=True, dir_okay=False) 68 | ) 69 | def fl_master(algorithm_config, ps_endpoint, config): 70 | logging.basicConfig( 71 | level=logging.DEBUG, format="%(asctime)s-%(levelname)s: %(message)s" 72 | ) 73 | 74 | logger = logging.getLogger(__name__) # noqa: F841 75 | with open(config) as f: 76 | config_json = json.load(f) 77 | worker_num = config_json["worker_num"] 78 | inner_step = config_json["inner_step"] 79 | 80 | model = CNN() 81 | job_generator = JobGenerator() 82 | job_generator.set_losses([model.loss]) 83 | job_generator.set_optimizer(fluid.optimizer.Adam(0.001)) 84 | job_generator.set_startup_program(model.startup_program) 85 | job_generator.set_infer_feed_and_target_names( 86 | [feed.name for feed in model.feeds], [target.name for target in model.targets] 87 | ) 88 | job_generator.set_feeds(model.feeds) 89 | 90 | strategy = FedAvgStrategy() 91 | strategy.fed_avg = True 92 | strategy._inner_step = inner_step 93 | 94 | endpoints = [ps_endpoint] 95 | output = "compile" 96 | job_generator.generate_fl_job( 97 | strategy, server_endpoints=endpoints, worker_num=worker_num, output=output 98 | ) 99 | 100 | 101 | if __name__ == "__main__": 102 | fl_master() 103 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/job.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | import sys 17 | from pathlib import Path 18 | from typing import List 19 | 20 | from fedvision import __logs_dir__ 21 | from fedvision.framework.abc.job import Job 22 | from fedvision.framework.cluster.executor import ProcessExecutor 23 | from fedvision.framework.protobuf import job_pb2, coordinator_pb2, cluster_pb2 24 | from fedvision.framework.utils.exception import FedvisionJobCompileException 25 | from fedvision.paddle_fl.protobuf import fl_job_pb2 26 | 27 | JOB_TYPE = "paddle_fl" 28 | 29 | 30 | class PaddleFLJob(Job): 31 | job_type = JOB_TYPE 32 | 33 | @classmethod 34 | def load(cls, job_id, config, algorithm_config) -> "PaddleFLJob": 35 | return PaddleFLJob( 36 | job_id=job_id, 37 | proposal_wait_time=config["proposal_wait_time"], 38 | worker_num=config["worker_num"], 39 | program=config["program"], 40 | config=config, 41 | algorithm_config=algorithm_config, 42 | ) 43 | 44 | def __init__( 45 | self, 46 | job_id, 47 | program, 48 | proposal_wait_time, 49 | worker_num, 50 | config, 51 | algorithm_config, 52 | ): 53 | super().__init__(job_id=job_id) 54 | self._proposal_wait_time = proposal_wait_time 55 | self._worker_num = worker_num 56 | self._program = program 57 | self._trainer_entrypoint = f"fedvision.ml.paddle.{self._program}.fl_trainer" 58 | 59 | self._config_string = json.dumps(config) 60 | self._algorithm_config = algorithm_config 61 | 62 | @property 63 | def resource_required(self): 64 | return cluster_pb2.TaskResourceRequire.REQ(num_endpoints=2) 65 | 66 | # noinspection PyAttributeOutsideInit 67 | def set_required_resource(self, response): 68 | self._server_endpoint = response.endpoints[0] 69 | self._aggregator_endpoint = response.endpoints[1] 70 | self._aggregator_assignee = response.worker_id 71 | 72 | @property 73 | def compile_path(self): 74 | return Path(__logs_dir__).joinpath(f"jobs/{self.job_id}/master") 75 | 76 | async def compile(self): 77 | executor = ProcessExecutor(self.compile_path) 78 | with self.compile_path.joinpath("algorithm_config.yaml").open("w") as f: 79 | f.write(self._algorithm_config) 80 | with self.compile_path.joinpath("config.json").open("w") as f: 81 | f.write(self._config_string) 82 | executable = sys.executable 83 | cmd = " ".join( 84 | [ 85 | f"{executable} -m fedvision.ml.paddle.{self._program}.fl_master", 86 | f"--ps-endpoint {self._server_endpoint}", 87 | f"--algorithm-config algorithm_config.yaml", 88 | f"--config config.json", 89 | f">{executor.stdout} 2>{executor.stderr}", 90 | ] 91 | ) 92 | returncode = await executor.execute(cmd) 93 | if returncode != 0: 94 | 95 | raise FedvisionJobCompileException("compile error") 96 | 97 | def generate_proposal_request(self) -> coordinator_pb2.Proposal.REQ: 98 | request = coordinator_pb2.Proposal.REQ( 99 | job_id=self.job_id, 100 | job_type=self.job_type, 101 | proposal_wait_time=self._proposal_wait_time, 102 | minimum_acceptance=self._worker_num, 103 | maximum_acceptance=self._worker_num, 104 | ) 105 | for i in range(self._worker_num): 106 | task = request.tasks.add() 107 | self._generate_trainer_task_pb(task, i) 108 | return request 109 | 110 | def generate_local_tasks(self) -> List[job_pb2.Task]: 111 | return [ 112 | self._generate_aggregator_task_pb(), 113 | ] 114 | 115 | def _generate_trainer_task_pb(self, task_pb, i): 116 | trainer_pb = fl_job_pb2.PaddleFLWorkerTask( 117 | scheduler_ep=self._aggregator_endpoint, 118 | trainer_id=i, 119 | trainer_ep=f"trainer_{i}", 120 | entrypoint=self._trainer_entrypoint, 121 | main_program=_load_program_bytes( 122 | self.compile_path.joinpath(f"compile/trainer{i}/trainer.main.program") 123 | ), 124 | startup_program=_load_program_bytes( 125 | self.compile_path.joinpath( 126 | f"compile/trainer{i}/trainer.startup.program" 127 | ) 128 | ), 129 | send_program=_load_program_bytes( 130 | self.compile_path.joinpath(f"compile/trainer{i}/trainer.send.program") 131 | ), 132 | recv_program=_load_program_bytes( 133 | self.compile_path.joinpath(f"compile/trainer{i}/trainer.recv.program") 134 | ), 135 | feed_names=_load_program_bytes( 136 | self.compile_path.joinpath(f"compile/trainer{i}/feed_names") 137 | ), 138 | target_names=_load_program_bytes( 139 | self.compile_path.joinpath(f"compile/trainer{i}/target_names") 140 | ), 141 | strategy=_load_program_bytes( 142 | self.compile_path.joinpath(f"compile/trainer{i}/strategy.pkl") 143 | ), 144 | feeds=_load_program_bytes( 145 | self.compile_path.joinpath(f"compile/trainer{1}/feeds.pkl") 146 | ), 147 | config_string=self._config_string, 148 | algorithm_config_string=self._algorithm_config, 149 | ) 150 | task_pb.job_id = self.job_id 151 | task_pb.task_id = f"trainer_{i}" 152 | task_pb.task_type = "fl_trainer" 153 | task_pb.task.Pack(trainer_pb) 154 | return task_pb 155 | 156 | def _generate_aggregator_task_pb(self): 157 | scheduler_pb = fl_job_pb2.PaddleFLAggregatorTask( 158 | scheduler_ep=self._aggregator_endpoint, 159 | ) 160 | scheduler_pb.main_program = _load_program_bytes( 161 | self.compile_path.joinpath(f"compile/server0/server.main.program") 162 | ) 163 | scheduler_pb.startup_program = _load_program_bytes( 164 | self.compile_path.joinpath(f"compile/server0/server.startup.program") 165 | ) 166 | scheduler_pb.config_string = self._config_string 167 | 168 | task_pb = job_pb2.Task( 169 | job_id=self.job_id, 170 | task_id=f"aggregator", 171 | task_type="fl_aggregator", 172 | assignee=self._aggregator_assignee, 173 | ) 174 | task_pb.task.Pack(scheduler_pb) 175 | return task_pb 176 | 177 | 178 | def _load_program_bytes(path: Path): 179 | with path.open("rb") as f: 180 | return f.read() 181 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/protobuf/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # files in this folder are Generated by the protocol buffer compiler. DO NOT EDIT! 16 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/cli/fl_server.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from pathlib import Path 16 | 17 | import click 18 | from paddle import fluid 19 | 20 | 21 | @click.command() 22 | @click.option( 23 | "--main-program", 24 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 25 | required=True, 26 | ) 27 | @click.option( 28 | "--startup-program", 29 | type=click.Path(exists=True, file_okay=True, dir_okay=False), 30 | required=True, 31 | ) 32 | def fl_server( 33 | startup_program, 34 | main_program, 35 | ): 36 | def _load_job_from_file(path): 37 | with Path(path).open("rb") as f: 38 | return fluid.Program.parse_from_string(f.read()) 39 | 40 | server_startup_program = _load_job_from_file(Path(startup_program)) 41 | server_main_program = _load_job_from_file(Path(main_program)) 42 | exe = fluid.Executor(fluid.CPUPlace()) 43 | exe.run(server_startup_program) 44 | exe.run(server_main_program) 45 | 46 | 47 | if __name__ == "__main__": 48 | fl_server() 49 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/task/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/task/aggregator.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | from fedvision.framework.abc.executor import Executor 18 | from fedvision.framework.abc.task import Task 19 | from fedvision.framework.protobuf import job_pb2 20 | from fedvision.framework.utils.exception import FedvisionWorkerException 21 | from fedvision.paddle_fl.protobuf import fl_job_pb2 22 | 23 | 24 | class FLAggregator(Task): 25 | task_type = "fl_aggregator" 26 | 27 | def __init__( 28 | self, 29 | job_id, 30 | task_id, 31 | scheduler_ep, 32 | main_program, 33 | startup_program, 34 | config_string, 35 | ): 36 | super().__init__(job_id=job_id, task_id=task_id) 37 | self._scheduler_ep = scheduler_ep 38 | self._main_program = main_program 39 | self._startup_program = startup_program 40 | self._config_string = config_string 41 | 42 | @classmethod 43 | def deserialize(cls, pb: job_pb2.Task) -> "FLAggregator": 44 | if pb.task_type != cls.task_type: 45 | raise FedvisionWorkerException( 46 | f"try to deserialize task_type {pb.task_type} by {cls.task_type}" 47 | ) 48 | scheduler_task_pb = fl_job_pb2.PaddleFLAggregatorTask() 49 | pb.task.Unpack(scheduler_task_pb) 50 | return FLAggregator( 51 | job_id=pb.job_id, 52 | task_id=pb.task_id, 53 | scheduler_ep=scheduler_task_pb.scheduler_ep, 54 | startup_program=scheduler_task_pb.startup_program, 55 | main_program=scheduler_task_pb.main_program, 56 | config_string=scheduler_task_pb.config_string, 57 | ) 58 | 59 | async def exec(self, executor: Executor): 60 | python_executable = sys.executable 61 | cmd = " ".join( 62 | [ 63 | f"{python_executable} -m fedvision.paddle_fl.tasks.cli.fl_scheduler", 64 | f"--scheduler-ep={self._scheduler_ep}", 65 | f"--startup-program=startup_program", 66 | f"--main-program=main_program", 67 | f"--config=config.json", 68 | f">{executor.stdout} 2>{executor.stderr}", 69 | ] 70 | ) 71 | with executor.working_dir.joinpath("main_program").open("wb") as f: 72 | f.write(self._main_program) 73 | with executor.working_dir.joinpath("startup_program").open("wb") as f: 74 | f.write(self._startup_program) 75 | with executor.working_dir.joinpath("config.json").open("w") as f: 76 | f.write(self._config_string) 77 | returncode = await executor.execute(cmd) 78 | if returncode != 0: 79 | raise FedvisionWorkerException( 80 | f"execute task: {self.task_id} failed, return code: {returncode}" 81 | ) 82 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/task/trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | from fedvision.framework.abc.executor import Executor 18 | from fedvision.framework.abc.task import Task 19 | from fedvision.framework.protobuf import job_pb2 20 | from fedvision.framework.utils.exception import FedvisionWorkerException 21 | from fedvision.paddle_fl.protobuf import fl_job_pb2 22 | 23 | 24 | class FLTrainer(Task): 25 | task_type = "fl_trainer" 26 | 27 | def __init__( 28 | self, 29 | job_id, 30 | task_id, 31 | scheduler_ep: str, 32 | trainer_id: int, 33 | trainer_ep: str, 34 | entrypoint, 35 | startup_program, 36 | main_program, 37 | send_program, 38 | recv_program, 39 | feed_names, 40 | target_names, 41 | strategy, 42 | feeds, 43 | config_string, 44 | algorithm_config_string, 45 | ): 46 | super().__init__(job_id=job_id, task_id=task_id) 47 | self._scheduler_ep = scheduler_ep 48 | self._trainer_id = trainer_id 49 | self._trainer_ep = trainer_ep 50 | self._entrypoint = entrypoint 51 | self._startup_program = startup_program 52 | self._main_program = main_program 53 | self._send_program = send_program 54 | self._recv_program = recv_program 55 | self._feed_names = feed_names 56 | self._target_names = target_names 57 | self._strategy = strategy 58 | self._feeds = feeds 59 | self._config_string = config_string 60 | self._algorithm_config_string = algorithm_config_string 61 | 62 | @classmethod 63 | def deserialize(cls, pb: job_pb2.Task) -> "FLTrainer": 64 | if pb.task_type != cls.task_type: 65 | raise FedvisionWorkerException( 66 | f"try to deserialize task_type {pb.task_type} by {cls.task_type}" 67 | ) 68 | worker_task_pb = fl_job_pb2.PaddleFLWorkerTask() 69 | pb.task.Unpack(worker_task_pb) 70 | return FLTrainer( 71 | job_id=pb.job_id, 72 | task_id=pb.task_id, 73 | scheduler_ep=worker_task_pb.scheduler_ep, 74 | trainer_id=worker_task_pb.trainer_id, 75 | trainer_ep=worker_task_pb.trainer_ep, 76 | entrypoint=worker_task_pb.entrypoint, 77 | startup_program=worker_task_pb.startup_program, 78 | main_program=worker_task_pb.main_program, 79 | send_program=worker_task_pb.send_program, 80 | recv_program=worker_task_pb.recv_program, 81 | feed_names=worker_task_pb.feed_names, 82 | target_names=worker_task_pb.target_names, 83 | strategy=worker_task_pb.strategy, 84 | feeds=worker_task_pb.feeds, 85 | config_string=worker_task_pb.config_string, 86 | algorithm_config_string=worker_task_pb.algorithm_config_string, 87 | ) 88 | 89 | async def exec(self, executor: Executor): 90 | python_executable = sys.executable 91 | cmd = " ".join( 92 | [ 93 | f"{python_executable} -m {self._entrypoint}", 94 | f"--scheduler-ep={self._scheduler_ep}", 95 | f"--trainer-id={self._trainer_id}", 96 | f"--trainer-ep={self._trainer_ep}", 97 | f"--startup-program=startup_program", 98 | f"--main-program=main_program", 99 | f"--send-program=send_program", 100 | f"--recv-program=recv_program", 101 | f"--feed-names=feed_names", 102 | f"--target-names=target_names", 103 | f"--feeds=feeds", 104 | f"--strategy=strategy", 105 | f"--config config.json", 106 | f"--algorithm-config algorithm_config.yaml" 107 | f">{executor.stdout} 2>{executor.stderr}", 108 | ] 109 | ) 110 | with executor.working_dir.joinpath("main_program").open("wb") as f: 111 | f.write(self._main_program) 112 | with executor.working_dir.joinpath("startup_program").open("wb") as f: 113 | f.write(self._startup_program) 114 | with executor.working_dir.joinpath("send_program").open("wb") as f: 115 | f.write(self._send_program) 116 | with executor.working_dir.joinpath("recv_program").open("wb") as f: 117 | f.write(self._recv_program) 118 | with executor.working_dir.joinpath("feed_names").open("wb") as f: 119 | f.write(self._feed_names) 120 | with executor.working_dir.joinpath("target_names").open("wb") as f: 121 | f.write(self._target_names) 122 | with executor.working_dir.joinpath("strategy").open("wb") as f: 123 | f.write(self._strategy) 124 | with executor.working_dir.joinpath("feeds").open("wb") as f: 125 | f.write(self._feeds) 126 | with executor.working_dir.joinpath("config.json").open("w") as f: 127 | f.write(self._config_string) 128 | with executor.working_dir.joinpath("algorithm_config.yaml").open("w") as f: 129 | f.write(self._algorithm_config_string) 130 | returncode = await executor.execute(cmd) 131 | if returncode is None or returncode != 0: 132 | raise FedvisionWorkerException( 133 | f"execute task: {self.task_id} failed, return code: {returncode}" 134 | ) 135 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from fedvision.paddle_fl.tasks.utils._trainer import FedAvgTrainer 16 | 17 | __all__ = ["FedAvgTrainer"] 18 | -------------------------------------------------------------------------------- /fedvision/paddle_fl/tasks/utils/_trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | import pickle 16 | from typing import Optional 17 | 18 | import grpc 19 | from paddle import fluid 20 | 21 | from fedvision.framework.utils.logger import Logger 22 | from fedvision.paddle_fl.protobuf import scheduler_pb2_grpc, scheduler_pb2 23 | from paddle_fl.paddle_fl.core.master.fl_job import FLJobBase 24 | 25 | 26 | class TrainerSchedulerAgent(Logger): 27 | def __init__(self, worker_name, scheduler_ep): 28 | self._worker_name = worker_name 29 | self._scheduler_ep = scheduler_ep 30 | 31 | self._channel: Optional[grpc.Channel] = None 32 | self._stub: Optional[scheduler_pb2_grpc.SchedulerStub] = None 33 | 34 | def start_channel(self): 35 | self._channel = grpc.insecure_channel(self._scheduler_ep) 36 | self._stub = scheduler_pb2_grpc.SchedulerStub(self._channel) 37 | 38 | self.debug(f"waiting channel ready") 39 | future = grpc.channel_ready_future(self._channel) 40 | future.result() 41 | self.debug(f"channel ready") 42 | return self 43 | 44 | def init_worker(self): 45 | self.debug(f"start to init") 46 | self._stub.Init(scheduler_pb2.Init.REQ(name=self._worker_name)) 47 | self.debug(f"init success") 48 | 49 | def join(self, step: int): 50 | self.debug("start to join") 51 | response = self._stub.WorkerJoin( 52 | scheduler_pb2.WorkerJoin.REQ(name=self._worker_name, step=step) 53 | ) 54 | self.debug(f"join success: {response.status}") 55 | return response.status == scheduler_pb2.WorkerJoin.ACCEPT 56 | 57 | def finish(self): 58 | self.debug("start to finish") 59 | status = self._stub.WorkerFinish( 60 | scheduler_pb2.WorkerFinish.REQ(name=self._worker_name) 61 | ) 62 | self.debug(f"finish success: {status}") 63 | return status == scheduler_pb2.WorkerFinish.DONE 64 | 65 | def close(self): 66 | self._channel.close() 67 | 68 | 69 | class FedAvgTrainer(FLJobBase): 70 | def __init__(self, scheduler_ep, trainer_ep): 71 | self._logger = logging.getLogger("FLTrainer") 72 | super(FedAvgTrainer, self).__init__() 73 | self._scheduler_ep = scheduler_ep 74 | self._trainer_ep = trainer_ep 75 | 76 | self.scheduler_agent: Optional[TrainerSchedulerAgent] = None 77 | self.exe: Optional[fluid.Executor] = None 78 | self.cur_step = 0 79 | 80 | def start(self, place): 81 | self.scheduler_agent = TrainerSchedulerAgent( 82 | scheduler_ep=self._scheduler_ep, worker_name=self._trainer_ep 83 | ) 84 | self.scheduler_agent.start_channel() 85 | self.scheduler_agent.init_worker() 86 | 87 | self.exe = fluid.Executor(place) 88 | self.exe.run(self._startup_program) 89 | 90 | def load_job( 91 | self, 92 | startup_program: str, 93 | main_program: str, 94 | send_program: str, 95 | recv_program: str, 96 | feed_names: str, 97 | target_names: str, 98 | strategy: str, 99 | ): 100 | self._startup_program = self._load_program(startup_program) 101 | self._main_program = self._load_program(main_program) 102 | self._send_program = self._load_program(send_program) 103 | self._recv_program = self._load_program(recv_program) 104 | 105 | self._step = self._load_strategy(strategy)._inner_step 106 | self._feed_names = self._load_str_list(feed_names) 107 | self._target_names = self._load_str_list(target_names) 108 | 109 | def load_feed_list(self, feeds_path): 110 | data = [] 111 | with open(feeds_path, "rb") as f: 112 | num = pickle.load(f) 113 | for _ in range(num): 114 | data.append(fluid.data(**pickle.load(f))) 115 | return data 116 | 117 | @staticmethod 118 | def _load_strategy(input_file): 119 | 120 | return pickle.load(open(input_file, "rb")) 121 | 122 | def reset(self): 123 | self.cur_step = 0 124 | 125 | def run_with_epoch(self, reader, feeder, fetch, num_epoch): 126 | self._logger.debug("begin to run recv program") 127 | self.exe.run(self._recv_program) 128 | self._logger.debug("recv done") 129 | epoch = 0 130 | for i in range(num_epoch): 131 | for data in reader(): 132 | acc = self.exe.run( 133 | self._main_program, feed=feeder.feed(data), fetch_list=fetch 134 | ) 135 | print(f"acc: {acc}") 136 | self.cur_step += 1 137 | epoch += 1 138 | self._logger.debug("begin to run send program") 139 | self.exe.run(self._send_program) 140 | 141 | def run(self, feed, fetch): 142 | self._logger.debug( 143 | f"begin to run FedAvgTrainer, cur_step={self.cur_step}, inner_step={self._step}" 144 | ) 145 | if self.cur_step % self._step == 0: 146 | self._logger.debug("run recv program start") 147 | self.exe.run(self._recv_program) 148 | self._logger.debug("run recv program done") 149 | 150 | self._logger.debug("run main program start") 151 | loss = self.exe.run(self._main_program, feed=feed, fetch_list=fetch) 152 | self._logger.debug("run main program done") 153 | 154 | if self.cur_step % self._step == 0: 155 | self._logger.debug("run send program start") 156 | self.exe.run(self._send_program) 157 | self._logger.debug("run send program done") 158 | self.cur_step += 1 159 | return loss 160 | 161 | def save_model(self, model_path): 162 | fluid.io.save_inference_model( 163 | dirname=model_path, 164 | feeded_var_names=self._feed_names, 165 | target_vars=[ 166 | self._main_program.global_block().var(fetch_var_name) 167 | for fetch_var_name in self._target_names 168 | ], 169 | executor=self.exe, 170 | main_program=self._main_program, 171 | ) 172 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: FedVision 2 | repo_name: FederatedAI/FedVision 3 | repo_url: https://github.com/FederatedAI/FedVision 4 | edit_uri: https://github.com/FederatedAI/FedVision/tree/main/docs 5 | site_url: http://github.com/FederatedAI/FedVision 6 | 7 | copyright: © Copyright 2020 WeBank Co., Ltd. | All Rights Reserved. 8 | 9 | theme: 10 | name: material 11 | features: 12 | tabs: true 13 | 14 | plugins: 15 | - search 16 | - mkdocstrings: 17 | default_handler: python 18 | handlers: 19 | python: 20 | rendering: 21 | show_source: true 22 | watch: 23 | - fedvision 24 | - mkdocs-jupyter 25 | 26 | markdown_extensions: 27 | - pymdownx.snippets 28 | - mdx_math 29 | - codehilite 30 | - admonition 31 | - codehilite: 32 | guess_lang: false 33 | linenums: false 34 | - toc: 35 | permalink: true 36 | - footnotes 37 | - meta 38 | - def_list 39 | - pymdownx.arithmatex 40 | - pymdownx.betterem: 41 | smart_enable: all 42 | - pymdownx.caret 43 | - pymdownx.critic 44 | - pymdownx.details 45 | - pymdownx.inlinehilite 46 | - pymdownx.magiclink 47 | - pymdownx.mark 48 | - pymdownx.smartsymbols 49 | - pymdownx.superfences 50 | - pymdownx.tasklist: 51 | custom_checkbox: true 52 | - pymdownx.tabbed 53 | - pymdownx.tilde 54 | 55 | extra_css: 56 | - 'css/termynal.css' 57 | - 'css/custom.css' 58 | 59 | extra_javascript: 60 | # - 'https://unpkg.com/mermaid@8.4.6/dist/mermaid.min.js' 61 | - 'js/termynal.js' 62 | - 'js/custom.js' 63 | - 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-MML-AM_CHTML' 64 | 65 | nav: 66 | - Welcome to Fedvision: index.md 67 | - Quickstart: quickstart/quickstart.md 68 | - Overview: 69 | - framework: framework/overview.md 70 | - paddledetection: framework/paddledetection.md 71 | - paddlefl: framework/paddlefl.md 72 | - Deploy: 73 | - fedvision-deploy-toolkit-cli: deploy/cli.md 74 | - Develop: 75 | - codestyle: develop/codestyle.md 76 | - API: 77 | - framework: 78 | - cluster: apis/cluster.md 79 | - coordinator: apis/coordinator.md 80 | - master: apis/master.md 81 | - proto buffer: apis/proto.md 82 | - Releases: 83 | - release notes: release/release.md 84 | - v0.1: release/v0.1.md 85 | -------------------------------------------------------------------------------- /proto/build_config.yaml: -------------------------------------------------------------------------------- 1 | proto: 2 | - "fedvision/framework/protobuf/cluster.proto" 3 | - "fedvision/framework/protobuf/coordinator.proto" 4 | - "fedvision/framework/protobuf/job.proto" 5 | - "fedvision/paddle_fl/protobuf/fl_job.proto" 6 | - "fedvision/paddle_fl/protobuf/scheduler.proto" 7 | 8 | grpc: 9 | - "fedvision/framework/protobuf/cluster.proto" 10 | - "fedvision/framework/protobuf/coordinator.proto" 11 | - "fedvision/framework/protobuf/job.proto" 12 | - "fedvision/paddle_fl/protobuf/fl_job.proto" 13 | - "fedvision/paddle_fl/protobuf/scheduler.proto" -------------------------------------------------------------------------------- /proto/fedvision/framework/protobuf/cluster.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | import "google/protobuf/any.proto"; 3 | import "fedvision/framework/protobuf/job.proto"; 4 | 5 | package fedvision.framework.cluster; 6 | 7 | 8 | //service in cluster manager called by worker 9 | service ClusterManager{ 10 | // service for worker: enroll and fetch tasks 11 | rpc Enroll(Enroll.REQ) returns (stream Enroll.REP) {} 12 | // service for worker: update status or heartbeat 13 | rpc UpdateTaskStatus(UpdateStatus.REQ) returns (UpdateStatus.REP) {} 14 | // service for master: submit task to cluster 15 | rpc TaskSubmit(TaskSubmit.REQ) returns (TaskSubmit.REP) {} 16 | rpc TaskResourceRequire(TaskResourceRequire.REQ) returns (TaskResourceRequire.REP) {} 17 | } 18 | 19 | message Enroll { 20 | enum Status { 21 | UNKNOWN = 0; 22 | ENROLL_SUCCESS = 1; 23 | ALREADY_ENROLL = 2; 24 | TASK_READY = 3; 25 | } 26 | message REQ { 27 | string worker_id = 1; 28 | string worker_ip = 2; 29 | int32 max_tasks = 3; 30 | int32 port_start = 4; 31 | int32 port_end = 5; 32 | } 33 | message REP { 34 | Status status = 1; 35 | fedvision.framework.Task task = 2; 36 | } 37 | } 38 | 39 | message UpdateStatus { 40 | enum TaskStatus { 41 | TASK_UNKNOWN = 0; 42 | TASK_CANCEL = 1; 43 | TASK_EXCEPTION = 2; 44 | TASK_FINISH = 3; 45 | } 46 | enum Status { 47 | UNKNOWN = 0; 48 | FAILED = 1; 49 | SUCCESS = 2; 50 | } 51 | message REQ { 52 | string worker_id = 1; 53 | string job_id = 2; 54 | string task_id = 3; 55 | TaskStatus task_status = 4; 56 | string exception_id = 5; 57 | string exception = 6; 58 | google.protobuf.Any exec_result = 7; 59 | } 60 | message REP { 61 | Status status = 1; 62 | } 63 | } 64 | 65 | message TaskSubmit { 66 | enum Status { 67 | UNKNOWN = 0; 68 | FAILED = 1; 69 | SUCCESS = 2; 70 | } 71 | message REQ { 72 | fedvision.framework.Task task = 1; 73 | } 74 | message REP { 75 | Status status = 1; 76 | } 77 | } 78 | 79 | 80 | message TaskResourceRequire { 81 | enum Status { 82 | UNKNOWN = 0; 83 | FAILED = 1; 84 | SUCCESS = 2; 85 | } 86 | message REQ { 87 | int32 num_endpoints = 1; 88 | } 89 | message REP { 90 | Status status = 1; 91 | string worker_id = 2; 92 | repeated string endpoints = 3; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /proto/fedvision/framework/protobuf/coordinator.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "fedvision/framework/protobuf/job.proto"; 4 | 5 | package fedvision.framework.coordinator; 6 | 7 | service Coordinator { 8 | rpc Subscribe(Subscribe.REQ) returns (stream Subscribe.REP) {} 9 | rpc Proposal(Proposal.REQ) returns (Proposal.REP) {} 10 | rpc FetchTask(FetchTask.REQ) returns (FetchTask.REP) {} 11 | rpc Leave(Leave.REQ) returns (Leave.REP) {} 12 | } 13 | 14 | message Subscribe { 15 | enum Status { 16 | DUPLICATE_ENROLL = 0; 17 | NOT_SERVING = 1; 18 | SUCCESS = 2; 19 | } 20 | 21 | message REQ { 22 | string party_id = 1; 23 | repeated string job_types = 2; 24 | string credential = 3; // preserve 25 | } 26 | 27 | message REP { 28 | Status status = 1; 29 | string proposal_id = 2; 30 | string job_type = 3; 31 | } 32 | } 33 | 34 | message FetchTask{ 35 | enum Status { 36 | NOT_FOUND = 0; 37 | NOT_ALLOW = 1; 38 | TIMEOUT = 2; 39 | CANCELED = 3; 40 | RANDOM_OUT = 4; 41 | READY = 5; 42 | } 43 | message REQ { 44 | string party_id = 1; 45 | string proposal_id = 2; 46 | } 47 | message REP { 48 | Status status = 1; 49 | fedvision.framework.Task task = 2; 50 | } 51 | } 52 | 53 | 54 | message Proposal{ 55 | enum Status { 56 | UNKNOWN = 0; 57 | SUCCESS = 1; 58 | NOT_ENOUGH_RESPONDERS= 2; 59 | NOT_ENOUGH_SUBSCRIBERS = 3; 60 | REJECT = 4; 61 | } 62 | 63 | message REQ { 64 | string job_id = 1; 65 | string job_type = 2; 66 | repeated fedvision.framework.Task tasks = 4; 67 | uint32 proposal_wait_time = 5; 68 | uint32 minimum_acceptance = 6; 69 | uint32 maximum_acceptance = 7; 70 | } 71 | message REP { 72 | Status status = 1; 73 | } 74 | } 75 | 76 | 77 | message Leave{ 78 | enum Status { 79 | NOT_FOUND = 0; 80 | SUCCESS = 1; 81 | } 82 | message REQ { 83 | string party_id = 1; 84 | } 85 | message REP { 86 | Status status = 1; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /proto/fedvision/framework/protobuf/job.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | import "google/protobuf/any.proto"; 3 | 4 | package fedvision.framework; 5 | 6 | message Task { 7 | string job_id = 1; 8 | string task_id = 2; 9 | string task_type = 3; 10 | google.protobuf.Any task = 4; 11 | string assignee = 5; 12 | } 13 | 14 | -------------------------------------------------------------------------------- /proto/fedvision/paddle_fl/protobuf/fl_job.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package fedvision.paddle_fl; 3 | 4 | message PaddleFLAggregatorTask { 5 | string scheduler_ep = 1; 6 | 7 | bytes main_program = 2; 8 | bytes startup_program = 3; 9 | 10 | string config_string = 4; 11 | } 12 | 13 | 14 | message PaddleFLWorkerTask { 15 | string scheduler_ep = 1; 16 | uint32 trainer_id = 2; 17 | string trainer_ep = 3; 18 | string entrypoint = 4; 19 | 20 | bytes main_program = 5; 21 | bytes startup_program = 6; 22 | bytes send_program = 7; 23 | bytes recv_program = 8; 24 | bytes feed_names = 9; 25 | bytes target_names = 10; 26 | bytes strategy = 11; 27 | bytes feeds = 12; 28 | 29 | string config_string = 13; 30 | string algorithm_config_string = 14; 31 | } 32 | 33 | -------------------------------------------------------------------------------- /proto/fedvision/paddle_fl/protobuf/scheduler.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package fedvision.paddle_fl; 4 | 5 | service Scheduler { 6 | rpc Init(Init.REQ) returns (Init.REP) {} 7 | rpc WorkerJoin(WorkerJoin.REQ) returns (WorkerJoin.REP) {} 8 | rpc WorkerFinish(WorkerFinish.REQ) returns (WorkerFinish.REP) {} 9 | } 10 | 11 | message Init { 12 | enum Status { 13 | REJECT = 0; 14 | INIT = 1; 15 | } 16 | message REQ { 17 | string name = 1; 18 | } 19 | 20 | message REP { 21 | Status status = 1; 22 | } 23 | } 24 | 25 | message WorkerJoin { 26 | enum Status { 27 | REJECT = 0; 28 | NOT_SELECTED = 1; 29 | ACCEPT = 2; 30 | } 31 | message REQ { 32 | string name = 1; 33 | uint32 step = 2; 34 | } 35 | 36 | message REP { 37 | Status status = 1; 38 | } 39 | } 40 | 41 | message WorkerFinish { 42 | enum Status { 43 | REJECT = 0; 44 | DONE = 1; 45 | } 46 | message REQ { 47 | string name = 1; 48 | } 49 | 50 | message REP { 51 | Status status = 1; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | attr>=0.3.1 2 | grpcio>=1.33.2,!=1.34.0 3 | aiohttp>=3.7,<3.8 4 | loguru>=0.5 5 | protobuf==3.14.0 6 | jsonschema==3.2.0 7 | PyYAML>=5.3.1 8 | click==7.1.2 9 | paddlepaddle==1.8.5 10 | 11 | # enhance 12 | visualdl==2.0.5 -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pre-commit 2 | grpcio-tools>=1.33.2,!=1.34.0 3 | requests 4 | typer 5 | -------------------------------------------------------------------------------- /sbin/cluster_manager.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 18 | PROJECT_BASE=$(dirname "${DIR}") 19 | 20 | # shellcheck source=env.sh 21 | . "${PROJECT_BASE}/sbin/env.sh" 22 | # shellcheck source=service.sh 23 | . "${PROJECT_BASE}/sbin/service.sh" 24 | 25 | usage="Usage: cluster_manager.sh (start|stop) " 26 | if [ $# -le 1 ]; then 27 | echo "$usage" 28 | exit 1 29 | fi 30 | 31 | if [ -z "${FEDVISION_PYTHON_EXECUTABLE}" ]; then 32 | echo "fedvision python executable not set" 33 | exit 1 34 | fi 35 | 36 | start_cluster_manager() { 37 | local re='^[0-9]+$' 38 | if ! [[ $1 =~ $re ]]; then 39 | echo "error: port should be number" >&2 40 | echo "$usage" 41 | exit 1 42 | fi 43 | mkdir -p "$PROJECT_BASE/logs/nohup" 44 | nohup "${FEDVISION_PYTHON_EXECUTABLE}" -m fedvision.framework.cli.cluster_manager --port "${1}" >>"${PROJECT_BASE}/logs/nohup/manager" 2>&1 & 45 | } 46 | 47 | case "$1" in 48 | start) 49 | start_service "$2" clustermanager start_cluster_manager "$2" 50 | exit 0 51 | ;; 52 | stop) 53 | stop_service_by_port "$2" clustermanager 54 | exit 0 55 | ;; 56 | *) 57 | echo bad command 58 | echo "$usage" 59 | exit 1 60 | ;; 61 | esac 62 | -------------------------------------------------------------------------------- /sbin/cluster_worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 18 | PROJECT_BASE=$(dirname "${DIR}") 19 | 20 | # shellcheck source=env.sh 21 | . "${PROJECT_BASE}/sbin/env.sh" 22 | # shellcheck source=service.sh 23 | . "${PROJECT_BASE}/sbin/service.sh" 24 | 25 | usage="Usage: cluster_worker.sh (start|stop) [ ]" 26 | if [ $# -le 1 ]; then 27 | echo "$usage" 28 | exit 1 29 | fi 30 | 31 | if [ -z "${FEDVISION_PYTHON_EXECUTABLE}" ]; then 32 | echo "fedvision python executable not set" 33 | exit 1 34 | fi 35 | 36 | start_cluster_worker() { 37 | local pid 38 | pid=$( 39 | ps aux | grep "fedvision.framework.cli.cluster_worker" | grep "name ${1}" | grep -v grep | awk '{print $2}' 40 | ) 41 | if [[ -z ${pid} ]]; then 42 | mkdir -p "$PROJECT_BASE/logs/nohup" 43 | nohup "${FEDVISION_PYTHON_EXECUTABLE}" -m fedvision.framework.cli.cluster_worker --name "$1" --worker-ip "$2" --port-start "$3" --port-end "$4" --max-tasks "$5" --manager-address "$6" --data-base-dir "$7" >>"${PROJECT_BASE}/logs/nohup/worker" 2>&1 & 44 | for ((i = 1; i <= 20; i++)); do 45 | sleep 0.1 46 | pid=$( 47 | ps aux | grep "fedvision.framework.cli.cluster_worker" | grep "name ${1}" | grep -v grep | awk '{print $2}' 48 | ) 49 | if [[ -n ${pid} ]]; then 50 | echo "cluster worker service start successfully. pid: ${pid}" 51 | exit 0 52 | fi 53 | done 54 | echo "cluster worker service start failed" 55 | exit 1 56 | else 57 | echo "cluster worker service named <$1> already started. pid: $pid" 58 | exit 1 59 | fi 60 | } 61 | 62 | stop_cluster_worker() { 63 | local pid 64 | pid=$( 65 | ps aux | grep "fedvision.framework.cli.cluster_worker" | grep "name ${1}" | grep -v grep | awk '{print $2}' 66 | ) 67 | if [[ -n ${pid} ]]; then 68 | echo "killing: $(ps aux | grep "${pid}" | grep -v grep)" 69 | for ((i = 1; i <= 20; i++)); do 70 | sleep 0.1 71 | kill "${pid}" 72 | pid=$( 73 | ps aux | grep "fedvision.framework.cli.cluster_worker" | grep "name ${1}" | grep -v grep | awk '{print $2}' 74 | ) 75 | if [[ -z ${pid} ]]; then 76 | echo "killed by SIGTERM" 77 | exit 0 78 | fi 79 | done 80 | if [[ $(kill -9 "${pid}") -eq 0 ]]; then 81 | echo "killed by SIGKILL" 82 | exit 0 83 | else 84 | echo "Kill error" 85 | exit 1 86 | fi 87 | else 88 | echo "cluster worker named <${1}> service not running" 89 | exit 1 90 | fi 91 | } 92 | 93 | case "$1" in 94 | start) 95 | start_cluster_worker "$2" "$3" "$4" "$5" "$6" "$7" "$8" 96 | exit 0 97 | ;; 98 | stop) 99 | stop_cluster_worker "$2" 100 | exit 0 101 | ;; 102 | *) 103 | echo bad command 104 | echo "$usage" 105 | exit 1 106 | ;; 107 | esac 108 | -------------------------------------------------------------------------------- /sbin/coordinator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 18 | PROJECT_BASE=$(dirname "${DIR}") 19 | 20 | # shellcheck source=env.sh 21 | . "${PROJECT_BASE}/sbin/env.sh" 22 | # shellcheck source=service.sh 23 | . "${PROJECT_BASE}/sbin/service.sh" 24 | 25 | usage="Usage: [FEDVISION_PYTHON_EXECUTABLE=...] coordinator.sh (start|stop) " 26 | if [ $# -le 1 ]; then 27 | echo "$usage" 28 | exit 1 29 | fi 30 | 31 | if [ -z "${FEDVISION_PYTHON_EXECUTABLE}" ]; then 32 | echo "fedvision python executable not set" 33 | exit 1 34 | fi 35 | 36 | start_coordinator() { 37 | local re='^[0-9]+$' 38 | if ! [[ $1 =~ $re ]]; then 39 | echo "error: port should be number" >&2 40 | echo "$usage" 41 | exit 1 42 | fi 43 | mkdir -p "$PROJECT_BASE/logs/nohup" 44 | nohup "${FEDVISION_PYTHON_EXECUTABLE}" -m fedvision.framework.cli.coordinator --port "${1}" >>"${PROJECT_BASE}/logs/nohup/coordinator" 2>&1 & 45 | } 46 | 47 | case "$1" in 48 | start) 49 | start_service "$2" coordinator start_coordinator "$2" 50 | exit 0 51 | ;; 52 | stop) 53 | stop_service_by_port "$2" coordinator 54 | exit 0 55 | ;; 56 | *) 57 | echo bad command 58 | echo "$usage" 59 | exit 1 60 | ;; 61 | esac 62 | -------------------------------------------------------------------------------- /sbin/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # export Environment Variables 18 | # PYTHONPATH Python default search path for module files, add Fedvision, PaddleFL, PaddleDetection 19 | # FEDVISION_PYTHON_EXECUTABLE python executable path, such as python | python3 | venv/bin/python 20 | 21 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 22 | PROJECT_BASE=$(dirname "${DIR}") 23 | unset PYTHONPATH 24 | 25 | DEPS_DIR="${PROJECT_BASE}/deps" 26 | if [ -d "${DEPS_DIR}" ]; then 27 | # development layout 28 | export PYTHONPATH="${PROJECT_BASE}:${DEPS_DIR}/PaddleDetection:${DEPS_DIR}/PaddleFL/python:${PYTHONPATH}" 29 | else 30 | export PYTHONPATH="${PROJECT_BASE}:${PYTHONPATH}" 31 | fi 32 | 33 | if [ -z "${FEDVISION_PYTHON_EXECUTABLE}" ]; then 34 | export FEDVISION_PYTHON_EXECUTABLE= 35 | fi 36 | -------------------------------------------------------------------------------- /sbin/master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | DIR="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" 18 | PROJECT_BASE=$(dirname "${DIR}") 19 | 20 | # shellcheck source=env.sh 21 | . "${PROJECT_BASE}/sbin/env.sh" 22 | # shellcheck source=service.sh 23 | . "${PROJECT_BASE}/sbin/service.sh" 24 | 25 | usage="Usage: [FEDVISION_PYTHON_EXECUTABLE=...] master.sh (start|stop) [ ]" 26 | if [ $# -le 1 ]; then 27 | echo "$usage" 28 | exit 1 29 | fi 30 | 31 | if [ -z "${FEDVISION_PYTHON_EXECUTABLE}" ]; then 32 | echo "fedvision python executable not set" 33 | exit 1 34 | fi 35 | 36 | start_master() { 37 | local re='^[0-9]+$' 38 | if ! [[ $1 =~ $re ]]; then 39 | echo "error: port should be number" >&2 40 | echo "$usage" 41 | exit 1 42 | fi 43 | mkdir -p "$PROJECT_BASE/logs/nohup" 44 | nohup "${FEDVISION_PYTHON_EXECUTABLE}" -m fedvision.framework.cli.master --submitter-port "$1" --party-id "$2" --cluster-address "$3" --coordinator-address "$4" >>"${PROJECT_BASE}/logs/nohup/master" 2>&1 & 45 | } 46 | 47 | case "$1" in 48 | start) 49 | start_service "$2" master start_master "$2" "$3" "$4" "$5" 50 | exit 0 51 | ;; 52 | stop) 53 | stop_service_by_port "$2" master 54 | exit 0 55 | ;; 56 | *) 57 | echo bad command 58 | echo "$usage" 59 | exit 1 60 | ;; 61 | esac 62 | -------------------------------------------------------------------------------- /sbin/service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # start_service 18 | start_service() { 19 | local pid 20 | pid=$(lsof -i:"$1" | grep 'LISTEN' | awk '{print $2}' | uniq) 21 | if [[ -z ${pid} ]]; then 22 | $3 "${@:4}" 23 | for ((i = 1; i <= 20; i++)); do 24 | sleep 0.1 25 | pid=$(lsof -i:"$1" | grep 'LISTEN' | awk '{print $2}' | uniq) 26 | if [[ -n ${pid} ]]; then 27 | echo "$2 service start successfully. pid: ${pid}" 28 | exit 0 29 | fi 30 | done 31 | echo "$2 service start failed" 32 | exit 1 33 | else 34 | echo "$2 service already started at port $1. pid: $pid" 35 | exit 1 36 | fi 37 | } 38 | 39 | # stop_service 40 | stop_service_by_port() { 41 | local pid 42 | pid=$(lsof -i:"$1" | grep 'LISTEN' | awk '{print $2}' | uniq) 43 | if [[ -n ${pid} ]]; then 44 | echo "killing: $(ps aux | grep "${pid}" | grep -v grep)" 45 | for ((i = 1; i <= 20; i++)); do 46 | sleep 0.1 47 | kill "$pid" 48 | pid=$(lsof -i:"$1" | grep 'LISTEN' | awk '{print $2}' | uniq) 49 | if [[ -z ${pid} ]]; then 50 | echo "killed by SIGTERM" 51 | exit 0 52 | fi 53 | done 54 | echo $pid 55 | kill -9 "$pid" 56 | echo "killed by SIGKILL" 57 | exit 0 58 | else 59 | echo "$2 service not running" 60 | exit 1 61 | fi 62 | } 63 | -------------------------------------------------------------------------------- /schema/paddle_fl.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "title": "PaddleFL", 4 | "description": "PaddleFL job config", 5 | "type": "object", 6 | "properties": { 7 | "proposal_wait_time": { 8 | "type": "integer", 9 | "description": "how many secends to wait for other parties to accept proposal", 10 | "minimum": 1 11 | }, 12 | "program": { 13 | "type": "string", 14 | "description": "program to run" 15 | }, 16 | "worker_num": { 17 | "description": "number of worker", 18 | "type": "integer" 19 | }, 20 | "max_iter": { 21 | "description": "max number of iteration", 22 | "type": "integer" 23 | }, 24 | "inner_step": { 25 | "description": "inner step", 26 | "type": "integer" 27 | } 28 | }, 29 | "required": [ 30 | "proposal_wait_time", 31 | "worker_num", 32 | "program", 33 | "max_iter", 34 | "inner_step" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """We not distribute FedVision directly by pypi 16 | since we have a more proper toolkit: fedvision_deploy_toolkit to deploy|start|stop services distributed. 17 | This setup script will error a hint to tell users to install fedvision_deploy_toolkit 18 | instead of install FedVision directly. 19 | """ 20 | import os 21 | import sys 22 | 23 | import setuptools 24 | 25 | CLASSIFIERS = [ 26 | "Development Status :: 3 - Alpha", 27 | "Environment :: Console", 28 | "Environment :: GPU :: NVIDIA CUDA", 29 | "Intended Audience :: Developers", 30 | "Intended Audience :: Education", 31 | "Intended Audience :: Science/Research", 32 | "License :: OSI Approved :: Apache Software License", 33 | "Operating System :: MacOS", 34 | "Operating System :: POSIX :: Linux", 35 | "Programming Language :: Python :: 3 :: Only", 36 | "Programming Language :: Python :: 3.7", 37 | "Programming Language :: Python :: 3.8", 38 | "Programming Language :: Python :: 3.9", 39 | "Topic :: Scientific/Engineering :: Image Recognition", 40 | "Topic :: Security", 41 | ] 42 | 43 | 44 | def get_version(): 45 | from fedvision import __version__ 46 | 47 | return __version__ 48 | 49 | 50 | HINT = "Please install the fedvision_deploy_toolkit package with: pip install fedvision_deploy_toolkit" 51 | 52 | if "sdist" not in sys.argv and os.path.exists( 53 | os.path.join(__file__, os.path.pardir, ".gitignore") 54 | ): 55 | """build source distribution only""" 56 | raise RuntimeError(HINT) 57 | 58 | # build proto buffer 59 | if True: 60 | import tools.protobuf 61 | 62 | tools.protobuf.build() 63 | tools.protobuf.doc() 64 | 65 | setuptools.setup( 66 | name="fedvision", 67 | version=get_version(), 68 | description=HINT, 69 | author="The FedVision Authors", 70 | author_email="contact@FedAI.org", 71 | url="https://FedAI.org", 72 | license="Apache License 2.0", 73 | classifiers=CLASSIFIERS, 74 | ) 75 | -------------------------------------------------------------------------------- /tools/protobuf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020 The FedVision Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os.path 17 | import shutil 18 | import tarfile 19 | import tempfile 20 | 21 | import grpc_tools.protoc 22 | import pkg_resources 23 | import requests 24 | import typer 25 | import yaml 26 | 27 | app = typer.Typer() 28 | 29 | 30 | _base_path = os.path.abspath(os.path.join(__file__, os.path.pardir, os.path.pardir)) 31 | _proto_path = os.path.abspath(os.path.join(_base_path, "proto")) 32 | _config_path = os.path.abspath(os.path.join(_proto_path, "build_config.yaml")) 33 | 34 | 35 | @app.command() 36 | def build(): 37 | """ 38 | generate proto buffer files 39 | """ 40 | proto_include = pkg_resources.resource_filename("grpc_tools", "_proto") 41 | with open(_config_path) as f: 42 | config = yaml.safe_load(f) 43 | proto = set(config.get("proto", [])) 44 | grpc = set(config.get("grpc", [])) 45 | 46 | for filename in proto: 47 | args = [ 48 | "", 49 | f"-I{proto_include}", 50 | f"-I{_proto_path}", 51 | f"--python_out={_base_path}", 52 | ] 53 | if filename in grpc: 54 | args.append(f"--grpc_python_out={_base_path}") 55 | file_path = os.path.join(_proto_path, filename) 56 | args.append(file_path) 57 | if grpc_tools.protoc.main(args) != 0: 58 | raise Exception(f"build {file_path} failed") 59 | 60 | typer.echo(f"build success") 61 | 62 | 63 | @app.command() 64 | def clean(dry: bool = typer.Option(False, help="dry run")): 65 | """ 66 | clean generated files 67 | """ 68 | with open(_config_path) as f: 69 | config = yaml.safe_load(f) 70 | proto = set(config.get("proto", [])) 71 | grpc = set(config.get("grpc", [])) 72 | 73 | for filename in proto: 74 | path = os.path.join(_base_path, filename.replace(".proto", "_pb2.py")) 75 | if os.path.exists(path): 76 | if dry: 77 | typer.echo(f"remove {path}") 78 | else: 79 | os.remove(path) 80 | for filename in grpc: 81 | path = os.path.join(_base_path, filename.replace(".proto", "_pb2_grpc.py")) 82 | if os.path.exists(path): 83 | if dry: 84 | typer.echo(f"remove {path}") 85 | else: 86 | os.remove(path) 87 | typer.echo(f"clean success") 88 | 89 | 90 | @app.command() 91 | def doc(): 92 | """ 93 | build proto buffer docs 94 | """ 95 | 96 | url_base = "https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.3.2/" 97 | if os.uname().sysname == "Darwin": 98 | download_url = f"{url_base}/protoc-gen-doc-1.3.2.darwin-amd64.go1.12.6.tar.gz" 99 | elif os.uname().sysname == "Linux": 100 | download_url = f"{url_base}/protoc-gen-doc-1.3.2.linux-amd64.go1.12.6.tar.gz" 101 | else: 102 | raise Exception(f"{os.uname().sysname} not supported") 103 | 104 | # process in temp dir 105 | with tempfile.TemporaryDirectory() as d: 106 | # download 107 | req = requests.get(download_url, stream=True) 108 | if req.status_code != 200: 109 | raise RuntimeError( 110 | f"Downloading from {download_url} failed with code {req.status_code}!" 111 | ) 112 | target_name = os.path.join(d, "protoc-gen-doc.tar.gz") 113 | total_size = req.headers.get("content-length") 114 | with open(target_name, "wb") as f: 115 | if total_size: 116 | with typer.progressbar( 117 | req.iter_content(chunk_size=1024), 118 | length=(int(total_size) + 1023) // 1024, 119 | ) as bar: 120 | for chunk in bar: 121 | f.write(chunk) 122 | else: 123 | for chunk in req.iter_content(chunk_size=1024): 124 | if chunk: 125 | f.write(chunk) 126 | 127 | # extract 128 | with tarfile.open(os.path.join(d, target_name), "r:gz") as tar: 129 | for member in tar.getmembers(): 130 | if member.name.endswith("protoc-gen-doc"): 131 | tar.extract(member, path=d) 132 | shutil.move( 133 | os.path.join(d, member.name), os.path.join(d, "protoc-gen-doc") 134 | ) 135 | 136 | # generate doc 137 | proto_include = pkg_resources.resource_filename("grpc_tools", "_proto") 138 | with open(_config_path) as f: 139 | config = yaml.safe_load(f) 140 | proto = set(config.get("proto", [])) 141 | 142 | cmd = [ 143 | "", 144 | f"--plugin=protoc-gen-doc={os.path.join(d, 'protoc-gen-doc')}", 145 | f"--doc_out={os.path.join(_base_path, 'docs', 'apis')}", 146 | "--doc_opt=markdown,proto.md", 147 | f"-I{proto_include}", 148 | f"-I{_proto_path}", 149 | ] 150 | for filename in proto: 151 | cmd.append(os.path.join(_proto_path, filename)) 152 | if not grpc_tools.protoc.main(cmd) == 0: 153 | raise RuntimeError("generate proto doc failed") 154 | 155 | typer.echo("generate proto doc success") 156 | 157 | 158 | if __name__ == "__main__": 159 | app() 160 | --------------------------------------------------------------------------------