├── .github ├── ISSUE_TEMPLATE │ ├── doc_inprovement.yaml │ ├── feature_request.yaml │ └── issue_report.yaml ├── PULL_REQUEST_TEMPLATE │ └── pr_template.md └── workflows │ ├── deploy_doc.yaml │ ├── docker_publish.yaml │ ├── publish_pypi.yaml │ └── release.yaml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── README_zh.md ├── dev ├── format.sh ├── pack.sh ├── publish.sh └── test.sh ├── doc ├── README.md ├── README_zh.md ├── docs │ ├── about │ │ ├── changelog.md │ │ ├── changelog.zh.md │ │ ├── contact.md │ │ └── contact.zh.md │ ├── api │ │ ├── api_reference.md │ │ ├── api_reference.zh.md │ │ └── gen_ref_nav.py │ ├── css │ │ ├── iflearner.css │ │ ├── material.css │ │ └── style.css │ ├── faq │ │ ├── faq.md │ │ └── faq.zh.md │ ├── images │ │ ├── iFLearner框架设计.jpg │ │ └── visualdl.png │ ├── index.md │ ├── index.zh.md │ ├── quick_start │ │ ├── installation.md │ │ ├── installation.zh.md │ │ ├── quickstart_keras.md │ │ ├── quickstart_keras.zh.md │ │ ├── quickstart_mxnet.md │ │ ├── quickstart_mxnet.zh.md │ │ ├── quickstart_opacus.md │ │ ├── quickstart_opacus.zh.md │ │ ├── quickstart_pytorch.md │ │ ├── quickstart_pytorch.zh.md │ │ ├── quickstart_smpc.md │ │ ├── quickstart_smpc.zh.md │ │ ├── quickstart_tensorflow.md │ │ └── quickstart_tensorflow.zh.md │ └── tutorial │ │ ├── argument.md │ │ ├── argument.zh.md │ │ ├── contributor_guide.md │ │ ├── contributor_guide.zh.md │ │ ├── custom_aggregation_strategy.md │ │ ├── custom_aggregation_strategy.zh.md │ │ ├── metrics_visualization.md │ │ ├── metrics_visualization.zh.md │ │ ├── run_in_container.md │ │ ├── run_in_container.zh.md │ │ └── strategy │ │ ├── fednova.md │ │ ├── fednova.zh.md │ │ ├── fedopt.md │ │ ├── fedopt.zh.md │ │ ├── qfedavg.md │ │ └── qfedavg.zh.md ├── mkdocs.yml └── requirements.txt ├── enviroment ├── docker │ ├── README.md │ ├── README.zh.md │ ├── iflearner_base.Dockerfile │ ├── iflearner_keras2.9.0.Dockerfile │ ├── iflearner_mxnet1.9.1.Dockerfile │ ├── iflearner_tensorflow2.9.1.Dockerfile │ └── iflearner_torch1.7.1.Dockerfile └── jupyterlab │ ├── README.md │ ├── README.zh.md │ └── iflearner_jupyterlab_base.Dockerfile ├── examples ├── __init__.py ├── homo │ ├── FedOpt │ │ ├── README.md │ │ ├── iid-client01 │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── iid-client02 │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── iid-client03 │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── imagenet.py │ │ ├── noniid-client01 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── noniid-client02 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── noniid-client03 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ └── server.py │ ├── __init__.py │ ├── feddyn │ │ ├── README.md │ │ ├── feddyn_server.py │ │ ├── imagenet.py │ │ └── server.py │ ├── fednova │ │ ├── README.md │ │ ├── imagenet.py │ │ ├── noniid-client01 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── noniid-client02 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── noniid-client03 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ └── server.py │ ├── imagenet │ │ ├── Iid_Top1.png │ │ ├── Iid_Top5.png │ │ ├── Noniid_Top1.png │ │ ├── Noniid_Top5.png │ │ ├── README.md │ │ ├── __init__.py │ │ ├── imagenet.py │ │ ├── plot.py │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── jupyterlab │ │ ├── en │ │ │ ├── README.md │ │ │ ├── pytorch_mnist_client1.ipynb │ │ │ ├── pytorch_mnist_client2.ipynb │ │ │ └── pytorch_mnist_server.ipynb │ │ └── zh │ │ │ ├── README.md │ │ │ ├── pytorch_mnist_client1.zh.ipynb │ │ │ ├── pytorch_mnist_client2.zh.ipynb │ │ │ └── pytorch_mnist_server.zh.ipynb │ ├── mnist │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dataset.py │ │ └── mnist.py │ ├── qFedavg │ │ ├── imagenet.py │ │ ├── qnoniid-client01 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── qnoniid-client02 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── qnoniid-client03 │ │ │ ├── loss.png │ │ │ ├── top1.png │ │ │ └── top5.png │ │ ├── readme.md │ │ └── server.py │ ├── quickstart_keras │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_keras.py │ │ ├── requirements.txt │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── quickstart_mxnet │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_mxnet.py │ │ ├── requirements.txt │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── quickstart_opacus │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_pytorch.py │ │ ├── requirements.txt │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── quickstart_pytorch │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_pytorch.py │ │ ├── requirements.txt │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── quickstart_sklearn │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_sklearn.py │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── quickstart_smpc │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_pytorch.py │ │ ├── requirements.txt │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ ├── quickstart_tensorflow │ │ ├── README.md │ │ ├── README.zh.md │ │ ├── __init__.py │ │ ├── quickstart_tensorflow.py │ │ ├── requirements.txt │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ └── server.py │ └── word_language_model_pytorch │ │ ├── README.md │ │ ├── __init__.py │ │ ├── data.py │ │ ├── model.py │ │ ├── run_client.sh │ │ ├── run_server.sh │ │ ├── server.py │ │ └── word_language_client.py ├── mpc │ └── quickstart_piss │ │ ├── __init__.py │ │ ├── client_service.py │ │ ├── piss_data_test.csv │ │ ├── quickstart_piss.py │ │ └── server.py └── spilt_dataset.py ├── iflearner ├── __init__.py ├── business │ ├── __init__.py │ ├── hetero │ │ ├── builder │ │ │ ├── builders.py │ │ │ ├── demo_builder.py │ │ │ ├── lr_builder.py │ │ │ └── model_builder.py │ │ ├── driver.py │ │ ├── model │ │ │ ├── base_model.py │ │ │ ├── demo │ │ │ │ ├── demo_arbiter.py │ │ │ │ ├── demo_arbiter_flow.yaml │ │ │ │ ├── demo_guest.py │ │ │ │ ├── demo_guest_flow.yaml │ │ │ │ ├── demo_host.py │ │ │ │ └── demo_host_flow.yaml │ │ │ ├── logistic_regression │ │ │ │ ├── dataset.py │ │ │ │ ├── logistic_regression.py │ │ │ │ ├── lr_arbiter.py │ │ │ │ ├── lr_arbiter_flow.yaml │ │ │ │ ├── lr_guest.py │ │ │ │ ├── lr_guest_flow.yaml │ │ │ │ ├── lr_host.py │ │ │ │ └── lr_host_flow.yaml │ │ │ └── role.py │ │ ├── parser.py │ │ └── task.yaml │ ├── homo │ │ ├── Dockerfile │ │ ├── __init__.py │ │ ├── aggregate_server.py │ │ ├── argument.py │ │ ├── keras_trainer.py │ │ ├── mxnet_trainer.py │ │ ├── pytorch_trainer.py │ │ ├── sklearn_trainer.py │ │ ├── strategy │ │ │ ├── __init__.py │ │ │ ├── fedavg_client.py │ │ │ ├── fedavg_server.py │ │ │ ├── fednova_client.py │ │ │ ├── fednova_server.py │ │ │ ├── fedopt_client.py │ │ │ ├── fedopt_server.py │ │ │ ├── opt │ │ │ │ ├── __init__.py │ │ │ │ ├── fedadagrad.py │ │ │ │ ├── fedadam.py │ │ │ │ ├── fedavgm.py │ │ │ │ ├── fedopt.py │ │ │ │ └── fedyogi.py │ │ │ ├── qfedavg_client.py │ │ │ ├── qfedavg_server.py │ │ │ ├── stc_client.py │ │ │ ├── stc_server.py │ │ │ ├── strategy_client.py │ │ │ └── strategy_server.py │ │ ├── tensorflow_trainer.py │ │ ├── train_client.py │ │ └── trainer.py │ ├── mpc │ │ └── piss │ │ │ ├── argument.py │ │ │ ├── piss_aggregate_server.py │ │ │ ├── piss_client_controller.py │ │ │ ├── piss_startegy_base.py │ │ │ ├── piss_strategy_client.py │ │ │ └── piss_strategy_server.py │ └── util │ │ ├── __init__.py │ │ ├── metric.py │ │ ├── metric_dev.py │ │ └── metric_test.py ├── communication │ ├── __init__.py │ ├── base │ │ ├── __init__.py │ │ ├── base.proto │ │ ├── base_client.py │ │ ├── base_client_test.py │ │ ├── base_exception.py │ │ ├── base_pb2.py │ │ ├── base_pb2_grpc.py │ │ ├── base_server.py │ │ ├── base_server_test.py │ │ └── constant.py │ ├── build.sh │ ├── hetero │ │ ├── __init__.py │ │ ├── hetero.proto │ │ ├── hetero_client.py │ │ ├── hetero_network.py │ │ └── hetero_server.py │ ├── homo │ │ ├── __init__.py │ │ ├── homo.proto │ │ ├── homo_client.py │ │ ├── homo_exception.py │ │ ├── homo_exception_test.py │ │ ├── homo_pb2.py │ │ ├── homo_pb2_grpc.py │ │ ├── homo_server.py │ │ └── message_type.py │ ├── mpc │ │ └── piss │ │ │ ├── __init__.py │ │ │ ├── message_type.py │ │ │ ├── piss.proto │ │ │ ├── piss_base.py │ │ │ ├── piss_client.py │ │ │ ├── piss_client_services.py │ │ │ ├── piss_exception.py │ │ │ ├── piss_exception_test.py │ │ │ ├── piss_pb2.py │ │ │ ├── piss_pb2_grpc.py │ │ │ └── piss_server.py │ ├── peer │ │ ├── __init__.py │ │ ├── aes.py │ │ ├── diffie_hellman.py │ │ ├── diffie_hellman_inst.py │ │ ├── diffie_hellman_test.py │ │ ├── message_type.py │ │ ├── peer_client.py │ │ └── peer_server.py │ └── secureprotol │ │ ├── gmpy_math.py │ │ └── secretsharing │ │ └── feldman_verifiable_secretsharing │ │ └── feldman_verifiable_secret_sharing.py └── datasets │ ├── __init__.py │ ├── cifar.py │ ├── fl_dataset.py │ ├── mnist.py │ ├── readme.md │ ├── sampler.py │ ├── split_dataset.py │ └── utils.py ├── mypy.ini ├── pyproject.toml ├── requirements.txt └── setup.py /.github/ISSUE_TEMPLATE/doc_inprovement.yaml: -------------------------------------------------------------------------------- 1 | name: Documentation Inprovement 2 | description: As a user, I want to make an enhancement for iflearner docmument 3 | title: "[Enhancement]: " 4 | labels: [kind/enhancement] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to suggest an enhancement for iflearner document! 10 | - type: textarea 11 | attributes: 12 | label: Which chapter do you think needs improvement? 13 | description: A concise description of what you're expecting/suggesting. 14 | placeholder: | 15 | For example Quickstart/installation 16 | validations: 17 | required: true 18 | - type: textarea 19 | attributes: 20 | label: What information do you think needs to be added? 21 | description: A concise description of the reason/motivation 22 | placeholder: | 23 | For example Quickstart/installation 24 | validations: 25 | required: false -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yaml: -------------------------------------------------------------------------------- 1 | name: Feature request 2 | description: As a user, I want to request a feature for iflearner 3 | title: "[Feature Request]: " 4 | labels: [kind/feature] 5 | assignees: 6 | - 'dejavu6' 7 | - 'xs233' 8 | body: 9 | - type: markdown 10 | attributes: 11 | value: | 12 | Thanks for taking the time to request a feature for iflearner! 13 | - type: textarea 14 | attributes: 15 | label: Describe the type of feature and its functionality. 16 | description: A concise description of the problem you are facing or the motivetion behind this feature request.. 17 | validations: 18 | required: true 19 | - type: textarea 20 | attributes: 21 | label: Is there something else you want to add? 22 | validations: 23 | required: false 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue_report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: Create a report to help us reproduce and correct the bug 3 | labels: ['bug'] 4 | 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: > 9 | #### Thanks for taking the time to fill out this bug report! 10 | Before submitting a bug, please make sure the issue hasn't been already 11 | addressed by searching through [the past issues](https://github.com/iflytek/iflearner/issues). 12 | - type: textarea 13 | attributes: 14 | label: Describe the bug 15 | description: > 16 | A clear and concise description of what the bug is. 17 | validations: 18 | required: true 19 | - type: textarea 20 | id: logs 21 | attributes: 22 | label: Additional Information 23 | description: Please copy and paste any relevant Info output. 24 | render: shell 25 | validations: 26 | required: false 27 | - type: markdown 28 | attributes: 29 | value: > 30 | Thanks for contributing! -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pr_template.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | #### Reference Issues/PRs 10 | 11 | 17 | 18 | #### What does this implement/fix? Explain your changes. 19 | 20 | 25 | 26 | #### Any other comments? 27 | 28 | -------------------------------------------------------------------------------- /.github/workflows/deploy_doc.yaml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | - uses: actions/setup-python@v4 13 | with: 14 | python-version: 3.9 15 | - uses: actions/cache@v2 16 | with: 17 | key: ${{ github.ref }} 18 | path: .cache 19 | - run: export PYTHONPATH=$PYTHONPATH:../iflearner 20 | - run: pip install -Ur doc/requirements.txt 21 | - run: cd doc && mkdocs gh-deploy --force -------------------------------------------------------------------------------- /.github/workflows/docker_publish.yaml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | 6 | # GitHub recommends pinning actions to a commit SHA. 7 | # To get a newer version, you will need to update the SHA. 8 | # You can also reference a tag or branch, but the action may change without warning. 9 | 10 | name: Create and publish a Docker image 11 | 12 | on: 13 | push: 14 | tags: ["v*"] 15 | 16 | env: 17 | REGISTRY: ghcr.io 18 | IMAGE_NAME: ${{ github.repository }} 19 | 20 | jobs: 21 | build-and-push-image: 22 | runs-on: ubuntu-latest 23 | permissions: 24 | contents: read 25 | packages: write 26 | 27 | steps: 28 | - name: Checkout repository 29 | uses: actions/checkout@v3 30 | 31 | - name: Log in to the Container registry 32 | uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 33 | with: 34 | registry: ${{ env.REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: Extract metadata (tags, labels) for Docker 39 | id: meta 40 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 41 | with: 42 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 43 | 44 | - name: Build and push Docker image 45 | uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc 46 | with: 47 | context: . 48 | file: iflearner/business/homo/Dockerfile 49 | push: true 50 | tags: ${{ steps.meta.outputs.tags }} 51 | labels: ${{ steps.meta.outputs.labels }} 52 | -------------------------------------------------------------------------------- /.github/workflows/publish_pypi.yaml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | # action的名称 5 | name: Pypi 6 | 7 | on: 8 | push: 9 | # branches: 10 | # - main 11 | # 每次 push tag 时进行构建，不需要每次 push 都构建。使用通配符匹配每次 tag 的提交，记得 tag 名一定要以 v 开头 12 | tags: 13 | - v* 14 | # Triggers the workflow when a release is created 15 | # release: 16 | # types: [ created ] 17 | 18 | # 当发布时，触发action 19 | #release: 20 | # types: [created] 21 | 22 | jobs: 23 | deploy: 24 | name: publish python package to PYPI 25 | # 此作业在 Linux 上运行 26 | runs-on: ubuntu-latest 27 | 28 | steps: 29 | # 此步骤使用 GitHub 的 https://github.com/actions/checkout 30 | - uses: actions/checkout@v3 31 | 32 | # 设置python环境 33 | # 此步骤使用 GitHub 的 https://github.com/actions/setup-python 34 | - name: Set up Python 35 | uses: actions/setup-python@v3 36 | with: 37 | python-version: '3.9' 38 | 39 | # 安装依赖 40 | - name: Install dependencies 41 | run: | 42 | python -m pip install --upgrade pip 43 | pip install setuptools wheel twine 44 | 45 | # 构建和发布 46 | - name: Build and publish 47 | env: 48 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 49 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 50 | run: | 51 | python setup.py sdist bdist_wheel 52 | twine upload dist/* -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | release: 10 | name: Create Release 11 | runs-on: ubuntu-latest 12 | outputs: 13 | upload_url: ${{ steps.create_release.outputs.upload_url }} 14 | 15 | steps: 16 | # 此步骤使用 GitHub 的 https://github.com/actions/checkout 17 | - uses: actions/checkout@v3 18 | 19 | # 设置python环境 20 | # 此步骤使用 GitHub 的 https://github.com/actions/setup-python 21 | - name: Set up Python 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: '3.9' 25 | 26 | - name: Build Assets 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install setuptools wheel twine 30 | python setup.py sdist bdist_wheel 31 | ls -lh 32 | 33 | - name: Get tag 34 | id: tag 35 | uses: dawidd6/action-get-tag@v1 36 | with: 37 | # Optionally strip `v` prefix 38 | strip_v: true 39 | 40 | # - name: conventional changelog action 41 | # id: changelog 42 | # # https://github.com/TriPSs/conventional-changelog-action 43 | # uses: TriPSs/conventional-changelog-action@latest 44 | # with: 45 | # # you can also create separate token to trace action 46 | # github-token: "${{ secrets.GITHUB_TOKEN }}" 47 | 48 | - name: Create Release 49 | id: create_release 50 | uses: actions/create-release@v1 51 | env: 52 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 53 | with: 54 | tag_name: ${{ github.ref }} 55 | release_name: ${{ github.ref }} 56 | # body: ${{steps.changelog.outputs.clean_changelog}} 57 | draft: false 58 | prerelease: false 59 | 60 | - name: Upload assets 61 | id: upload-release-asset 62 | uses: actions/upload-release-asset@v1 63 | env: 64 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 65 | with: 66 | upload_url: ${{ steps.create_release.outputs.upload_url }} 67 | asset_path: ./dist/${{ github.event.repository.name }}-${{steps.tag.outputs.tag}}-py3-none-any.whl 68 | asset_name: ${{ github.event.repository.name }}-${{steps.tag.outputs.tag}}-py3-none-any.whl 69 | asset_content_type: application/zip -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IFLearner file 2 | log 3 | doc/build 4 | doc/docs/api/reference 5 | dist 6 | build 7 | iflearner.egg-info 8 | examples/homo/**/data 9 | examples/homo/**/t10* 10 | examples/homo/**/train-* 11 | data 12 | examples/homo/**/client* 13 | iflearner/business/util/client* 14 | .ipynb_checkpoints 15 | *.log 16 | 17 | # macOS 18 | .DS_Store 19 | 20 | # Editor 21 | .vscode 22 | .idea 23 | 24 | # Byte-compiled / optimized / DLL files 25 | *.pyc 26 | __pycache__/ 27 | *.py[cod] 28 | *$py.class -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt -------------------------------------------------------------------------------- /dev/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ 5 | 6 | # core 7 | python -m isort -rc iflearner/ 8 | python -m black --config pyproject.toml --line-length 88 iflearner/ 9 | python -m docformatter -i -r iflearner/ 10 | 11 | # examples 12 | python -m isort -rc examples/ 13 | python -m black examples/ 14 | python -m docformatter -i -r examples/ -------------------------------------------------------------------------------- /dev/pack.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ 4 | 5 | rm -rf dist iflearner.egg-info || true 6 | python setup.py sdist 7 | 8 | # whl 9 | ## python setup.py bdist_wheel -------------------------------------------------------------------------------- /dev/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ 4 | 5 | 6 | #First, Configture setuptools tools 7 | # Linux 8 | ## vim ~/.pypirc 9 | # Windows 10 | ## C:\Users\Username\.pypirc 11 | : <<'COMMENT' 12 | [distutils] 13 | index-servers=pypi 14 | 15 | [pypi] 16 | repository=https://upload.pypi.org/legacy/ 17 | username= 18 | password= 19 | COMMENT 20 | 21 | # Second, Upload to pypi 22 | twine upload dist/* -r pypi 23 | -------------------------------------------------------------------------------- /dev/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ 4 | 5 | echo $PWD 6 | echo "=== test.sh ===" 7 | 8 | python -m isort iflearner/ examples/ && echo "- isort: done" && 9 | python -m docformatter -rc iflearner/ examples/ && echo "- docformatter: done" && 10 | python -m black --check --line-length 88 iflearner/ examples/ && echo "- black: done" && 11 | python -m mypy --config-file mypy.ini iflearner/ && echo "- mypy: done" && 12 | python -m flake8 iflearner/ && echo "- flake8: done" && 13 | echo "- All Python checks passed" -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # introduce 2 | 3 | ## install dependencies 4 | ```shell 5 | pip install -r requirements.txt 6 | ```` 7 | 8 | ## Modify the content of the document 9 | Built with the mkdocs tool, the document structure is defined in the mkdocs.yml file, and the specific content of the document is in the docs directory. 10 | 11 | This document is edited in markdown syntax. If new documents need to be added, please edit mkdocs.yaml and add chapters. 12 | 13 | ## Local debugging documentation 14 | ```shell 15 | mkdocs serve -a 127.0.0.1:8030 16 | ```` 17 | After executing the above command, you can view the content of the generated document through the http://127.0.0.1:8030 address. 18 | > When the document is modified, the page content will be updated automatically. 19 | 20 | ## Local build documentation 21 | ```shell 22 | mkdocs build 23 | ```` 24 | After executing the above command, the static files of the documentation site will be generated in the site directory, and the generated static files can be accessed by proxy. -------------------------------------------------------------------------------- /doc/README_zh.md: -------------------------------------------------------------------------------- 1 | # 介绍 2 | 3 | ## 安装依赖 4 | ```shell 5 | pip install -r requirements.txt 6 | ``` 7 | 8 | ## 修改文档内容 9 | 采用mkdocs工具构建，文档结构定义在 mkdocs.yml 文件中，文档的具体内容均在 docs 目录中。 10 | 11 | 本文档采用 markdown 语法编辑，如有新的文档需要添加，请编辑 mkdocs.yaml 添加章节即可。 12 | 13 | ## 本地调试文档 14 | ```shell 15 | mkdocs serve -a 127.0.0.1:8030 16 | ``` 17 | 执行上述命令后，可通过 http://127.0.0.1:8030 地址查看生成的文档内容. 18 | > 当修改文档后，页面内容会自动更新。 19 | 20 | ## 本地构建文档 21 | ```shell 22 | mkdocs build 23 | ``` 24 | 执行上述命令后，会在 site 目录下生成文档站点的静态文件, 可将生成的静态文件进行代理访问。 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /doc/docs/about/changelog.md: -------------------------------------------------------------------------------- 1 | # changelog 2 | All notable changes to this project will be recorded in this file. 3 | 4 | ## iflearner 5 | ## [Unreleased] 6 | #### Add 7 | - Support the underlying communication protocol of GRPC, and complete the abstraction of the upper layer protocol 8 | - Support access to deep learning frameworks such as Tensorflow, Pytorch, Mxnet, Keras, etc. 9 | - Support common aggregation strategies, and support users to customize their own aggregation strategies 10 | - Support SMPC, differential privacy security encryption strategy -------------------------------------------------------------------------------- /doc/docs/about/changelog.zh.md: -------------------------------------------------------------------------------- 1 | # 版本日志 2 | 此项目的所有显着更改都将记录在此文件中。 3 | 4 | ## iflearner 5 | ## [Unreleased] 6 | #### Add 7 | - 支持GRPC底层通信协议，并完成上层协议抽象化 8 | - 支持Tensorflow、Pytorch、Mxnet、Keras等深度学习框架的接入 9 | - 支持常见的聚合策略，并支持用户自定义实现自己的聚合策略 10 | - 支持SMPC、差分隐私的安全加密策略 -------------------------------------------------------------------------------- /doc/docs/about/contact.md: -------------------------------------------------------------------------------- 1 | # contact us 2 | 3 | If you have any questions you want to solve or want to join us to build a project together, you can contact us through the following contact channels, 4 | We very much welcome you to join us. 5 | 6 | ## contact path 7 | - You can join our WeChat group. 8 | - For frequently asked questions, we provide you with the corresponding `FAQ` document. 9 | - Please use `issues` to file bugs. 10 | - Please use `pull requests` to submit and contribute code. -------------------------------------------------------------------------------- /doc/docs/about/contact.zh.md: -------------------------------------------------------------------------------- 1 | # 联系我们 2 | 3 | 如果您有任何问题想解决或者想参与我们一起共同构建项目, 可以通过下面的联系途径联系到我们， 4 | 我们非常欢迎您的加入。 5 | 6 | ## 联系路径 7 | - 可以加入我们的微信群。 8 | - 对于常见问题, 我们为您提供了对应的`FAQ`文档。 9 | - 请使用 `issues` 提交BUG。 10 | - 请使用 `pull requests` 提交、贡献代码。 -------------------------------------------------------------------------------- /doc/docs/api/gen_ref_nav.py: -------------------------------------------------------------------------------- 1 | """Generate the code reference pages and navigation.""" 2 | 3 | from pathlib import Path 4 | 5 | import mkdocs_gen_files 6 | 7 | nav = mkdocs_gen_files.Nav() 8 | 9 | for path in sorted(Path("../iflearner").glob("**/*.py")): 10 | path = Path(path) 11 | module_path = path.relative_to("../").with_suffix("") 12 | doc_path = path.relative_to("../", "iflearner").with_suffix(".md") 13 | full_doc_path = Path("docs/api/reference", doc_path) 14 | parts = list(module_path.parts) 15 | if parts[-1] == "__init__": 16 | parts = parts[:-1] 17 | doc_path = doc_path.with_name("index.md") 18 | full_doc_path = full_doc_path.with_name("index.md") 19 | elif parts[-1] == "__main__": 20 | continue 21 | elif parts[-1].endswith("_test"): 22 | continue 23 | nav[parts] = doc_path 24 | 25 | with mkdocs_gen_files.open(full_doc_path.absolute(), "w+") as fd: 26 | ident = ".".join(parts) 27 | fd.write("::: " + ident) 28 | mkdocs_gen_files.set_edit_path(full_doc_path, path) 29 | 30 | # with mkdocs_gen_files.open(Path("docs/api/reference/SUMMARY.md").absolute(), "w+") as nav_file: 31 | # nav_file.writelines(nav.build_literate_nav()) 32 | -------------------------------------------------------------------------------- /doc/docs/css/iflearner.css: -------------------------------------------------------------------------------- 1 | /* Indentation. */ 2 | div.doc-contents:not(.first) { 3 | padding-left: 25px; 4 | border-left: 4px solid rgba(230, 230, 230); 5 | margin-bottom: 80px; 6 | } 7 | 8 | /* Avoid breaking parameters name, etc. in table cells. */ 9 | td code { 10 | word-break: normal !important; 11 | } 12 | -------------------------------------------------------------------------------- /doc/docs/css/material.css: -------------------------------------------------------------------------------- 1 | /* More space at the bottom of the page. */ 2 | .md-main__inner { 3 | margin-bottom: 1.5rem; 4 | } 5 | -------------------------------------------------------------------------------- /doc/docs/css/style.css: -------------------------------------------------------------------------------- 1 | /* Increase logo size */ 2 | .md-header__button.md-logo { 3 | padding-bottom: 0.2rem; 4 | padding-right: 0; 5 | } 6 | .md-header__button.md-logo img { 7 | height: 1.5rem; 8 | } 9 | 10 | /* Mark external links as such (also in nav) */ 11 | a.external:hover::after, a.md-nav__link[href^="https:"]:hover::after { 12 | /* https://primer.style/octicons/link-external-16 */ 13 | background-image: url('data:image/svg+xml,'); 14 | height: 0.8em; 15 | width: 0.8em; 16 | margin-left: 0.2em; 17 | content: ' '; 18 | display: inline-block; 19 | } 20 | 21 | /* More space at the bottom of the page */ 22 | .md-main__inner { 23 | margin-bottom: 1.5rem; 24 | } 25 | -------------------------------------------------------------------------------- /doc/docs/faq/faq.md: -------------------------------------------------------------------------------- 1 | ## FAQ -------------------------------------------------------------------------------- /doc/docs/faq/faq.zh.md: -------------------------------------------------------------------------------- 1 | ## FAQ -------------------------------------------------------------------------------- /doc/docs/images/iFLearner框架设计.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/doc/docs/images/iFLearner框架设计.jpg -------------------------------------------------------------------------------- /doc/docs/images/visualdl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/doc/docs/images/visualdl.png -------------------------------------------------------------------------------- /doc/docs/index.md: -------------------------------------------------------------------------------- 1 | ![](https://img.shields.io/badge/language-python-blue.svg) 2 | ![](https://img.shields.io/badge/license-Apache-000000.svg) 3 | # iFLearner - A Powerfule and Lightweight Federated Learning Framework 4 | 5 | iFLearner is a federated learning framework, which provides a secure computing framework based on 6 | data privacy security protection, mainly for federated modeling in deep learning scenarios. Its security bottom 7 | layer supports various encryption technologies such as homomorphic encryption, secret sharing, and differential 8 | privacy. The algorithm layer supports various deep learning network models, and supports mainstream frameworks 9 | such as Tensorflow, Mxnet, and Pytorch. 10 | 11 | ## Architecture 12 | ![iFLeaner Arch](./images/iFLearner框架设计.jpg) 13 | 14 | The design of iFLearner is based on a few guiding principles: 15 | 16 | * **Event-driven mechanism**: Use an event-driven programming paradigm to build federated learning, that is, 17 | to regard federated learning as the process of sending and receiving messages between participants, 18 | and describe the federated learning process by defining message types and the behavior of processing messages. 19 | 20 | * **Training framework abstraction**: Abstract deep learning backend, compatible with support for multiple 21 | types of framework backends such as Tensorflow and Pytorch. 22 | 23 | * **High scalability: modular design**, users can customize aggregation strategies, encryption modules, 24 | and support algorithms in various scenarios. 25 | 26 | * **Lightweight and simple**: The framework is Lib level, light enough, and users can simply transform their deep 27 | learning algorithms into federated learning algorithms. 28 | -------------------------------------------------------------------------------- /doc/docs/index.zh.md: -------------------------------------------------------------------------------- 1 | 2 | ![](https://img.shields.io/badge/language-python-blue.svg) 3 | ![](https://img.shields.io/badge/license-Apache-000000.svg) 4 | # iFLearner - 一个强大且轻量的联邦学习框架 5 | 6 | iFLearner是一个强大且轻量的联邦学习框架，提供了一种基于数据隐私安全保护的安全计算框架， 7 | 主要针对深度学习场景下的联邦建模。其安全底层支持同态加密、秘密共享、差分隐私等多种加密技术， 8 | 算法层支持各类深度学习网络模型，并且同时支持Tensorflow、Mxnet、Pytorch等主流框架。 9 | 10 | ## 架构 11 | ![iFLeaner Arch](./images/iFLearner框架设计.jpg) 12 | 13 | iFLearner主要基于以下原则进行设计: 14 | 15 | * **事件驱动机制**: 使用事件驱动的编程范式来构建联邦学习，即将联邦学习看成是参与方之间收发消息的过程， 16 | 通过定义消息类型以及处理消息的行为来描述联邦学习过程。 17 | 18 | * **训练框架抽象**: 抽象深度学习后端，兼容支持Tensorflow、Pytorch等多类框架后端。 19 | 20 | * **扩展性高**：模块式设计，用户可以自定义聚合策略，加密模块，同时支持各类场景下的算法。 21 | 22 | * **轻量且简单**：该框架Lib级别，足够轻量，同时用户可以简单改造自己的深度学习算法为联邦学习算法。 23 | 24 | -------------------------------------------------------------------------------- /doc/docs/quick_start/installation.md: -------------------------------------------------------------------------------- 1 | ## Installation IFLearner 2 | 3 | ### Python Version 4 | IFLearner requires Python 3.7 or above. 5 | 6 | ### Install 7 | You can execute the following command for quick installation: 8 | ```shell 9 | pip install iflearner 10 | ``` 11 | 12 | Once the installation is complete, you can verify that the installation was successful by 13 | running the following command: 14 | ```shell 15 | python -c "import iflearner;print(iflearner.__version__)" 16 | ``` 17 | -------------------------------------------------------------------------------- /doc/docs/quick_start/installation.zh.md: -------------------------------------------------------------------------------- 1 | ## 安装IFLearner 2 | 3 | ### Python版本 4 | IFLearner 需要 Python 3.7 或更高版本。 5 | 6 | ### 安装 7 | 您可以执行以下命令进行快速安装： 8 | ```shell 9 | pip install iflearner 10 | ``` 11 | 12 | 安装完成后，您可以通过运行以下命令验证是否已成功安装： 13 | ```shell 14 | python -c "import iflearner;print(iflearner.__version__)" 15 | ``` -------------------------------------------------------------------------------- /doc/docs/tutorial/argument.md: -------------------------------------------------------------------------------- 1 | ## Startup Options 2 | 3 | ### Client 4 | You need import `parser` from `iflearner.business.homo.argument` firstly and then call `parser.parse_args`, because we provided some common arguments that 5 | need to be parsered. If you want to add addtional arguments for yourself, you can call `parser.add_argument` repeatedly to add them before `parser.parse_args` 6 | has been called. After parsered arguments, you can create your trainer instance base on previous implemented class, and put it with `args` to `train_client.Controller`. 7 | In the end, you just need call `controller.run` to run your client. 8 | 9 | The list of client default options is as follows: 10 | 11 | | option | type | describe | default | 12 | | :-----| :---- | :---- |:----: | 13 | | name | str | name of client | client | 14 | | epochs | int | number of total epochs to run | 10 | 15 | | server | str | the address of connecting aggerating server | localhost:50001 | 16 | | enable-ll | int | enable local training (1 、 0) | 0 | 17 | | peers | str | enabled SMPC if the argument had specified. all clients' addresses and use semicolon separate all addresses. First one is your own address. | | 18 | | cert | str | path of server SSL cert. use secure channel to connect to server if not none| | 19 | 20 | ### Server 21 | 22 | The list of server options is as follows: 23 | 24 | | option | type | describe | default | 25 | | :-----| :---- | :---- |:----: | 26 | | num | int | the number of all clients | 0 | 27 | | epochs | int | the total epoch | | 28 | | addr | str |The aggregation server itself listens to the address (used for client connections) | "0.0.0.0:50001" | 29 | | http_addr | str |Federation training status listening address (for viewing federation training status) | "0.0.0.0:50002" | 30 | | strategy | str |the aggregation starategy (FedAvg、Scaffold、FedOpt、qFedAvg、FedNova) | FedAvg | 31 | | strategy_params | dict | specify the params of strategy | {} | -------------------------------------------------------------------------------- /doc/docs/tutorial/argument.zh.md: -------------------------------------------------------------------------------- 1 | ## 启动参数 2 | 3 | ### 客户端 4 | 您需要从`iflearner.business.homo`导入`parser`， 然后调用`parser.parse_args`， 因为我们提供了一些需要解析的常见参数。 5 | 如果您自己添加其他参数，可以调用`parser.add_argument`将其添加到`parser.parse_args`之前。在解析参数后，您可以基于之前实现的类创建trainer实例，并将其与`args`传递到`train_client.Controller`函数中。最后，你只需要调用 6 | `controller.run`来启动你的客户端进程。 7 | 8 | 下面是客户端默认的参数: 9 | 10 | | option | type | describe | default | 11 | | :-----| :---- | :---- |:----: | 12 | | name | str | 客户端名称(必须唯一) | client | 13 | | epochs | int | 总的训练轮数 | 10 | 14 | | server | str | 链接聚合服务端的地址 | localhost:50001 | 15 | | enable-ll | int | 启动本地训练进行对比 (1 、 0), 1代表开启 | 0 | 16 | | peers | str | 如果参数已指定，则启用 SMPC。 所有客户端的地址并使用分号分隔所有地址。 第一个是你自己的地址。 | | 17 | | cert | str | 服务器 SSL 证书的路径。 如果指定，则使用安全通道连接到服务器| | 18 | 19 | ### 服务端 20 | 21 | 服务端参数列表如下: 22 | 23 | | 选项 | 类型 | 描述 | 默认值 | 24 | | :-----| :---- | :---- |:----: | 25 | | num | int | 客户端数目 | 0 | 26 | | epochs | int | 总的聚合轮数 | | 27 | | addr | str | 聚合服务端本身监听地址(用于客户端链接) | "0.0.0.0:50001" | 28 | | http_addr | str |联邦训练状态监听地址(用于查看联邦训练状态) | "0.0.0.0:50002" | 29 | | strategy | str | 聚合策略 (FedAvg、Scaffold、FedOpt、qFedAvg、FedNova) | FedAvg | 30 | | strategy_params | dict | 聚合策略参数 | {} | 31 | -------------------------------------------------------------------------------- /doc/docs/tutorial/contributor_guide.md: -------------------------------------------------------------------------------- 1 | ## Contributor Guide 2 | 3 | ### Prerequisites 4 | **Python 3.7** or **above** 5 | 6 | ### Development Environment Setup 7 | First, clone the IFLearner repository from GitHub: 8 | ```shell 9 | $ git clone https://github.com/iflytek/iflearner.git 10 | $ cd iflearner 11 | ``` 12 | 13 | Then, you need to create a python virtual environment using virtual tools like conda, pyenv, etc. Below is an example 14 | of a command to create a virtual environment using conda: 15 | ```shell 16 | $ conda create -n iflearner python==3.9 17 | $ conda activate iflearner 18 | ``` 19 | 20 | Finally, you need to install the dependencies required by iflearner: 21 | ```shell 22 | $ pip install -r requirements.txt 23 | ``` 24 | 25 | ### Development Script 26 | We provide some development scripts, you can find them in the ./dev directory. 27 | 28 | ### Code Auto-Format And Auto-Test 29 | First, execute the script to automatically format: 30 | ```shell 31 | $ ./dev/format.sh 32 | ``` 33 | Second, execute the test script. Then, you should follow the code guidelines and make code adjustments as prompted. 34 | ```shell 35 | $ ./dev/test.sh 36 | ``` 37 | 38 | ### Build Documentation 39 | IFLearner uses mkdocs to build documentation, you can go to the ./doc directory and follow the readme tutorial to build documentation. 40 | 41 | ### Pack Whl Release 42 | IFLearner uses setup to pack release. You can use the following command to package: 43 | ```shell 44 | python setup.py bdist_wheel 45 | ``` 46 | The iflearner-*.whl and iflearner-*.tar.gz packages will be stored in the ./dist subdirectory. 47 | 48 | ### Publish Release 49 | If you have permission to publish release packages, you can do so as follows. 50 | 51 | First, configure the pypirc file. 52 | ```shell 53 | #Linux 54 | ## vim ~/.pypirc 55 | # Windows 56 | ## C:\Users\Username\.pypirc 57 | : <<'COMMENT' 58 | [distutils] 59 | index-servers=pypi 60 | 61 | [pypi] 62 | repository=https://upload.pypi.org/legacy/ 63 | username= 64 | password= 65 | COMMENT 66 | ``` 67 | Then, package release: 68 | ```shell 69 | python setup.py sdist 70 | ```` 71 | Finally, to publish release: 72 | ```shell 73 | twine upload dist/* -r pypi 74 | ```` -------------------------------------------------------------------------------- /doc/docs/tutorial/contributor_guide.zh.md: -------------------------------------------------------------------------------- 1 | ## 贡献者指南 2 | 3 | ### 先决条件 4 | **Python 3.7** 或 **以上** 5 | 6 | ### 开发环境设置 7 | 首先，从 GitHub 克隆 IFLearner 库： 8 | ```shell 9 | $ git clone https://github.com/iflytek/iflearner.git 10 | $ cd iflearner 11 | ``` 12 | 13 | 然后，您需要使用 conda、pyenv 等虚拟工具创建一个 python 虚拟环境。下面是一个使用 conda 创建虚拟环境的示例： 14 | ```shell 15 | $ conda create -n iflearner python==3.9 16 | $ conda activate iflearner 17 | ``` 18 | 19 | 最后，需要安装iflearner所需的依赖： 20 | ```shell 21 | $ pip install -r requirements.txt 22 | ``` 23 | 24 | ### 开发脚本 25 | 我们提供了一些开发脚本，您可以在 ./dev 目录中找到它们。 26 | 27 | ### 代码自动格式化和自动测试 28 | 首先，执行脚本自动格式化： 29 | ```shell 30 | $ ./dev/format.sh 31 | ``` 32 | 其次，执行测试脚本。然后，您应该遵循代码规范，根据提示进行代码调整。 33 | ```shell 34 | $ ./dev/test.sh 35 | ``` 36 | 37 | ### 构建文档 38 | IFLearner 使用 mkdocs 构建文档，您可以进入 ./doc 目录并按照readme教程构建文档。 39 | 40 | ### Whl打包 41 | IFLearner 使用 setup 来进行打包： 42 | ```shell 43 | python setup.py bdist_wheel 44 | ``` 45 | iflearner-*.whl 和 iflearner-*.tar.gz 包将存储在 ./dist 子目录中。 46 | 47 | ### 发布Pypi 48 | 如果你有权限发布版本包，你可以按下述操作进行发布. 49 | 50 | 首先, 配置pypirc文件. 51 | ```shell 52 | # Linux 53 | ## vim ~/.pypirc 54 | # Windows 55 | ## C:\Users\Username\.pypirc 56 | : <<'COMMENT' 57 | [distutils] 58 | index-servers=pypi 59 | 60 | [pypi] 61 | repository=https://upload.pypi.org/legacy/ 62 | username= 63 | password= 64 | COMMENT 65 | ``` 66 | 然后, 进行打包: 67 | ```shell 68 | python setup.py sdist 69 | ``` 70 | 最后, 进行发布: 71 | ```shell 72 | twine pload dist/* -r pypi 73 | ``` -------------------------------------------------------------------------------- /doc/docs/tutorial/metrics_visualization.md: -------------------------------------------------------------------------------- 1 | ## Metrics Visualization 2 | 3 | We have integrated [VisualDL](https://github.com/PaddlePaddle/VisualDL) inside iflearner, so you can visualize training metrics with it. When you have finished training, there will be a directory called "metric" under your training code directory. You can start [VisualDL](https://github.com/PaddlePaddle/VisualDL) with the following command: 4 | 5 | ``` 6 | visualdl --logdir ./metric --host 127.0.0.1 --port 8082 7 | ``` 8 | 9 | Then, open the link http://127.0.0.1:8082 in your browser. 10 | 11 | ![VisualDL](docs/../../images/visualdl.png) 12 | 13 | -------------------------------------------------------------------------------- /doc/docs/tutorial/metrics_visualization.zh.md: -------------------------------------------------------------------------------- 1 | ## 指标可视化 2 | 3 | 我们在iflearner中集成了[VisualDL](https://github.com/PaddlePaddle/VisualDL)来可视化训练指标，当你完成训练的时候，在你的训练代码目录下会有一个名为metric的目录，可以通过以下的命令来启动[VisualDL](https://github.com/PaddlePaddle/VisualDL)： 4 | 5 | ``` 6 | visualdl --logdir ./metric --host 127.0.0.1 --port 8082 7 | ``` 8 | 9 | 然后，你可以在浏览器中打开链接 。 10 | 11 | ![VisualDL](docs/../../images/visualdl.png) 12 | -------------------------------------------------------------------------------- /doc/docs/tutorial/run_in_container.md: -------------------------------------------------------------------------------- 1 | ## Run IFLearner in the container 2 | We can use containerization technology to run IFLearner in Docker and Jupyterlab. 3 | 4 | ### Docker 5 | Docker is a package of Linux containers that provides an easy-to-use container interface. It is currently the most popular Linux container solution. 6 | 7 | Docker packages the application and the program's dependencies in a single file. Running this file will generate a virtual container. Programs run in this virtual container as if they were running on a real physical machine. With Docker, you don't have to worry about the environment. 8 | 9 | **You can find some of our examples and tutorials in iflearner project's ./enviroment/docker directory.** 10 | 11 | ### Jupyterlab 12 | JupyterLab is Jupyter's latest data science production tool, and in a sense, it appeared to replace Jupyter Notebook. But don't worry that Jupyter Notebook will disappear, JupyterLab includes all the features of Jupyter Notebook. 13 | 14 | JupyterLab is a web-based integrated development environment, you can use it to write notebooks, operate terminals, edit markdown text, open interactive mode, view csv files and pictures and other functions. 15 | 16 | **You can find some of our examples and tutorials in iflearner project's ./enviroment/jupyterlab directory.** -------------------------------------------------------------------------------- /doc/docs/tutorial/run_in_container.zh.md: -------------------------------------------------------------------------------- 1 | ## 在容器中运行IFLearner 2 | 我们可以使用容器化技术，将IFLearner在Docker和Jupyterlab中进行运行。 3 | 4 | 您可以在项目的 ./enviroment 目录下找到我们提供的一些示例和教程。 5 | 6 | ### Docker 7 | Docker 属于 Linux 容器的一种封装，提供简单易用的容器使用接口。它是目前最流行的 Linux 容器解决方案。 8 | 9 | Docker 将应用程序与该程序的依赖，打包在一个文件里面。运行这个文件，就会生成一个虚拟容器。程序在这个虚拟容器里运行，就好像在真实的物理机上运行一样。有了 Docker，就不用担心环境问题。 10 | 11 | **您可以在项目的 ./enviroment/docker 目录下找到我们提供的一些示例和教程。** 12 | 13 | ### Jupyterlab 14 | JupyterLab是Jupyter主打的最新数据科学生产工具，某种意义上，它的出现是为了取代Jupyter Notebook。不过不用担心Jupyter Notebook会消失，JupyterLab包含了Jupyter Notebook所有功能。 15 | 16 | JupyterLab作为一种基于web的集成开发环境，你可以使用它编写notebook、操作终端、编辑markdown文本、打开交互模式、查看csv文件及图片等功能。 17 | 18 | **您可以在项目的 ./enviroment/jupyterlab 目录下找到我们提供的一些示例和教程。** -------------------------------------------------------------------------------- /doc/docs/tutorial/strategy/fednova.md: -------------------------------------------------------------------------------- 1 | ## FedNova 2 | According to the article [***Tackling the Objective Inconsistency Problem in Heterogeneous Federated Optimization*** 3 | ](https://arxiv.org/abs/2007.07481), we implement FedNova aggregation algorithm. 4 | 5 | The complete source code 6 | reference for this example [fednova](https://git.iflytek.com/TURING/iflearner/-/tree/master/examples/homo/fednova). 7 | ### To start a FedNova server 8 | 9 | ```python 10 | strategy = message_type.STRATEGY_FEDNOVA # define server type 11 | 12 | server = AggregateServer(args.addr, strategy, args.num) 13 | server.run() 14 | ``` 15 | or 16 | ```shell 17 | python iflearner/business/homo/aggregate_server.py -n 2 --strategy FedNova 18 | ``` 19 | 20 | ### To start a client 21 | 22 | See [how to use](../../api/api_reference.md) 23 | 24 | FedNova involves the number of samples on the client side (`sample_num`) and the number of times each round of training optimizations (`batch_num`), so the `Trainer.config` method needs to be override to return these two values 25 | 26 | ```python 27 | def config(self) -> dict(): 28 | 29 | return { 30 | "batch_num": len(self._train_loader), 31 | "sample_num": len(self._train_loader) * self._train_loader.batch_size, 32 | } 33 | ``` 34 | 35 | FedNova also needs to override the `Trainer.get` method to return the difference between the client current model and the previous round model. 36 | 37 | ```python 38 | def get(self, param_type=""): 39 | parameters = dict() 40 | for name, p in self._model.named_parameters(): 41 | if p.requires_grad: 42 | parameters[name.replace("module.", "")] = ( 43 | p.cpu().detach().numpy() 44 | - self._old_weights[name].cpu().detach().numpy() 45 | ) 46 | 47 | return parameters 48 | 49 | ``` -------------------------------------------------------------------------------- /doc/docs/tutorial/strategy/fednova.zh.md: -------------------------------------------------------------------------------- 1 | ## FedNova 2 | 根据论文 [***Tackling the Objective Inconsistency Problem in Heterogeneous Federated Optimization*** 3 | ](https://arxiv.org/abs/2007.07481), 我们实现了FedNova聚合算法 4 | 5 | 完整的该示例的源代码参考[FedNova](https://git.iflytek.com/TURING/iflearner/-/tree/master/examples/homo/fednova)。 6 | 7 | ### To start a FedNova server 8 | 9 | ```python 10 | strategy = message_type.STRATEGY_FEDNOVA # define server type 11 | 12 | server = AggregateServer(args.addr, strategy, args.num) 13 | server.run() 14 | ``` 15 | or 16 | ```shell 17 | python iflearner/business/homo/aggregate_server.py -n 2 --strategy FedNova 18 | ``` 19 | 20 | ### To start a client 21 | 22 | See [how to use](../../api/api_reference.md) 23 | 24 | FedNova涉及客户端的样本数（`sample_num`）和每轮训练优化的次数（`batch_num`），因此需要覆盖`Trainer.config`方法以返回这两个值 25 | 26 | 27 | ```python 28 | def config(self) -> dict(): 29 | 30 | return { 31 | "batch_num": len(self._train_loader), 32 | "sample_num": len(self._train_loader) * self._train_loader.batch_size, 33 | } 34 | ``` 35 | 36 | FedNova 还需要重写 `Trainer.get`方法，以返回客户端当前模型和上一轮模型之间的差异。 37 | 38 | 39 | 40 | ```python 41 | def get(self, param_type=""): 42 | parameters = dict() 43 | for name, p in self._model.named_parameters(): 44 | if p.requires_grad: 45 | parameters[name.replace("module.", "")] = ( 46 | p.cpu().detach().numpy() 47 | - self._old_weights[name].cpu().detach().numpy() 48 | ) 49 | 50 | return parameters 51 | 52 | ``` -------------------------------------------------------------------------------- /doc/docs/tutorial/strategy/qfedavg.zh.md: -------------------------------------------------------------------------------- 1 | ## qFedAvg 2 | 根据文章[***FAIR RESOURCE ALLOCATION IN FEDERATED LEARNING***](https://openreview.net/pdf?id=ByexElSYDr)，我们实现了qfedavg聚合算法。 3 | 4 | 完整的该示例的源代码参考[qFedAvg](https://git.iflytek.com/TURING/iflearner/-/tree/master/examples/homo/qFedAvg)。 5 | 6 | ### To start a qfedav server 7 | 8 | ```python 9 | strategy = message_type.STRATEGY_qFEDAVG # define server type 10 | 11 | server = AggregateServer(args.addr, strategy, args.num, q=.2, learning_rate = 1) 12 | server.run() 13 | ``` 14 | 或者 15 | ```shell 16 | python iflearner/business/homo/aggregate_server.py -n 2 --strategy qFedAvg --strategy_params {"q":.2, "learning_rate":1} 17 | ``` 18 | 19 | ### To start a client 20 | 请参阅[如何使用](../../api/api_reference.md) 21 | 22 | 需要注意的是，在客户端开始拟合之前，需要获取当前模型在训练数据上的损失值`loss`。 然后，您应该在客户端重写 `Trainer.get` 方法，并将 loss 关键字添加到上传的参数中。 23 | 24 | ```python 25 | def evaluate_traindata(self): 26 | batch_time = AverageMeter("Time", ":6.3f", Summary.AVERAGE) 27 | losses = AverageMeter("Loss", ":.4e", Summary.AVERAGE) 28 | top1 = AverageMeter("Acc@1", ":6.2f", Summary.AVERAGE) 29 | top5 = AverageMeter("Acc@5", ":6.2f", Summary.AVERAGE) 30 | progress = ProgressMeter( 31 | len(self._train_loader), [batch_time, losses, top1, top5], prefix="Test on training data: " 32 | ) 33 | 34 | with torch.no_grad(): 35 | end = time.time() 36 | for i, (images, target) in enumerate(self._train_loader): 37 | if self._args.gpu is not None: 38 | images = images.cuda(self._args.gpu, non_blocking=True) 39 | if torch.cuda.is_available(): 40 | target = target.cuda(self._args.gpu, non_blocking=True) 41 | 42 | # compute output 43 | output = self._model(images) 44 | loss = self._criterion(output, target) 45 | 46 | # measure accuracy and record loss 47 | acc1, acc5 = accuracy(output, target, topk=(1, 5)) 48 | losses.update(loss.item(), images.size(0)) 49 | top1.update(acc1[0], images.size(0)) 50 | top5.update(acc5[0], images.size(0)) 51 | 52 | # measure elapsed time 53 | batch_time.update(time.time() - end) 54 | end = time.time() 55 | 56 | if i % self._args.print_freq == 0: 57 | progress.display(i) 58 | 59 | progress.display_summary() 60 | self._fs = losses.avg 61 | 62 | def get(self, param_type=''): 63 | parameters = dict() 64 | parameters['loss'] = np.array([self._fs]) 65 | for name, p in self._model.named_parameters(): 66 | if p.requires_grad: 67 | parameters[name.replace('module.', '') 68 | ] = p.cpu().detach().numpy() 69 | 70 | return parameters 71 | ``` 72 | 73 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs==1.3.0 2 | mdx_gh_links==0.2 3 | markdown==3.3.7 4 | lightgallery==0.5 5 | mkdocs-material==8.3.8 6 | mkdocs-static-i18n==0.46 7 | mkdocs-awesome-pages-plugin==2.7.0 8 | mkdocs-render-swagger-plugin==0.0.3 9 | mkdocs-swagger-ui-tag==0.3.0 10 | mkdocs-gen-files==0.3.5 11 | mkdocstrings==0.19.0 12 | mkdocstrings-python==0.7.1 13 | mkdocs-include-markdown-plugin==3.6.1 -------------------------------------------------------------------------------- /enviroment/docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker 2 | 3 | ## Introduction 4 | We provide Dockerfiles for IFLearner. 5 | You need to install docker first, and if you need to use gpu training, please install nvidia driver and nvidia-docker first. 6 | 7 | ## Dockerfile 8 | - `iflearner_base.Dockerfile`: Only install iflearner library. 9 | - `iflearner_torch1.7.1.Dockerfile`: Install iflearner、torch1.7.1、torchvision0.8.2 library. 10 | - `iflearner_tensorflow2.9.1.Dockerfile`: Install iflearner、tensorflow2.9.1 library. 11 | - `iflearner_mxnet1.9.1.Dockerfile`: Install iflearner、mxnet1.9.1 library. 12 | - `iflearner_kera2.9.0.Dockerfile`: Install iflearner、keras2.9.0 library. 13 | 14 | > Here, we just provide a few simple environment installation examples. Of course, you can also customize the Dockerfile, 15 | > install iflearner and various dependencies 16 | 17 | > If you need to use gpu, you need to install the corresponding cuda in the image or download a cuda version image from 18 | > dockerhub as the base image. 19 | 20 | ## Build Image 21 | 22 | In the current directory, you can specify the corresponding Dockerfile to build different images, 23 | here is an example of a build command: 24 | ```shell 25 | docker build -f iflearner_torch1.7.1.Dockerfile -t iflearner_torch1.7.1 . 26 | ``` -------------------------------------------------------------------------------- /enviroment/docker/README.zh.md: -------------------------------------------------------------------------------- 1 | # Docker 2 | 3 | #＃ 介绍 4 | 我们为 IFLearner 提供 Dockerfiles。 5 | 需要先安装docker，如果需要使用gpu训练，请先安装nvidia驱动和nvidia-docker。 6 | 7 | #＃ Dockerfile 8 | - `iflearner_base.Dockerfile`：仅安装 iflearner 库。 9 | - `iflearner_torch1.7.1.Dockerfile`：安装iflearner、torch1.7.1、torchvision0.8.2库。 10 | - `iflearner_tensorflow2.9.1.Dockerfile`：安装iflearner、tensorflow2.9.1库。 11 | - `iflearner_mxnet1.9.1.Dockerfile`：安装iflearner、mxnet1.9.1库。 12 | - `iflearner_kera2.9.0.Dockerfile`：安装iflearner、keras2.9.0库。 13 | 14 | > 这里只提供几个简单的环境安装示例。当然你也可以自定义Dockerfile， 安装 iflearner 和各种依赖项. 15 | > 如果你需要使用gpu, 则需要在镜像中安装对应的cuda或者从dockerhub下载一个cuda版本镜像作为基础镜像。 16 | 17 | ## 构建镜像 18 | 在当前目录下，可以指定对应的Dockerfile来构建不同的镜像， 下面是一个构建示例： 19 | ```shell 20 | docker build -f iflearner_torch1.7.1.Dockerfile -t iflearner_torch1.7.1 . 21 | ``` -------------------------------------------------------------------------------- /enviroment/docker/iflearner_base.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | RUN pip3 install --upgrade pip && groupadd -r iflearner && useradd -r -g iflearner iflearner && \ 4 | pip3 install iflearner --index-url http://pypi.douban.com/simple --trusted-host pypi.douban.com 5 | 6 | USER iflearner 7 | 8 | -------------------------------------------------------------------------------- /enviroment/docker/iflearner_keras2.9.0.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | 4 | RUN pip3 install --upgrade pip && groupadd -r iflearner && useradd -r -g iflearner iflearner && \ 5 | pip3 install iflearner keras==2.9.0 --index-url http://pypi.douban.com/simple --trusted-host pypi.douban.com 6 | 7 | USER iflearner -------------------------------------------------------------------------------- /enviroment/docker/iflearner_mxnet1.9.1.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | RUN pip3 install --upgrade pip && groupadd -r iflearner && useradd -r -g iflearner iflearner && \ 4 | pip3 install iflearner mxnet==1.9.1 --index-url http://pypi.douban.com/simple --trusted-host pypi.douban.com 5 | 6 | USER iflearner -------------------------------------------------------------------------------- /enviroment/docker/iflearner_tensorflow2.9.1.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | RUN pip3 install --upgrade pip && groupadd -r iflearner && useradd -r -g iflearner iflearner && \ 4 | pip3 install iflearner tensorflow==2.9.1 --index-url http://pypi.douban.com/simple --trusted-host pypi.douban.com 5 | 6 | USER iflearner -------------------------------------------------------------------------------- /enviroment/docker/iflearner_torch1.7.1.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | RUN pip3 install --upgrade pip && groupadd -r iflearner && useradd -r -g iflearner iflearner && \ 4 | pip3 install iflearner tensorflow==2.9.1 torchvision==0.8.2 --index-url http://pypi.douban.com/simple --trusted-host pypi.douban.com 5 | 6 | USER iflearner -------------------------------------------------------------------------------- /enviroment/jupyterlab/README.md: -------------------------------------------------------------------------------- 1 | # Jupyterlab 2 | 3 | ## Introduction 4 | We provide Dockerfiles for build IFLearner with jupyterlab. 5 | 6 | ## Dockerfile 7 | - `iflearner_jupyterlab_base.Dockerfile`: Only install jupyterlab3.0.5 and iflearner library. 8 | 9 | > Here, we just provide a few simple environment installation examples. Of course, you can also customize the Dockerfile, 10 | > install iflearner and various dependencies 11 | 12 | > If you need to use gpu, you need to install a jupyterlab gpu image from dockerhub as the base image. 13 | 14 | ## Build Image 15 | 16 | In the current directory, you can specify the corresponding Dockerfile to build different images, 17 | here is an example of a build command: 18 | ```shell 19 | docker build -f iflearner_jupyterlab_base.Dockerfile -t iflearner_jupyterlab_base:latest . 20 | ``` 21 | 22 | ## Start the jupytelab image 23 | ```shell 24 | docker run -d --name iflearner_jupyterlab -p 8888:8888 -v ${pwd}:/opt/notebooks iflearner_jupyterlab_base:latest 25 | ```` 26 | 27 | ## Visit jupyterlab 28 | 1. First start to get token 29 | ```shell 30 | docker logs iflearner_jupyterlab 31 | ```` 32 | 2. Find a field similar to http://049ac86d1ad0:8888/lab?token=0dd1ed0ee2e3ca15b5c433f57f477b8ba005a494d865f56e in the log, 33 | The part after token= is the jupyterlab login token. You can also copy the entire paragraph and enter it into the browser address bar to log in directly. 34 | 35 | 3. Enter http://{ip}:8888 in the browser to access 36 | -------------------------------------------------------------------------------- /enviroment/jupyterlab/README.zh.md: -------------------------------------------------------------------------------- 1 | # Jupyterlab 2 | 3 | #＃ 介绍 4 | 我们为使用 jupyterlab 构建 IFLearner 提供 Dockerfile。 5 | 6 | ## Dockerfile 7 | - `iflearner_jupyterlab_base.Dockerfile`：仅安装 jupyterlab3.0.5 和 iflearner 库。 8 | 9 | > 这里只提供几个简单的环境安装示例。 当然你也可以自定义Dockerfile， 10 | > 安装 iflearner 和各种依赖项 11 | 12 | > 如果需要使用gpu，需要从dockerhub下载一个jupyterlab的gpu版本的镜像作为基础镜像。 13 | 14 | ## 构建镜像 15 | 16 | 在当前目录下，可以指定对应的Dockerfile来构建不同的镜像， 17 | 这是构建命令的示例： 18 | ```shell 19 | docker build -f iflearner_jupyterlab_base.Dockerfile -t iflearner_jupyterlab_base:latest . 20 | ``` 21 | 22 | ## 启动jupytelab镜像 23 | ```shell 24 | docker run -d --name iflearner_jupyterlab -p 8888:8888 -v (pwd):/opt/notebooks iflearner_jupyterlab_base:latest 25 | ``` 26 | 27 | ## 访问jupyterlab 28 | 1. 首次启动获取token 29 | ```shell 30 | docker logs iflearner_jupyterlab 31 | ``` 32 | 2. 找到日志中类似http://049ac86d1ad0:8888/lab?token=0dd1ed0ee2e3ca15b5c433f57f477b8ba005a494d865f56e字段， 33 | 其中token=后面的部分为jupyterlab登录token，你也可以复制整段，输入到浏览器地址栏来直接登录。 34 | 35 | 3. 在浏览器输入http://{ip}:8888即可进行访问 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /enviroment/jupyterlab/iflearner_jupyterlab_base.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM captainji/jupyterlab:3.0.5 2 | 3 | RUN pip3 install --upgrade pip && pip3 install iflearner \ 4 | --extra-index-url http://pypi.douban.com/simple \ 5 | --trusted-host pypi.douban.com -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/__init__.py -------------------------------------------------------------------------------- /examples/homo/FedOpt/README.md: -------------------------------------------------------------------------------- 1 | # FedOpt example 2 | 3 | In the example, we will introduce how to train imagenet with Ifleaner FedOpt and PyTorch. 4 | 5 | ### Server 6 | 7 | ```cmd 8 | python server.py -n 3 9 | ``` 10 | 11 | ### Client 12 | 13 | ```cmd 14 | python imagenet.py --data your_data_path1 --name iid-client01 --epochs 60 --server "0.0.0.0:50001" 15 | python imagenet.py --data your_data_path2 --name iid-client02 --epochs 60 --server "0.0.0.0:50001" 16 | python imagenet.py --data your_data_path3 --name iid-client03 --epochs 60 --server "0.0.0.0:50001" 17 | ``` 18 | 19 | ### Result 20 | 21 | We train the example [imagenet](../imagenet/README.md) by FedAdam with (learning rate = 1, adaptivity=0.1) and FedAvgM with (learning rate = 1, momentum = 0.9). The results is shown in the table. 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 |
Fedopt typeepochclient nameTop 1 accTop 5 acc
FedAdam60iid-client0155.88079.550
iid-client0255.92079.540
iid-client03 55.73079.410
FedAvgM60iid-client0155.96079.600
iid-client0255.92079.660
iid-client03 56.11080.060
69 | -------------------------------------------------------------------------------- /examples/homo/FedOpt/iid-client01/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/iid-client01/top1.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/iid-client01/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/iid-client01/top5.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/iid-client02/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/iid-client02/top1.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/iid-client02/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/iid-client02/top5.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/iid-client03/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/iid-client03/top1.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/iid-client03/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/iid-client03/top5.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client01/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client01/loss.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client01/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client01/top1.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client01/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client01/top5.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client02/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client02/loss.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client02/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client02/top1.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client02/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client02/top5.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client03/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client03/loss.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client03/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client03/top1.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/noniid-client03/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/FedOpt/noniid-client03/top5.png -------------------------------------------------------------------------------- /examples/homo/FedOpt/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from iflearner.business.homo.aggregate_server import AggregateServer 5 | from iflearner.communication.homo import message_type 6 | 7 | if __name__ == "__main__": 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument( 10 | "-n", "--num", help="the number of all clients", type=int, default=1 11 | ) 12 | 13 | parser.add_argument( 14 | "--addr", help="the server address", default="0.0.0.0:50001", type=str 15 | ) 16 | parser.add_argument( 17 | "--strategy_params", 18 | help="specify the params of strategy", 19 | type=json.loads, 20 | default='{"learning_rate":1, "betas":[0.9,0.99], "t":0.1, "opt":"FedAdam"}', 21 | ) 22 | 23 | args = parser.parse_args() 24 | 25 | strategy = message_type.STRATEGY_FEDOPT 26 | 27 | server = AggregateServer(args.addr, strategy, args.num, args.strategy_params) 28 | server.run() 29 | -------------------------------------------------------------------------------- /examples/homo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/__init__.py -------------------------------------------------------------------------------- /examples/homo/feddyn/README.md: -------------------------------------------------------------------------------- 1 | # FedDyn example 2 | 3 | In the example, we will introduce how to train imagenet with Ifleaner [FedDyn](https://openreview.net/pdf?id=B7v4QMR6Z9w) and PyTorch. 4 | We customize an aggregation strategy [`feddyn_server.py`](./feddyn_server.py), 5 | ### Server 6 | 7 | ```cmd 8 | python server.py -n 2 9 | ``` 10 | 11 | ### Client 12 | 13 | ```cmd 14 | python imagenet.py --data your_data_path1 --name iid-client01 --epochs 60 --server "0.0.0.0:50001" 15 | python imagenet.py --data your_data_path2 --name iid-client02 --epochs 60 --server "0.0.0.0:50001" 16 | ``` 17 | 18 | ### Result 19 | 20 | We train the example [imagenet](../imagenet/README.md) by FedAdam with (learning rate = 1, adaptivity=0.1) and FedAvgM with (learning rate = 1, momentum = 0.9). The results is shown in the table. 21 | 22 |
23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 |
Fedopt typeepochclient nameTop 1 accTop 5 acc
FedAdam60iid-client0155.88079.550
iid-client0255.92079.540
iid-client03 55.73079.410
noniid-client0150.57076.380
noniid-client0251.39076.880
noniid-client03 51.69076.720
FedAvgM60iid-client0155.96079.600
iid-client0255.92079.660
iid-client03 56.11080.060
82 | -------------------------------------------------------------------------------- /examples/homo/feddyn/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from statistics import mode 3 | 4 | from feddyn_server import FedDynServer 5 | from torchvision import models 6 | 7 | from iflearner.business.homo.aggregate_server import AggregateServer 8 | from iflearner.business.homo.strategy.opt import FedAdam 9 | from iflearner.communication.homo import message_type 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument( 14 | "-n", "--num", help="the number of all clients", type=int, default=1 15 | ) 16 | 17 | parser.add_argument( 18 | "--addr", help="the server address", default="0.0.0.0:50001", type=str 19 | ) 20 | 21 | args = parser.parse_args() 22 | 23 | model = models.__dict__["resnet18"](pretrained=False) 24 | model.train() 25 | params = dict() 26 | for name, param in model.named_parameters(): 27 | if param.requires_grad: 28 | params[name] = param.numpy() 29 | 30 | strategy = FedDynServer(args.num, learning_rate=1, alpha=1, params=params) 31 | 32 | server = AggregateServer(args.addr, strategy, args.num) 33 | server.run() 34 | -------------------------------------------------------------------------------- /examples/homo/fednova/README.md: -------------------------------------------------------------------------------- 1 | # FedNova example 2 | 3 | In the example, we will introduce how to train imagenet with Ifleaner FedNova and PyTorch. 4 | 5 | ### Server 6 | 7 | ```cmd 8 | python server.py -n 3 9 | ``` 10 | 11 | ### Client 12 | 13 | ```cmd 14 | python imagenet.py --data your_data_path1 --name iid-client01 --epochs 60 --server "0.0.0.0:50001" 15 | python imagenet.py --data your_data_path2 --name iid-client02 --epochs 60 --server "0.0.0.0:50001" 16 | python imagenet.py --data your_data_path3 --name iid-client03 --epochs 60 --server "0.0.0.0:50001" 17 | ``` 18 | 19 | The client only use vanilla SGD as a local solve, so this situation is about equal to Fedavg. If you want to use a full reproduced method of origin paper, you can modify the Optimizer referring to the author's open source [code](https://github.com/JYWa/FedNova.git) by yourself. 20 | 21 | ### Result 22 | 23 | We train the example [imagenet](../imagenet/README.md) by FedNova. The results is shown in the table. 24 | 25 |
26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 |
Fedopt typeepochclient nameTop 1 accTop 5 acc
FedNova60noniid-client0146.13073.580
noniid-client0246.70073.480
noniid-client0346.60073.640
53 | -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client01/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client01/loss.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client01/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client01/top1.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client01/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client01/top5.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client02/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client02/loss.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client02/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client02/top1.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client02/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client02/top5.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client03/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client03/loss.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client03/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client03/top1.png -------------------------------------------------------------------------------- /examples/homo/fednova/noniid-client03/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/fednova/noniid-client03/top5.png -------------------------------------------------------------------------------- /examples/homo/fednova/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from iflearner.business.homo.aggregate_server import AggregateServer 4 | from iflearner.communication.homo import message_type 5 | 6 | if __name__ == "__main__": 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument( 9 | "-n", "--num", help="the number of all clients", type=int, default=1 10 | ) 11 | 12 | parser.add_argument( 13 | "--addr", help="the server address", default="0.0.0.0:50001", type=str 14 | ) 15 | 16 | args = parser.parse_args() 17 | 18 | strategy = message_type.STRATEGY_FEDNOVA 19 | 20 | server = AggregateServer(args.addr, strategy, args.num) 21 | server.run() 22 | -------------------------------------------------------------------------------- /examples/homo/imagenet/Iid_Top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/imagenet/Iid_Top1.png -------------------------------------------------------------------------------- /examples/homo/imagenet/Iid_Top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/imagenet/Iid_Top5.png -------------------------------------------------------------------------------- /examples/homo/imagenet/Noniid_Top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/imagenet/Noniid_Top1.png -------------------------------------------------------------------------------- /examples/homo/imagenet/Noniid_Top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/imagenet/Noniid_Top5.png -------------------------------------------------------------------------------- /examples/homo/imagenet/README.md: -------------------------------------------------------------------------------- 1 | # Imagenet 2 | 3 | 原始训练代码来自pytorch官方开源的[imagenet示例](https://github.com/pytorch/examples/tree/main/imagenet)中，根据我们的联邦学习训练框架进行了改造，以支持联邦学习的训练。 4 | 5 | 6 | ## 如何改造原始训练代码 7 | 8 | 参见我们联邦学习框架的集成开发文档 9 | 10 | 11 | ## 准备数据集 12 | 13 | 由于完整的imagenet数据集过大，我们这里准备了一份几百兆的小数据集。包含200个分类，每个分类训练集和验证集分别为500张图片和50张图片。我们将训练集按照独立同分布和非独立同分布两种方式各分成3份等量的训练集，其中非独立同分布按照每个参与方50、100和350张图片在200个分类中循环。 14 | 15 | 16 | ## 启动服务端 17 | 18 | ``` 19 | PYTHONPATH=$pwd python iflearner/business/homo/aggregate_server.py -a 3 20 | ``` 21 | 22 | `-a 3`: 表示有三个参与方 23 | 24 | 25 | ## 启动客户端 26 | 27 | 分别启动3个客户端 28 | 29 | ``` 30 | PYTHONPATH=/data1/shengxu8/projects/iflytek/iflearner python imagenet.py -a resnet18 /data1/shengxu8/projects/iflytek/data/tiny-imagenet-200-iid-0 --name iid-client01 --epochs 60 --server "0.0.0.0:50001" --peers "0.0.0.0:50012;0.0.0.0:50013;0.0.0.0:50014" 31 | 32 | PYTHONPATH=/data1/shengxu8/projects/iflytek/iflearner python imagenet.py -a resnet18 /data1/shengxu8/projects/iflytek/data/tiny-imagenet-200-iid-1 --name iid-client02 --epochs 60 --server "0.0.0.0:50001" --peers "0.0.0.0:50013;0.0.0.0:50012;0.0.0.0:50014" 33 | 34 | PYTHONPATH=/data1/shengxu8/projects/iflytek/iflearner python imagenet.py -a resnet18 /data1/shengxu8/projects/iflytek/data/tiny-imagenet-200-iid-2 --name iid-client03 --epochs 60 --server "0.0.0.0:50001" --peers "0.0.0.0:50014;0.0.0.0:50012;0.0.0.0:50013" 35 | ``` 36 | 37 | `--name`: 客户端名称 38 | `--epochs`: 训练轮数 39 | `--server`: 需要连接的服务端地址 40 | `--peers`: 启动SMPC聚合方式，需要指定所有的客户端地址，其中第一个是自己监听的地址 41 | 42 | ## 结果分析 43 | 44 | 按照两种数据分布，分别统计聚合数据的训练、联邦学习训练以及单参与方训练下的Top1和Top3的acc值。 45 | 46 | 47 | * 数据非独立同分布 48 | 49 | ![Noniid Top1](Noniid_Top1.png) 50 | 51 | ![Noniid Top1](Noniid_Top5.png) 52 | 53 | 可以看出无论Top1或者Top5的acc值，聚合数据的训练>联邦学习训练>单参与方训练，总体结果符合预期。 54 | 55 | * 数据独立同分布 56 | 57 | ![Iid Top1](Iid_Top1.png) 58 | 59 | ![Iid Top5](Iid_Top5.png) 60 | 61 | 可以看出无论Top1或者Top5的acc值，聚合数据的训练>联邦学习训练>单参与方训练，但是在联邦学习场景下，数据独立同分布的结果要好于数据非独立同分布，总体结果符合预期。 -------------------------------------------------------------------------------- /examples/homo/imagenet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/imagenet/__init__.py -------------------------------------------------------------------------------- /examples/homo/imagenet/plot.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from cProfile import label 3 | 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def extract(path): 8 | with open(path, "rb") as f: 9 | inst = pickle.load(f) 10 | return (inst._x_list, inst._y_list) 11 | 12 | 13 | def draw(all_top, client01_top, client02_top, client03_top, single_top, title, dis): 14 | plt.clf() 15 | plt.plot(all_top[0], all_top[1], label="nonfl-all") 16 | plt.plot(client01_top[0], client01_top[1], label=f"fl-{dis}-party1") 17 | plt.plot(client02_top[0], client02_top[1], label=f"fl-{dis}-party2") 18 | plt.plot(client03_top[0], client03_top[1], label=f"fl-{dis}-party3") 19 | plt.plot(single_top[0], single_top[1], label=f"nonfl-{dis}-party1") 20 | plt.xlabel("Epoch") 21 | plt.ylabel("Acc") 22 | plt.title(title) 23 | plt.legend() 24 | plt.savefig(f"{title}.png") 25 | 26 | 27 | all_top1 = extract("all/top1.pkl") 28 | all_top5 = extract("all/top5.pkl") 29 | 30 | client01_top1 = extract("client01/top1.pkl") 31 | client01_top5 = extract("client01/top5.pkl") 32 | 33 | client02_top1 = extract("client02/top1.pkl") 34 | client02_top5 = extract("client02/top5.pkl") 35 | 36 | client03_top1 = extract("client03/top1.pkl") 37 | client03_top5 = extract("client03/top5.pkl") 38 | 39 | single_top1 = extract("single/top1.pkl") 40 | single_top5 = extract("single/top5.pkl") 41 | 42 | draw( 43 | all_top1, 44 | client01_top1, 45 | client02_top1, 46 | client03_top1, 47 | single_top1, 48 | "Noniid Top1", 49 | "noniid", 50 | ) 51 | draw( 52 | all_top5, 53 | client01_top5, 54 | client02_top5, 55 | client03_top5, 56 | single_top5, 57 | "Noniid Top5", 58 | "noniid", 59 | ) 60 | 61 | iid_client01_top1 = extract("iid-client01/top1.pkl") 62 | iid_client01_top5 = extract("iid-client01/top5.pkl") 63 | 64 | iid_client02_top1 = extract("iid-client02/top1.pkl") 65 | iid_client02_top5 = extract("iid-client02/top5.pkl") 66 | 67 | iid_client03_top1 = extract("iid-client03/top1.pkl") 68 | iid_client03_top5 = extract("iid-client03/top5.pkl") 69 | 70 | iid_single_top1 = extract("iid-single/top1.pkl") 71 | iid_single_top5 = extract("iid-single/top5.pkl") 72 | 73 | draw( 74 | all_top1, 75 | iid_client01_top1, 76 | iid_client02_top1, 77 | iid_client03_top1, 78 | iid_single_top1, 79 | "Iid Top1", 80 | "iid", 81 | ) 82 | draw( 83 | all_top5, 84 | iid_client01_top5, 85 | iid_client02_top5, 86 | iid_client03_top5, 87 | iid_single_top5, 88 | "Iid Top5", 89 | "iid", 90 | ) 91 | -------------------------------------------------------------------------------- /examples/homo/imagenet/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for i in {1..3}; do 3 | echo "Starting client $i" 4 | python imagenet.py -a resnet18 /data1/shengxu8/projects/iflytek/data/tiny-imagenet-200-iid-$i --name iid-client0$i --epochs 60 --server "0.0.0.0:50001" --peers "0.0.0.0:50012;0.0.0.0:50013;0.0.0.0:50014" --pretrained --gpu 0 & 5 | done 6 | 7 | # This will allow you to use CTRL+C to stop all background processes 8 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 9 | # Wait for all background processes to complete 10 | wait 11 | -------------------------------------------------------------------------------- /examples/homo/imagenet/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 3 4 | # This will allow you to use CTRL+C to stop all background processes 5 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 6 | # Wait for all background processes to complete 7 | wait -------------------------------------------------------------------------------- /examples/homo/imagenet/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from iflearner.business.homo.aggregate_server import AggregateServer 5 | 6 | if __name__ == "__main__": 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument( 9 | "-n", "--num", help="the number of all clients", default=0, type=int 10 | ) 11 | parser.add_argument("--epochs", help="the total epoch", type=int) 12 | parser.add_argument( 13 | "--addr", 14 | help="The aggregation server itself listens to the address (used for client connections)", 15 | default="0.0.0.0:50001", 16 | type=str, 17 | ) 18 | parser.add_argument( 19 | "--http_addr", 20 | help="Federation training status listening address (for viewing federation training status)", 21 | default="0.0.0.0:50002", 22 | type=str, 23 | ) 24 | parser.add_argument( 25 | "--strategy", 26 | help="the aggregation starategy (FedAvg | Scaffold | FedOpt | qFedAvg | FedNova)", 27 | default="FedAvg", 28 | type=str, 29 | ) 30 | parser.add_argument( 31 | "--strategy_params", 32 | help="specify the params of strategy", 33 | default={}, 34 | type=json.loads, 35 | ) 36 | 37 | args = parser.parse_args() 38 | 39 | server = AggregateServer( 40 | args.addr, args.strategy, args.num, args.strategy_params, args.epochs 41 | ) 42 | server.run() 43 | -------------------------------------------------------------------------------- /examples/homo/jupyterlab/en/README.md: -------------------------------------------------------------------------------- 1 | # Jupyterlab 2 | 3 | JupyterLab is Jupyter's latest data science production tool, and in a sense, it appeared to replace Jupyter Notebook. But don't worry about Jupyter Notebook disappearing, 4 | JupyterLab contains all the features of Jupyter Notebook. 5 | 6 | JupyterLab is a web-based integrated development environment, you can use it to write notebooks, operate terminals, edit markdown text, open interactive mode, view csv files and pictures and other functions. 7 | 8 | ## Example 9 | This example mainly demonstrates how to install and run the iflearner library in jupyterlab, and start one server and two clients to complete the federated learning task of image classification under the mnist dataset. 10 | 11 | The installation startup sequence is: 1. Server side 2. Client side. 12 | 13 | ### 1. Server side 14 | Please refer to `pytorch_mnist_server.ipynb` for server installation and startup 15 | 16 | ### 2. Client 17 | 1. Please refer to `pytorch_mnist_client1.ipynb` to install and start client1 18 | 2. Please refer to `pytorch_mnist_client2.ipynb` to install and start client2 19 | 20 | When both clients start to register, the federated training task will be started automatically. -------------------------------------------------------------------------------- /examples/homo/jupyterlab/zh/README.md: -------------------------------------------------------------------------------- 1 | # Jupyterlab 2 | 3 | JupyterLab是Jupyter主打的最新数据科学生产工具，某种意义上，它的出现是为了取代Jupyter Notebook。不过不用担心Jupyter Notebook会消失， 4 | JupyterLab包含了Jupyter Notebook所有功能。 5 | 6 | JupyterLab作为一种基于web的集成开发环境，你可以使用它编写notebook、操作终端、编辑markdown文本、打开交互模式、查看csv文件及图片等功能。 7 | 8 | ## 示例 9 | 该实例主要演示如何在jupyterlab中安装和运行iflearner库，并且启动一个server端和两个client端，完成在mnist数据集下的图像分类的联邦学习任务。 10 | 11 | 安装启动顺序为: 1. Server端 2. Client端。 12 | 13 | ### 1. Server端 14 | 请参阅`pytorch_mnist_server.ipynb`，进行server的安装和启动 15 | 16 | ### 2. Client端 17 | 1. 请参阅`pytorch_mnist_client1.ipynb`，进行client1的安装和启动 18 | 2. 请参阅`pytorch_mnist_client2.ipynb`，进行client2的安装和启动 19 | 20 | 当两个client都启动注册后，将自动开启联邦训练任务。 -------------------------------------------------------------------------------- /examples/homo/mnist/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/mnist/README.md -------------------------------------------------------------------------------- /examples/homo/mnist/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/mnist/__init__.py -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client01/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client01/loss.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client01/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client01/top1.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client01/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client01/top5.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client02/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client02/loss.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client02/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client02/top1.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client02/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client02/top5.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client03/loss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client03/loss.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client03/top1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client03/top1.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/qnoniid-client03/top5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/qFedavg/qnoniid-client03/top5.png -------------------------------------------------------------------------------- /examples/homo/qFedavg/readme.md: -------------------------------------------------------------------------------- 1 | # qFedavg example 2 | 3 | In the example, we will introduce how to train imagenet with Ifleaner qFedavg and PyTorch. More details about qFedavg can be seen [here](../../../doc/source/qfedavg.md) 4 | 5 | ### Server 6 | 7 | ```cmd 8 | python server.py -n 2 --server "0.0.0.0:50001" 9 | ``` 10 | 11 | ### Client 12 | 13 | ```cmd 14 | python imagenet.py --data your_data_path1 --name client01 --epochs 60 --server "0.0.0.0:50001" 15 | python imagenet.py --data your_data_path2 --name client02 --epochs 60 --server "0.0.0.0:50001" 16 | ``` 17 | -------------------------------------------------------------------------------- /examples/homo/qFedavg/server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from iflearner.business.homo.aggregate_server import AggregateServer 5 | from iflearner.communication.homo import message_type 6 | 7 | if __name__ == "__main__": 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument( 10 | "-n", "--num", help="the number of all clients", type=int, default=1 11 | ) 12 | 13 | parser.add_argument( 14 | "--addr", help="the server address", default="0.0.0.0:50001", type=str 15 | ) 16 | 17 | parser.add_argument( 18 | "--strategy_params", 19 | help="specify the params of strategy", 20 | type=json.loads, 21 | default='{"learning_rate":1, "q":0.0001}', 22 | ) 23 | 24 | args = parser.parse_args() 25 | strategy = message_type.STRATEGY_qFEDAVG 26 | server = AggregateServer(args.addr, strategy, args.num, args.strategy_params) 27 | server.run() 28 | -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Keras 2 | 3 | This introductory example will help you understand how to run a federated task under keras 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 quickstart_keras.py --name "client1" --epochs 10 30 | ``` 31 | 32 | Start client 2 in the second terminal: 33 | 34 | ```shell 35 | python3 quickstart_keras.py --name "client2" --epochs 10 36 | ``` 37 | 38 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 39 | 40 | start server in the first terminal. 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | start multiple clients in the second terminal. 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Keras 2 | 3 | 该示例将帮助您了解如何在 keras 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_keras.py --name "client1" --epochs 10 30 | ``` 31 | 32 | 在第二个终端启动客户端2： 33 | 34 | ```shell 35 | python3 quickstart_keras.py --name "client2" --epochs 10 36 | ``` 37 | 38 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 39 | 40 | 在第一个终端启动服务端。 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | 在第二个终端中启动多个客户端。 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_keras/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/requirements.txt: -------------------------------------------------------------------------------- 1 | keras==2.9.0 -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in $(seq 2); do 4 | echo "Starting client $i" 5 | python quickstart_keras.py --name "client$i" --epochs 3 & 6 | done 7 | 8 | # This will allow you to use CTRL+C to stop all background processes 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 10 | # Wait for all background processes to complete 11 | wait 12 | -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | 5 | # This will allow you to use CTRL+C to stop all background processes 6 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 7 | # Wait for all background processes to complete 8 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_keras/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Mxnet 2 | 3 | This introductory example will help you understand how to run a federated task under mxnet 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip3 install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 quickstart_mxnet.py --name "client1" --epochs 10 30 | ``` 31 | 32 | Start client 2 in the second terminal: 33 | 34 | ```shell 35 | python3 quickstart_mxnet.py --name "client2" --epochs 10 36 | ``` 37 | 38 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 39 | 40 | start server in the first terminal. 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | start multiple clients in the second terminal. 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Mxnet 2 | 3 | 该示例将帮助您了解如何在 mxnet 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_mxnet.py --name "client1" --epochs 10 30 | ``` 31 | 32 | 在第二个终端启动客户端2： 33 | 34 | ```shell 35 | python3 quickstart_mxnet.py --name "client2" --epochs 10 36 | ``` 37 | 38 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 39 | 40 | 在第一个终端启动服务端。 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | 在第二个终端中启动多个客户端。 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_mxnet/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/requirements.txt: -------------------------------------------------------------------------------- 1 | mxnet==1.9.1 -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in $(seq 2); do 4 | echo "Starting client $i" 5 | python quickstart_mxnet.py --name "client$i" --epochs 3 & 6 | done 7 | 8 | # This will allow you to use CTRL+C to stop all background processes 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 10 | # Wait for all background processes to complete 11 | wait 12 | -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | 5 | # This will allow you to use CTRL+C to stop all background processes 6 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 7 | # Wait for all background processes to complete 8 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_mxnet/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner DP using Opacus 2 | 3 | This example will help you understand how to run federated tasks under pytorch using Opacus library combined 4 | with differential privacy encryption technology 5 | 6 | ## Preconditions 7 | First, you need to install iflearner, and then install the corresponding dependencies 8 | ```shell 9 | pip install iflearner 10 | ``` 11 | 12 | install corresponding dependencies. 13 | ```shell 14 | pip install -r requirements.txt 15 | ``` 16 | 17 | ## Project Setup 18 | 19 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 20 | ```shell 21 | python3 server.py -n 2 22 | ``` 23 | > -n: The parameter n represents accept n clients to connect and you can modify it 24 | 25 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 26 | 27 | Start client 1 in the first terminal: 28 | 29 | ```shell 30 | python3 quickstart_pytorch.py --name "client1" --epochs 10 31 | ``` 32 | 33 | Start client 2 in the second terminal: 34 | 35 | ```shell 36 | python3 quickstart_pytorch.py --name "client2" --epochs 10 37 | ``` 38 | 39 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 40 | 41 | start server in the first terminal. 42 | ```shell 43 | bash run_server.sh 44 | ``` 45 | 46 | start multiple clients in the second terminal. 47 | ```shell 48 | bash run_client.sh 49 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner DP using Opacus 2 | 3 | 该示例将帮助您了解结合差分隐私加密技术， 使用Opacus库，在 pytorch 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_pytorch.py --name "client1" --epochs 10 30 | ``` 31 | 32 | 在第二个终端启动客户端2： 33 | 34 | ```shell 35 | python3 quickstart_pytorch.py --name "client2" --epochs 10 36 | ``` 37 | 38 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 39 | 40 | 在第一个终端启动服务端。 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | 在第二个终端中启动多个客户端。 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_opacus/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/requirements.txt: -------------------------------------------------------------------------------- 1 | opacus==1.1.3 2 | torch==1.8.1 3 | torchvision==0.9.1 4 | -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in $(seq 2); do 4 | echo "Starting client $i" 5 | python quickstart_pytorch.py --name "client$i" --epochs 10 & 6 | done 7 | 8 | # This will allow you to use CTRL+C to stop all background processes 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 10 | # Wait for all background processes to complete 11 | wait 12 | -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | 5 | # This will allow you to use CTRL+C to stop all background processes 6 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 7 | # Wait for all background processes to complete 8 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_opacus/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using PyTorch 2 | 3 | This introductory example will help you understand how to run a federated task under pytorch 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 quickstart_pytorch.py --name "client1" --epochs 10 30 | ``` 31 | 32 | Start client 2 in the second terminal: 33 | 34 | ```shell 35 | python3 quickstart_pytorch.py --name "client2" --epochs 10 36 | ``` 37 | 38 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 39 | 40 | start server in the first terminal. 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | start multiple clients in the second terminal. 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Pytorch 2 | 3 | 该示例将帮助您了解如何在 pytorch 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_pytorch.py --name "client1" --epochs 10 30 | ``` 31 | 32 | 在第二个终端启动客户端2： 33 | 34 | ```shell 35 | python3 quickstart_pytorch.py --name "client2" --epochs 10 36 | ``` 37 | 38 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 39 | 40 | 在第一个终端启动服务端。 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | 在第二个终端中启动多个客户端。 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_pytorch/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.7.1 2 | torchvision==0.8.2 -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in $(seq 2); do 4 | echo "Starting client $i" 5 | python quickstart_pytorch.py --name "client$i" --epochs 10 & 6 | done 7 | 8 | # This will allow you to use CTRL+C to stop all background processes 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 10 | # Wait for all background processes to complete 11 | wait 12 | -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | 5 | # This will allow you to use CTRL+C to stop all background processes 6 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 7 | # Wait for all background processes to complete 8 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_pytorch/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/quickstart_sklearn/README.md: -------------------------------------------------------------------------------- 1 | iFlearner Example using Sklearn 2 | 3 | This introductory example will help you understand how to run a federated task under sklearn 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip3 install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 quickstart_sklearn.py --name "client1" --epochs 10 30 | ``` 31 | 32 | Start client 2 in the second terminal: 33 | 34 | ```shell 35 | python3 quickstart_sklearn.py --name "client2" --epochs 10 36 | ``` 37 | 38 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 39 | 40 | start server in the first terminal. 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | start multiple clients in the second terminal. 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_sklearn/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Sklearn 2 | 3 | 该示例将帮助您了解如何在 Sklearn 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_sklearn.py --name "client1" --epochs 10 30 | ``` 31 | 32 | 在第二个终端启动客户端2： 33 | 34 | ```shell 35 | python3 quickstart_sklearn.py --name "client2" --epochs 10 36 | ``` 37 | 38 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 39 | 40 | 在第一个终端启动服务端。 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | 在第二个终端中启动多个客户端。 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_sklearn/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_sklearn/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_sklearn/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in $(seq 2); do 4 | echo "Starting client $i" 5 | python quickstart_sklearn.py --name "client$i" --epochs 10 & 6 | done 7 | 8 | # This will allow you to use CTRL+C to stop all background processes 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 10 | # Wait for all background processes to complete 11 | wait 12 | -------------------------------------------------------------------------------- /examples/homo/quickstart_sklearn/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | python iflearner. -n 2 5 | 6 | # This will allow you to use CTRL+C to stop all background processes 7 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 8 | # Wait for all background processes to complete 9 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_sklearn/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner SMPC Example using PyTorch 2 | 3 | This introductory example will help smpc you understand how to run a federated task under pytorch 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 quickstart_pytorch.py --name "client1" --epochs 10 --server "0.0.0.0:50001" --peers "0.0.0.0:50012;0.0.0.0:50013" 30 | ``` 31 | > Configure peers to use smpc, peers are configured as the listening address of all clients, and the first address is the listening address of the client 32 | 33 | 34 | Start client 2 in the second terminal: 35 | 36 | ```shell 37 | python3 quickstart_pytorch.py --name "client2" --epochs 10 --server "0.0.0.0:50001" --peers "0.0.0.0:50013;0.0.0.0:50012" 38 | ``` 39 | > Configure peers to use smpc, peers are configured as the listening address of all clients, and the first address is the listening address of the client 40 | 41 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 42 | 43 | start server in the first terminal. 44 | ```shell 45 | bash run_server.sh 46 | ``` 47 | 48 | start multiple clients in the second terminal. 49 | ```shell 50 | bash run_client.sh 51 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner SMPC Example using Pytorch 2 | 3 | 该示例将帮助您了解 SMPC 如何在 pytorch 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_pytorch.py --name "client1" --epochs 10 --server "0.0.0.0:50001" --peers "0.0.0.0:50012;0.0.0.0:50013" 30 | ``` 31 | > 配置peers即使用smpc, peers配置为所有客户端的监听地址, 第一个地址为该客户端的监听地址 32 | 33 | 在第二个终端启动客户端2： 34 | 35 | ```shell 36 | python3 quickstart_pytorch.py --name "client2" --epochs 10 --server "0.0.0.0:50001" --peers "0.0.0.0:50013;0.0.0.0:50012" 37 | ``` 38 | > 配置peers即使用smpc, peers配置为所有客户端的监听地址, 第一个地址为该客户端的监听地址 39 | 40 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 41 | 42 | 在第一个终端启动服务端。 43 | ```shell 44 | bash run_server.sh 45 | ``` 46 | 47 | 在第二个终端中启动多个客户端。 48 | ```shell 49 | bash run_client.sh 50 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_smpc/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.7.1 2 | torchvision==0.8.2 -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | python quickstart_pytorch.py --name "client1" --epochs 10 --server "0.0.0.0:50001" --peers "0.0.0.0:50012;0.0.0.0:50013" & 5 | python quickstart_pytorch.py --name "client2" --epochs 10 --server "0.0.0.0:50001" --peers "0.0.0.0:50013;0.0.0.0:50012" & 6 | 7 | # This will allow you to use CTRL+C to stop all background processes 8 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 9 | # Wait for all background processes to complete 10 | wait 11 | -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | 5 | # This will allow you to use CTRL+C to stop all background processes 6 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 7 | # Wait for all background processes to complete 8 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_smpc/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using PyTorch 2 | 3 | This introductory example will help you understand how to run a federated task under tensorflow. 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip3 install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 quickstart_tensorflow.py --name "client1" --epochs 10 30 | ``` 31 | 32 | Start client 2 in the second terminal: 33 | 34 | ```shell 35 | python3 quickstart_tensorflow.py --name "client2" --epochs 10 36 | ``` 37 | 38 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 39 | 40 | start server in the first terminal. 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | start multiple clients in the second terminal. 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/README.zh.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using Tensorflow 2 | 3 | 该示例将帮助您了解如何在 tensorflow 下运行联邦任务 4 | 5 | ## 前提条件 6 | 首先需要安装iflearner，然后安装对应的依赖 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | 安装相应的依赖项。 12 | ```shell 13 | pip install -r requirements.txt 14 | ``` 15 | 16 | ## 项目安装 17 | 18 | 首先，您已准备好启动 iflearner 服务端和客户端。您可以简单地在终端中启动服务端，如下所示：。 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n：参数n代表接受n个客户端连接，可以修改 23 | 24 | 现在您已准备好启动将参与学习的 iflearner 客户端。为此，只需再打开两个终端窗口并运行以下命令。 25 | 26 | 在第一个终端启动客户端1： 27 | 28 | ```shell 29 | python3 quickstart_tensorflow.py --name "client1" --epochs 10 30 | ``` 31 | 32 | 在第二个终端启动客户端2： 33 | 34 | ```shell 35 | python3 quickstart_tensorflow.py --name "client2" --epochs 10 36 | ``` 37 | 38 | 当然你也可以快速修改脚本`run_server.sh`和`run_client.sh`，然后使用下面的命令快速启动一个体验demo。 39 | 40 | 在第一个终端启动服务端。 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | 在第二个终端中启动多个客户端。 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/quickstart_tensorflow/__init__.py -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow==2.9.1 -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in $(seq 2); do 4 | echo "Starting client $i" 5 | python quickstart_tensorflow.py --name "client$i" --epochs 10 & 6 | done 7 | 8 | # This will allow you to use CTRL+C to stop all background processes 9 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 10 | # Wait for all background processes to complete 11 | wait 12 | -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | 5 | # This will allow you to use CTRL+C to stop all background processes 6 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 7 | # Wait for all background processes to complete 8 | wait -------------------------------------------------------------------------------- /examples/homo/quickstart_tensorflow/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/homo/word_language_model_pytorch/README.md: -------------------------------------------------------------------------------- 1 | # iFlearner Example using PyTorch 2 | 3 | This introductory example will help you understand how to run a federated task `word_language` under pytorch 4 | 5 | ## Preconditions 6 | First, you need to install iflearner, and then install the corresponding dependencies 7 | ```shell 8 | pip install iflearner 9 | ``` 10 | 11 | install corresponding dependencies. 12 | ```shell 13 | pip3 install -r requirements.txt 14 | ``` 15 | 16 | ## Project Setup 17 | 18 | First you are ready to start the iflearner server as well as the clients. You can simply start the server in a terminal as follows:. 19 | ```shell 20 | python3 server.py -n 2 21 | ``` 22 | > -n: The parameter n represents accept n clients to connect and you can modify it 23 | 24 | Now you are ready to start the iflearner clients which will participate in the learning. To do so simply open two more terminal windows and run the following commands. 25 | 26 | Start client 1 in the first terminal: 27 | 28 | ```shell 29 | python3 word_language_client.py --name "client1" --epochs 10 30 | ``` 31 | 32 | Start client 2 in the second terminal: 33 | 34 | ```shell 35 | python3 word_language_client.py --name "client2" --epochs 10 36 | ``` 37 | 38 | Of course, you can also quickly modify the script `run_server.sh` and `run_client.sh`, and then use the following command to quickly start an experience demo. 39 | 40 | start server in the first terminal. 41 | ```shell 42 | bash run_server.sh 43 | ``` 44 | 45 | start multiple clients in the second terminal. 46 | ```shell 47 | bash run_client.sh 48 | ``` -------------------------------------------------------------------------------- /examples/homo/word_language_model_pytorch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/homo/word_language_model_pytorch/__init__.py -------------------------------------------------------------------------------- /examples/homo/word_language_model_pytorch/data.py: -------------------------------------------------------------------------------- 1 | import os 2 | from io import open 3 | 4 | import torch 5 | 6 | 7 | class Dictionary(object): 8 | def __init__(self): 9 | self.word2idx = {} 10 | self.idx2word = [] 11 | 12 | def add_word(self, word): 13 | if word not in self.word2idx: 14 | self.idx2word.append(word) 15 | self.word2idx[word] = len(self.idx2word) - 1 16 | return self.word2idx[word] 17 | 18 | def __len__(self): 19 | return len(self.idx2word) 20 | 21 | 22 | class Corpus(object): 23 | def __init__(self, path): 24 | self.dictionary = Dictionary() 25 | self.train = self.tokenize(os.path.join(path, "train.txt")) 26 | self.valid = self.tokenize(os.path.join(path, "valid.txt")) 27 | self.test = self.tokenize(os.path.join(path, "test.txt")) 28 | 29 | def tokenize(self, path): 30 | """Tokenizes a text file.""" 31 | assert os.path.exists(path) 32 | # Add words to the dictionary 33 | with open(path, "r", encoding="utf8") as f: 34 | for line in f: 35 | words = line.split() + [""] 36 | for word in words: 37 | self.dictionary.add_word(word) 38 | 39 | # Tokenize file content 40 | with open(path, "r", encoding="utf8") as f: 41 | idss = [] 42 | for line in f: 43 | words = line.split() + [""] 44 | ids = [] 45 | for word in words: 46 | ids.append(self.dictionary.word2idx[word]) 47 | idss.append(torch.tensor(ids).type(torch.int64)) 48 | ids = torch.cat(idss) 49 | 50 | return ids 51 | -------------------------------------------------------------------------------- /examples/homo/word_language_model_pytorch/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | for i in $(seq 2); do 3 | echo "Starting client $i" 4 | python word_language_client.py --name "client$i" --epochs 20 & 5 | done 6 | 7 | # This will allow you to use CTRL+C to stop all background processes 8 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 9 | # Wait for all background processes to complete 10 | wait 11 | -------------------------------------------------------------------------------- /examples/homo/word_language_model_pytorch/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python server.py -n 2 4 | # This will allow you to use CTRL+C to stop all background processes 5 | trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM 6 | # Wait for all background processes to complete 7 | wait -------------------------------------------------------------------------------- /examples/homo/word_language_model_pytorch/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.homo.aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/mpc/quickstart_piss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/examples/mpc/quickstart_piss/__init__.py -------------------------------------------------------------------------------- /examples/mpc/quickstart_piss/client_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from importlib import import_module 17 | from iflearner.business.mpc.piss.piss_client_controller import PissClientServicesController 18 | from iflearner.business.mpc.piss.argument import parser 19 | 20 | 21 | if __name__ == "__main__": 22 | args = parser.parse_args() 23 | print(args) 24 | controller = PissClientServicesController(args) 25 | controller.run() 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /examples/mpc/quickstart_piss/piss_data_test.csv: -------------------------------------------------------------------------------- 1 | ID,Age,Money,Height,Credit_value 2 | 10001,18,6.1,1.88,98.6 3 | 10002,19,-99.25,1.76,87 4 | 10003,20,8.6,1.68,99.5 5 | -------------------------------------------------------------------------------- /examples/mpc/quickstart_piss/quickstart_piss.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import argparse 16 | import json 17 | from importlib import import_module 18 | from typing import Any, Dict, Union 19 | 20 | from iflearner.business.mpc.piss.piss_client_controller import PissClientController 21 | from iflearner.business.mpc.piss.argument import parser 22 | 23 | if __name__ == "__main__": 24 | 25 | parser = argparse.ArgumentParser() 26 | 27 | parser.add_argument( 28 | "--data", 29 | default='examples/mpc/quickstart_piss/piss_data_test.csv', 30 | type=str, 31 | help="path of data") 32 | 33 | parser.add_argument( 34 | "--param", 35 | default={'10001':'Age', '10002':'Money'}, 36 | type=json.loads, 37 | help="encryption param" 38 | ) 39 | parser.add_argument( 40 | "--server", 41 | default="127.0.0.1:45551", 42 | type=str, 43 | help="address of client service" 44 | ) 45 | parser.add_argument( 46 | "--name", 47 | default="client_querty", 48 | type=str, 49 | help="querty client name" 50 | ) 51 | 52 | parser.add_argument( 53 | "--cert", 54 | default=None, 55 | type=str, 56 | help="path of server SSL cert" 57 | """use secure channel to connect to server if not none""" 58 | ) 59 | 60 | args = parser.parse_args() 61 | print(args) 62 | controller = PissClientController(args) 63 | #controller.init_data() 64 | controller.start_querty() 65 | print(controller.get_secrets_sum()) -------------------------------------------------------------------------------- /examples/mpc/quickstart_piss/server.py: -------------------------------------------------------------------------------- 1 | from iflearner.business.mpc.piss.piss_aggregate_server import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /examples/spilt_dataset.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | from iflearner.datasets.mnist import MNIST 4 | from iflearner.datasets.sampler import Sampler 5 | 6 | 7 | clients = ['party1', 'party2', 'party3'] 8 | dataset = MNIST('./data', True) 9 | sampler = Sampler(dataset.train_labels, clients, 'dirichlet', alpha=2) 10 | clients_index = sampler.client_index 11 | 12 | d = {} 13 | for name, index in clients_index.items(): 14 | index = list(index) 15 | p = pd.Series(dataset.train_labels[index].astype('int64')) 16 | 17 | print(name+'各个类别：') 18 | print(p.value_counts()) 19 | print() -------------------------------------------------------------------------------- /iflearner/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | __version__ = "0.1.0" 16 | -------------------------------------------------------------------------------- /iflearner/business/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /iflearner/business/hetero/builder/builders.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from typing import Dict 17 | from iflearner.business.hetero.model.base_model import BaseModel 18 | from iflearner.business.hetero.builder.demo_builder import DemoBuilder 19 | from iflearner.business.hetero.builder.lr_builder import LRBuilder 20 | 21 | Builders: Dict[str, BaseModel] = { 22 | "demo": DemoBuilder(), 23 | "logistic_regression": LRBuilder(), 24 | } 25 | -------------------------------------------------------------------------------- /iflearner/business/hetero/builder/demo_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from iflearner.business.hetero.model.role import Role 17 | from iflearner.business.hetero.model.base_model import BaseModel 18 | from iflearner.business.hetero.model.demo import demo_guest, demo_host, demo_arbiter 19 | 20 | from iflearner.business.hetero.builder.model_builder import ModelBuilder 21 | 22 | 23 | class DemoBuilder(ModelBuilder): 24 | 25 | def create_role_model_instance(self, role: str) -> BaseModel: 26 | """Create a model instance base on specific role. 27 | 28 | Args: 29 | role (str): The role name. 30 | 31 | Returns: 32 | BaseModel: Return the base class. 33 | """ 34 | if role == Role.guest: 35 | return demo_guest.DemoGuest() 36 | elif role == Role.host: 37 | return demo_host.DemoHost() 38 | elif role == Role.arbiter: 39 | return demo_arbiter.DemoArbiter() 40 | 41 | raise Exception(f"{role} is not existed.") 42 | 43 | def get_role_model_flow_file(self, role: str) -> str: 44 | """Get model flow file by role name. 45 | 46 | Args: 47 | role (str): The role name. 48 | 49 | Returns: 50 | str: Return the filename. 51 | """ 52 | if role == Role.guest: 53 | return "demo_guest_flow.yaml" 54 | elif role == Role.host: 55 | return "demo_host_flow.yaml" 56 | elif role == Role.arbiter: 57 | return "demo_arbiter_flow.yaml" 58 | 59 | raise Exception(f"{role} is not existed.") 60 | -------------------------------------------------------------------------------- /iflearner/business/hetero/builder/lr_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from iflearner.business.hetero.model.role import Role, Guest, Host, Arbiter 17 | from iflearner.business.hetero.model.base_model import BaseModel 18 | from iflearner.business.hetero.model.logistic_regression import lr_guest, lr_host, lr_arbiter 19 | 20 | from iflearner.business.hetero.builder.model_builder import ModelBuilder 21 | 22 | 23 | class LRBuilder(ModelBuilder): 24 | 25 | def create_role_model_instance(self, role: Role) -> BaseModel: 26 | """Create a model instance base on specific role. 27 | 28 | Args: 29 | role (Role): The role name. 30 | 31 | Returns: 32 | BaseModel: Return the base class. 33 | """ 34 | if isinstance(role, Guest): 35 | return lr_guest.LRGuest() 36 | elif isinstance(role, Host): 37 | return lr_host.LRHost() 38 | elif isinstance(role, Arbiter): 39 | return lr_arbiter.LRArbiter() 40 | 41 | raise Exception(f"{role} is not existed.") 42 | 43 | def get_role_model_flow_file(self, role: Role) -> str: 44 | """Get model flow file by role name. 45 | 46 | Args: 47 | role (Role): The role name. 48 | 49 | Returns: 50 | str: Return the filename. 51 | """ 52 | if isinstance(role, Guest): 53 | return "lr_guest_flow.yaml" 54 | elif isinstance(role, Host): 55 | return "lr_host_flow.yaml" 56 | elif isinstance(role, Arbiter): 57 | return "lr_arbiter_flow.yaml" 58 | 59 | raise Exception(f"{role} is not existed.") 60 | -------------------------------------------------------------------------------- /iflearner/business/hetero/builder/model_builder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from abc import ABC, abstractmethod 17 | from iflearner.business.hetero.model.role import Role 18 | from iflearner.business.hetero.model.base_model import BaseModel 19 | 20 | 21 | class ModelBuilder(ABC): 22 | """Build a model instance base on the role you specify. 23 | """ 24 | 25 | @abstractmethod 26 | def create_role_model_instance(self, role: Role) -> BaseModel: 27 | """Create a model instance base on specific role. 28 | 29 | Args: 30 | role (Role): The role name. 31 | 32 | Returns: 33 | BaseModel: Return the base class. 34 | """ 35 | pass 36 | 37 | @abstractmethod 38 | def get_role_model_flow_file(self, role: Role) -> str: 39 | """Get model flow file by role name. 40 | 41 | Args: 42 | role (Role): The role name. 43 | 44 | Returns: 45 | str: Return the filename. 46 | """ 47 | pass 48 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/demo/demo_arbiter.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from loguru import logger 17 | from typing import List, Tuple 18 | from iflearner.business.hetero.model.role import Role 19 | from iflearner.business.hetero.model.base_model import BaseModel 20 | 21 | 22 | class DemoArbiter(BaseModel): 23 | """A demo showing the model development. 24 | """ 25 | 26 | def __init__(self) -> None: 27 | super().__init__() 28 | 29 | self._register_own_step("step1", self.handle_own_step1) 30 | self._register_own_step("step2", self.handle_own_step2) 31 | 32 | self._register_another_step( 33 | Role.host, "step2", self.handle_host_step2) 34 | 35 | def handle_own_step1(self) -> Tuple[str, bytes]: 36 | logger.info("Arbiter step1") 37 | return Role.guest, "Arbiter step1 completed.".encode("utf-8") 38 | 39 | def handle_own_step2(self): 40 | logger.info("Arbiter step2") 41 | return Role.guest, "Arbiter step2 completed.".encode("utf-8") 42 | 43 | def handle_host_step2(self, data: List[Tuple[str, bytes]]): 44 | for item in data: 45 | logger.info(f"{item[0]}, {item[1].decode('utf-8')}") 46 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/demo/demo_arbiter_flow.yaml: -------------------------------------------------------------------------------- 1 | role: arbiter 2 | steps: 3 | - name: step1 4 | upstreams: null 5 | - name: step2 6 | upstreams: 7 | - role: host 8 | step: step2 9 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/demo/demo_guest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from loguru import logger 17 | from typing import List, Tuple 18 | from iflearner.business.hetero.model.role import Role 19 | from iflearner.business.hetero.model.base_model import BaseModel 20 | 21 | 22 | class DemoGuest(BaseModel): 23 | """A demo showing the model development. 24 | """ 25 | 26 | def __init__(self) -> None: 27 | super().__init__() 28 | 29 | self._register_own_step("step1", self.handle_own_step1) 30 | self._register_own_step("step2", self.handle_own_step2) 31 | 32 | self._register_another_step( 33 | Role.host, "step1", self.handle_host_step1) 34 | self._register_another_step( 35 | Role.arbiter, "step1", self.handle_arbiter_step1) 36 | self._register_another_step( 37 | Role.arbiter, "step2", self.handle_arbiter_step2) 38 | 39 | def handle_own_step1(self) -> Tuple[str, bytes]: 40 | logger.info("Guest step1") 41 | return Role.host, "Guest step1 completed.".encode("utf-8") 42 | 43 | def handle_own_step2(self): 44 | logger.info("Guest step2") 45 | 46 | def handle_host_step1(self, data: List[Tuple[str, bytes]]): 47 | for item in data: 48 | logger.info(f"{item[0]}, {item[1].decode('utf-8')}") 49 | 50 | def handle_arbiter_step1(self, data: List[Tuple[str, bytes]]): 51 | for item in data: 52 | logger.info(f"{item[0]}, {item[1].decode('utf-8')}") 53 | 54 | def handle_arbiter_step2(self, data: List[Tuple[str, bytes]]): 55 | for item in data: 56 | logger.info(f"{item[0]}, {item[1].decode('utf-8')}") 57 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/demo/demo_guest_flow.yaml: -------------------------------------------------------------------------------- 1 | role: guest 2 | steps: 3 | - name: step1 4 | upstreams: 5 | - role: host 6 | step: step1 7 | - role: arbiter 8 | step: step1 9 | - name: step2 10 | upstreams: 11 | - role: arbiter 12 | step: step2 13 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/demo/demo_host.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from loguru import logger 17 | from typing import List, Tuple 18 | from iflearner.business.hetero.model.role import Role 19 | from iflearner.business.hetero.model.base_model import BaseModel 20 | 21 | 22 | class DemoHost(BaseModel): 23 | """A demo showing the model development. 24 | """ 25 | 26 | def __init__(self) -> None: 27 | super().__init__() 28 | 29 | self._register_own_step("step1", self.handle_own_step1) 30 | self._register_own_step("step2", self.handle_own_step2) 31 | 32 | self._register_another_step( 33 | Role.guest, "step1", self.handle_guest_step1) 34 | 35 | def handle_own_step1(self) -> Tuple[str, bytes]: 36 | logger.info("Host step1") 37 | return Role.guest, "Host step1 completed.".encode("utf-8") 38 | 39 | def handle_own_step2(self): 40 | logger.info("Host step2") 41 | return Role.arbiter, "Host step2 completed.".encode("utf-8") 42 | 43 | def handle_guest_step1(self, data: List[Tuple[str, bytes]]): 44 | for item in data: 45 | logger.info(f"{item[0]}, {item[1].decode('utf-8')}") 46 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/demo/demo_host_flow.yaml: -------------------------------------------------------------------------------- 1 | role: host 2 | steps: 3 | - name: step1 4 | upstreams: null 5 | - name: step2 6 | upstreams: 7 | - role: guest 8 | step: step1 9 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/logistic_regression/dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import numpy as np 17 | from sklearn.preprocessing import StandardScaler 18 | from sklearn.datasets import load_breast_cancer 19 | from sklearn.model_selection import train_test_split 20 | 21 | 22 | def load_data(): 23 | breast = load_breast_cancer() 24 | x_train, x_test, y_train, y_test = train_test_split( 25 | breast.data, breast.target, random_state=1) 26 | std = StandardScaler() 27 | x_train = std.fit_transform(x_train) 28 | x_test = std.transform(x_test) 29 | return x_train, y_train, x_test, y_test 30 | 31 | 32 | def vertically_partition_data(X, X_test, A_idx, B_idx): 33 | XA = X[:, A_idx] 34 | XB = X[:, B_idx] 35 | # print(X.shape[0], np.ones(X.shape[0])) 36 | # print(X.shape[1], np.ones(X.shape[1])) 37 | XB = np.c_[np.ones(X.shape[0]), XB] 38 | XA_test = X_test[:, A_idx] 39 | XB_test = X_test[:, B_idx] 40 | XB_test = np.c_[np.ones(XB_test.shape[0]), XB_test] 41 | return XA, XB, XA_test, XB_test 42 | 43 | 44 | def get_guest_data(): 45 | x, y, x_test, y_test = load_data() 46 | XA, XB, XA_test, XB_test = vertically_partition_data(x, x_test, [ 47 | 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) 48 | return XB, y 49 | 50 | 51 | def get_host_data(): 52 | x, y, x_test, y_test = load_data() 53 | XA, XB, XA_test, XB_test = vertically_partition_data(x, x_test, [ 54 | 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) 55 | return XA 56 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/logistic_regression/lr_arbiter_flow.yaml: -------------------------------------------------------------------------------- 1 | role: arbiter 2 | init_steps: 3 | - name: generate_he_keypair 4 | upstreams: null 5 | steps: 6 | - name: decrypt_guest_data 7 | upstreams: 8 | - role: guest 9 | step: calc_final_result_with_host 10 | - name: decrypt_host_data 11 | upstreams: 12 | - role: host 13 | step: calc_final_result_with_guest 14 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/logistic_regression/lr_guest_flow.yaml: -------------------------------------------------------------------------------- 1 | role: guest 2 | init_steps: 3 | - name: get_he_public_key 4 | upstreams: 5 | - role: arbiter 6 | step: generate_he_keypair 7 | virtual: true 8 | steps: 9 | - name: calc_guest_partial_result 10 | upstreams: null 11 | - name: calc_final_result_with_host 12 | upstreams: 13 | - role: host 14 | step: calc_host_partial_result 15 | - name: update_weights 16 | upstreams: 17 | - role: arbiter 18 | step: decrypt_guest_data 19 | virtual: true 20 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/logistic_regression/lr_host_flow.yaml: -------------------------------------------------------------------------------- 1 | role: host 2 | init_steps: 3 | - name: get_he_public_key 4 | upstreams: 5 | - role: arbiter 6 | step: generate_he_keypair 7 | virtual: true 8 | steps: 9 | - name: calc_host_partial_result 10 | upstreams: null 11 | - name: calc_final_result_with_guest 12 | upstreams: 13 | - role: guest 14 | step: calc_guest_partial_result 15 | - name: update_weights 16 | upstreams: 17 | - role: arbiter 18 | step: decrypt_host_data 19 | virtual: true 20 | -------------------------------------------------------------------------------- /iflearner/business/hetero/model/role.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from abc import ABC, abstractmethod 17 | 18 | 19 | class Role(ABC): 20 | 21 | @abstractmethod 22 | def __str__(self) -> str: 23 | pass 24 | 25 | 26 | class Guest(Role): 27 | def __str__(self) -> str: 28 | return "guest" 29 | 30 | 31 | class Host(Role): 32 | def __str__(self) -> str: 33 | return "host" 34 | 35 | 36 | class Arbiter(Role): 37 | def __str__(self) -> str: 38 | return "arbiter" 39 | 40 | 41 | guest = Guest() 42 | host = Host() 43 | arbiter = Arbiter() 44 | 45 | 46 | def role_class(name: str) -> Role: 47 | if name == str(guest): 48 | return guest 49 | elif name == str(host): 50 | return host 51 | elif name == str(arbiter): 52 | return arbiter 53 | else: 54 | raise Exception(f"Role {name} is not existed.") 55 | -------------------------------------------------------------------------------- /iflearner/business/hetero/task.yaml: -------------------------------------------------------------------------------- 1 | guest: 2 | - name: guest-1 3 | addr: 127.0.0.1:50050 4 | host: 5 | - name: host-1 6 | addr: 127.0.0.1:50051 7 | arbiter: 8 | - name: arbiter-1 9 | addr: 127.0.0.1:50052 10 | -------------------------------------------------------------------------------- /iflearner/business/homo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | ARG IFLEARNER_DIR=/iflearner 4 | COPY requirements.txt $IFLEARNER_DIR/ 5 | COPY iflearner $IFLEARNER_DIR/iflearner 6 | COPY examples $IFLEARNER_DIR/examples 7 | ENV PYTHONPATH=$IFLEARNER_DIR 8 | WORKDIR $IFLEARNER_DIR 9 | RUN pip install -r requirements.txt -i https://pypi.douban.com/simple 10 | RUN pip install torch==1.8.1+cpu torchvision==0.9.1+cpu -f https://download.pytorch.org/whl/torch_stable.html 11 | -------------------------------------------------------------------------------- /iflearner/business/homo/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /iflearner/business/homo/argument.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import argparse 16 | 17 | parser = argparse.ArgumentParser() 18 | 19 | parser.add_argument( 20 | "--name", 21 | default="client", 22 | type=str, 23 | help="name of client", 24 | ) 25 | 26 | parser.add_argument( 27 | "--epochs", 28 | default=10, 29 | type=int, 30 | help="number of total epochs to run", 31 | ) 32 | 33 | parser.add_argument( 34 | "--server", 35 | default="localhost:50001", 36 | type=str, 37 | help="address of aggerating server", 38 | ) 39 | 40 | parser.add_argument( 41 | "--cert", 42 | default=None, 43 | type=str, 44 | help="path of server SSL cert" 45 | """use secure channel to connect to server if not none""", 46 | ) 47 | 48 | parser.add_argument( 49 | "--enable-ll", 50 | default=0, 51 | type=int, 52 | help="enable local training (1 | 0)", 53 | ) 54 | 55 | parser.add_argument( 56 | "--peers", 57 | default=None, 58 | type=str, 59 | help="enabled SMPC if the argument had specified " 60 | """all clients' addresses and use semicolon separate all addresses """ 61 | "First one is your own address. ", 62 | ) 63 | 64 | parser.add_argument( 65 | "--peer-cert", 66 | default=None, 67 | type=str, 68 | help="path of party SSL cert" 69 | """use secure channel to connect to other parties if not none""", 70 | ) 71 | -------------------------------------------------------------------------------- /iflearner/business/homo/mxnet_trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from typing import Dict 16 | 17 | import mxnet as mx 18 | import numpy as np 19 | import numpy.typing as npt 20 | 21 | from iflearner.business.homo.trainer import Trainer 22 | 23 | 24 | class MxnetTrainer(Trainer): 25 | """implement the 'get' and 'set' function for the usual mxnet trainer.""" 26 | 27 | def __init__(self, model: mx.gluon.nn.Sequential) -> None: 28 | self._model = model 29 | 30 | def get( 31 | self, param_type: Trainer.ParameterType = Trainer.ParameterType.ParameterModel 32 | ) -> Dict[str, npt.NDArray[np.float32]]: # type: ignore 33 | """get parameters form the client, maybe the model parameter or 34 | gradient. 35 | 36 | Args: 37 | param_type: Param_type is ParameterModel or ParameterGradient, default is ParameterModel. 38 | 39 | Returns: 40 | dict, k: str (the parameter name), v: np.ndarray (the parameter value) 41 | """ 42 | parameters = dict() 43 | for key, val in self._model.collect_params(".*weight").items(): 44 | p = val.data().asnumpy() 45 | parameters[key] = p 46 | return parameters 47 | 48 | def set( 49 | self, 50 | parameters: Dict[str, npt.NDArray[np.float32]], # type: ignore 51 | param_type: Trainer.ParameterType = Trainer.ParameterType.ParameterModel, 52 | ) -> None: 53 | """set parameters to the client, maybe the model parameter or gradient. 54 | 55 | Args: 56 | parameters: Parameters is the same as the return of 'get' function. 57 | param_type: Param_type is ParameterModel or ParameterGradient, default is ParameterModel. 58 | 59 | Returns: None 60 | """ 61 | for key, value in parameters.items(): 62 | self._model.collect_params().setattr(key, value) 63 | -------------------------------------------------------------------------------- /iflearner/business/homo/sklearn_trainer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from typing import Any, Dict 16 | 17 | import numpy as np 18 | import numpy.typing as npt 19 | 20 | from iflearner.business.homo.trainer import Trainer 21 | 22 | 23 | class SklearnTrainer(Trainer): 24 | """implement the 'get' and 'set' function for the usual sklearn trainer.""" 25 | 26 | def __init__(self, model: Any) -> None: 27 | self._model: Any = model 28 | super().__init__() 29 | 30 | def get( 31 | self, param_type: Trainer.ParameterType = Trainer.ParameterType.ParameterModel 32 | ) -> Dict[str, npt.NDArray[np.float32]]: # type: ignore 33 | """get parameters form the client, maybe the model parameter or 34 | gradient. 35 | 36 | Args: 37 | param_type: Param_type is ParameterModel or ParameterGradient, default is ParameterModel. 38 | 39 | Returns: 40 | dict, k: str (the parameter name), v: np.ndarray (the parameter value) 41 | """ 42 | pass 43 | 44 | def set( 45 | self, 46 | parameters: Dict[str, npt.NDArray[np.float32]], # type: ignore 47 | param_type: Trainer.ParameterType = Trainer.ParameterType.ParameterModel, 48 | ) -> None: 49 | """set parameters to the client, maybe the model parameter or gradient. 50 | 51 | Args: 52 | parameters: Parameters is the same as the return of 'get' function. 53 | param_type: Param_type is ParameterModel or ParameterGradient, default is ParameterModel. 54 | 55 | Returns: None 56 | """ 57 | pass 58 | -------------------------------------------------------------------------------- /iflearner/business/homo/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from .fedavg_server import FedavgServer 16 | from .fedopt_server import FedoptServer 17 | from .opt import * 18 | from .qfedavg_server import qFedavgServer 19 | 20 | __all__ = [ 21 | "FedoptServer", 22 | "FedavgServer", 23 | "FedAdam", 24 | "FedYogi", 25 | "FedAdagrad", 26 | "qFedavgServer", 27 | ] 28 | -------------------------------------------------------------------------------- /iflearner/business/homo/strategy/fednova_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from iflearner.business.homo.strategy import strategy_client 16 | 17 | 18 | class FedNovaClient(strategy_client.StrategyClient): 19 | """Implement the strategy of fednova on client side.""" 20 | 21 | def __init__(self) -> None: 22 | super().__init__() 23 | -------------------------------------------------------------------------------- /iflearner/business/homo/strategy/fedopt_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from iflearner.business.homo.strategy import strategy_client 16 | 17 | 18 | class FedoptClient(strategy_client.StrategyClient): 19 | """Implement the strategy of fedopt on client side.""" 20 | 21 | def __init__(self) -> None: 22 | super().__init__() 23 | -------------------------------------------------------------------------------- /iflearner/business/homo/strategy/opt/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from .fedadagrad import FedAdagrad 16 | from .fedadam import FedAdam 17 | from .fedavgm import FedAvgm 18 | from .fedyogi import FedYogi 19 | 20 | __all__ = ["FedAdagrad", "FedAdam", "FedYogi", "FedAvgm"] 21 | -------------------------------------------------------------------------------- /iflearner/business/homo/strategy/opt/fedopt.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from typing import Dict, Tuple 16 | 17 | import numpy as np 18 | import numpy.typing as npt 19 | 20 | 21 | class FedOpt: 22 | """Implementation based on https://arxiv.org/abs/2003.00295. 23 | 24 | Attributes: 25 | learning_rate (float, optional): learning rate. Defaults to 0.1. 26 | betas (Tuple[float, float], optional): coefficients used for computing 27 | running averages of gradient and its square. Defaults to (0.9, 0.999). 28 | t (float, optional): adaptivity parameter. Defaults to 0.001. 29 | """ 30 | 31 | def __init__( 32 | self, 33 | learning_rate: float = 0.1, 34 | betas: Tuple[float, float] = (0.9, 0.999), 35 | t: float = 0.001, 36 | ) -> None: 37 | self._lr = learning_rate 38 | self._beta1 = betas[0] 39 | self._beta2 = betas[1] 40 | self._adaptivity = t 41 | self._params: dict = {} 42 | 43 | def step( 44 | self, 45 | pseudo_gradient: Dict[str, npt.NDArray[np.float32]], 46 | ) -> Dict[str, npt.NDArray[np.float32]]: 47 | """a step to optimize parameters of server model with pseudo gradient. 48 | 49 | Args: 50 | pseudo_gradient (Dict[str, npt.NDArray[np.float32]]): the pseudo gradient of server model 51 | 52 | Returns: 53 | Dict[str, npt.NDArray[np.float32]]: parameters of server model after step 54 | """ 55 | pass 56 | 57 | def set_params(self, params): 58 | """set params to self._params. 59 | 60 | Args: 61 | params (_type_): parameters of server model 62 | """ 63 | self._params = params 64 | -------------------------------------------------------------------------------- /iflearner/business/homo/strategy/qfedavg_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from iflearner.business.homo.strategy import strategy_client 16 | 17 | 18 | class qFedavgClient(strategy_client.StrategyClient): 19 | """Implement the strategy of qfedavg on client side.""" 20 | 21 | def __init__(self) -> None: 22 | super().__init__() 23 | -------------------------------------------------------------------------------- /iflearner/business/mpc/piss/argument.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import argparse 16 | 17 | parser = argparse.ArgumentParser() 18 | 19 | 20 | parser.add_argument( 21 | "--server", 22 | default="127.0.0.1:15554", 23 | type=str, 24 | help="address of aggerating server", 25 | ) 26 | 27 | parser.add_argument( 28 | "--name", 29 | default="client003", 30 | type=str, 31 | help="party name of client", 32 | ) 33 | 34 | parser.add_argument( 35 | "--addr", 36 | default="127.0.0.1:45553", 37 | type=str, 38 | help="address of client service", 39 | ) 40 | 41 | parser.add_argument( 42 | "--data", 43 | default='examples/mpc/quickstart_piss/piss_data_test.csv', 44 | type=str, 45 | help="path of data") 46 | 47 | parser.add_argument( 48 | "--cert", 49 | default=None, 50 | type=str, 51 | help="path of server SSL cert" 52 | """use secure channel to connect to server if not none""" 53 | ) 54 | 55 | -------------------------------------------------------------------------------- /iflearner/business/mpc/piss/piss_aggregate_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import argparse 16 | from importlib import import_module 17 | from typing import Any, Dict, Union 18 | 19 | from iflearner.communication.mpc.piss import piss_server 20 | from iflearner.communication.base import base_server 21 | 22 | class PissAggregateServer: 23 | 24 | def __init__(self, addr: str ,party_name: str) -> None: 25 | self._addr = addr 26 | self._party_name = party_name 27 | self._piss_server_inst = piss_server.PissServer(party_name= party_name) 28 | 29 | def run(self) -> None: 30 | """start piss server""" 31 | base_server.start_server(self._addr,self._piss_server_inst) 32 | 33 | def main(): 34 | parser = argparse.ArgumentParser() 35 | 36 | 37 | parser.add_argument( 38 | "--addr", help="the server address", default="127.0.0.1:20001", type=str 39 | ) 40 | parser.add_argument( 41 | "--name", help="the server name", default="server", type=str 42 | ) 43 | args = parser.parse_args() 44 | 45 | global server 46 | server = PissAggregateServer(addr = args.addr, party_name= args.name) 47 | server.run() 48 | 49 | if __name__ == "__main__": 50 | main() 51 | 52 | -------------------------------------------------------------------------------- /iflearner/business/mpc/piss/piss_startegy_base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from abc import ABC 17 | from typing import Any, Dict 18 | import numpy as np 19 | 20 | from importlib import import_module 21 | from typing import Any, Dict, Union 22 | from loguru import logger 23 | import grpc 24 | from iflearner.communication.mpc import piss 25 | from iflearner.communication.base import base_pb2_grpc 26 | 27 | 28 | class PissStrategyBase(ABC): 29 | def __init__(self ,cert_path: str, party_name: str,options) -> None: 30 | 31 | self._cert_path = cert_path 32 | self._party_name = party_name 33 | self._options = options 34 | 35 | self._routes: dict = dict() 36 | self._stubs: dict = dict() 37 | self._party_name_list = [] 38 | 39 | self._initiator_party_name: str = str() 40 | self._initiator_route: str = str() 41 | self._initiator_stub = None 42 | 43 | def generate_stub(self, destination_addr: str): 44 | 45 | if self._cert_path is None: 46 | channel = grpc.insecure_channel(destination_addr, options = self._options) 47 | else: 48 | with open(self._cert_path, "rb") as f: 49 | cert_bytes = f.read() 50 | 51 | channel = grpc.secure_channel( 52 | destination_addr, grpc.ssl_channel_credentials(cert_bytes), options = self._options 53 | ) 54 | stub = base_pb2_grpc.BaseStub(channel) 55 | return stub 56 | -------------------------------------------------------------------------------- /iflearner/business/util/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from .metric_dev import ( 16 | AccuracyMetric, 17 | BaseMetric, 18 | F1Metric, 19 | LossMetric, 20 | Metrics, 21 | Scalar, 22 | TrainType, 23 | ) 24 | 25 | __all__ = [ 26 | "Metrics", 27 | "BaseMetric", 28 | "LossMetric", 29 | "AccuracyMetric", 30 | "F1Metric", 31 | ] 32 | -------------------------------------------------------------------------------- /iflearner/business/util/metric.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from typing import Any, Dict 16 | from visualdl import LogWriter 17 | 18 | 19 | class Metric: 20 | """Integrate visualdl to visualize metrics. 21 | """ 22 | 23 | def __init__(self, logdir: str) -> None: 24 | """Init class with log directory.""" 25 | 26 | self._tag_prefix = "train" 27 | self._logdir = logdir 28 | self._writers: Dict[str, LogWriter] = dict() 29 | self._figs: Dict[str, Any] = dict() 30 | 31 | def add(self, name: str, label: str, x: Any, y: Any) -> None: 32 | """Add a point. 33 | 34 | Args: 35 | name: The name of metric, eg: acc, loss. 36 | label: The label of metric, eg: local learning, federated learning. 37 | x: The x of point, eg: 1, 2, 3... 38 | y: The y of point, eg: 95.5, 96.0, 96.5... 39 | """ 40 | 41 | if label not in self._writers: 42 | self._writers[label] = LogWriter( 43 | logdir=f"{self._logdir}/{label}", display_name=label) 44 | 45 | self._writers[label].add_scalar(f"{self._tag_prefix}/{name}", y, x) 46 | -------------------------------------------------------------------------------- /iflearner/communication/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /iflearner/communication/base/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /iflearner/communication/base/base.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | service Base { 4 | // Use this function to transport information synchronously. 5 | rpc send (BaseRequest) returns (BaseResponse) {} 6 | 7 | // Use this function to transport information asynchronously. 8 | rpc post (BaseRequest) returns (BaseResponse) {} 9 | 10 | // Use this function to wait for server information. 11 | rpc callback (BaseRequest) returns (BaseResponse) {} 12 | } 13 | 14 | // Base information for all rpc request. 15 | message BaseRequest { 16 | string party_name = 1; 17 | string type = 2; // The type of data. 18 | bytes data = 3; 19 | } 20 | 21 | // Base information for all rpc response. 22 | message BaseResponse { 23 | int32 code = 1; 24 | string message = 2; 25 | string type = 3; // The type of data. 26 | bytes data = 4; 27 | } 28 | 29 | 30 | -------------------------------------------------------------------------------- /iflearner/communication/base/base_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import grpc 16 | 17 | from iflearner.communication.base import base_pb2, base_pb2_grpc, constant 18 | 19 | 20 | class BaseClient: 21 | """Provides methods that implement functionality of base client.""" 22 | 23 | def __init__(self, addr: str, cert_path: str = None) -> None: 24 | options = [ 25 | ("grpc.max_message_length", constant.MAX_MSG_LENGTH), 26 | ("grpc.max_send_message_length", constant.MAX_MSG_LENGTH), 27 | ("grpc.max_receive_message_length", constant.MAX_MSG_LENGTH), 28 | ] 29 | 30 | if cert_path is None: 31 | channel = grpc.insecure_channel(addr, options=options) 32 | else: 33 | with open(cert_path, "rb") as f: 34 | cert_bytes = f.read() 35 | 36 | channel = grpc.secure_channel( 37 | addr, grpc.ssl_channel_credentials(cert_bytes), options=options 38 | ) 39 | 40 | self._stub: base_pb2_grpc.BaseStub = base_pb2_grpc.BaseStub(channel) 41 | 42 | def _send(self, req: base_pb2.BaseRequest) -> base_pb2.BaseResponse: 43 | """Call send function.""" 44 | 45 | resp = self._stub.send(req) 46 | return resp 47 | 48 | def _post(self, req: base_pb2.BaseRequest) -> base_pb2.BaseResponse: 49 | """Call post function.""" 50 | 51 | resp = self._stub.post(req) 52 | return resp 53 | 54 | def _callback(self, req: base_pb2.BaseRequest) -> base_pb2.BaseResponse: 55 | """Call callback function.""" 56 | 57 | resp = self._stub.callback(req) 58 | return resp 59 | -------------------------------------------------------------------------------- /iflearner/communication/base/base_client_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import unittest 16 | 17 | from iflearner.communication.base import base_client, base_pb2 18 | 19 | 20 | class TestBaseClient(unittest.TestCase): 21 | def test_send(self) -> None: 22 | client = base_client.BaseClient("localhost:50051") 23 | print(client._send(base_pb2.BaseRequest())) 24 | -------------------------------------------------------------------------------- /iflearner/communication/base/base_exception.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | class BaseException(Exception): 16 | """Define the base exception class. 17 | 18 | You can inherit the base class and implement your business logic. 19 | 20 | Attributes: 21 | code: error code 22 | message: error details 23 | """ 24 | 25 | def __init__(self, code: int, message: str) -> None: 26 | self.code = code 27 | self.message = message 28 | 29 | def __str__(self) -> str: 30 | return f"code: {self.code}, message: {self.message}" 31 | -------------------------------------------------------------------------------- /iflearner/communication/base/base_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: base.proto 4 | """Generated protocol buffer code.""" 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import descriptor_pool as _descriptor_pool 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | 16 | 17 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nbase.proto\"=\n\x0b\x42\x61seRequest\x12\x12\n\nparty_name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"I\n\x0c\x42\x61seResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x32\x7f\n\x04\x42\x61se\x12%\n\x04send\x12\x0c.BaseRequest\x1a\r.BaseResponse\"\x00\x12%\n\x04post\x12\x0c.BaseRequest\x1a\r.BaseResponse\"\x00\x12)\n\x08\x63\x61llback\x12\x0c.BaseRequest\x1a\r.BaseResponse\"\x00\x62\x06proto3') 18 | 19 | 20 | 21 | _BASEREQUEST = DESCRIPTOR.message_types_by_name['BaseRequest'] 22 | _BASERESPONSE = DESCRIPTOR.message_types_by_name['BaseResponse'] 23 | BaseRequest = _reflection.GeneratedProtocolMessageType('BaseRequest', (_message.Message,), { 24 | 'DESCRIPTOR' : _BASEREQUEST, 25 | '__module__' : 'base_pb2' 26 | # @@protoc_insertion_point(class_scope:BaseRequest) 27 | }) 28 | _sym_db.RegisterMessage(BaseRequest) 29 | 30 | BaseResponse = _reflection.GeneratedProtocolMessageType('BaseResponse', (_message.Message,), { 31 | 'DESCRIPTOR' : _BASERESPONSE, 32 | '__module__' : 'base_pb2' 33 | # @@protoc_insertion_point(class_scope:BaseResponse) 34 | }) 35 | _sym_db.RegisterMessage(BaseResponse) 36 | 37 | _BASE = DESCRIPTOR.services_by_name['Base'] 38 | if _descriptor._USE_C_DESCRIPTORS == False: 39 | 40 | DESCRIPTOR._options = None 41 | _BASEREQUEST._serialized_start=14 42 | _BASEREQUEST._serialized_end=75 43 | _BASERESPONSE._serialized_start=77 44 | _BASERESPONSE._serialized_end=150 45 | _BASE._serialized_start=152 46 | _BASE._serialized_end=279 47 | # @@protoc_insertion_point(module_scope) 48 | -------------------------------------------------------------------------------- /iflearner/communication/base/base_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from abc import ABC, abstractmethod 16 | from concurrent import futures 17 | from typing import Any 18 | 19 | import grpc 20 | 21 | from iflearner.communication.base import base_pb2, base_pb2_grpc, constant 22 | 23 | 24 | class BaseServer(base_pb2_grpc.BaseServicer, ABC): 25 | """Provides methods that implement functionality of base server.""" 26 | 27 | @abstractmethod 28 | def send(self, request: base_pb2.BaseRequest, context: Any) -> None: 29 | pass 30 | 31 | @abstractmethod 32 | def post(self, request: base_pb2.BaseRequest, context: Any) -> None: 33 | pass 34 | 35 | @abstractmethod 36 | def callback(self, request: base_pb2.BaseRequest, context: Any) -> None: 37 | pass 38 | 39 | 40 | def start_server(addr: str, servicer: BaseServer) -> None: 41 | """Start server at the address.""" 42 | server = grpc.server( 43 | futures.ThreadPoolExecutor(max_workers=10), 44 | options=[ 45 | ("grpc.max_message_length", constant.MAX_MSG_LENGTH), 46 | ("grpc.max_send_message_length", constant.MAX_MSG_LENGTH), 47 | ("grpc.max_receive_message_length", constant.MAX_MSG_LENGTH), 48 | ], 49 | ) 50 | base_pb2_grpc.add_BaseServicer_to_server(servicer, server) 51 | server.add_insecure_port(addr) 52 | server.start() 53 | server.wait_for_termination() 54 | -------------------------------------------------------------------------------- /iflearner/communication/base/base_server_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import unittest 16 | 17 | from iflearner.communication.base import base_server 18 | 19 | 20 | class TestBaseServer(unittest.TestCase): 21 | def test_serve(self) -> None: 22 | server = base_server.BaseServer("[::]:50051") # type: ignore 23 | server.serve() # type: ignore 24 | -------------------------------------------------------------------------------- /iflearner/communication/base/constant.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | MAX_MSG_LENGTH = 1024 * 1024 * 1024 16 | -------------------------------------------------------------------------------- /iflearner/communication/build.sh: -------------------------------------------------------------------------------- 1 | python -m grpc_tools.protoc --proto_path=base --python_out=base --grpc_python_out=base base.proto 2 | sed -i 's/^import .*_pb2 as/from iflearner.communication.base \0/' base/base_pb2_grpc.py 3 | 4 | python -m grpc_tools.protoc --proto_path=homo --python_out=homo --grpc_python_out=homo homo.proto 5 | -------------------------------------------------------------------------------- /iflearner/communication/hetero/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /iflearner/communication/hetero/hetero.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message CommonRequest { 4 | } 5 | -------------------------------------------------------------------------------- /iflearner/communication/hetero/hetero_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from loguru import logger 17 | from iflearner.communication.base import base_client, base_pb2 18 | 19 | 20 | class HeteroClient(base_client.BaseClient): 21 | def __init__(self, addr: str, party_name: str, cert_path: str = None) -> None: 22 | super().__init__(addr, cert_path) 23 | self._party_name = party_name 24 | 25 | def post(self, type: str, data: bytes): 26 | """Post a specific type of data to the server. 27 | 28 | Args: 29 | type (str): Sign the data with type. 30 | data (bytes): The binary data. 31 | """ 32 | req = base_pb2.BaseRequest( 33 | party_name=self._party_name, type=type, data=data) 34 | self._post(req) 35 | -------------------------------------------------------------------------------- /iflearner/communication/hetero/hetero_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import pickle 17 | from loguru import logger 18 | from typing import Any, Dict, Tuple, List 19 | from iflearner.communication.base import base_pb2, base_server 20 | 21 | 22 | class HeteroServer(base_server.BaseServer): 23 | """Implement the server which saves the client requests. 24 | 25 | Attributes: 26 | messages (Dict[str, Dict[str, bytes]]): Save the client requests. 27 | """ 28 | 29 | def __init__(self, party_with_role: Dict[str, str]) -> None: 30 | self.messages: Dict[str, Dict[str, bytes]] = dict() 31 | self._party_with_role = party_with_role 32 | 33 | def send(self, request: base_pb2.BaseRequest, context: Any) -> base_pb2.BaseResponse: 34 | pass 35 | 36 | def callback(self, request: base_pb2.BaseRequest, context: Any) -> base_pb2.BaseResponse: 37 | pass 38 | 39 | def post(self, request: base_pb2.BaseRequest, context: Any) -> base_pb2.BaseResponse: 40 | """Save the received data according to the party name and type. 41 | 42 | Args: 43 | request (base_pb2.BaseRequest): Request details. 44 | context (Any): No use. 45 | 46 | Returns: 47 | base_pb2.BaseResponse: Empty response. 48 | """ 49 | role = self._party_with_role[request.party_name] 50 | logger.info( 51 | f"Receive message, role: {role}, party: {request.party_name}, step: {request.type}, data length: {len(request.data)}") 52 | 53 | data = pickle.loads(request.data) 54 | if f"{role}.{request.type}" in self.messages: 55 | self.messages[f"{role}.{request.type}"][request.party_name] = data 56 | else: 57 | self.messages[f"{role}.{request.type}"] = { 58 | request.party_name: data} 59 | return base_pb2.BaseResponse() 60 | -------------------------------------------------------------------------------- /iflearner/communication/homo/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /iflearner/communication/homo/homo.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message RegistrationResponse { 4 | string strategy = 1; // The name of strategy. 5 | map parameters = 2; // All parameters of client 6 | } 7 | 8 | // The request of MSG_UPLOAD_PARAM. 9 | message UploadParam { 10 | int32 epoch = 1; // The epoch of training. 11 | map parameters = 2; // All parameters of client. 12 | map metrics = 3; // The client metrics. 13 | } 14 | 15 | // The response of MSG_AGGREGATE_RESULT. 16 | message AggregateResult { 17 | map parameters = 1; // Aggregated parameter results. 18 | } 19 | 20 | message Parameter { 21 | repeated int32 shape = 1; 22 | repeated double values = 2; 23 | bytes custom_values = 3; 24 | } 25 | 26 | message RegistrationInfo { 27 | int32 sample_num = 1; 28 | int32 step_num = 2; 29 | } 30 | -------------------------------------------------------------------------------- /iflearner/communication/homo/homo_exception.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from enum import IntEnum, auto 16 | 17 | from iflearner.communication.base import base_exception 18 | 19 | 20 | class HomoException(base_exception.BaseException): 21 | class HomoResponseCode(IntEnum): 22 | """Define response code""" 23 | 24 | BadRequest = auto() 25 | Unauthorized = auto() 26 | Forbidden = auto() 27 | Conflict = auto() 28 | InternalError = auto() 29 | 30 | def __init__(self, code: HomoResponseCode, message: str) -> None: 31 | super().__init__( 32 | code.value, f"{code.__class__.__name__}.{code.name} - {message}" 33 | ) 34 | -------------------------------------------------------------------------------- /iflearner/communication/homo/homo_exception_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from iflearner.communication.homo.homo_exception import HomoException 16 | 17 | if __name__ == "__main__": 18 | try: 19 | raise HomoException(HomoException.HomoResponseCode.InternalError, "abc") 20 | except HomoException as e: 21 | print(e) 22 | -------------------------------------------------------------------------------- /iflearner/communication/homo/homo_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | -------------------------------------------------------------------------------- /iflearner/communication/homo/message_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Define the heartbeat interval between client and server.""" 16 | MSG_HEARTBEAT_INTERVAL = 1 17 | 18 | """Define the message type for communication. (From client to server)""" 19 | MSG_REGISTER = "msg_register" 20 | MSG_CLIENT_READY = "msg_client_ready" 21 | MSG_UPLOAD_PARAM = "msg_upload_param" 22 | MSG_COMPLETE = "msg_complete" 23 | 24 | """Define the message type for communication. (From server to client)""" 25 | MSG_AGGREGATE_RESULT = "msg_aggregate_result" 26 | MSG_NOTIFY_TRAINING = "msg_notify_training" 27 | 28 | """Define the name of strategy.""" 29 | STRATEGY_FEDAVG = "FedAvg" 30 | STRATEGY_SCAFFOLD = "Scaffold" 31 | STRATEGY_STC = "STC" 32 | STRATEGY_FEDOPT = "FedOpt" 33 | STRATEGY_qFEDAVG = "qFedAvg" 34 | STRATEGY_FEDNOVA = "FedNova" 35 | -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # 隐私信息统计求和(private information statistical summation) -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/message_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Define the message type for communication. (From client to server)""" 16 | MSG_REGISTER = "msg_register" 17 | MSG_START_QUERY = "msg_start_query" 18 | MSG_PARTICIPANTS_READY = "msg_participants_ready" 19 | 20 | """Define the message type for communication. (From server to client)""" 21 | MSG_PARTICIPANTS_ROUTES = "msg_participants_routes" 22 | 23 | """Define the message type for communication. (Between clients)""" 24 | MSG_SHARE_ENCRYPTION_PARAM = "msg_share_encryption_parameters" 25 | MSG_SHARE_ENCRYPTED_SECRETS = "msg_share_encrypted_secrets" 26 | MSG_RETURN_ENCRYPTED_DATA_SUM = "msg_return_encrypted_data_sum" 27 | 28 | MSG_INIT_DATA = "msg_init_data" 29 | MSG_GET_SUM_SECRETS = "msg_get_sum_secrets" 30 | MSG_END_QUERY = "msg_end_query" 31 | 32 | """Define the heartbeat interval between client and client service.""" 33 | MSG_HEARTBEAT_INTERVAL = 10 34 | 35 | """Define the heartbeat interval of Sum time.""" 36 | MSG_SUM_TIME = 2 37 | 38 | MSG_INITIATOR = "msg_initiator" 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/piss.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pissPKG; 3 | 4 | message RegistrationInfo { 5 | string route = 1; 6 | } 7 | 8 | message InitData { 9 | string data_path = 1; 10 | } 11 | 12 | //from server to all clients 13 | message ParticipantsRoutes{ 14 | map routes = 1; 15 | string initiator_route = 2; 16 | string initiator_party_name = 3; 17 | } 18 | 19 | //from init to clients 20 | message ShareEncryptionParam{ 21 | map encryption_param = 1; 22 | string route = 2; 23 | } 24 | 25 | //form client to clients 26 | message ShareEncryptedSecrets{ 27 | map sub_keys = 1; 28 | map commitments = 2; 29 | string recv_party_name = 3; 30 | } 31 | 32 | message SubSecretsSUM{ 33 | map sub_keys_sum= 1; 34 | } 35 | 36 | message SecretsSUM{ 37 | string secrets_sum= 1; 38 | } 39 | 40 | message CallBack{ 41 | string call_back_msg = 1; 42 | } 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/piss_base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from email import message 16 | from operator import imod 17 | from tkinter import NO 18 | from typing import Any 19 | from loguru import logger 20 | from iflearner.communication.base import base_pb2,base_server,constant 21 | 22 | 23 | class PissBase(base_server.BaseServer): 24 | def __init__(self, party_name: str,route: str = None,cert_path: str = None) -> None: 25 | self._options = [ 26 | ("grpc.max_message_length", constant.MAX_MSG_LENGTH), 27 | ("grpc.max_send_message_length", constant.MAX_MSG_LENGTH), 28 | ("grpc.max_receive_message_length", constant.MAX_MSG_LENGTH), 29 | ] 30 | self._party_name = party_name 31 | self._route = route 32 | self._cert_path = cert_path 33 | 34 | def _send(self, stub, req: base_pb2.BaseRequest) -> base_pb2.BaseResponse: 35 | """Call send function.""" 36 | 37 | resp = stub.send(req) 38 | return resp 39 | 40 | def _post(self, stub, req: base_pb2.BaseRequest) -> base_pb2.BaseResponse: 41 | """Call post function.""" 42 | 43 | resp = stub.post(req) 44 | return resp 45 | 46 | def _callback(self, stub, req: base_pb2.BaseRequest) -> base_pb2.BaseResponse: 47 | """Call callback function.""" 48 | 49 | resp = stub.callback(req) 50 | return resp -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/piss_exception.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from enum import IntEnum, auto 16 | 17 | from iflearner.communication.base import base_exception 18 | 19 | 20 | class PissException(base_exception.BaseException): 21 | class PissResponseCode(IntEnum): 22 | """Define response code""" 23 | 24 | VerifySubkeyFailed = auto() 25 | VerifySumkeyFailed = auto() 26 | NotInitiator = auto() 27 | AlreadyExistsPartyName = auto() 28 | Unregistered = auto() 29 | InsufficientParty = auto() 30 | IllegalQuery = auto() 31 | InternalError = auto() 32 | 33 | def __init__(self, code: PissResponseCode, message: str) -> None: 34 | super().__init__( 35 | code.value, f"{code.__class__.__name__}.{code.name} - {message}" 36 | ) 37 | -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/piss_exception_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from iflearner.communication.mpc.piss.piss_exception import PissException 16 | 17 | if __name__ == "__main__": 18 | try: 19 | raise PissException(PissException.PissResponseCode.InternalError, "abc") 20 | except PissException as e: 21 | print(e) 22 | -------------------------------------------------------------------------------- /iflearner/communication/mpc/piss/piss_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | -------------------------------------------------------------------------------- /iflearner/communication/peer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iflytek/iflearner/040693a31784f5c6eeed17c95983f83b523518a3/iflearner/communication/peer/__init__.py -------------------------------------------------------------------------------- /iflearner/communication/peer/aes.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import hashlib 3 | from typing import Any, List 4 | 5 | from Crypto import Random 6 | from Crypto.Cipher import AES 7 | 8 | 9 | class AESCipher(object): 10 | def __init__(self, key: str) -> None: 11 | self.bs = AES.block_size 12 | self.key = hashlib.sha256(key.encode()).digest() 13 | 14 | def encrypt(self, raw: str) -> bytes: 15 | """Encrypt string to bytes""" 16 | 17 | raw = self._pad(raw) 18 | iv = Random.new().read(AES.block_size) 19 | cipher = AES.new(self.key, AES.MODE_CBC, iv) 20 | return base64.b64encode(iv + cipher.encrypt(raw.encode())) 21 | 22 | def decrypt(self, enc: List) -> Any: 23 | """Decrypt data""" 24 | 25 | enc = base64.b64decode(enc) 26 | iv = enc[: AES.block_size] 27 | cipher = AES.new(self.key, AES.MODE_CBC, iv) 28 | return self._unpad(cipher.decrypt(enc[AES.block_size :])).decode("utf-8") 29 | 30 | def _pad(self, s: str) -> Any: 31 | return s + (self.bs - len(s) % self.bs) * chr(self.bs - len(s) % self.bs) 32 | 33 | @staticmethod 34 | def _unpad(s) -> Any: 35 | return s[: -ord(s[len(s) - 1 :])] 36 | -------------------------------------------------------------------------------- /iflearner/communication/peer/diffie_hellman_inst.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import random 16 | import typing 17 | from datetime import datetime 18 | 19 | from numpy import byte 20 | 21 | from iflearner.communication.peer import diffie_hellman 22 | 23 | p, g = ( 24 | 179769313486231590770839156793787453197860296048756011706444423684197180216158519368947833795864925541502180565485980503646440548199239100050792877003355816639229553136239076508735759914822574862575007425302077447712589550957937778424442426617334727629299387668709205606050270810842907692932019128194467627007, 25 | 2, 26 | ) 27 | random.seed(datetime.now()) 28 | r = random.randint(1, 10000000) 29 | DH_public_key = diffie_hellman.DiffieHellman.encrypt(g, r, p) 30 | 31 | 32 | class DiffieHellmanInst(object): 33 | """The Diffie-Hellman instance for generating public key and secret.""" 34 | 35 | @staticmethod 36 | def generate_public_key() -> typing.List[byte]: 37 | return DH_public_key 38 | 39 | @staticmethod 40 | def generate_secret(data) -> str: 41 | return str(diffie_hellman.DiffieHellman.decrypt(data, r, p)) 42 | -------------------------------------------------------------------------------- /iflearner/communication/peer/diffie_hellman_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | from .diffie_hellman_inst import DiffieHellmanInst 16 | 17 | print("------------") 18 | 19 | DiffieHellmanInst() 20 | DiffieHellmanInst() 21 | 22 | DiffieHellmanInst() 23 | -------------------------------------------------------------------------------- /iflearner/communication/peer/message_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """The public key of diffie hellman .""" 16 | MSG_DH_PUBLIC_KEY = "msg_dh_public_key" 17 | 18 | """The random key in SMPC.""" 19 | MSG_SMPC_RANDOM_KEY = "msg_smpc_random_key" 20 | -------------------------------------------------------------------------------- /iflearner/communication/peer/peer_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 iFLYTEK. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import time 16 | from typing import List 17 | 18 | from loguru import logger 19 | 20 | from iflearner.communication.base import base_client, base_pb2 21 | from iflearner.communication.peer import aes, message_type 22 | from iflearner.communication.peer.diffie_hellman_inst import DiffieHellmanInst 23 | 24 | 25 | class PeerClient(base_client.BaseClient): 26 | """The client for peer party communication. 27 | """ 28 | 29 | def __init__(self, server_addr: str, party_name: str, peer_cert: str = None) -> None: 30 | super().__init__(server_addr, peer_cert) 31 | self._party_name = party_name 32 | 33 | def get_DH_public_key(self) -> List: 34 | """Get Diffie-Hellman public key from specified server. 35 | 36 | Returns: 37 | The public key. 38 | """ 39 | 40 | while True: 41 | try: 42 | req = base_pb2.BaseRequest( 43 | party_name=self._party_name, 44 | type=message_type.MSG_DH_PUBLIC_KEY, 45 | data=DiffieHellmanInst.generate_public_key(), 46 | ) 47 | resp = self._send(req) 48 | public_key = resp.data 49 | 50 | logger.info(f"Public key: {public_key}") 51 | return public_key 52 | except Exception as e: 53 | logger.info(e) 54 | time.sleep(3) 55 | 56 | def get_SMPC_random_key(self, key: str) -> float: 57 | """Get random value from the other party. 58 | 59 | Returns: 60 | A float value. 61 | """ 62 | 63 | req = base_pb2.BaseRequest( 64 | party_name=self._party_name, type=message_type.MSG_SMPC_RANDOM_KEY 65 | ) 66 | resp = self._send(req) 67 | 68 | random_float = float(aes.AESCipher(key).decrypt(resp.data)) 69 | logger.info(f"Random float: {random_float}") 70 | 71 | return random_float 72 | -------------------------------------------------------------------------------- /iflearner/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from cifar import * 2 | from mnist import * 3 | 4 | SUPPORT_DATASET = ['MNIST', 'FashionMNIST', 'KMNIST', 'EMNIST', 'CIFAR10', 'CIFAR100'] 5 | -------------------------------------------------------------------------------- /iflearner/datasets/fl_dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | class FLDateset: 5 | def __init__(self) -> None: 6 | self.train_x = None 7 | self.test_x = None 8 | self.test_targets = None 9 | self.train_targets = None 10 | 11 | def download(self): 12 | pass 13 | 14 | @property 15 | def train_labels(self): 16 | return self.train_targets 17 | 18 | @property 19 | def test_labels(self): 20 | return self.test_targets 21 | 22 | @property 23 | def train_data(self): 24 | return self.train_x 25 | 26 | @property 27 | def test_data(self): 28 | return self.test_x 29 | -------------------------------------------------------------------------------- /iflearner/datasets/readme.md: -------------------------------------------------------------------------------- 1 | ## How to use split_dataset 2 | 3 | Run ``python iflearner/datasets/split_dataset.py -h`` to see the help of the command. 4 | 5 | ``` 6 | optional arguments: 7 | -h, --help show this help message and exit 8 | --config CONFIG The path of config file 9 | --iid IID if to split dataset as iid 10 | --noniid NONIID the kind of noniid type 11 | --alpha ALPHA the parameter to control the noniid degree, only using when iid is false 12 | --save_path SAVE_PATH 13 | the path of saving the splitted dataset 14 | --dataset DATASET the name of dataset 15 | --data_path DATA_PATH 16 | the whole data in .npy format, only using when dataset is None 17 | --label_path LABEL_PATH 18 | the whole label in .npy format, only using when dataset is None 19 | --save_test_set_path SAVE_TEST_SET_PATH 20 | if save the test dataset in 21 | --clients CLIENTS [CLIENTS ...] 22 | clients names, eg. client1 client2 23 | ``` 24 | 25 | If the `config` argument is not None, all other argument must be set in the config file with ``yaml`` formart. It is like: 26 | 27 | ```yaml 28 | iid: False 29 | noniid: "noniid" 30 | alpha: 1 31 | save_path: "data_train" 32 | dataset: "MNIST" 33 | data_path: "/data/hanyuhu/iflearber-github/data/0-train_data.npy" 34 | label_path: "/data/hanyuhu/iflearber-github/data/0-train_label.npy" 35 | save_test_set_path: "data_test" 36 | clients: 37 | - client1 38 | - client2 39 | ``` 40 | 41 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options: 2 | 3 | [mypy] 4 | ignore_missing_imports = true 5 | strict = false 6 | show_error_codes = true 7 | ignore_errors = false 8 | warn_unused_configs = false 9 | exclude = (?x)( 10 | ^diffie_hellman\.py$ 11 | ) 12 | 13 | # Per-module options: 14 | [mypy-google.*] 15 | ignore_errors = true 16 | ignore_missing_imports = true 17 | 18 | [mypy-iflearner.communication.peer.*] 19 | ignore_errors = true -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.isort] 2 | line_length = 88 3 | indent = " " 4 | multi_line_output = 3 5 | include_trailing_comma = true 6 | force_grid_wrap = 0 7 | use_parentheses = true 8 | 9 | [tool.black] 10 | line-length = 88 11 | target-version = ["py36", "py37", "py38", "py39"] 12 | 13 | [tool.pylint."MESSAGES CONTROL"] 14 | disable = "bad-continuation,duplicate-code,too-few-public-methods,useless-import-alias" 15 | 16 | [tool.pytest.ini_options] 17 | minversion = "6.2" 18 | addopts = "-qq" 19 | testpaths = [ 20 | ] -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gmpy2==2.1.2 2 | grpcio==1.46.3 3 | grpcio-tools==1.46.3 4 | loguru==0.6.0 5 | flask==2.2.0 6 | matplotlib==3.3.4 7 | numpy==1.21.6 8 | protobuf==3.19.4 9 | pycryptodome==3.14.1 10 | recommonmark==0.7.1 11 | urllib3==1.26.8 12 | isort==5.10.1 13 | black==22.3.0 14 | docformatter==1.4 15 | mypy==0.960 16 | flake8==4.0.1 17 | mkdocs==1.3.0 18 | mdx_gh_links==0.2 19 | markdown==3.3.7 20 | lightgallery==0.5 21 | mkdocs-material==8.3.8 22 | mkdocs-static-i18n==0.46 23 | mkdocs-awesome-pages-plugin==2.7.0 24 | mkdocs-render-swagger-plugin==0.0.3 25 | mkdocs-swagger-ui-tag==0.3.0 26 | mkdocs-gen-files==0.3.5 27 | mkdocstrings==0.19.0 28 | mkdocstrings-python==0.7.1 29 | setuptools==62.3.2 30 | twine==4.0.1 31 | visualdl==2.3.0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | from __future__ import division 4 | 5 | import setuptools 6 | import os 7 | 8 | def _process_requirements(): 9 | packages = open('requirements.txt').read().strip().split('\n') 10 | requires = [] 11 | for pkg in packages: 12 | if pkg.startswith('git+ssh'): 13 | return_code = os.system('pip install {}'.format(pkg)) 14 | assert return_code == 0, 'error, status_code is: {}, exit!'.format(return_code) 15 | else: 16 | requires.append(pkg) 17 | return requires 18 | 19 | with open("README.md", "r") as fh: 20 | long_description = fh.read() 21 | 22 | setuptools.setup( 23 | name="iflearner", 24 | version="0.1.0", 25 | author="The iFLYTEK Turing Group", 26 | author_email="", 27 | description="Federated learning package", 28 | long_description=long_description, 29 | long_description_content_type="text/markdown", 30 | url="https://github.com/iflytek/iflearner", 31 | packages=[ 32 | package for package in setuptools.find_packages() 33 | if package.startswith('iflearner') 34 | ], 35 | entry_points={ 36 | "console_scripts": [ 37 | "ifl_server = iflearner.business.homo.aggregate_server:main", 38 | ] 39 | }, 40 | install_requires=_process_requirements(), 41 | setup_requires=[], 42 | license="Apache License 2.0", 43 | classifiers=[ 44 | 'Development Status :: 3 - Alpha', 45 | "License :: OSI Approved :: Apache Software License", 46 | "Operating System :: OS Independent", 47 | "Operating System :: MacOS :: MacOS X", 48 | "Operating System :: POSIX :: Linux", 49 | "Programming Language :: Python", 50 | "Programming Language :: Python :: 3", 51 | "Programming Language :: Python :: 3 :: Only", 52 | "Programming Language :: Python :: 3.7", 53 | "Programming Language :: Python :: 3.8", 54 | "Programming Language :: Python :: 3.9", 55 | "Programming Language :: Python :: 3.10", 56 | "Programming Language :: Python :: 3.11", 57 | "Programming Language :: Python :: Implementation :: CPython", 58 | ], 59 | python_requires='>=3.7', 60 | ) 61 | --------------------------------------------------------------------------------