├── .gitignore ├── LICENSE ├── README.md ├── README_cn.md ├── assets ├── GFL-base-framework.png └── GFL-core-framework.png ├── gfl ├── __init__.py ├── __main__.py ├── abc │ ├── __init__.py │ ├── aggregate.py │ ├── dataset.py │ └── train.py ├── api │ ├── __init__.py │ ├── net.py │ └── node.py ├── core │ ├── __init__.py │ ├── data.proto │ ├── data_pb2.py │ ├── db │ │ ├── __init__.py │ │ ├── db.py │ │ └── entities.py │ ├── fl_dataset.py │ ├── fl_job.py │ ├── fs │ │ ├── __init__.py │ │ ├── lfs.py │ │ └── path.py │ ├── net │ │ ├── __init__.py │ │ └── rpc │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ ├── gfl.proto │ │ │ ├── gfl_pb2.py │ │ │ ├── gfl_pb2_grpc.py │ │ │ └── server.py │ ├── node.py │ └── scheduler │ │ ├── __init__.py │ │ ├── aggregate_scheduler.py │ │ ├── scheduler.py │ │ └── train_scheduler.py ├── data │ ├── __init__.py │ ├── computing_resource.py │ ├── config.py │ ├── constants.py │ ├── data.py │ ├── meta.py │ ├── pramas.py │ └── strategy.py ├── env_detect.py ├── runtime │ ├── __init__.py │ ├── action.py │ ├── app.py │ ├── cli_main.py │ ├── config.py │ ├── constants.py │ ├── log.py │ ├── manager │ │ ├── __init__.py │ │ ├── client_manager.py │ │ ├── resource_manager.py │ │ └── server_manager.py │ └── utils.py ├── shell │ ├── __init__.py │ ├── __main__.py │ ├── ipython.py │ └── ipython_startup.py ├── utils │ ├── __init__.py │ ├── sys_utils.py │ └── zip_utils.py └── version.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── fl_job └── __init__.py ├── test.py ├── test_db.py ├── test_db2.py └── test_server_manager.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # project 132 | .idea 133 | /data/ 134 | /dataset/ 135 | /conf/ 136 | /res/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 The GFL Authors. All rights reserved. 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS 179 | 180 | APPENDIX: How to apply the Apache License to your work. 181 | 182 | To apply the Apache License to your work, attach the following 183 | boilerplate notice, with the fields enclosed by brackets "[]" 184 | replaced with your own identifying information. (Don't include 185 | the brackets!) The text should be enclosed in the appropriate 186 | comment syntax for the file format. We also recommend that a 187 | file or class name and description of purpose be included on the 188 | same "printed page" as the copyright notice for easier 189 | identification within third-party archives. 190 | 191 | Copyright [yyyy] [name of copyright owner] 192 | 193 | Licensed under the Apache License, Version 2.0 (the "License"); 194 | you may not use this file except in compliance with the License. 195 | You may obtain a copy of the License at 196 | 197 | http://www.apache.org/licenses/LICENSE-2.0 198 | 199 | Unless required by applicable law or agreed to in writing, software 200 | distributed under the License is distributed on an "AS IS" BASIS, 201 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202 | See the License for the specific language governing permissions and 203 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GFL Framework 2 | 3 | --------------------------------------------------- 4 | 5 | English | [简体中文](./README_cn.md) 6 | 7 | **Galaxy Federated Learning Framework(GFL)** is a decentralized federated learning framework based on blockchain. GFL builds a decentralized communication network based on Ethereum, and executes key operations that requires credibility in FL through smart contracts. 8 | 9 | ## Quick Start 10 | 11 | ### 1. System Envs Required 12 | 13 | a) GFL only supports Python3， please make sure your python version is no less than `3.4`. 14 | 15 | b) GFL is based on `Pytorch`, so, `torch>=1.4.0` and `torchvision>=0.5.0` is required before using GFL. [Pytorch installation tutorial](https://pytorch.org/get-started/locally/) 16 | 17 | ### 2. Install 18 | 19 | ```shell 20 | pip install gfl_p 21 | ``` 22 | 23 | ### 3. Usage 24 | 25 | The available commands of GFL. 26 | 27 | ``` 28 | usage: GFL [-h] {init,app,attach} ... 29 | 30 | optional arguments: 31 | -h, --help show this help message and exit 32 | 33 | actions: 34 | {init,app,attach} 35 | init init gfl env 36 | run startup gfl 37 | attach connect to gfl node 38 | ``` 39 | 40 | Init GFL node in `datadir` directory. 41 | 42 | ```shell 43 | python -m gfl_p init --home datadir 44 | ``` 45 | 46 | Start GFL node(start in standalone mode by default). If you need to open console when starting node, use the `--console`` argument. 47 | 48 | ```shell 49 | python -m gfl_p run --home datadir 50 | ``` 51 | 52 | Open console for operating GFL node. The following three methods can be used to connect to the node started in the previous step. 53 | 54 | ``` 55 | python -m gfl attach # connect to http://localhost:9434 in default 56 | python -m gfl attach -H 127.0.0.1 -P 9434 57 | python -m gfl attach --home datadir 58 | ``` 59 | 60 | ## GFL base design 61 | 62 | ![image-20210903165315547](./assets/GFL-base-framework.png) 63 | 64 | The GFL framework is divided into two parts: 65 | 66 | **Job Generator** 67 | 68 | > Used to create a job that can be executed in the GFL network. Developers can use the interface provided by GFL to generate a Job for various configuration parameters and distribute them to the network for training. 69 | 70 | **Run-Time Network** 71 | 72 | > Several running nodes build GFL's decentralized training network, and each GFL node is also a node in the blockchain. These nodes continuously process the jobs to be trained in the network according to user commands. 73 | 74 | ## GFL core arch 75 | 76 | ![image-20210903213928765](./assets/GFL-core-framework.png) 77 | 78 | + **Manager Layer** 79 | 80 | + The start/stop/status operation of node 81 | + Provide communication interface for nodes 82 | + Sync job 83 | 84 | + **Scheduler Layer** 85 | 86 | + Manage the execution process of each job 87 | + Synchronize parameter files among nodes 88 | + Schedule the execution order of multiple jobs on the node 89 | 90 | + **FL Layer** 91 | 92 | + Configure the running environment of the job 93 | + Perform training/aggregation tasks 94 | + Provide the interfaces of user-defined action 95 | -------------------------------------------------------------------------------- /README_cn.md: -------------------------------------------------------------------------------- 1 | # GFL Framework 2 | 3 | ------- 4 | 5 | [English](./README.md) | 简体中文 6 | 7 | **Galaxy Federated Learning Framework(GFL)** 是一个基于区块链的去中心化联邦学习框架。GFL以Ethereum为基础建立去中心化通信网络，通过智能合约完成联邦学习中要求可信的关键功能。同时，GFL简化了联邦学习建模过程，让开发者可以快速建模和验证。 8 | 9 | ## 快速开始 10 | 11 | ### 1. 环境要求 12 | 13 | a) GFL只支持Python3， 请确定你的Python版本不低于Python3.4。 14 | 15 | b) GFL基于Pytorch实现模型训练，在使用GFL前需要安装`torch>=1.4.0`和`torchvision>=0.5.0`。[Pytorch安装教程](https://pytorch.org/get-started/locally/) 16 | 17 | ### 2. GFL安装 18 | 19 | ```shell 20 | pip install gfl_p 21 | ``` 22 | 23 | ### 3. GFL用法 24 | 25 | GFL可用的命令。 26 | 27 | ``` 28 | usage: GFL [-h] {init,app,attach} ... 29 | 30 | optional arguments: 31 | -h, --help show this help message and exit 32 | 33 | actions: 34 | {init,app,attach} 35 | init init gfl env 36 | run startup gfl 37 | attach connect to gfl node 38 | ``` 39 | 40 | 在datadir目录下初始化GFL节点。 41 | 42 | ```shell 43 | python -m gfl_p init --home datadir 44 | ``` 45 | 46 | 启动节点（默认以standalone模式启动），如果需要在启动节点的同时打开交互式命令行，使用`--console`参数。 47 | 48 | ```shell 49 | python -m gfl_p run --home datadir 50 | ``` 51 | 52 | 打开交互式命令行以对节点执行操作。以下三种方式都可以连接到上一步中启动的节点。 53 | 54 | ``` 55 | python -m gfl attach # 默认连接http://localhost:9434 56 | python -m gfl attach -H 127.0.0.1 -P 9434 57 | python -m gfl attach --home datadir 58 | ``` 59 | 60 | ## GFL基础设计 61 | 62 | ![image-20210903165315547](./assets/GFL-base-framework.png) 63 | 64 | GFL框架分为两部分： 65 | 66 | **Job Generator** 67 | 68 | > *用于创建可以在GFL网络中执行的Job。开发者可以通过GFL提供的接口， 将各项配置参数生成一个Job分发到网络中以待训练。* 69 | 70 | **Run-Time Network** 71 | 72 | > *若干运行中的节点组成了GFL的去中心化训练网络， 每个GFL节点同时也是区块链中的一个节点。这些节点根据用户命令不停的处理网络中待训练的Job。* 73 | 74 | ## GFL核心框架结构 75 | 76 | ![image-20210903213928765](./assets/GFL-core-framework.png) 77 | 78 | + **Manager Layer** 79 | 80 | + 节点的启动/停止/状态探测 81 | + 提供节点的进程间通信接口 82 | + 同步Job 83 | 84 | + **Scheduler Layer** 85 | 86 | + 管理每个Job的执行流程 87 | + 在节点间同步参数文件 88 | + 调度节点上的多Job执行顺序 89 | 90 | + **FL Layer** 91 | 92 | + 配置Job的运行环境 93 | + 执行训练/聚合任务 94 | + 提供用户自定义动作接口 95 | -------------------------------------------------------------------------------- /assets/GFL-base-framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GalaxyLearning/GFL/007174933f4bb2cb35dd864758a9c4a02a3f46a5/assets/GFL-base-framework.png -------------------------------------------------------------------------------- /assets/GFL-core-framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GalaxyLearning/GFL/007174933f4bb2cb35dd864758a9c4a02a3f46a5/assets/GFL-core-framework.png -------------------------------------------------------------------------------- /gfl/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import gfl.env_detect 16 | 17 | 18 | def __version__(): 19 | return "0.2.0" 20 | -------------------------------------------------------------------------------- /gfl/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from gfl.runtime.cli_main import main 16 | 17 | if __name__ == "__main__": 18 | main() 19 | -------------------------------------------------------------------------------- /gfl/abc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | Abstract Base Classes 17 | """ 18 | 19 | from gfl.abc.aggregate import FLAggregator 20 | from gfl.abc.dataset import FLDataset 21 | from gfl.abc.train import FLTrainer 22 | -------------------------------------------------------------------------------- /gfl/abc/aggregate.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "FLAggregator" 17 | ] 18 | 19 | 20 | class FLAggregator(object): 21 | 22 | def __init__(self): 23 | super(FLAggregator, self).__init__() 24 | 25 | def aggregate(self): 26 | pass 27 | -------------------------------------------------------------------------------- /gfl/abc/dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "FLDataset" 17 | ] 18 | 19 | 20 | class FLDataset(object): 21 | 22 | def __init__(self, root): 23 | super(FLDataset, self).__init__() 24 | self.root = root 25 | -------------------------------------------------------------------------------- /gfl/abc/train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "FLTrainer" 17 | ] 18 | 19 | class FLTrainer(object): 20 | 21 | def __init__(self, job): 22 | super(FLTrainer, self).__init__() 23 | 24 | def train(self): 25 | pass 26 | 27 | def test(self): 28 | pass 29 | -------------------------------------------------------------------------------- /gfl/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/api/net.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import google.protobuf.wrappers_pb2 as wrappers_pb2 16 | import gfl.core.data_pb2 as data_pb2 17 | import gfl.core.net.rpc.gfl_pb2 as gfl_pb2 18 | from gfl.core.fs import FS 19 | from gfl.core.net.rpc.client import build_client 20 | from gfl.core.node import GflNode 21 | from gfl.runtime.config import GflConfig 22 | 23 | 24 | class Net(object): 25 | 26 | def __init__(self, home): 27 | super(Net, self).__init__() 28 | self.__fs = FS(home) 29 | self.__config = GflConfig.load(self.__fs.path.config_file()) 30 | self.__node = GflNode.load_node(self.__fs.path.key_file()) 31 | self.__client = None 32 | self.__init_client() 33 | 34 | def __init_client(self): 35 | self.__client = build_client(self.__config.node.rpc.server_host, self.__config.node.rpc.server_port) 36 | self.__client.SendNodeInfo(gfl_pb2.NodeInfo( 37 | address=self.__node.address, 38 | pub_key=self.__node.pub_key 39 | )) 40 | 41 | def get_pub_key(self, address): 42 | resp = self.__client.GetPubKey(wrappers_pb2.StringValue(value=address)) 43 | print(f"Resp: {resp.value}") 44 | -------------------------------------------------------------------------------- /gfl/api/node.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from gfl.core.fs import FS 16 | from gfl.core.node import GflNode 17 | 18 | 19 | class Node(object): 20 | 21 | def __init__(self, home): 22 | super(Node, self).__init__() 23 | self.__fs = FS(home) 24 | self.__gfl_node = GflNode.load_node(self.__fs.path.key_file()) 25 | -------------------------------------------------------------------------------- /gfl/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/core/data.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package gfl.core; 4 | 5 | message JobMeta { 6 | string job_id = 1; 7 | string owner = 2; 8 | int64 create_time = 3; 9 | int32 status = 4; 10 | repeated string dataset_ids = 5; 11 | } 12 | 13 | message JobConfig { 14 | string trainer = 1; 15 | string aggregator = 2; 16 | } 17 | 18 | message TrainConfig { 19 | string model = 1; 20 | string optimizer = 2; 21 | string criterion = 3; 22 | string lr_scheduler = 4; 23 | int32 epoch = 5; 24 | int32 batch_size = 6; 25 | } 26 | 27 | message AggregateConfig { 28 | int32 global_epoch = 1; 29 | } 30 | 31 | message JobData { 32 | JobMeta meta = 1; 33 | string content = 2; 34 | JobConfig job_config = 3; 35 | TrainConfig train_config = 4; 36 | AggregateConfig aggregate_config = 5; 37 | bytes package_data = 6; 38 | } 39 | 40 | message JobTrace { 41 | string job_id = 1; 42 | int64 begin_timepoint = 2; 43 | int64 end_timepoint = 3; 44 | int32 ready_time = 4; 45 | int32 aggregate_running_time = 5; 46 | int32 aggregate_waiting_time = 6; 47 | int32 train_running_time = 7; 48 | int32 train_waiting_time = 8; 49 | int32 comm_time = 9; 50 | int32 used_time = 10; 51 | } 52 | 53 | message DatasetMeta { 54 | string dataset_id = 1; 55 | string owner = 2; 56 | int64 create_time = 3; 57 | int32 status = 4; 58 | int32 type = 5; 59 | int32 size = 6; 60 | int32 request_cnt = 7; 61 | int32 used_cnt = 8; 62 | } 63 | 64 | message DatasetConfig { 65 | string dataset = 1; 66 | string val_dataset = 2; 67 | float val_rate = 3; 68 | } 69 | 70 | message DatasetData { 71 | DatasetMeta meta = 1; 72 | string content = 2; 73 | DatasetConfig dataset_config = 3; 74 | bytes package_data = 4; 75 | } 76 | 77 | message DatasetTrace { 78 | string dataset_id = 1; 79 | string job_id = 2; 80 | bool confirmed = 3; 81 | float score = 4; 82 | } 83 | 84 | message ModelParams { 85 | string job_id = 1; 86 | string node_address = 2; 87 | string dataset_id = 3; 88 | int32 step = 4; 89 | string path = 5; 90 | float loss = 6; 91 | string metric_name = 7; 92 | float metric_value = 8; 93 | float score = 9; 94 | bool is_aggregate = 10; 95 | bytes data = 11; 96 | } 97 | 98 | message NodeInfo { 99 | string address = 1; 100 | string pub_key = 2; 101 | } 102 | -------------------------------------------------------------------------------- /gfl/core/data_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: gfl/core/data.proto 4 | """Generated protocol buffer code.""" 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import descriptor_pool as _descriptor_pool 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | 16 | 17 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gfl/core/data.proto\x12\x08gfl.core\"b\n\x07JobMeta\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\r\n\x05owner\x18\x02 \x01(\t\x12\x13\n\x0b\x63reate_time\x18\x03 \x01(\x03\x12\x0e\n\x06status\x18\x04 \x01(\x05\x12\x13\n\x0b\x64\x61taset_ids\x18\x05 \x03(\t\"0\n\tJobConfig\x12\x0f\n\x07trainer\x18\x01 \x01(\t\x12\x12\n\naggregator\x18\x02 \x01(\t\"{\n\x0bTrainConfig\x12\r\n\x05model\x18\x01 \x01(\t\x12\x11\n\toptimizer\x18\x02 \x01(\t\x12\x11\n\tcriterion\x18\x03 \x01(\t\x12\x14\n\x0clr_scheduler\x18\x04 \x01(\t\x12\r\n\x05\x65poch\x18\x05 \x01(\x05\x12\x12\n\nbatch_size\x18\x06 \x01(\x05\"\'\n\x0f\x41ggregateConfig\x12\x14\n\x0cglobal_epoch\x18\x01 \x01(\x05\"\xc6\x01\n\x07JobData\x12\x1f\n\x04meta\x18\x01 \x01(\x0b\x32\x11.gfl.core.JobMeta\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\t\x12\'\n\njob_config\x18\x03 \x01(\x0b\x32\x13.gfl.core.JobConfig\x12+\n\x0ctrain_config\x18\x04 \x01(\x0b\x32\x15.gfl.core.TrainConfig\x12\x33\n\x10\x61ggregate_config\x18\x05 \x01(\x0b\x32\x19.gfl.core.AggregateConfig\"\xfc\x01\n\x08JobTrace\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x17\n\x0f\x62\x65gin_timepoint\x18\x02 \x01(\x03\x12\x15\n\rend_timepoint\x18\x03 \x01(\x03\x12\x12\n\nready_time\x18\x04 \x01(\x05\x12\x1e\n\x16\x61ggregate_running_time\x18\x05 \x01(\x05\x12\x1e\n\x16\x61ggregate_waiting_time\x18\x06 \x01(\x05\x12\x1a\n\x12train_running_time\x18\x07 \x01(\x05\x12\x1a\n\x12train_waiting_time\x18\x08 \x01(\x05\x12\x11\n\tcomm_time\x18\t \x01(\x05\x12\x11\n\tused_time\x18\n \x01(\x05\"\x98\x01\n\x0b\x44\x61tasetMeta\x12\x12\n\ndataset_id\x18\x01 \x01(\t\x12\r\n\x05owner\x18\x02 \x01(\t\x12\x13\n\x0b\x63reate_time\x18\x03 \x01(\x03\x12\x0e\n\x06status\x18\x04 \x01(\x05\x12\x0c\n\x04type\x18\x05 \x01(\x05\x12\x0c\n\x04size\x18\x06 \x01(\x05\x12\x13\n\x0brequest_cnt\x18\x07 \x01(\x05\x12\x10\n\x08used_cnt\x18\x08 \x01(\x05\"G\n\rDatasetConfig\x12\x0f\n\x07\x64\x61taset\x18\x01 \x01(\t\x12\x13\n\x0bval_dataset\x18\x02 \x01(\t\x12\x10\n\x08val_rate\x18\x03 \x01(\x02\"t\n\x0b\x44\x61tasetData\x12#\n\x04meta\x18\x01 \x01(\x0b\x32\x15.gfl.core.DatasetMeta\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\t\x12/\n\x0e\x64\x61taset_config\x18\x03 \x01(\x0b\x32\x17.gfl.core.DatasetConfig\"T\n\x0c\x44\x61tasetTrace\x12\x12\n\ndataset_id\x18\x01 \x01(\t\x12\x0e\n\x06job_id\x18\x02 \x01(\t\x12\x11\n\tconfirmed\x18\x03 \x01(\x08\x12\r\n\x05score\x18\x04 \x01(\x02\"\xcf\x01\n\x0bModelParams\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x14\n\x0cnode_address\x18\x02 \x01(\t\x12\x12\n\ndataset_id\x18\x03 \x01(\t\x12\x0c\n\x04step\x18\x04 \x01(\x05\x12\x0c\n\x04path\x18\x05 \x01(\t\x12\x0c\n\x04loss\x18\x06 \x01(\x02\x12\x13\n\x0bmetric_name\x18\x07 \x01(\t\x12\x14\n\x0cmetric_value\x18\x08 \x01(\x02\x12\r\n\x05score\x18\t \x01(\x02\x12\x14\n\x0cis_aggregate\x18\n \x01(\x08\x12\x0c\n\x04\x64\x61ta\x18\x0b \x01(\x0c\",\n\x08NodeInfo\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x0f\n\x07pub_key\x18\x02 \x01(\tb\x06proto3') 18 | 19 | 20 | 21 | _JOBMETA = DESCRIPTOR.message_types_by_name['JobMeta'] 22 | _JOBCONFIG = DESCRIPTOR.message_types_by_name['JobConfig'] 23 | _TRAINCONFIG = DESCRIPTOR.message_types_by_name['TrainConfig'] 24 | _AGGREGATECONFIG = DESCRIPTOR.message_types_by_name['AggregateConfig'] 25 | _JOBDATA = DESCRIPTOR.message_types_by_name['JobData'] 26 | _JOBTRACE = DESCRIPTOR.message_types_by_name['JobTrace'] 27 | _DATASETMETA = DESCRIPTOR.message_types_by_name['DatasetMeta'] 28 | _DATASETCONFIG = DESCRIPTOR.message_types_by_name['DatasetConfig'] 29 | _DATASETDATA = DESCRIPTOR.message_types_by_name['DatasetData'] 30 | _DATASETTRACE = DESCRIPTOR.message_types_by_name['DatasetTrace'] 31 | _MODELPARAMS = DESCRIPTOR.message_types_by_name['ModelParams'] 32 | _NODEINFO = DESCRIPTOR.message_types_by_name['NodeInfo'] 33 | JobMeta = _reflection.GeneratedProtocolMessageType('JobMeta', (_message.Message,), { 34 | 'DESCRIPTOR' : _JOBMETA, 35 | '__module__' : 'gfl.core.data_pb2' 36 | # @@protoc_insertion_point(class_scope:gfl.core.JobMeta) 37 | }) 38 | _sym_db.RegisterMessage(JobMeta) 39 | 40 | JobConfig = _reflection.GeneratedProtocolMessageType('JobConfig', (_message.Message,), { 41 | 'DESCRIPTOR' : _JOBCONFIG, 42 | '__module__' : 'gfl.core.data_pb2' 43 | # @@protoc_insertion_point(class_scope:gfl.core.JobConfig) 44 | }) 45 | _sym_db.RegisterMessage(JobConfig) 46 | 47 | TrainConfig = _reflection.GeneratedProtocolMessageType('TrainConfig', (_message.Message,), { 48 | 'DESCRIPTOR' : _TRAINCONFIG, 49 | '__module__' : 'gfl.core.data_pb2' 50 | # @@protoc_insertion_point(class_scope:gfl.core.TrainConfig) 51 | }) 52 | _sym_db.RegisterMessage(TrainConfig) 53 | 54 | AggregateConfig = _reflection.GeneratedProtocolMessageType('AggregateConfig', (_message.Message,), { 55 | 'DESCRIPTOR' : _AGGREGATECONFIG, 56 | '__module__' : 'gfl.core.data_pb2' 57 | # @@protoc_insertion_point(class_scope:gfl.core.AggregateConfig) 58 | }) 59 | _sym_db.RegisterMessage(AggregateConfig) 60 | 61 | JobData = _reflection.GeneratedProtocolMessageType('JobData', (_message.Message,), { 62 | 'DESCRIPTOR' : _JOBDATA, 63 | '__module__' : 'gfl.core.data_pb2' 64 | # @@protoc_insertion_point(class_scope:gfl.core.JobData) 65 | }) 66 | _sym_db.RegisterMessage(JobData) 67 | 68 | JobTrace = _reflection.GeneratedProtocolMessageType('JobTrace', (_message.Message,), { 69 | 'DESCRIPTOR' : _JOBTRACE, 70 | '__module__' : 'gfl.core.data_pb2' 71 | # @@protoc_insertion_point(class_scope:gfl.core.JobTrace) 72 | }) 73 | _sym_db.RegisterMessage(JobTrace) 74 | 75 | DatasetMeta = _reflection.GeneratedProtocolMessageType('DatasetMeta', (_message.Message,), { 76 | 'DESCRIPTOR' : _DATASETMETA, 77 | '__module__' : 'gfl.core.data_pb2' 78 | # @@protoc_insertion_point(class_scope:gfl.core.DatasetMeta) 79 | }) 80 | _sym_db.RegisterMessage(DatasetMeta) 81 | 82 | DatasetConfig = _reflection.GeneratedProtocolMessageType('DatasetConfig', (_message.Message,), { 83 | 'DESCRIPTOR' : _DATASETCONFIG, 84 | '__module__' : 'gfl.core.data_pb2' 85 | # @@protoc_insertion_point(class_scope:gfl.core.DatasetConfig) 86 | }) 87 | _sym_db.RegisterMessage(DatasetConfig) 88 | 89 | DatasetData = _reflection.GeneratedProtocolMessageType('DatasetData', (_message.Message,), { 90 | 'DESCRIPTOR' : _DATASETDATA, 91 | '__module__' : 'gfl.core.data_pb2' 92 | # @@protoc_insertion_point(class_scope:gfl.core.DatasetData) 93 | }) 94 | _sym_db.RegisterMessage(DatasetData) 95 | 96 | DatasetTrace = _reflection.GeneratedProtocolMessageType('DatasetTrace', (_message.Message,), { 97 | 'DESCRIPTOR' : _DATASETTRACE, 98 | '__module__' : 'gfl.core.data_pb2' 99 | # @@protoc_insertion_point(class_scope:gfl.core.DatasetTrace) 100 | }) 101 | _sym_db.RegisterMessage(DatasetTrace) 102 | 103 | ModelParams = _reflection.GeneratedProtocolMessageType('ModelParams', (_message.Message,), { 104 | 'DESCRIPTOR' : _MODELPARAMS, 105 | '__module__' : 'gfl.core.data_pb2' 106 | # @@protoc_insertion_point(class_scope:gfl.core.ModelParams) 107 | }) 108 | _sym_db.RegisterMessage(ModelParams) 109 | 110 | NodeInfo = _reflection.GeneratedProtocolMessageType('NodeInfo', (_message.Message,), { 111 | 'DESCRIPTOR' : _NODEINFO, 112 | '__module__' : 'gfl.core.data_pb2' 113 | # @@protoc_insertion_point(class_scope:gfl.core.NodeInfo) 114 | }) 115 | _sym_db.RegisterMessage(NodeInfo) 116 | 117 | if _descriptor._USE_C_DESCRIPTORS == False: 118 | 119 | DESCRIPTOR._options = None 120 | _JOBMETA._serialized_start=33 121 | _JOBMETA._serialized_end=131 122 | _JOBCONFIG._serialized_start=133 123 | _JOBCONFIG._serialized_end=181 124 | _TRAINCONFIG._serialized_start=183 125 | _TRAINCONFIG._serialized_end=306 126 | _AGGREGATECONFIG._serialized_start=308 127 | _AGGREGATECONFIG._serialized_end=347 128 | _JOBDATA._serialized_start=350 129 | _JOBDATA._serialized_end=548 130 | _JOBTRACE._serialized_start=551 131 | _JOBTRACE._serialized_end=803 132 | _DATASETMETA._serialized_start=806 133 | _DATASETMETA._serialized_end=958 134 | _DATASETCONFIG._serialized_start=960 135 | _DATASETCONFIG._serialized_end=1031 136 | _DATASETDATA._serialized_start=1033 137 | _DATASETDATA._serialized_end=1149 138 | _DATASETTRACE._serialized_start=1151 139 | _DATASETTRACE._serialized_end=1235 140 | _MODELPARAMS._serialized_start=1238 141 | _MODELPARAMS._serialized_end=1445 142 | _NODEINFO._serialized_start=1447 143 | _NODEINFO._serialized_end=1491 144 | # @@protoc_insertion_point(module_scope) 145 | -------------------------------------------------------------------------------- /gfl/core/db/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .db import DB 16 | from .entities import init_sqlite 17 | -------------------------------------------------------------------------------- /gfl/core/db/db.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "DB" 17 | ] 18 | 19 | import logging 20 | 21 | from sqlalchemy import create_engine 22 | from sqlalchemy.orm import sessionmaker 23 | 24 | from .entities import * 25 | from .. import data_pb2 26 | from gfl.data import DatasetStatus, MIN_HOT_CNT 27 | 28 | 29 | logger = logging.getLogger("gfl.db") 30 | 31 | 32 | class DB(object): 33 | 34 | """ 35 | 36 | """ 37 | 38 | def __init__(self, sqlite_path): 39 | super(DB, self).__init__() 40 | self.__sqlite_path = sqlite_path 41 | self.__engine = create_engine(f"sqlite:///{self.__sqlite_path}") 42 | self.__db_session = sessionmaker(bind=self.__engine) 43 | 44 | def add_node(self, address, pub_key): 45 | """ 46 | 47 | :param address: 48 | :param pub_key: 49 | :return: 50 | """ 51 | with self.__db_session() as s: 52 | if s.query(NodeTable).filter_by(address=address).first() is None: 53 | s.add(NodeTable(address=address, pub_key=pub_key)) 54 | s.commit() 55 | 56 | def get_pub_key(self, address): 57 | """ 58 | 59 | :param address: 60 | :return: 61 | """ 62 | with self.__db_session() as s: 63 | node = s.query(NodeTable).filter_by(address=address).one_or_none() 64 | return node.pub_key if node is not None else None 65 | 66 | def add_job(self, job: data_pb2.JobMeta): 67 | """ 68 | 69 | :param job: 70 | :return: 71 | """ 72 | with self.__db_session() as s: 73 | if s.query(JobTable).filter_by(job_id=job.job_id).first() is None: 74 | s.add(JobTable(job_id=job.job_id, owner=job.owner, create_time=job.create_time, status=job.status)) 75 | s.add(JobTraceTable(job_id=job.job_id)) 76 | for dataset_id in job.dataset_ids: 77 | s.add(DatasetTraceTable(dataset_id=dataset_id, job_id=job.job_id)) 78 | s.commit() 79 | 80 | def update_job(self, job_id, status): 81 | """ 82 | 83 | :param job_id: 84 | :param status: 85 | :return: 86 | """ 87 | def update_fn(job): 88 | job.status = status 89 | 90 | return 1 == self._update_one(JobTable, update_fn, job_id=job_id) 91 | 92 | def get_job(self, job_id): 93 | """ 94 | 95 | :param job_id: 96 | :return: 97 | """ 98 | with self.__db_session() as s: 99 | job = s.query(JobTable).filter_by(job_id=job_id).one_or_none() 100 | if job is None: 101 | return None 102 | dataset_traces = s.query(DatasetTraceTable).filter_by(job_id=job_id).all() 103 | dataset_ids = [dt.dataset_id for dt in dataset_traces] 104 | return data_pb2.JobMeta(job_id=job.job_id, owner=job.owner, create_time=job.create_time, status=job.status, 105 | dataset_ids=dataset_ids) 106 | 107 | def add_dataset(self, dataset: data_pb2.DatasetMeta): 108 | """ 109 | 110 | :param dataset: 111 | :return: 112 | """ 113 | with self.__db_session() as s: 114 | if s.query(DatasetTable).filter_by(dataset_id=dataset.dataset_id).first() is None: 115 | s.add(DatasetTable(dataset_id=dataset.dataset_id, 116 | owner=dataset.owner, 117 | create_time=dataset.create_time, 118 | type=dataset.type, 119 | size=dataset.size)) 120 | s.commit() 121 | 122 | def update_dataset(self, dataset_id, *, inc_request_cnt=None, inc_used_cnt=None): 123 | """ 124 | 125 | :param dataset_id: 126 | :param inc_request_cnt: 127 | :param inc_used_cnt: 128 | :return: 129 | """ 130 | def update_fn(dataset): 131 | if inc_request_cnt is not None: 132 | dataset.request_cnt = dataset.request_cnt + inc_request_cnt 133 | if inc_used_cnt is not None: 134 | dataset.used_cnt = dataset.used_cnt + inc_used_cnt 135 | if dataset.used_cnt >= MIN_HOT_CNT: 136 | dataset.status = DatasetStatus.HOT.value 137 | 138 | return 1 == self._update_one(DatasetTable, update_fn, dataset_id=dataset_id) 139 | 140 | def get_dataset(self, dataset_id): 141 | """ 142 | 143 | :param dataset_id: 144 | :return: 145 | """ 146 | with self.__db_session() as s: 147 | dataset = s.query(DatasetTable).filter_by(dataset_id=dataset_id).one_or_none() 148 | if dataset is None: 149 | return None 150 | return data_pb2.DatasetMeta(dataset_id=dataset.dataset_id, 151 | owner=dataset.owner, 152 | create_time=dataset.create_time, 153 | status=dataset.status, 154 | type=dataset.type, 155 | size=dataset.size, 156 | request_cnt=dataset.request_cnt, 157 | used_cnt=dataset.used_cnt) 158 | 159 | def update_dataset_trace(self, dataset_id, job_id, *, confirmed=None, score=None, keys=None): 160 | """ 161 | 162 | :param dataset_id: 163 | :param job_id: 164 | :param confirmed: 165 | :param score: 166 | :param keys: 167 | :return: 168 | """ 169 | if keys is None: 170 | keys = [] 171 | 172 | def update_fn(dataset): 173 | if confirmed is not None or "confirmed" in keys: 174 | dataset.confirmed = confirmed 175 | if score is not None or "score" in keys: 176 | dataset.score = score 177 | 178 | return 1 == self._update_one(DatasetTraceTable, update_fn, dataset_id=dataset_id, job_id=job_id) 179 | 180 | def update_job_trace(self, job_id, *, 181 | begin_timepoint=None, 182 | end_timepoint=None, 183 | inc_ready_time=None, 184 | inc_aggregate_running_time=None, 185 | inc_aggregate_waiting_time=None, 186 | inc_train_running_time=None, 187 | inc_train_waiting_time=None, 188 | inc_comm_time=None): 189 | """ 190 | 191 | :param job_id: 192 | :param begin_timepoint: 193 | :param end_timepoint: 194 | :param inc_ready_time: 195 | :param inc_aggregate_running_time: 196 | :param inc_aggregate_waiting_time: 197 | :param inc_train_running_time: 198 | :param inc_train_waiting_time: 199 | :param inc_comm_time: 200 | :return: 201 | """ 202 | def update_fn(job_trace): 203 | if begin_timepoint is not None: 204 | job_trace.begin_timepoint = begin_timepoint 205 | if end_timepoint is not None: 206 | job_trace.end_timepoint = end_timepoint 207 | job_trace.used_time = (job_trace.end_timepoint - job_trace.begin_timepoint) // 1000 208 | 209 | if inc_ready_time is not None: 210 | job_trace.ready_time = job_trace.ready_time + inc_ready_time 211 | if inc_aggregate_running_time is not None: 212 | job_trace.aggregate_running_time = job_trace.aggregate_running_time + inc_aggregate_running_time 213 | if inc_aggregate_waiting_time is not None: 214 | job_trace.aggregate_waiting_time = job_trace.aggregate_waiting_time + inc_aggregate_waiting_time 215 | if inc_train_running_time is not None: 216 | job_trace.train_running_time = job_trace.train_running_time + inc_train_running_time 217 | if inc_train_waiting_time is not None: 218 | job_trace.train_waiting_time = job_trace.train_waiting_time + inc_train_waiting_time 219 | if inc_comm_time is not None: 220 | job_trace.comm_time = job_trace.comm_time + inc_comm_time 221 | 222 | return 1 == self._update_one(JobTraceTable, update_fn, job_id=job_id) 223 | 224 | def get_dataset_trace(self, dataset_id=None, job_id=None): 225 | """ 226 | 227 | :param dataset_id: 228 | :param job_id: 229 | :return: 230 | """ 231 | if dataset_id is None and job_id is None: 232 | return [] 233 | with self.__db_session() as s: 234 | if dataset_id is not None and job_id is not None: 235 | traces = s.query(DatasetTraceTable).filter_by(dataset_id=dataset_id, job_id=job_id).all() 236 | elif dataset_id is None: 237 | traces = s.query(DatasetTraceTable).filter_by(job_id=job_id) 238 | else: 239 | traces = s.query(DatasetTraceTable).filter_by(dataset_id=dataset_id) 240 | ret = [] 241 | for t in traces: 242 | ret.append(data_pb2.DatasetTrace(dataset_id=t.dataset_id, 243 | job_id=t.job_id, 244 | confirmed=t.confirmed, 245 | score=t.score)) 246 | return ret 247 | 248 | def get_job_trace(self, job_id) -> data_pb2.JobTrace: 249 | """ 250 | 251 | :param job_id: 252 | :return: 253 | """ 254 | with self.__db_session() as s: 255 | job_trace: JobTraceTable = s.query(JobTraceTable).filter_by(job_id=job_id).one_or_none() 256 | if job_trace is None: 257 | return None 258 | return data_pb2.JobTrace(job_id=job_trace.job_id, 259 | begin_timepoint=job_trace.begin_timepoint, 260 | end_timepoint=job_trace.end_timepoint, 261 | ready_time=job_trace.ready_time, 262 | aggregate_running_time=job_trace.aggregate_running_time, 263 | aggregate_waiting_time=job_trace.aggregate_waiting_time, 264 | train_running_time=job_trace.train_running_time, 265 | train_waiting_time=job_trace.train_waiting_time, 266 | comm_time=job_trace.comm_time, 267 | used_time=job_trace.used_time) 268 | 269 | def add_params(self, params: data_pb2.ModelParams): 270 | """ 271 | 272 | :param params: 273 | :return: 274 | """ 275 | with self.__db_session() as s: 276 | if s.query(ParamsTable).filter_by(job_id=params.job_id, 277 | node_address=params.node_address, 278 | dataset_id=params.dataset_id, 279 | step=params.step, 280 | is_aggregate=params.is_aggregate).first() is None: 281 | s.add(ParamsTable(job_id=params.job_id, 282 | node_address=params.node_address, 283 | dataset_id=params.dataset_id if params.dataset_id is not None else "", 284 | step=params.step, 285 | path=params.path, 286 | loss=params.loss, 287 | metric_name=params.metric_name, 288 | metric_value=params.metric_value, 289 | score=params.score, 290 | is_aggregate=params.is_aggregate)) 291 | s.commit() 292 | 293 | def update_params(self, job_id, node_address, dataset_id, step, is_aggregate, *, 294 | path=None, 295 | loss=None, 296 | metric_name=None, 297 | metric_value=None, 298 | score=None, 299 | keys=None): 300 | """ 301 | 302 | :param job_id: 303 | :param node_address: 304 | :param dataset_id: 305 | :param step: 306 | :param is_aggregate: 307 | :param path: 308 | :param loss: 309 | :param metric_name: 310 | :param metric_value: 311 | :param score: 312 | :param keys: 313 | :return: 314 | """ 315 | if dataset_id is None: 316 | dataset_id = "" 317 | if keys is None: 318 | keys = [] 319 | 320 | def update_fn(params): 321 | if path is not None or "path" in keys: 322 | params.path = path 323 | if loss is not None or "loss" in keys: 324 | params.loss = loss 325 | if metric_name is not None or "metric_name" in keys: 326 | params.metric_name = metric_name 327 | if metric_value is not None or "metric_value" in keys: 328 | params.metric_value = metric_value 329 | if score is not None or "score" in keys: 330 | params.score = score 331 | 332 | return 1 == self._update_one(ParamsTable, update_fn, 333 | job_id=job_id, node_address=node_address, dataset_id=dataset_id, step=step, 334 | is_aggregate=is_aggregate) 335 | 336 | def get_params(self, job_id, node_address, dataset_id, step, is_aggregate): 337 | """ 338 | 339 | :param job_id: 340 | :param node_address: 341 | :param dataset_id: 342 | :param step: 343 | :param is_aggregate: 344 | :return: 345 | """ 346 | if dataset_id is None: 347 | dataset_id = "" 348 | with self.__db_session() as s: 349 | params = s.query(ParamsTable).filter_by(job_id=job_id, 350 | node_address=node_address, 351 | dataset_id=dataset_id, 352 | step=step, 353 | is_aggregate=is_aggregate).one_or_none() 354 | if params is None: 355 | return None 356 | return data_pb2.ModelParams(job_id=params.job_id, 357 | node_address=params.node_address, 358 | dataset_id=params.dataset_id, 359 | step=params.step, 360 | path=params.path, 361 | loss=params.loss, 362 | metric_name=params.metric_name, 363 | metric_value=params.metric_value, 364 | score=params.score, 365 | is_aggregate=params.is_aggregate) 366 | 367 | def _update_one(self, entity, fn, **filter_dict): 368 | try: 369 | with self.__db_session() as s: 370 | ret = 0 371 | obj = s.query(entity).filter_by(**filter_dict).with_for_update().one_or_none() 372 | if obj is not None: 373 | fn(obj) 374 | ret += 1 375 | s.commit() 376 | return ret 377 | except Exception as e: 378 | filter_str = "&".join([f"{k}={v}" for k, v in filter_dict.items()]) 379 | logging.error(f"An error occured when compute and update {entity.__tablename__}, filter={filter_str}") 380 | return 0 381 | 382 | def _update_all(self, entity, fn, **filter_dict): 383 | try: 384 | with self.__db_session() as s: 385 | ret = 0 386 | objs = s.query(entity).filter_by(**filter_dict).with_for_update().all() 387 | for obj in objs: 388 | fn(obj) 389 | ret += 1 390 | s.commit() 391 | return ret 392 | except Exception as e: 393 | filter_str = "&".join([f"{k}={v}" for k, v in filter_dict.items()]) 394 | logging.error(f"An error occured when compute and update {entity.__tablename__}, filter={filter_str}") 395 | return 0 396 | -------------------------------------------------------------------------------- /gfl/core/db/entities.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "NodeTable", 17 | "JobTable", 18 | "JobTraceTable", 19 | "DatasetTable", 20 | "DatasetTraceTable", 21 | "ParamsTable", 22 | "init_sqlite" 23 | ] 24 | 25 | from sqlalchemy import Column, String, Integer, BigInteger, Float, Boolean, create_engine 26 | from sqlalchemy.ext.declarative import declarative_base 27 | 28 | from gfl.data.constants import JobStatus, DatasetStatus 29 | 30 | 31 | Base = declarative_base() 32 | 33 | 34 | class NodeTable(Base): 35 | 36 | __tablename__ = "node" 37 | 38 | address = Column("address", String(42), primary_key=True, nullable=False) 39 | pub_key = Column("pub_key", String(130), nullable=False) 40 | 41 | def __init__(self, address, pub_key): 42 | super(NodeTable, self).__init__() 43 | self.address = address 44 | self.pub_key = pub_key 45 | 46 | 47 | class JobTable(Base): 48 | 49 | __tablename__ = "job" 50 | 51 | job_id = Column("job_id", String(38), primary_key=True, nullable=False) 52 | owner = Column("owner", String(42), nullable=False) 53 | create_time = Column("create_time", BigInteger, nullable=False) 54 | status = Column("status", Integer, nullable=False, default=JobStatus.NEW.value) 55 | 56 | def __init__(self, job_id, owner, create_time, status=JobStatus.NEW.value): 57 | super(JobTable, self).__init__() 58 | self.job_id = job_id 59 | self.owner = owner 60 | self.create_time = create_time 61 | self.status = status 62 | 63 | 64 | class JobTraceTable(Base): 65 | 66 | __tablename__ = "job_trace" 67 | 68 | job_id = Column("job_id", String(38), primary_key=True, nullable=False) 69 | begin_timepoint = Column("begin_timepoint", BigInteger, nullable=False, default=0) 70 | end_timepoint = Column("end_timepoint", BigInteger, nullable=False, default=0) 71 | ready_time = Column("ready_time", Integer, nullable=False, default=0) 72 | aggregate_running_time = Column("aggregate_running_time", Integer, nullable=False, default=0) 73 | aggregate_waiting_time = Column("aggregate_waiting_time", Integer, nullable=False, default=0) 74 | train_running_time = Column("train_running_time", Integer, nullable=False, default=0) 75 | train_waiting_time = Column("train_waiting_time", Integer, nullable=False, default=0) 76 | comm_time = Column("comm_time", Integer, nullable=False, default=0) 77 | used_time = Column("used_time", Integer, nullable=False, default=0) 78 | 79 | def __init__(self, job_id, 80 | begin_timepoint=0, 81 | end_timepoint=0, 82 | ready_time=0, 83 | aggregate_running_time=0, 84 | aggregate_waiting_time=0, 85 | train_running_time=0, 86 | train_waiting_time=0, 87 | comm_time=0, 88 | used_time=0): 89 | super(JobTraceTable, self).__init__() 90 | self.job_id = job_id 91 | self.begin_timepoint = begin_timepoint 92 | self.end_timepoint = end_timepoint 93 | self.ready_time = ready_time 94 | self.aggregate_running_time = aggregate_running_time 95 | self.aggregate_waiting_time = aggregate_waiting_time 96 | self.train_running_time = train_running_time 97 | self.train_waiting_time = train_waiting_time 98 | self.comm_time = comm_time 99 | self.used_time = used_time 100 | 101 | 102 | class DatasetTable(Base): 103 | 104 | __tablename__ = "dataset" 105 | 106 | dataset_id = Column("dataset_id", String(38), primary_key=True, nullable=False) 107 | owner = Column("owner", String(42), nullable=False) 108 | create_time = Column("create_time", BigInteger, nullable=False) 109 | type = Column("type", Integer, nullable=False) 110 | size = Column("size", BigInteger, nullable=False) 111 | status = Column("status", Integer, nullable=False, default=DatasetStatus.NEW.value) 112 | request_cnt = Column("request_cnt", Integer, nullable=False, default=0) 113 | used_cnt = Column("used_cnt", Integer, nullable=False, default=0) 114 | 115 | def __init__(self, dataset_id, owner, create_time, type, size, 116 | status=DatasetStatus.NEW.value, request_cnt=0, used_cnt=0): 117 | super(DatasetTable, self).__init__() 118 | self.dataset_id = dataset_id 119 | self.owner = owner 120 | self.create_time = create_time 121 | self.type = type 122 | self.size = size 123 | self.status = status 124 | self.request_cnt = request_cnt 125 | self.used_cnt = used_cnt 126 | 127 | 128 | class DatasetTraceTable(Base): 129 | 130 | __tablename__ = "dataset_trace" 131 | 132 | id = Column("id", Integer, primary_key=True, nullable=False, autoincrement=True) 133 | dataset_id = Column("dataset_id", String(38), nullable=False) 134 | job_id = Column("job_id", String(38), nullable=False) 135 | confirmed = Column("confirmed", Boolean, nullable=False, default=False) 136 | score = Column("score", Float, nullable=False, default=0) 137 | 138 | def __init__(self, dataset_id, job_id, confirmed=False, score=0): 139 | super(DatasetTraceTable, self).__init__() 140 | self.dataset_id = dataset_id 141 | self.job_id = job_id 142 | self.confirmed = confirmed 143 | self.score = score 144 | 145 | 146 | class ParamsTable(Base): 147 | 148 | __tablename__ = "params" 149 | 150 | id = Column("id", Integer, primary_key=True, nullable=False, autoincrement=True) 151 | job_id = Column("job_id", String(38), nullable=False) 152 | node_address = Column("node_address", String(42), nullable=False) 153 | dataset_id = Column("dataset_id", String(38), nullable=True, default="") 154 | step = Column("step", Integer, nullable=False) 155 | path = Column("path", String(1024), nullable=False) 156 | loss = Column("loss", Float, nullable=False) 157 | metric_name = Column("metric_name", String(64), nullable=False) 158 | metric_value = Column("metric_value", Float, nullable=False) 159 | score = Column("score", Float, nullable=False) 160 | is_aggregate = Column("is_aggregate", Boolean, nullable=False, default=False) 161 | 162 | def __init__(self, 163 | job_id, 164 | node_address, 165 | dataset_id, 166 | step, 167 | path, 168 | loss, 169 | metric_name, 170 | metric_value, 171 | score, 172 | is_aggregate=False): 173 | super(ParamsTable, self).__init__() 174 | self.job_id = job_id 175 | self.node_address = node_address 176 | self.dataset_id = dataset_id 177 | self.step = step 178 | self.path = path 179 | self.loss = loss 180 | self.metric_name = metric_name 181 | self.metric_value = metric_value 182 | self.score = score 183 | self.is_aggregate = is_aggregate 184 | 185 | 186 | def init_sqlite(path): 187 | engine = create_engine(f"sqlite:///{path}") 188 | Base.metadata.create_all(engine) 189 | -------------------------------------------------------------------------------- /gfl/core/fl_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "FLDataset" 17 | ] 18 | 19 | import json 20 | import os 21 | import shutil 22 | 23 | import zcommons as zc 24 | 25 | from gfl.core.fs.path import DatasetPath 26 | from gfl.data import * 27 | from gfl.utils import ZipUtils 28 | 29 | 30 | class FLDataset(object): 31 | 32 | def __init__(self, dataset_path: DatasetPath, dataset_data: DatasetData): 33 | super(FLDataset, self).__init__() 34 | self._path = dataset_path 35 | self._data = dataset_data 36 | 37 | @property 38 | def data(self): 39 | return self._data 40 | 41 | @property 42 | def id(self): 43 | return self._data.meta.id 44 | 45 | @classmethod 46 | def load(cls, dataset_path: DatasetPath, dataset_id: str): 47 | path = dataset_path.dataset_dir(dataset_id) 48 | if not os.path.exists(path) or not os.path.isdir(path) or len(os.listdir(path)) == 0: 49 | raise ValueError(f"Dataset({dataset_id}) not exists.") 50 | with open(dataset_path.meta_file(dataset_id), "r") as f: 51 | dataset_meta = zc.dataclass.asobj(DatasetMeta, json.loads(f.read())) 52 | with open(dataset_path.config_file(dataset_id), "r") as f: 53 | config = json.loads(f.read()) 54 | dataset_config = config["dataset"] 55 | dataset_data = DatasetData( 56 | meta=dataset_meta, 57 | dataset_config=dataset_config 58 | ) 59 | return FLDataset(dataset_path, dataset_data) 60 | 61 | def save(self, package_data: bytes, overwrite=False): 62 | path = self._path.dataset_dir(self.id) 63 | if os.path.exists(path) and os.path.isdir(path) and len(os.listdir(path)) > 0: 64 | if overwrite: 65 | shutil.rmtree(path) 66 | else: 67 | raise ValueError(f"Dataset({self.id}) exists.") 68 | os.makedirs(path, exist_ok=True) 69 | with open(self._path.meta_file(self.id), "w") as f: 70 | f.write(json.dumps(zc.dataclass.asdict(self._data.meta))) 71 | with open(self._path.config_file(self.id), "w") as f: 72 | f.write(json.dumps(zc.dataclass.asdict(self._data.config))) 73 | ZipUtils.extract_data(package_data, self._path.module_dir(self.id)) 74 | -------------------------------------------------------------------------------- /gfl/core/fl_job.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "FLJob" 17 | ] 18 | 19 | import json 20 | import os 21 | import shutil 22 | 23 | import zcommons as zc 24 | 25 | from gfl.core.fs.path import JobPath 26 | from gfl.data import * 27 | from gfl.utils import ZipUtils 28 | 29 | 30 | class FLJob(object): 31 | 32 | def __init__(self, job_path: JobPath, job_data: JobData = None): 33 | super(FLJob, self).__init__() 34 | self._path = job_path 35 | self._data = job_data 36 | 37 | @property 38 | def data(self): 39 | return self._data 40 | 41 | @property 42 | def id(self): 43 | return self._data.meta.id 44 | 45 | @classmethod 46 | def load(cls, job_path: JobPath, job_id: str): 47 | path = job_path.job_dir(job_id) 48 | if not os.path.exists(path) or not os.path.isdir(path) or len(os.listdir(path)) == 0: 49 | raise ValueError(f"Job({job_id}) not exists") 50 | with open(job_path.meta_file(job_id), "r") as f: 51 | job_meta = zc.dataclass.asobj(JobMeta, json.loads(f.read())) 52 | with open(job_path.config_file(job_id), "r") as f: 53 | config = json.loads(f.read()) 54 | job_config = zc.dataclass.asobj(JobConfig, config["job"]) 55 | train_config = zc.dataclass.asobj(TrainConfig, config["train"]) 56 | aggregate_config = zc.dataclass.asobj(AggregateConfig, config["aggregate"]) 57 | job_data = JobData( 58 | meta=job_meta, 59 | job_config=job_config, 60 | train_config=train_config, 61 | aggregate_config=aggregate_config 62 | ) 63 | return FLJob(job_path, job_data) 64 | 65 | def save(self, package_data: bytes, overwrite=False): 66 | path = self._path.job_dir(self.id) 67 | if os.path.exists(path) and os.path.isdir(path) and len(os.listdir(path)) > 0: 68 | if overwrite: 69 | shutil.rmtree(path) 70 | else: 71 | raise ValueError(f"Job({self.id}) exists.") 72 | # make dirs 73 | os.makedirs(path, exist_ok=True) 74 | with open(self._path.meta_file(self.id), "w") as f: 75 | f.write(self._data.meta.to_json()) 76 | with open(self._path.config_file(self.id), "w") as f: 77 | f.write(json.dumps(self._data.config)) 78 | os.makedirs(self._path.params_dir(self.id)) 79 | os.makedirs(self._path.metrics_dir(self.id)) 80 | os.makedirs(self._path.reports_dir(self.id)) 81 | ZipUtils.extract_data(package_data, self._path.module_dir(self.id)) 82 | -------------------------------------------------------------------------------- /gfl/core/fs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from gfl.core.fs.lfs import Lfs as FS 16 | -------------------------------------------------------------------------------- /gfl/core/fs/lfs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from .path import Path 18 | 19 | 20 | class Lfs(object): 21 | 22 | def __init__(self, home): 23 | super(Lfs, self).__init__() 24 | self.__path = Path(home) 25 | 26 | @property 27 | def home(self): 28 | return str(self.__path.home()) 29 | 30 | @property 31 | def path(self): 32 | return self.__path 33 | 34 | def init(self, overwrite=False): 35 | if not self.__path.home().exists(): 36 | self.__path.home().makedirs() 37 | if len(os.listdir(self.__path.home())) > 0: 38 | if overwrite: 39 | self.__path.home().rm() 40 | self.__path.home().makedirs() 41 | else: 42 | raise ValueError(f"home dir{self.home} is exists and not empty") 43 | self.__path.data_dir().makedirs() 44 | self.__path.logs_dir().makedirs() 45 | self.__path.job.root_dir().makedirs() 46 | self.__path.dataset.root_dir().makedirs() 47 | -------------------------------------------------------------------------------- /gfl/core/fs/path.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "PathLike", 17 | "Path", 18 | "JobPath", 19 | "DatasetPath" 20 | ] 21 | 22 | import os 23 | import shutil 24 | from functools import wraps 25 | from pathlib import PurePosixPath 26 | 27 | 28 | class PathLike(os.PathLike): 29 | # TODO: Add protocol support, such as file://, ipfs://, grpc:// 30 | 31 | def __init__(self, path): 32 | super(PathLike, self).__init__() 33 | self.path = path 34 | 35 | def abspath(self): 36 | return os.path.abspath(self.path) 37 | 38 | def exists(self): 39 | return os.path.exists(self.path) 40 | 41 | def makedirs(self, exist_ok=True): 42 | os.makedirs(self.path, exist_ok=exist_ok) 43 | 44 | def rm(self): 45 | shutil.rmtree(str(self)) 46 | 47 | def as_posix(self): 48 | posix_path = PurePosixPath(self.path) 49 | return posix_path.as_posix() 50 | 51 | def __fspath__(self): 52 | return str(self) 53 | 54 | def __repr__(self): 55 | return self.path 56 | 57 | def __str__(self): 58 | return self.path 59 | 60 | 61 | def path_like(func): 62 | 63 | @wraps(func) 64 | def wrapper(*args, **kwargs) -> PathLike: 65 | return PathLike(func(*args, **kwargs)) 66 | 67 | return wrapper 68 | 69 | 70 | class JobPath(object): 71 | 72 | def __init__(self, job_root): 73 | super(JobPath, self).__init__() 74 | self.__job_root = job_root 75 | 76 | @path_like 77 | def root_dir(self): 78 | return self.__job_root 79 | 80 | @path_like 81 | def meta_file(self, job_id): 82 | return os.path.join(self.__job_root, job_id, "meta.json") 83 | 84 | @path_like 85 | def sqlite_file(self, job_id): 86 | return os.path.join(self.__job_root, job_id, "job.sqlite") 87 | 88 | @path_like 89 | def config_file(self, job_id): 90 | return os.path.join(self.__job_root, job_id, "config.json") 91 | 92 | @path_like 93 | def job_dir(self, job_id): 94 | return os.path.join(self.__job_root, job_id) 95 | 96 | @path_like 97 | def module_dir(self, job_id): 98 | return os.path.join(self.__job_root, job_id, "fl_job") 99 | 100 | @path_like 101 | def params_dir(self, job_id): 102 | return os.path.join(self.__job_root, job_id, "params") 103 | 104 | @path_like 105 | def metrics_dir(self, job_id): 106 | return os.path.join(self.__job_root, job_id, "metrics") 107 | 108 | @path_like 109 | def reports_dir(self, job_id): 110 | return os.path.join(self.__job_root, job_id, "reports") 111 | 112 | @path_like 113 | def train_params_dir(self, job_id, step, address): 114 | return os.path.join(self.__job_root, job_id, "params", str(step), "train", address) 115 | 116 | @path_like 117 | def aggregate_params_dir(self, job_id, step, address): 118 | return os.path.join(self.__job_root, job_id, "params", str(step), "aggregate", address) 119 | 120 | 121 | class DatasetPath(object): 122 | 123 | def __init__(self, dataset_root): 124 | super(DatasetPath, self).__init__() 125 | self.__dataset_root = dataset_root 126 | 127 | @path_like 128 | def root_dir(self): 129 | return self.__dataset_root 130 | 131 | @path_like 132 | def meta_file(self, dataset_id): 133 | return os.path.join(self.__dataset_root, dataset_id, "meta.json") 134 | 135 | @path_like 136 | def config_file(self, dataset_id): 137 | return os.path.join(self.__dataset_root, dataset_id, "config.json") 138 | 139 | @path_like 140 | def dataset_dir(self, dataset_id): 141 | return os.path.join(self.__dataset_root, dataset_id) 142 | 143 | @path_like 144 | def module_dir(self, dataset_id): 145 | return os.path.join(self.__dataset_root, dataset_id, "fl_dataset") 146 | 147 | 148 | class Path(object): 149 | 150 | def __init__(self, home): 151 | super(Path, self).__init__() 152 | self.__home = os.path.abspath(home) 153 | self.__pid_file = os.path.join(self.__home, "proc.pid") 154 | self.__sqlite_file = os.path.join(self.__home, "node.sqlite") 155 | self.__config_file = os.path.join(self.__home, "config.json") 156 | self.__key_file = os.path.join(self.__home, "key.json") 157 | self.__data_dir = os.path.join(self.__home, "data") 158 | self.__logs_dir = os.path.join(self.__home, "logs") 159 | self.__job_path = JobPath(os.path.join(self.__data_dir, "jobs")) 160 | self.__dataset_path = DatasetPath(os.path.join(self.__data_dir, "datasets")) 161 | 162 | @path_like 163 | def home(self): 164 | return self.__home 165 | 166 | @path_like 167 | def pid_file(self): 168 | return self.__pid_file 169 | 170 | @path_like 171 | def sqlite_file(self): 172 | return self.__sqlite_file 173 | 174 | @path_like 175 | def config_file(self): 176 | return self.__config_file 177 | 178 | @path_like 179 | def key_file(self): 180 | return self.__key_file 181 | 182 | @path_like 183 | def data_dir(self): 184 | return self.__data_dir 185 | 186 | @path_like 187 | def logs_dir(self): 188 | return self.__logs_dir 189 | 190 | @property 191 | def job(self): 192 | return self.__job_path 193 | 194 | @property 195 | def dataset(self): 196 | return self.__dataset_path 197 | -------------------------------------------------------------------------------- /gfl/core/net/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/core/net/rpc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/core/net/rpc/client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import grpc 16 | 17 | import gfl.core.net.rpc.gfl_pb2_grpc as gfl_pb2_grpc 18 | 19 | 20 | def build_client(host, poSrt) -> gfl_pb2_grpc.GflStub: 21 | channel = rpc.insecure_channel(f"{host}:{port}") 22 | stub = gfl_pb2_grpc.GflStub(channel) 23 | return stub 24 | -------------------------------------------------------------------------------- /gfl/core/net/rpc/gfl.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/empty.proto"; 4 | import "google/protobuf/wrappers.proto"; 5 | import "gfl/core/data.proto"; 6 | 7 | package gfl.core.net.rpc; 8 | 9 | message ComputingResource { 10 | int32 cpu_utilization = 1; 11 | int32 cpu_cores = 2; 12 | int32 memory_used = 3; 13 | int32 memory_total = 4; 14 | int32 gpu_memory_used = 5; 15 | int32 gpu_memory_total = 6; 16 | } 17 | 18 | message Health { 19 | int32 running_job_count = 1; 20 | ComputingPower power = 2; 21 | } 22 | 23 | message JobMetaList { 24 | repeated JobMeta metas = 1; 25 | } 26 | 27 | message DatasetMetaList { 28 | repeated DatasetMeta metas = 1; 29 | } 30 | 31 | message JobJoinRequest { 32 | string job_id = 1; 33 | string node_id = 2; 34 | string dataset_id = 3; 35 | } 36 | 37 | message ParamsFetchRequest { 38 | string job_id = 1; 39 | string node_id = 2; 40 | int32 step = 3; 41 | } 42 | 43 | service Gfl { 44 | 45 | rpc SendNodeInfo(gfl.core.NodeInfo) returns (google.protobuf.BoolValue); 46 | 47 | rpc GetPubKey(google.protobuf.StringValue) returns (google.protobuf.StringValue); 48 | 49 | rpc SendHealth(Health) returns (google.protobuf.BoolValue); 50 | 51 | rpc GetNetComputingPower(google.protobuf.Empty) returns(ComputingPower); 52 | 53 | rpc GetJobComputingPower(google.protobuf.StringValue) returns(ComputingPower); 54 | 55 | rpc FetchJobMetas(google.protobuf.Int32Value) returns(JobMetaList); 56 | 57 | rpc FetchJob(google.protobuf.StringValue) returns(gfl.core.JobData); 58 | 59 | rpc PushJob(gfl.core.JobData) returns(google.protobuf.BoolValue); 60 | 61 | rpc JoinJob(JobJoinRequest) returns(google.protobuf.BoolValue); 62 | 63 | rpc FetchDatasetMetas(google.protobuf.Empty) returns(DatasetMetaList); 64 | 65 | rpc FetchDataset(google.protobuf.StringValue) returns(gfl.core.DatasetData); 66 | 67 | rpc PushDataset(gfl.core.DatasetData) returns(google.protobuf.BoolValue); 68 | 69 | rpc FetchParams(ParamsFetchRequest) returns(ModelParams); 70 | 71 | rpc PushParams(ModelParams) returns(google.protobuf.BoolValue); 72 | 73 | } 74 | -------------------------------------------------------------------------------- /gfl/core/net/rpc/gfl_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # source: gfl/core/net/rpc/gfl.proto 4 | """Generated protocol buffer code.""" 5 | from google.protobuf import descriptor as _descriptor 6 | from google.protobuf import descriptor_pool as _descriptor_pool 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | # @@protoc_insertion_point(imports) 11 | 12 | _sym_db = _symbol_database.Default() 13 | 14 | 15 | from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 16 | from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 17 | from gfl.core import data_pb2 as gfl_dot_core_dot_data__pb2 18 | 19 | 20 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1agfl/core/net/rpc/gfl.proto\x12\x10gfl.core.net.rpc\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x13gfl/core/data.proto\"\x9a\x01\n\x0e\x43omputingPower\x12\x17\n\x0f\x63pu_utilization\x18\x01 \x01(\x05\x12\x11\n\tcpu_cores\x18\x02 \x01(\x05\x12\x13\n\x0bmemory_used\x18\x03 \x01(\x05\x12\x14\n\x0cmemory_total\x18\x04 \x01(\x05\x12\x17\n\x0fgpu_memory_used\x18\x05 \x01(\x05\x12\x18\n\x10gpu_memory_total\x18\x06 \x01(\x05\"T\n\x06Health\x12\x19\n\x11running_job_count\x18\x01 \x01(\x05\x12/\n\x05power\x18\x02 \x01(\x0b\x32 .gfl.core.net.rpc.ComputingPower\"/\n\x0bJobMetaList\x12 \n\x05metas\x18\x01 \x03(\x0b\x32\x11.gfl.core.JobMeta\"7\n\x0f\x44\x61tasetMetaList\x12$\n\x05metas\x18\x01 \x03(\x0b\x32\x15.gfl.core.DatasetMeta\"E\n\x0eJobJoinRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\t\x12\x12\n\ndataset_id\x18\x03 \x01(\t\"C\n\x12ParamsFetchRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x0f\n\x07node_id\x18\x02 \x01(\t\x12\x0c\n\x04step\x18\x03 \x01(\x05\x32\xed\x07\n\x03Gfl\x12>\n\x0cSendNodeInfo\x12\x12.gfl.core.NodeInfo\x1a\x1a.google.protobuf.BoolValue\x12G\n\tGetPubKey\x12\x1c.google.protobuf.StringValue\x1a\x1c.google.protobuf.StringValue\x12\x42\n\nSendHealth\x12\x18.gfl.core.net.rpc.Health\x1a\x1a.google.protobuf.BoolValue\x12P\n\x14GetNetComputingPower\x12\x16.google.protobuf.Empty\x1a .gfl.core.net.rpc.ComputingPower\x12V\n\x14GetJobComputingPower\x12\x1c.google.protobuf.StringValue\x1a .gfl.core.net.rpc.ComputingPower\x12K\n\rFetchJobMetas\x12\x1b.google.protobuf.Int32Value\x1a\x1d.gfl.core.net.rpc.JobMetaList\x12;\n\x08\x46\x65tchJob\x12\x1c.google.protobuf.StringValue\x1a\x11.gfl.core.JobData\x12\x38\n\x07PushJob\x12\x11.gfl.core.JobData\x1a\x1a.google.protobuf.BoolValue\x12G\n\x07JoinJob\x12 .gfl.core.net.rpc.JobJoinRequest\x1a\x1a.google.protobuf.BoolValue\x12N\n\x11\x46\x65tchDatasetMetas\x12\x16.google.protobuf.Empty\x1a!.gfl.core.net.rpc.DatasetMetaList\x12\x43\n\x0c\x46\x65tchDataset\x12\x1c.google.protobuf.StringValue\x1a\x15.gfl.core.DatasetData\x12@\n\x0bPushDataset\x12\x15.gfl.core.DatasetData\x1a\x1a.google.protobuf.BoolValue\x12J\n\x0b\x46\x65tchParams\x12$.gfl.core.net.rpc.ParamsFetchRequest\x1a\x15.gfl.core.ModelParams\x12?\n\nPushParams\x12\x15.gfl.core.ModelParams\x1a\x1a.google.protobuf.BoolValueb\x06proto3') 21 | 22 | 23 | 24 | _COMPUTINGPOWER = DESCRIPTOR.message_types_by_name['ComputingPower'] 25 | _HEALTH = DESCRIPTOR.message_types_by_name['Health'] 26 | _JOBMETALIST = DESCRIPTOR.message_types_by_name['JobMetaList'] 27 | _DATASETMETALIST = DESCRIPTOR.message_types_by_name['DatasetMetaList'] 28 | _JOBJOINREQUEST = DESCRIPTOR.message_types_by_name['JobJoinRequest'] 29 | _PARAMSFETCHREQUEST = DESCRIPTOR.message_types_by_name['ParamsFetchRequest'] 30 | ComputingPower = _reflection.GeneratedProtocolMessageType('ComputingPower', (_message.Message,), { 31 | 'DESCRIPTOR' : _COMPUTINGPOWER, 32 | '__module__' : 'gfl.core.net.rpc.gfl_pb2' 33 | # @@protoc_insertion_point(class_scope:gfl.core.net.rpc.ComputingPower) 34 | }) 35 | _sym_db.RegisterMessage(ComputingPower) 36 | 37 | Health = _reflection.GeneratedProtocolMessageType('Health', (_message.Message,), { 38 | 'DESCRIPTOR' : _HEALTH, 39 | '__module__' : 'gfl.core.net.rpc.gfl_pb2' 40 | # @@protoc_insertion_point(class_scope:gfl.core.net.rpc.Health) 41 | }) 42 | _sym_db.RegisterMessage(Health) 43 | 44 | JobMetaList = _reflection.GeneratedProtocolMessageType('JobMetaList', (_message.Message,), { 45 | 'DESCRIPTOR' : _JOBMETALIST, 46 | '__module__' : 'gfl.core.net.rpc.gfl_pb2' 47 | # @@protoc_insertion_point(class_scope:gfl.core.net.rpc.JobMetaList) 48 | }) 49 | _sym_db.RegisterMessage(JobMetaList) 50 | 51 | DatasetMetaList = _reflection.GeneratedProtocolMessageType('DatasetMetaList', (_message.Message,), { 52 | 'DESCRIPTOR' : _DATASETMETALIST, 53 | '__module__' : 'gfl.core.net.rpc.gfl_pb2' 54 | # @@protoc_insertion_point(class_scope:gfl.core.net.rpc.DatasetMetaList) 55 | }) 56 | _sym_db.RegisterMessage(DatasetMetaList) 57 | 58 | JobJoinRequest = _reflection.GeneratedProtocolMessageType('JobJoinRequest', (_message.Message,), { 59 | 'DESCRIPTOR' : _JOBJOINREQUEST, 60 | '__module__' : 'gfl.core.net.rpc.gfl_pb2' 61 | # @@protoc_insertion_point(class_scope:gfl.core.net.rpc.JobJoinRequest) 62 | }) 63 | _sym_db.RegisterMessage(JobJoinRequest) 64 | 65 | ParamsFetchRequest = _reflection.GeneratedProtocolMessageType('ParamsFetchRequest', (_message.Message,), { 66 | 'DESCRIPTOR' : _PARAMSFETCHREQUEST, 67 | '__module__' : 'gfl.core.net.rpc.gfl_pb2' 68 | # @@protoc_insertion_point(class_scope:gfl.core.net.rpc.ParamsFetchRequest) 69 | }) 70 | _sym_db.RegisterMessage(ParamsFetchRequest) 71 | 72 | _GFL = DESCRIPTOR.services_by_name['Gfl'] 73 | if _descriptor._USE_C_DESCRIPTORS == False: 74 | 75 | DESCRIPTOR._options = None 76 | _COMPUTINGPOWER._serialized_start=131 77 | _COMPUTINGPOWER._serialized_end=285 78 | _HEALTH._serialized_start=287 79 | _HEALTH._serialized_end=371 80 | _JOBMETALIST._serialized_start=373 81 | _JOBMETALIST._serialized_end=420 82 | _DATASETMETALIST._serialized_start=422 83 | _DATASETMETALIST._serialized_end=477 84 | _JOBJOINREQUEST._serialized_start=479 85 | _JOBJOINREQUEST._serialized_end=548 86 | _PARAMSFETCHREQUEST._serialized_start=550 87 | _PARAMSFETCHREQUEST._serialized_end=617 88 | _GFL._serialized_start=620 89 | _GFL._serialized_end=1625 90 | # @@protoc_insertion_point(module_scope) 91 | -------------------------------------------------------------------------------- /gfl/core/net/rpc/gfl_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | from gfl.core import data_pb2 as gfl_dot_core_dot_data__pb2 6 | from gfl.core.net.rpc import gfl_pb2 as gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2 7 | from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 8 | from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 9 | 10 | 11 | class GflStub(object): 12 | """Missing associated documentation comment in .proto file.""" 13 | 14 | def __init__(self, channel): 15 | """Constructor. 16 | 17 | Args: 18 | channel: A grpc.Channel. 19 | """ 20 | self.SendNodeInfo = channel.unary_unary( 21 | '/gfl.core.net.rpc.Gfl/SendNodeInfo', 22 | request_serializer=gfl_dot_core_dot_data__pb2.NodeInfo.SerializeToString, 23 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 24 | ) 25 | self.GetPubKey = channel.unary_unary( 26 | '/gfl.core.net.rpc.Gfl/GetPubKey', 27 | request_serializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 28 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.FromString, 29 | ) 30 | self.SendHealth = channel.unary_unary( 31 | '/gfl.core.net.rpc.Gfl/SendHealth', 32 | request_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.Health.SerializeToString, 33 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 34 | ) 35 | self.GetNetComputingPower = channel.unary_unary( 36 | '/gfl.core.net.rpc.Gfl/GetNetComputingPower', 37 | request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, 38 | response_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ComputingPower.FromString, 39 | ) 40 | self.GetJobComputingPower = channel.unary_unary( 41 | '/gfl.core.net.rpc.Gfl/GetJobComputingPower', 42 | request_serializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 43 | response_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ComputingPower.FromString, 44 | ) 45 | self.FetchJobMetas = channel.unary_unary( 46 | '/gfl.core.net.rpc.Gfl/FetchJobMetas', 47 | request_serializer=google_dot_protobuf_dot_wrappers__pb2.Int32Value.SerializeToString, 48 | response_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.JobMetaList.FromString, 49 | ) 50 | self.FetchJob = channel.unary_unary( 51 | '/gfl.core.net.rpc.Gfl/FetchJob', 52 | request_serializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 53 | response_deserializer=gfl_dot_core_dot_data__pb2.JobData.FromString, 54 | ) 55 | self.PushJob = channel.unary_unary( 56 | '/gfl.core.net.rpc.Gfl/PushJob', 57 | request_serializer=gfl_dot_core_dot_data__pb2.JobData.SerializeToString, 58 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 59 | ) 60 | self.JoinJob = channel.unary_unary( 61 | '/gfl.core.net.rpc.Gfl/JoinJob', 62 | request_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.JobJoinRequest.SerializeToString, 63 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 64 | ) 65 | self.FetchDatasetMetas = channel.unary_unary( 66 | '/gfl.core.net.rpc.Gfl/FetchDatasetMetas', 67 | request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, 68 | response_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.DatasetMetaList.FromString, 69 | ) 70 | self.FetchDataset = channel.unary_unary( 71 | '/gfl.core.net.rpc.Gfl/FetchDataset', 72 | request_serializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 73 | response_deserializer=gfl_dot_core_dot_data__pb2.DatasetData.FromString, 74 | ) 75 | self.PushDataset = channel.unary_unary( 76 | '/gfl.core.net.rpc.Gfl/PushDataset', 77 | request_serializer=gfl_dot_core_dot_data__pb2.DatasetData.SerializeToString, 78 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 79 | ) 80 | self.FetchParams = channel.unary_unary( 81 | '/gfl.core.net.rpc.Gfl/FetchParams', 82 | request_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ParamsFetchRequest.SerializeToString, 83 | response_deserializer=gfl_dot_core_dot_data__pb2.ModelParams.FromString, 84 | ) 85 | self.PushParams = channel.unary_unary( 86 | '/gfl.core.net.rpc.Gfl/PushParams', 87 | request_serializer=gfl_dot_core_dot_data__pb2.ModelParams.SerializeToString, 88 | response_deserializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 89 | ) 90 | 91 | 92 | class GflServicer(object): 93 | """Missing associated documentation comment in .proto file.""" 94 | 95 | def SendNodeInfo(self, request, context): 96 | """Missing associated documentation comment in .proto file.""" 97 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 98 | context.set_details('Method not implemented!') 99 | raise NotImplementedError('Method not implemented!') 100 | 101 | def GetPubKey(self, request, context): 102 | """Missing associated documentation comment in .proto file.""" 103 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 104 | context.set_details('Method not implemented!') 105 | raise NotImplementedError('Method not implemented!') 106 | 107 | def SendHealth(self, request, context): 108 | """Missing associated documentation comment in .proto file.""" 109 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 110 | context.set_details('Method not implemented!') 111 | raise NotImplementedError('Method not implemented!') 112 | 113 | def GetNetComputingPower(self, request, context): 114 | """Missing associated documentation comment in .proto file.""" 115 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 116 | context.set_details('Method not implemented!') 117 | raise NotImplementedError('Method not implemented!') 118 | 119 | def GetJobComputingPower(self, request, context): 120 | """Missing associated documentation comment in .proto file.""" 121 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 122 | context.set_details('Method not implemented!') 123 | raise NotImplementedError('Method not implemented!') 124 | 125 | def FetchJobMetas(self, request, context): 126 | """Missing associated documentation comment in .proto file.""" 127 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 128 | context.set_details('Method not implemented!') 129 | raise NotImplementedError('Method not implemented!') 130 | 131 | def FetchJob(self, request, context): 132 | """Missing associated documentation comment in .proto file.""" 133 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 134 | context.set_details('Method not implemented!') 135 | raise NotImplementedError('Method not implemented!') 136 | 137 | def PushJob(self, request, context): 138 | """Missing associated documentation comment in .proto file.""" 139 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 140 | context.set_details('Method not implemented!') 141 | raise NotImplementedError('Method not implemented!') 142 | 143 | def JoinJob(self, request, context): 144 | """Missing associated documentation comment in .proto file.""" 145 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 146 | context.set_details('Method not implemented!') 147 | raise NotImplementedError('Method not implemented!') 148 | 149 | def FetchDatasetMetas(self, request, context): 150 | """Missing associated documentation comment in .proto file.""" 151 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 152 | context.set_details('Method not implemented!') 153 | raise NotImplementedError('Method not implemented!') 154 | 155 | def FetchDataset(self, request, context): 156 | """Missing associated documentation comment in .proto file.""" 157 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 158 | context.set_details('Method not implemented!') 159 | raise NotImplementedError('Method not implemented!') 160 | 161 | def PushDataset(self, request, context): 162 | """Missing associated documentation comment in .proto file.""" 163 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 164 | context.set_details('Method not implemented!') 165 | raise NotImplementedError('Method not implemented!') 166 | 167 | def FetchParams(self, request, context): 168 | """Missing associated documentation comment in .proto file.""" 169 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 170 | context.set_details('Method not implemented!') 171 | raise NotImplementedError('Method not implemented!') 172 | 173 | def PushParams(self, request, context): 174 | """Missing associated documentation comment in .proto file.""" 175 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 176 | context.set_details('Method not implemented!') 177 | raise NotImplementedError('Method not implemented!') 178 | 179 | 180 | def add_GflServicer_to_server(servicer, server): 181 | rpc_method_handlers = { 182 | 'SendNodeInfo': grpc.unary_unary_rpc_method_handler( 183 | servicer.SendNodeInfo, 184 | request_deserializer=gfl_dot_core_dot_data__pb2.NodeInfo.FromString, 185 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.SerializeToString, 186 | ), 187 | 'GetPubKey': grpc.unary_unary_rpc_method_handler( 188 | servicer.GetPubKey, 189 | request_deserializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.FromString, 190 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 191 | ), 192 | 'SendHealth': grpc.unary_unary_rpc_method_handler( 193 | servicer.SendHealth, 194 | request_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.Health.FromString, 195 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.SerializeToString, 196 | ), 197 | 'GetNetComputingPower': grpc.unary_unary_rpc_method_handler( 198 | servicer.GetNetComputingPower, 199 | request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, 200 | response_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ComputingPower.SerializeToString, 201 | ), 202 | 'GetJobComputingPower': grpc.unary_unary_rpc_method_handler( 203 | servicer.GetJobComputingPower, 204 | request_deserializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.FromString, 205 | response_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ComputingPower.SerializeToString, 206 | ), 207 | 'FetchJobMetas': grpc.unary_unary_rpc_method_handler( 208 | servicer.FetchJobMetas, 209 | request_deserializer=google_dot_protobuf_dot_wrappers__pb2.Int32Value.FromString, 210 | response_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.JobMetaList.SerializeToString, 211 | ), 212 | 'FetchJob': grpc.unary_unary_rpc_method_handler( 213 | servicer.FetchJob, 214 | request_deserializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.FromString, 215 | response_serializer=gfl_dot_core_dot_data__pb2.JobData.SerializeToString, 216 | ), 217 | 'PushJob': grpc.unary_unary_rpc_method_handler( 218 | servicer.PushJob, 219 | request_deserializer=gfl_dot_core_dot_data__pb2.JobData.FromString, 220 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.SerializeToString, 221 | ), 222 | 'JoinJob': grpc.unary_unary_rpc_method_handler( 223 | servicer.JoinJob, 224 | request_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.JobJoinRequest.FromString, 225 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.SerializeToString, 226 | ), 227 | 'FetchDatasetMetas': grpc.unary_unary_rpc_method_handler( 228 | servicer.FetchDatasetMetas, 229 | request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, 230 | response_serializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.DatasetMetaList.SerializeToString, 231 | ), 232 | 'FetchDataset': grpc.unary_unary_rpc_method_handler( 233 | servicer.FetchDataset, 234 | request_deserializer=google_dot_protobuf_dot_wrappers__pb2.StringValue.FromString, 235 | response_serializer=gfl_dot_core_dot_data__pb2.DatasetData.SerializeToString, 236 | ), 237 | 'PushDataset': grpc.unary_unary_rpc_method_handler( 238 | servicer.PushDataset, 239 | request_deserializer=gfl_dot_core_dot_data__pb2.DatasetData.FromString, 240 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.SerializeToString, 241 | ), 242 | 'FetchParams': grpc.unary_unary_rpc_method_handler( 243 | servicer.FetchParams, 244 | request_deserializer=gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ParamsFetchRequest.FromString, 245 | response_serializer=gfl_dot_core_dot_data__pb2.ModelParams.SerializeToString, 246 | ), 247 | 'PushParams': grpc.unary_unary_rpc_method_handler( 248 | servicer.PushParams, 249 | request_deserializer=gfl_dot_core_dot_data__pb2.ModelParams.FromString, 250 | response_serializer=google_dot_protobuf_dot_wrappers__pb2.BoolValue.SerializeToString, 251 | ), 252 | } 253 | generic_handler = grpc.method_handlers_generic_handler( 254 | 'gfl.core.net.rpc.Gfl', rpc_method_handlers) 255 | server.add_generic_rpc_handlers((generic_handler,)) 256 | 257 | 258 | # This class is part of an EXPERIMENTAL API. 259 | class Gfl(object): 260 | """Missing associated documentation comment in .proto file.""" 261 | 262 | @staticmethod 263 | def SendNodeInfo(request, 264 | target, 265 | options=(), 266 | channel_credentials=None, 267 | call_credentials=None, 268 | insecure=False, 269 | compression=None, 270 | wait_for_ready=None, 271 | timeout=None, 272 | metadata=None): 273 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/SendNodeInfo', 274 | gfl_dot_core_dot_data__pb2.NodeInfo.SerializeToString, 275 | google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 276 | options, channel_credentials, 277 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 278 | 279 | @staticmethod 280 | def GetPubKey(request, 281 | target, 282 | options=(), 283 | channel_credentials=None, 284 | call_credentials=None, 285 | insecure=False, 286 | compression=None, 287 | wait_for_ready=None, 288 | timeout=None, 289 | metadata=None): 290 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/GetPubKey', 291 | google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 292 | google_dot_protobuf_dot_wrappers__pb2.StringValue.FromString, 293 | options, channel_credentials, 294 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 295 | 296 | @staticmethod 297 | def SendHealth(request, 298 | target, 299 | options=(), 300 | channel_credentials=None, 301 | call_credentials=None, 302 | insecure=False, 303 | compression=None, 304 | wait_for_ready=None, 305 | timeout=None, 306 | metadata=None): 307 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/SendHealth', 308 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.Health.SerializeToString, 309 | google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 310 | options, channel_credentials, 311 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 312 | 313 | @staticmethod 314 | def GetNetComputingPower(request, 315 | target, 316 | options=(), 317 | channel_credentials=None, 318 | call_credentials=None, 319 | insecure=False, 320 | compression=None, 321 | wait_for_ready=None, 322 | timeout=None, 323 | metadata=None): 324 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/GetNetComputingPower', 325 | google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, 326 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ComputingPower.FromString, 327 | options, channel_credentials, 328 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 329 | 330 | @staticmethod 331 | def GetJobComputingPower(request, 332 | target, 333 | options=(), 334 | channel_credentials=None, 335 | call_credentials=None, 336 | insecure=False, 337 | compression=None, 338 | wait_for_ready=None, 339 | timeout=None, 340 | metadata=None): 341 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/GetJobComputingPower', 342 | google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 343 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ComputingPower.FromString, 344 | options, channel_credentials, 345 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 346 | 347 | @staticmethod 348 | def FetchJobMetas(request, 349 | target, 350 | options=(), 351 | channel_credentials=None, 352 | call_credentials=None, 353 | insecure=False, 354 | compression=None, 355 | wait_for_ready=None, 356 | timeout=None, 357 | metadata=None): 358 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/FetchJobMetas', 359 | google_dot_protobuf_dot_wrappers__pb2.Int32Value.SerializeToString, 360 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.JobMetaList.FromString, 361 | options, channel_credentials, 362 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 363 | 364 | @staticmethod 365 | def FetchJob(request, 366 | target, 367 | options=(), 368 | channel_credentials=None, 369 | call_credentials=None, 370 | insecure=False, 371 | compression=None, 372 | wait_for_ready=None, 373 | timeout=None, 374 | metadata=None): 375 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/FetchJob', 376 | google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 377 | gfl_dot_core_dot_data__pb2.JobData.FromString, 378 | options, channel_credentials, 379 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 380 | 381 | @staticmethod 382 | def PushJob(request, 383 | target, 384 | options=(), 385 | channel_credentials=None, 386 | call_credentials=None, 387 | insecure=False, 388 | compression=None, 389 | wait_for_ready=None, 390 | timeout=None, 391 | metadata=None): 392 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/PushJob', 393 | gfl_dot_core_dot_data__pb2.JobData.SerializeToString, 394 | google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 395 | options, channel_credentials, 396 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 397 | 398 | @staticmethod 399 | def JoinJob(request, 400 | target, 401 | options=(), 402 | channel_credentials=None, 403 | call_credentials=None, 404 | insecure=False, 405 | compression=None, 406 | wait_for_ready=None, 407 | timeout=None, 408 | metadata=None): 409 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/JoinJob', 410 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.JobJoinRequest.SerializeToString, 411 | google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 412 | options, channel_credentials, 413 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 414 | 415 | @staticmethod 416 | def FetchDatasetMetas(request, 417 | target, 418 | options=(), 419 | channel_credentials=None, 420 | call_credentials=None, 421 | insecure=False, 422 | compression=None, 423 | wait_for_ready=None, 424 | timeout=None, 425 | metadata=None): 426 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/FetchDatasetMetas', 427 | google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, 428 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.DatasetMetaList.FromString, 429 | options, channel_credentials, 430 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 431 | 432 | @staticmethod 433 | def FetchDataset(request, 434 | target, 435 | options=(), 436 | channel_credentials=None, 437 | call_credentials=None, 438 | insecure=False, 439 | compression=None, 440 | wait_for_ready=None, 441 | timeout=None, 442 | metadata=None): 443 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/FetchDataset', 444 | google_dot_protobuf_dot_wrappers__pb2.StringValue.SerializeToString, 445 | gfl_dot_core_dot_data__pb2.DatasetData.FromString, 446 | options, channel_credentials, 447 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 448 | 449 | @staticmethod 450 | def PushDataset(request, 451 | target, 452 | options=(), 453 | channel_credentials=None, 454 | call_credentials=None, 455 | insecure=False, 456 | compression=None, 457 | wait_for_ready=None, 458 | timeout=None, 459 | metadata=None): 460 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/PushDataset', 461 | gfl_dot_core_dot_data__pb2.DatasetData.SerializeToString, 462 | google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 463 | options, channel_credentials, 464 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 465 | 466 | @staticmethod 467 | def FetchParams(request, 468 | target, 469 | options=(), 470 | channel_credentials=None, 471 | call_credentials=None, 472 | insecure=False, 473 | compression=None, 474 | wait_for_ready=None, 475 | timeout=None, 476 | metadata=None): 477 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/FetchParams', 478 | gfl_dot_core_dot_net_dot_rpc_dot_gfl__pb2.ParamsFetchRequest.SerializeToString, 479 | gfl_dot_core_dot_data__pb2.ModelParams.FromString, 480 | options, channel_credentials, 481 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 482 | 483 | @staticmethod 484 | def PushParams(request, 485 | target, 486 | options=(), 487 | channel_credentials=None, 488 | call_credentials=None, 489 | insecure=False, 490 | compression=None, 491 | wait_for_ready=None, 492 | timeout=None, 493 | metadata=None): 494 | return grpc.experimental.unary_unary(request, target, '/gfl.core.net.rpc.Gfl/PushParams', 495 | gfl_dot_core_dot_data__pb2.ModelParams.SerializeToString, 496 | google_dot_protobuf_dot_wrappers__pb2.BoolValue.FromString, 497 | options, channel_credentials, 498 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 499 | -------------------------------------------------------------------------------- /gfl/core/net/rpc/server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from concurrent import futures 16 | 17 | import grpc 18 | 19 | from gfl.data import * 20 | import google.protobuf.wrappers_pb2 as wrappers_pb2 21 | import gfl.core.data_pb2 as data_pb2 22 | import gfl.core.net.rpc.gfl_pb2_grpc as gfl_pb2_grpc 23 | from gfl.core.node import GflNode 24 | from gfl.runtime.manager.server_manager import ServerManager 25 | 26 | 27 | class GflServicer(gfl_pb2_grpc.GflServicer): 28 | 29 | def __init__(self, manager: ServerManager): 30 | super(GflServicer, self).__init__() 31 | self._manager = manager 32 | self.nodes = {} 33 | 34 | def SendNodeInfo(self, request, context): 35 | self.nodes[context.peer()] = GflNode(address=request.address, pub_key=request.pub_key) 36 | return wrappers_pb2.BoolValue(value=True) 37 | 38 | def GetPubKey(self, request, context): 39 | pub_key = "" 40 | for _, node in self.nodes.items(): 41 | if node.address == request.address.value: 42 | pub_key = node.pub_key 43 | break 44 | return wrappers_pb2.StringValue(value=pub_key) 45 | 46 | def SendHealth(self, request, context): 47 | cp = ComputingResource() 48 | cp.mem_used = request.computing_power.mem_used 49 | cp.mem_total 50 | node = self.nodes[context.peer()] 51 | self._manager.update_resource(node.address, request.running_job_count, cp) 52 | return wrappers_pb2.BoolValue(value=True) 53 | 54 | def GetNetComputingPower(self, request, context): 55 | return self._manager.get_net_resource() 56 | 57 | def GetJobComputingPower(self, request, context): 58 | return self._manager.get_node_resource(request.value) 59 | 60 | def FetchJobMetas(self, request, context): 61 | pass 62 | 63 | def FetchJob(self, request, context): 64 | pass 65 | 66 | def PushJob(self, request, context): 67 | pass 68 | 69 | def JoinJob(self, request, context): 70 | pass 71 | 72 | def FetchDatasetMetas(self, request, context): 73 | pass 74 | 75 | def FetchDataset(self, request, context): 76 | pass 77 | 78 | def PushDataset(self, request, context): 79 | pass 80 | 81 | def FetchParams(self, request, context): 82 | pass 83 | 84 | def PushParams(self, request, context): 85 | pass 86 | 87 | 88 | def startup(manager: ServerManager): 89 | print(f"Startup gRPC") 90 | rpc_config = manager.config.node.rpc 91 | bind_host, bind_port = rpc_config.server_host, rpc_config.server_port 92 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=rpc_config.max_workers)) 93 | gfl_pb2_grpc.add_GflServicer_to_server(GflServicer(manager), server) 94 | server.add_insecure_port(f"{bind_host}:{bind_port}") 95 | server.start() 96 | res = server.wait_for_termination() 97 | print(f"Res: {res}") 98 | -------------------------------------------------------------------------------- /gfl/core/node.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import base64 16 | import json 17 | import uuid 18 | 19 | import ecies 20 | from eth_account.messages import encode_defunct 21 | from eth_keys import keys 22 | from web3 import Web3 23 | 24 | w3 = Web3() 25 | 26 | 27 | __global_node: "GflNode" 28 | 29 | 30 | def check_empty(s: str, name: str): 31 | if not isinstance(s, str): 32 | raise ValueError(f"{name}({s}) is not instance of str") 33 | if s is None: 34 | raise ValueError(f"{name} cannot not be None") 35 | if s == "": 36 | raise ValueError(f"{name} cannot be empty str") 37 | 38 | 39 | def _set_global_node(node: "GflNode"): 40 | global __global_node 41 | check_empty(node.address, "address") 42 | check_empty(node.pub_key, "pub_key") 43 | check_empty(node.priv_key, "priv_key") 44 | __global_node = node 45 | 46 | 47 | def encode(bs: bytes, encoding: str): 48 | if encoding == "bytes": 49 | return bs 50 | elif encoding == "base64": 51 | return base64.b64encode(bs) 52 | elif encoding == "hex": 53 | return bs.hex().encode("ascii") 54 | else: 55 | raise ValueError(f"Unsupported encoding({encoding})") 56 | 57 | 58 | def decode(bs, encoding: str) -> bytes: 59 | if encoding == "bytes": 60 | return bs 61 | elif encoding == "base64": 62 | return base64.b64decode(bs) 63 | elif encoding == "hex": 64 | return bytes.fromhex(bs.decode("ascii")) 65 | else: 66 | raise ValueError(f"Unsupported encoding({encoding})") 67 | 68 | 69 | def _get_global_node() -> "GflNode": 70 | global __global_node 71 | return __global_node 72 | 73 | 74 | class GflNode(object): 75 | 76 | def __init__(self, address, pub_key, priv_key=None): 77 | super(GflNode, self).__init__() 78 | self.__address = address 79 | self.__pub_key = pub_key 80 | self.__priv_key = priv_key 81 | 82 | @property 83 | def address(self): 84 | return self.__address 85 | 86 | @property 87 | def pub_key(self): 88 | return self.__pub_key 89 | 90 | @property 91 | def priv_key(self): 92 | return self.__priv_key 93 | 94 | def sign(self, message: bytes) -> str: 95 | """ 96 | 97 | :param message: 98 | :return: 99 | """ 100 | if type(message) != bytes: 101 | raise TypeError("message must be bytes.") 102 | encoded_message = encode_defunct(hexstr=message.hex()) 103 | signed_message = w3.eth.account.sign_message(encoded_message, self.__priv_key) 104 | return signed_message.signature.hex() 105 | 106 | def recover(self, message: bytes, signature: str) -> str: 107 | """ 108 | Get the address of the manager that signed the given message. 109 | 110 | :param message: the message that was signed 111 | :param signature: the signature of the message 112 | :return: the address of the manager 113 | """ 114 | if self.__priv_key is None: 115 | raise ValueError(f"private key is None") 116 | if type(message) != bytes: 117 | raise TypeError("message must be bytes.") 118 | encoded_message = encode_defunct(message) 119 | return w3.eth.account.recover_message(encoded_message, signature=signature) 120 | 121 | def verify(self, message: bytes, signature: str, source_address: str) -> bool: 122 | """ 123 | Verify whether the message is signed by source address 124 | 125 | :param message: the message that was signed 126 | :param signature: the signature of the message 127 | :param source_address: the message sent from 128 | :return: True or False 129 | """ 130 | rec_addr = self.recover(message, signature) 131 | return rec_addr[2:].lower() == source_address.lower() 132 | 133 | def encrypt(self, plain: bytes, encoding="hex") -> bytes: 134 | """ 135 | Encrypt with receiver's public key 136 | 137 | :param plain: data to encrypt 138 | :param encoding: the encoding type of encrypted data, only can be 'bytes', 'base64', or 'hex' 139 | :return: encrypted data 140 | """ 141 | if type(plain) != bytes: 142 | raise TypeError("message must be bytes.") 143 | cipher = ecies.encrypt(self.__pub_key, plain) 144 | return encode(cipher, encoding) 145 | 146 | def decrypt(self, cipher: bytes, encoding="hex") -> bytes: 147 | """ 148 | Decrypt with private key 149 | 150 | :param cipher: encrypted data 151 | :param encoding: the encoding type of encrypted data, only can be 'bytes', 'base64', or 'hex' 152 | :return: 153 | """ 154 | if self.__priv_key is None: 155 | raise ValueError(f"private key is None") 156 | if type(cipher) != bytes: 157 | raise TypeError("cipher only support bytes.") 158 | cipher = decode(cipher, encoding) 159 | return ecies.decrypt(self.__priv_key, cipher) 160 | 161 | def as_alobal(self): 162 | _set_global_node(self) 163 | 164 | @classmethod 165 | def global_instance(cls): 166 | return _get_global_node() 167 | 168 | @classmethod 169 | def new_node(cls): 170 | account = w3.eth.account.create() 171 | priv_key = keys.PrivateKey(account.key) 172 | pub_key = priv_key.public_key 173 | return GflNode(account.address[2:], 174 | pub_key.to_hex()[2:], 175 | priv_key.to_hex()[2:]) 176 | 177 | @classmethod 178 | def load_node(cls, path): 179 | with open(path, "r") as f: 180 | keyjson = json.loads(f.read()) 181 | return GflNode(keyjson.get("address", None), 182 | keyjson.get("pub_key", None), 183 | keyjson.get("priv_key", None)) 184 | 185 | @classmethod 186 | def save_node(cls, node, path): 187 | node.save(path) 188 | 189 | def save(self, path): 190 | with open(path, "w") as f: 191 | f.write(json.dumps({ 192 | "address": self.__address, 193 | "pub_key": self.__pub_key, 194 | "priv_key": self.__priv_key 195 | }, indent=4)) 196 | 197 | 198 | GflNode.new_node().as_alobal() 199 | -------------------------------------------------------------------------------- /gfl/core/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/core/scheduler/aggregate_scheduler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .scheduler import Scheduler 16 | 17 | 18 | class AggregateScheduler(Scheduler): 19 | 20 | def __init__(self): 21 | super(AggregateScheduler, self).__init__() 22 | -------------------------------------------------------------------------------- /gfl/core/scheduler/scheduler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class Scheduler(object): 17 | 18 | def __init__(self): 19 | super(Scheduler, self).__init__() 20 | -------------------------------------------------------------------------------- /gfl/core/scheduler/train_scheduler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .scheduler import Scheduler 16 | from ..fl_job import FLJob 17 | 18 | 19 | class TrainScheduler(Scheduler): 20 | 21 | def __init__(self, job: FLJob): 22 | super(TrainScheduler, self).__init__() 23 | 24 | -------------------------------------------------------------------------------- /gfl/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "GPUResource", 17 | "ComputingResource" 18 | ] + [ 19 | "JobConfig", 20 | "TrainConfig", 21 | "AggregateConfig", 22 | "DatasetConfig" 23 | ] + [ 24 | "JobStatus", 25 | "DatasetStatus", 26 | "DatasetType", 27 | "MIN_HOT_CNT" 28 | ] + [ 29 | "JobData", 30 | "DatasetData" 31 | ] + [ 32 | "JobMeta", 33 | "DatasetMeta" 34 | ] + [ 35 | "ModelParams" 36 | ] 37 | 38 | from .computing_resource import * 39 | from .config import * 40 | from .constants import * 41 | from .meta import * 42 | from .data import * 43 | from .pramas import * 44 | -------------------------------------------------------------------------------- /gfl/data/computing_resource.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "GPUResource", 17 | "ComputingResource" 18 | ] 19 | 20 | 21 | import typing 22 | from dataclasses import dataclass, field 23 | 24 | from zcommons.dataclass import DataMixin 25 | 26 | # the unit of below memory field is Byte 27 | 28 | @dataclass() 29 | class GPUResource(DataMixin): 30 | 31 | gpu_mem_used: int = 0 32 | gpu_mem_total: int = 0 33 | gpu_utilization: int = 0 # [0, 100], means percentage 34 | 35 | 36 | @dataclass() 37 | class ComputingResource(DataMixin): 38 | 39 | running_job_number: int = 0 40 | cpu_utilization: int = 0 # [0, 100], means percentage 41 | cpu_cores: int = 0 # the logical cpu cores 42 | mem_used: int = 0 43 | mem_total: int = 0 44 | gpu_number: int = 0 45 | gpu_mem_used: int = 0 46 | gpu_mem_total: int = 0 47 | gpu_utilization: int = 0 # [0, 100], means percentage 48 | -------------------------------------------------------------------------------- /gfl/data/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "JobConfig", 17 | "TrainConfig", 18 | "AggregateConfig", 19 | "DatasetConfig" 20 | ] 21 | 22 | from dataclasses import dataclass 23 | 24 | from zcommons.dataclass import DataMixin 25 | 26 | 27 | @dataclass() 28 | class JobConfig(DataMixin): 29 | 30 | trainer: str 31 | aggregator: str 32 | 33 | 34 | @dataclass() 35 | class TrainConfig(DataMixin): 36 | 37 | model: str 38 | optimizer: str 39 | criterion: str 40 | lr_scheduler: str = None 41 | epoch: int = 10 42 | batch_size: int = 32 43 | 44 | 45 | @dataclass() 46 | class AggregateConfig(DataMixin): 47 | 48 | global_epoch: int = 50 49 | 50 | 51 | @dataclass() 52 | class DatasetConfig(DataMixin): 53 | 54 | dataset: str 55 | val_dataset: str = None 56 | val_rate: float = 0.2 57 | -------------------------------------------------------------------------------- /gfl/data/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "JobStatus", 17 | "DatasetStatus", 18 | "DatasetType", 19 | "MIN_HOT_CNT" 20 | ] 21 | 22 | from enum import Enum 23 | 24 | 25 | class JobStatus(Enum): 26 | 27 | # just created job 28 | NEW = 0 29 | 30 | # for aggregate : send all requests to train nodes 31 | # for train : send confirm response and waiting others 32 | READY = 1 33 | 34 | # for aggregate : running aggregate algorithm 35 | # for train : running train algorithm 36 | RUNNING = 2 37 | 38 | # for aggregate : waiting the local update params of train nodes 39 | # for train : waiting aggregated params of aggregate node 40 | WAITING = 3 41 | 42 | # job finished 43 | FINISHED = 4 44 | 45 | # job not finished and terminated by user or error 46 | ABORTED = 5 47 | 48 | 49 | class DatasetStatus(Enum): 50 | 51 | # just published dataset, not used(also means validated) by others 52 | NEW = 0 53 | 54 | # the dataset has been used by others at least once, this means it is validated by others. 55 | USED = 1 56 | 57 | # the dataset has been used by others at least ten times 58 | HOT = 2 59 | 60 | # the dataset has been destroyed by its owner 61 | DESTROYED = 3 62 | 63 | 64 | class DatasetType(Enum): 65 | 66 | IMAGE = 0 67 | VIDEO = 1 68 | AUDIO = 2 69 | TEXT = 3 70 | STRUCTURE = 4 71 | 72 | 73 | MIN_HOT_CNT = 10 74 | -------------------------------------------------------------------------------- /gfl/data/data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "JobData", 17 | "DatasetData" 18 | ] 19 | 20 | from dataclasses import dataclass 21 | 22 | from zcommons.dataclass import DataMixin 23 | 24 | from .config import * 25 | from .meta import * 26 | 27 | 28 | @dataclass() 29 | class JobData(DataMixin): 30 | 31 | meta: JobMeta 32 | job_config: JobConfig 33 | train_config: TrainConfig 34 | aggregate_config: AggregateConfig 35 | 36 | @property 37 | def config(self): 38 | return { 39 | "job": self.job_config.to_json_dict(), 40 | "train": self.train_config.to_json_dict(), 41 | "aggregate_config": self.aggregate_config.to_json_dict() 42 | } 43 | 44 | def to_json_dict(self) -> dict: 45 | return { 46 | "meta": self.meta.to_json_dict(), 47 | "job_config": self.job_config.to_json_dict(), 48 | "train_config": self.train_config.to_json_dict(), 49 | "aggregate_config": self.aggregate_config 50 | } 51 | 52 | @classmethod 53 | def from_json_dict(cls, json_dict: dict) -> "JobData": 54 | return JobData( 55 | JobMeta.from_json_dict(json_dict["meta"]), 56 | JobConfig.from_json_dict(json_dict["job_config"]), 57 | TrainConfig.from_json_dict(json_dict["train_config"]), 58 | AggregateConfig.from_json_dict(json_dict["aggregate_config"]) 59 | ) 60 | 61 | 62 | @dataclass() 63 | class DatasetData(DataMixin): 64 | 65 | meta: DatasetMeta 66 | dataset_config: DatasetConfig 67 | 68 | @property 69 | def config(self): 70 | return { 71 | "dataset": self.dataset_config.to_json_dict() 72 | } 73 | 74 | def to_json_dict(self) -> dict: 75 | return { 76 | "meta": self.meta.to_json_dict(), 77 | "dataset_config": self.dataset_config.to_json_dict() 78 | } 79 | 80 | @classmethod 81 | def from_json_dict(cls, json_dict: dict) -> "DataMixin": 82 | return DatasetData( 83 | DatasetMeta.from_json_dict(json_dict["meta"]), 84 | DatasetConfig.from_json_dict(json_dict["dataset_config"]) 85 | ) 86 | -------------------------------------------------------------------------------- /gfl/data/meta.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "JobMeta", 17 | "DatasetMeta" 18 | ] 19 | 20 | from dataclasses import dataclass 21 | from typing import List 22 | 23 | from zcommons.dataclass import DataMixin 24 | 25 | from .constants import * 26 | 27 | 28 | @dataclass() 29 | class Metadata(DataMixin): 30 | 31 | id: str = None 32 | owner: str = None 33 | create_time: int = None 34 | content: str = None 35 | 36 | 37 | @dataclass() 38 | class JobMeta(Metadata, DataMixin): 39 | 40 | status: JobStatus = JobStatus.NEW 41 | datasets: List[str] = None 42 | 43 | def to_json_dict(self) -> dict: 44 | return { 45 | "id": self.id, 46 | "owner": self.owner, 47 | "create_time": self.create_time, 48 | "content": self.content, 49 | "status": self.status.value, 50 | "datasets": self.datasets 51 | } 52 | 53 | @classmethod 54 | def from_json_dict(cls, json_dict: dict) -> "JobMeta": 55 | return JobMeta( 56 | json_dict["id"], 57 | json_dict["owner"], 58 | json_dict["create_time"], 59 | json_dict["content"], 60 | JobStatus(json_dict["status"]), 61 | json_dict["datasets"] 62 | ) 63 | 64 | 65 | @dataclass() 66 | class DatasetMeta(Metadata, DataMixin): 67 | 68 | type: DatasetType = DatasetType.IMAGE 69 | status: DatasetStatus = DatasetStatus.NEW 70 | size: int = 0 71 | used_cnt: int = 0 72 | request_cnt: int = 0 73 | 74 | def to_json_dict(self) -> dict: 75 | return { 76 | "id": self.id, 77 | "owner": self.owner, 78 | "create_time": self.create_time, 79 | "content": self.content, 80 | "type": self.type.value, 81 | "status": self.status.value, 82 | "size": self.size, 83 | "used_cnt": self.used_cnt, 84 | "request_cnt": self.request_cnt 85 | } 86 | 87 | @classmethod 88 | def from_json_dict(cls, json_dict: dict) -> "DatasetMeta": 89 | return DatasetMeta( 90 | json_dict["id"], 91 | json_dict["owner"], 92 | json_dict["create_time"], 93 | json_dict["content"], 94 | DatasetType(json_dict["type"]), 95 | DatasetStatus(json_dict["status"]), 96 | json_dict["size"], 97 | json_dict["used_cnt"], 98 | json_dict["request_cnt"] 99 | ) 100 | -------------------------------------------------------------------------------- /gfl/data/pramas.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "ModelParams" 17 | ] 18 | 19 | from dataclasses import dataclass 20 | 21 | from zcommons.dataclass import DataMixin 22 | 23 | 24 | @dataclass() 25 | class ModelParams(DataMixin): 26 | 27 | job_id: str = None 28 | node_address: str = None 29 | dataset_id: str = None 30 | step: int = 0 31 | path: str = None 32 | loss: float = 0.0 33 | metric_name: str = None 34 | metric_value: float = 0.0 35 | score: float = 0.0 36 | is_aggregate: bool = False 37 | -------------------------------------------------------------------------------- /gfl/data/strategy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/env_detect.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [] 16 | -------------------------------------------------------------------------------- /gfl/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import gfl.runtime.log # enable default logging config 16 | -------------------------------------------------------------------------------- /gfl/runtime/action.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import shutil 17 | 18 | from ..shell import startup as shell_startup 19 | from .app import GflApplication 20 | from .config import GflConfig 21 | from .utils import default_home_path 22 | 23 | 24 | def __clean(path): 25 | if not os.path.exists(path): 26 | return 27 | if not os.path.isdir(path): 28 | raise ValueError(f"{path} is not a directory.") 29 | shutil.rmtree(path) 30 | 31 | 32 | def gfl_init(home, 33 | gfl_config, 34 | force): 35 | if not home: 36 | home = default_home_path() 37 | if gfl_config is None or gfl_config == "": 38 | config = GflConfig() 39 | else: 40 | config = GflConfig.load(gfl_config) 41 | 42 | app = GflApplication(home) 43 | app.init(config, overwrite=force) 44 | 45 | 46 | def gfl_start(home, 47 | no_webui, 48 | no_daemon, 49 | shell): 50 | if not home: 51 | home = default_home_path() 52 | 53 | app = GflApplication(home) 54 | app.start(not no_daemon) 55 | 56 | 57 | def gfl_attach(shell_type, 58 | home, 59 | http_ip, 60 | http_port): 61 | shell_startup(shell_type, home, http_ip, http_port) 62 | -------------------------------------------------------------------------------- /gfl/runtime/app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from .config import GflConfig 17 | from gfl.core.db import init_sqlite 18 | from gfl.core.fs import FS 19 | from gfl.core.node import GflNode 20 | 21 | 22 | class GflApplication(object): 23 | 24 | def __init__(self, home): 25 | super(GflApplication, self).__init__() 26 | self.__fs = FS(home) 27 | self.__node = GflNode.global_instance() 28 | self.__config = GflConfig() 29 | 30 | @property 31 | def home(self) -> str: 32 | return self.__fs.home 33 | 34 | @property 35 | def config(self) -> GflConfig: 36 | return self.__config 37 | 38 | @property 39 | def fs(self) -> FS: 40 | return self.__fs 41 | 42 | @property 43 | def node(self) -> GflNode: 44 | return self.__node 45 | 46 | def init(self, config, *, overwrite): 47 | if isinstance(config, GflConfig): 48 | self.__config = config 49 | elif isinstance(config, str): 50 | self.__config = GflConfig.load(config) 51 | else: 52 | self.__config = GflConfig(config) 53 | self.__fs.init(overwrite) 54 | self.node.save(self.__fs.path.key_file()) 55 | self.config.save(self.__fs.path.config_file()) 56 | init_sqlite(self.__fs.path.sqlite_file()) 57 | 58 | def start(self, daemon=False): 59 | self.__node = GflNode.load_node(self.__fs.path.key_file()) 60 | self.__config = GflConfig.load(self.__fs.path.config_file()) 61 | if self.__config.node.rpc.as_server: 62 | self.__start_server(daemon) 63 | else: 64 | self.__start_client(daemon) 65 | 66 | def __start_server(self, daemon): 67 | from ..core.net.rpc.server import startup 68 | from .manager.server_manager import ServerManager 69 | manager = ServerManager(self.__fs, self.__node, self.__config) 70 | startup(manager) 71 | 72 | def __start_client(self, daemon): 73 | from ..core.net.rpc.client import build_client 74 | from .manager.client_manager import ClientManager 75 | client = build_client(self.__config.node.rpc.server_host, self.__config.node.rpc.server_port) 76 | manager = ClientManager(self.__fs, self.__node, self.__config, client) 77 | manager.startup() 78 | -------------------------------------------------------------------------------- /gfl/runtime/cli_main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import logging 17 | import time 18 | 19 | from .action import gfl_init, gfl_start, gfl_attach 20 | 21 | 22 | logger = logging.getLogger("gfl.runtime") 23 | 24 | 25 | def parse_args(args=None): 26 | parser = argparse.ArgumentParser(prog="GFL", description="") 27 | subparsers = parser.add_subparsers(dest="action", title="action") 28 | 29 | init_parser = subparsers.add_parser("init", help="init gfl env") 30 | init_parser.add_argument("--home", type=str, required=False, help="gfl home directory") 31 | init_parser.add_argument("--gfl-config", "--gfl_config", type=str, required=False, help="gfl config file path") 32 | init_parser.add_argument("--force", action="store_true", help="overwrite exists directory") 33 | 34 | start_parser = subparsers.add_parser("start", help="start gfl node") 35 | start_parser.add_argument("--home", type=str, required=False, help="gfl home directory") 36 | start_parser.add_argument("--allow-remote", action="store_true", help="whether allow remote connection") 37 | start_parser.add_argument("--no-webui", action="store_true", help="disable webui") 38 | start_parser.add_argument("--no-daemon", action="store_true", help="start without daemon") 39 | start_parser.add_argument("--shell", action="store_true", help="start shell with main process") 40 | 41 | attach_parser = subparsers.add_parser("attach", help="attach to gfl node") 42 | attach_parser.add_argument("--shell-type", type=str, default="ipython", help="shell type") 43 | attach_parser.add_argument("--home", type=str, required=False, help="gfl home directory") 44 | attach_parser.add_argument("--node-ip", type=str, required=False, help="gfl node ip") 45 | attach_parser.add_argument("--node-port", type=str, required=False, help="gfl node port") 46 | 47 | return parser.parse_args(args) 48 | 49 | 50 | def main(args=None): 51 | args = parse_args(args) 52 | if args.action == "init": 53 | gfl_init(home=args.home, 54 | gfl_config=args.gfl_config, 55 | force=args.force) 56 | elif args.action == "start": 57 | gfl_start(home=args.home, 58 | no_webui=args.no_webui, 59 | no_daemon=args.no_daemon, 60 | shell=args.shell) 61 | elif args.action == "attach": 62 | gfl_attach(shell_type=args.shell_type, 63 | home=args.home, 64 | http_ip=args.node_ip, 65 | http_port=args.node_ip) 66 | pass 67 | else: 68 | raise ValueError(f"Unsupported action {args.action}") 69 | -------------------------------------------------------------------------------- /gfl/runtime/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | import yaml 17 | from traits.api import HasTraits, Str, Int, Bool, Instance 18 | 19 | 20 | def _load_dict(path, ext): 21 | if ext == "yaml": 22 | return yaml.load(path, Loader=yaml.SafeLoader) 23 | elif ext == "json": 24 | with open(path) as f: 25 | return json.loads(f.read(), encoding="utf8") 26 | else: 27 | raise ValueError(f"Expected yaml or json ext. Received: {ext}") 28 | 29 | 30 | def _check_int_between(_v, _min, _max, msg): 31 | if not isinstance(_v, int) or _v < _min or _v > _max: 32 | raise ValueError(f"") 33 | 34 | 35 | class LogConfig(HasTraits): 36 | level = Str 37 | root = Str 38 | 39 | def __init__(self, config_dict:dict=None): 40 | super(LogConfig, self).__init__() 41 | if config_dict is None: 42 | config_dict = {} 43 | self.level = config_dict.get("level", "INFO") 44 | self.path = config_dict.get("root", "logs") 45 | 46 | @property 47 | def config_dict(self): 48 | return { 49 | "level": self.level, 50 | "root": self.root 51 | } 52 | 53 | 54 | class HttpRpcConfig(HasTraits): 55 | enabled = Bool 56 | as_server = Bool 57 | server_host = Str 58 | server_port = Int 59 | max_workers = Int 60 | 61 | def __init__(self, config_dict:dict=None): 62 | super(HttpRpcConfig, self).__init__() 63 | if config_dict is None: 64 | config_dict = {} 65 | self.enabled = config_dict.get("enabled", False) 66 | self.as_server = config_dict.get("as_server", False) 67 | self.server_host = config_dict.get("server_host", "127.0.0.1") 68 | self.server_port = config_dict.get("server_port", 10702) 69 | self.max_workers = config_dict.get("max_workers", 3) 70 | 71 | @property 72 | def config_dict(self): 73 | return { 74 | "enabled": self.enabled, 75 | "as_server": self.as_server, 76 | "server_host": self.server_host, 77 | "server_port": self.server_port 78 | } 79 | 80 | 81 | class EthConfig(HasTraits): 82 | enabled = Bool 83 | eth_host = Str 84 | eth_port = Int 85 | contract_address = Str 86 | 87 | def __init__(self, config_dict:dict=None): 88 | super(EthConfig, self).__init__() 89 | if config_dict is None: 90 | config_dict = {} 91 | self.enabled = config_dict.get("enabled", False) 92 | self.eth_host = config_dict.get("eth_host", "127.0.0.1") 93 | self.eth_port = config_dict.get("eth_port", 8545) 94 | self.contract_address = config_dict.get("contract_address", "") 95 | 96 | @property 97 | def config_dict(self): 98 | return { 99 | "enabled": self.enabled, 100 | "eth_host": self.eth_host, 101 | "eth_port": self.eth_port, 102 | "contract_address": self.contract_address 103 | } 104 | 105 | 106 | class NodeConfig(HasTraits): 107 | http = Instance(HttpRpcConfig) 108 | rpc = Instance(HttpRpcConfig) 109 | eth = Instance(EthConfig) 110 | 111 | def __init__(self, config_dict: dict = None): 112 | super(NodeConfig, self).__init__() 113 | if config_dict is None: 114 | config_dict = {} 115 | self.http = HttpRpcConfig(config_dict.get("http", {})) 116 | self.rpc = HttpRpcConfig(config_dict.get("rpc", {})) 117 | self.eth = EthConfig(config_dict.get("eth", {})) 118 | 119 | @property 120 | def config_dict(self): 121 | return { 122 | "http": self.http.config_dict, 123 | "rpc": self.rpc.config_dict, 124 | "eth": self.eth.config_dict 125 | } 126 | 127 | 128 | class AppConfig(HasTraits): 129 | shell = Str 130 | 131 | def __init__(self, config_dict): 132 | super(AppConfig, self).__init__() 133 | if config_dict is None: 134 | config_dict = {} 135 | self.shell = config_dict.get("shell", "ipython") 136 | 137 | @property 138 | def config_dict(self): 139 | return { 140 | "shell": self.shell 141 | } 142 | 143 | 144 | class GflConfig(HasTraits): 145 | 146 | app = Instance(AppConfig) 147 | node = Instance(NodeConfig) 148 | log = Instance(LogConfig) 149 | 150 | def __init__(self, config_dict: dict = None): 151 | super(GflConfig, self).__init__() 152 | if config_dict is None: 153 | config_dict = {} 154 | self.app = AppConfig(config_dict.get("app", {})) 155 | self.node = NodeConfig(config_dict.get("node", {})) 156 | self.log = LogConfig(config_dict.get("log", {})) 157 | 158 | @property 159 | def config_dict(self): 160 | return { 161 | "app": self.app.config_dict, 162 | "node": self.node.config_dict, 163 | "log": self.log.config_dict 164 | } 165 | 166 | def save(self, path): 167 | with open(path, "w") as f: 168 | f.write(json.dumps(self.config_dict, indent=4)) 169 | 170 | @classmethod 171 | def load(cls, path, ext="json"): 172 | config_dict = _load_dict(path, ext) 173 | return GflConfig(config_dict) 174 | -------------------------------------------------------------------------------- /gfl/runtime/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | 18 | __FILE_ABSPATH = os.path.abspath(__file__) 19 | GFL_PATH = os.path.dirname(os.path.dirname(__FILE_ABSPATH)) 20 | GFL_CONFIG_FILENAME = "config.json" 21 | 22 | # os envs 23 | ENV_GFL_HOME = "GFL_HOME" 24 | -------------------------------------------------------------------------------- /gfl/runtime/log.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging.config 16 | import os 17 | from logging import Filter 18 | 19 | import zcommons as zc 20 | 21 | 22 | class ColorFilter(Filter): 23 | 24 | colors = { 25 | logging.DEBUG: zc.FORE_CYAN, 26 | logging.INFO: zc.FORE_GREEN, 27 | logging.WARN: zc.FORE_YELLOW, 28 | logging.WARNING: zc.FORE_YELLOW, 29 | logging.ERROR: zc.FORE_RED, 30 | logging.CRITICAL: zc.FORE_MAGENTA 31 | } 32 | 33 | def __init__(self): 34 | super(ColorFilter, self).__init__() 35 | 36 | def filter(self, record) -> bool: 37 | color = self.colors.get(record.levelno, None) 38 | if color: 39 | record.levelname = f"{color}{record.levelname}{' ' * (8 - len(record.levelname))}{zc.FORE_RESET}" 40 | return True 41 | 42 | 43 | def logging_config(log_level="INFO", log_root="logs", color=True, terminal_only=False): 44 | if isinstance(log_level, int): 45 | log_level = { 46 | logging.DEBUG: "DEBUG", 47 | logging.INFO: "INFO", 48 | logging.WARNING: "WARNING", 49 | logging.WARN: "WARNING", 50 | logging.ERROR: "ERROR", 51 | logging.CRITICAL: "CRITICAL" 52 | }[log_level] 53 | log_root = os.path.abspath(log_root) 54 | 55 | handlers = { 56 | "console": { 57 | "class": "logging.StreamHandler", 58 | "level": "INFO", 59 | "formatter": "common", 60 | "stream": "ext://sys.stdout", 61 | "filters": ["color"] if color else [] 62 | } 63 | } 64 | if not terminal_only: 65 | handlers.update({ 66 | "file": { 67 | "class": "logging.FileHandler", 68 | "level": "INFO", 69 | "formatter": "common", 70 | "filename": os.path.join(log_root, "root.log") 71 | }, 72 | "core": { 73 | "class": "logging.FileHandler", 74 | "level": "INFO", 75 | "formatter": "common", 76 | "filename": os.path.join(log_root, "core.log") 77 | }, 78 | "error": { 79 | "class": "logging.FileHandler", 80 | "level": "ERROR", 81 | "formatter": "common", 82 | "filename": os.path.join(log_root, "error.log") 83 | } 84 | }) 85 | 86 | loggers = { 87 | "gfl": { 88 | "level": log_level, 89 | "handlers": ["console", "file"] if not terminal_only else ["console"], 90 | "propagate": "no" 91 | } 92 | } 93 | if not terminal_only: 94 | loggers.update({ 95 | "fedflow.core": { 96 | "level": log_level, 97 | "handlers": ["core"], 98 | "propagate": "yes" 99 | } 100 | }) 101 | 102 | return { 103 | "version": 1, 104 | "formatters": { 105 | "common": { 106 | "format": "%(asctime)s %(process)5d %(name)16s [%(levelname)-5s] %(message)s" 107 | } 108 | }, 109 | "filters": { 110 | "color": { 111 | "()": ColorFilter 112 | } 113 | }, 114 | "handlers": handlers, 115 | "loggers": loggers 116 | } 117 | 118 | 119 | def update_logging_config(log_level="INFO", log_root="logs", color=True, terminal_only=False): 120 | config = logging_config(log_level, log_root, color, terminal_only) 121 | logging.config.dictConfig(config) 122 | 123 | 124 | def set_level(log_level): 125 | update_logging_config(log_level=log_level) 126 | 127 | 128 | def set_root(log_root): 129 | update_logging_config(log_root=log_root) 130 | 131 | 132 | def set_color(use_color): 133 | update_logging_config(color=use_color) 134 | 135 | 136 | update_logging_config(terminal_only=True) 137 | -------------------------------------------------------------------------------- /gfl/runtime/manager/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /gfl/runtime/manager/client_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | class ClientManager(object): 17 | 18 | def __init__(self, fs, node, config, client): 19 | super(ClientManager, self).__init__() 20 | self.__fs = fs 21 | self.__node = node 22 | self.__config = config 23 | self.__client = client 24 | 25 | def startup(self): 26 | pass 27 | -------------------------------------------------------------------------------- /gfl/runtime/manager/resource_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import copy 16 | from threading import Lock 17 | 18 | import zcommons.time as time 19 | 20 | from gfl.data.computing_resource import ComputingResource 21 | 22 | 23 | class _Resource(object): 24 | 25 | def __init__(self, **kwargs): 26 | super(_Resource, self).__init__() 27 | self._resource = ComputingResource(**kwargs) 28 | self._update_timestamp = time.time_ms() 29 | 30 | @property 31 | def running_job_number(self): 32 | return self._resource.running_job_number 33 | 34 | @property 35 | def cpu_utilization(self): 36 | return self._resource.cpu_utilization 37 | 38 | @property 39 | def cpu_cores(self): 40 | return self._resource.cpu_cores 41 | 42 | @property 43 | def memory_used(self): 44 | return self._resource.mem_used 45 | 46 | @property 47 | def memory_total(self): 48 | return self._resource.mem_total 49 | 50 | @property 51 | def gpu_number(self): 52 | return self._resource.gpu_number 53 | 54 | @property 55 | def gpu_memory_used(self): 56 | return self._resource.gpu_mem_used 57 | 58 | @property 59 | def gpu_memory_total(self): 60 | return self._resource.gpu_mem_total 61 | 62 | @property 63 | def gpu_utilization(self): 64 | return self._resource.gpu_utilization 65 | 66 | @property 67 | def update_timestamp(self): 68 | return self._update_timestamp 69 | 70 | def update(self, resource: ComputingResource): 71 | self._resource = copy.deepcopy(resource) 72 | self._update_timestamp = time.time_ms() 73 | 74 | def add(self, resource: '_Resource'): 75 | self._resource.running_job_number += resource.running_job_number 76 | used_cpu_cores = (self.cpu_utilization * self.cpu_cores + resource.cpu_utilization * resource.cpu_cores) / 100 77 | self._resource.cpu_utilization = int(100 * used_cpu_cores / (self.cpu_cores + resource.cpu_cores)) 78 | self._resource.cpu_cores += resource.cpu_cores 79 | self._resource.mem_used += resource.memory_used 80 | self._resource.mem_total += resource.memory_total 81 | self._resource.gpu_number += resource.gpu_number 82 | used_gpu = (self.gpu_utilization * self.gpu_number + resource.gpu_utilization * resource.gpu_number) / 100 83 | self._resource.gpu_utilization = int(100 * used_gpu / (self.gpu_number + resource.gpu_number)) 84 | self._resource.gpu_number += resource.gpu_number 85 | self._resource.gpu_mem_used += resource.gpu_memory_used 86 | self._resource.gpu_mem_total += resource.gpu_memory_total 87 | 88 | @property 89 | def computing_resource(self): 90 | return copy.deepcopy(self._resource) 91 | 92 | 93 | class ResourceManager(object): 94 | 95 | def __init__(self): 96 | super(ResourceManager, self).__init__() 97 | self.__resources = dict() 98 | self.__resources_lock = Lock() 99 | 100 | def update_resource(self, node_address: str, computing_resource: ComputingResource): 101 | with self.__resources_lock: 102 | if node_address not in self.__resources: 103 | self.__resources[node_address] = _Resource() 104 | resource = self.__resources[node_address] 105 | resource.update(computing_resource) 106 | 107 | def get_resource(self, node_address: str) -> ComputingResource: 108 | resource: _Resource = self.__resources.get(node_address, None) 109 | return resource.computing_resource if resource is not None else None 110 | 111 | def get_net_resource(self) -> ComputingResource: 112 | resource = _Resource() 113 | for _, res in self.__resources.items(): 114 | resource.add(res) 115 | return resource.computing_resource if resource is not None else None 116 | -------------------------------------------------------------------------------- /gfl/runtime/manager/server_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import typing 17 | 18 | import zcommons as zc 19 | 20 | from ..config import GflConfig 21 | from gfl.core.db import DB 22 | from gfl.core.fs import FS 23 | from gfl.core.fl_dataset import FLDataset 24 | from gfl.core.fl_job import FLJob 25 | from gfl.core.node import GflNode 26 | from gfl.data import * 27 | from gfl.runtime.manager.resource_manager import ResourceManager 28 | 29 | 30 | class ServerManager(object): 31 | 32 | def __init__(self, fs: FS, node: GflNode, config: GflConfig): 33 | super(ServerManager, self).__init__() 34 | self.__fs = fs 35 | self.__node = node 36 | self.__config = config 37 | self.__db = DB(fs.path.sqlite_file()) 38 | self.__resource_manager = ResourceManager() 39 | 40 | @property 41 | def config(self) -> GflConfig: 42 | return self.__config 43 | 44 | def update_resource(self, node_address, computing_resource): 45 | self.__resource_manager.update_resource(node_address, computing_resource) 46 | 47 | def get_node_resource(self, node_address): 48 | return self.__resource_manager.get_resource(node_address) 49 | 50 | def get_net_resource(self): 51 | return self.__resource_manager.get_net_resource() 52 | 53 | def push_job(self, job_data: JobData, package_data: bytes): 54 | job = FLJob(job_path=self.__fs.path.job, 55 | job_data=job_data) 56 | job.save(package_data, overwrite=False) 57 | # self.__db.add_job(job_data) 58 | 59 | def push_dataset(self, dataset_data: DatasetData, package_data: bytes): 60 | dataset = FLDataset(dataset_path=self.__fs.path.dataset, 61 | dataset_data=dataset_data) 62 | dataset.save(package_data, overwrite=False) 63 | self.__db.add_dataset(dataset_data) 64 | 65 | def fetch_job_metas(self, status) -> typing.List[JobMeta]: 66 | pass 67 | 68 | def fetch_job(self, job_id: str) -> JobData: 69 | job = FLJob.load(self.__fs.path.job, job_id) 70 | return job.data 71 | 72 | def fetch_dataset_metas(self, status) -> typing.List[DatasetMeta]: 73 | pass 74 | 75 | def fetch_dataset(self, dataset_id) -> DatasetData: 76 | dataset = FLDataset.load(self.__fs.path.dataset, dataset_id) 77 | return dataset.data 78 | 79 | def fetch_params(self): 80 | pass 81 | 82 | def push_params(self): 83 | pass 84 | -------------------------------------------------------------------------------- /gfl/runtime/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from .constants import ENV_GFL_HOME, GFL_CONFIG_FILENAME 18 | 19 | 20 | def default_home_path(): 21 | home = os.getenv(ENV_GFL_HOME, None) 22 | if home is None: 23 | home = os.path.join(os.path.expanduser("~"), ".gfl_p") 24 | return home 25 | 26 | 27 | def default_config_path(): 28 | config_path = os.path.join(default_home_path(), GFL_CONFIG_FILENAME) 29 | if os.path.exists(config_path): 30 | return config_path 31 | else: 32 | return None 33 | 34 | 35 | def check_home(home, is_init, is_attach): 36 | if home is None: 37 | if is_attach: 38 | return 39 | home = default_home_path() 40 | if not is_init: 41 | config_path = os.path.join(home, GFL_CONFIG_FILENAME) 42 | if not os.path.exists(config_path): 43 | raise ValueError(f"Cannot find gfl_config.yaml file in gfl home path.") 44 | -------------------------------------------------------------------------------- /gfl/shell/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | from .ipython import startup as ipython_startup 18 | 19 | 20 | def startup(shell_type, home=None, node_ip=None, node_port=None): 21 | if home is not None: 22 | os.environ["__GFL_HOME__"] = home 23 | if node_ip is not None: 24 | os.environ["__GFL_NODE_IP__"] = node_ip 25 | if node_port is not None: 26 | os.environ["__GFL_NODE_PORT__"] = node_port 27 | 28 | if shell_type is None or not isinstance(shell_type, str): 29 | raise ValueError(f"shell_type should be a str") 30 | if shell_type.lower() == "ipython": 31 | ipython_startup() 32 | else: 33 | raise ValueError(f"{shell_type} is not supported") 34 | -------------------------------------------------------------------------------- /gfl/shell/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from gfl.shell import startup 16 | 17 | 18 | if __name__ == "__main__": 19 | startup("ipython") 20 | -------------------------------------------------------------------------------- /gfl/shell/ipython.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | 17 | import IPython 18 | from traitlets.config import Config 19 | 20 | 21 | def startup(): 22 | c = Config() 23 | 24 | c.InteractiveShellApp.exec_lines = [ 25 | "from gfl.shell.ipython_startup import *" 26 | ] 27 | c.InteractiveShell.colors = 'Neutral' 28 | c.InteractiveShell.confirm_exit = False 29 | c.TerminalIPythonApp.display_banner = False 30 | 31 | # Now we start ipython with our configuration 32 | IPython.start_ipython(argv=[], config=c) 33 | -------------------------------------------------------------------------------- /gfl/shell/ipython_startup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __all__ = [ 16 | "gfl", 17 | "net", 18 | "node", 19 | "version" 20 | ] 21 | 22 | import os 23 | import time 24 | import warnings 25 | 26 | import gfl 27 | from gfl.api.net import Net 28 | from gfl.api.node import Node 29 | from gfl.core.fs.path import Path 30 | 31 | 32 | _path = Path(os.environ["__GFL_HOME__"]) 33 | 34 | try: 35 | net = Net(_path.home()) 36 | except: 37 | warnings.warn(f"Server node has not started, 'net' object is unavailable") 38 | net = None 39 | 40 | node = Node(_path.home()) 41 | 42 | version = "0.2.0" 43 | 44 | 45 | def welcome(): 46 | global version 47 | print(f"GFL {version} ({time.asctime()})") 48 | 49 | 50 | welcome() 51 | -------------------------------------------------------------------------------- /gfl/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from gfl.utils.zip_utils import ZipUtils 16 | -------------------------------------------------------------------------------- /gfl/utils/sys_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import psutil 16 | import pynvml 17 | import os 18 | 19 | 20 | try: 21 | pynvml.nvmlInit() 22 | _NVML_INITIALIZED = True 23 | except: 24 | _NVML_INITIALIZED = False 25 | 26 | 27 | class SysUtils(object): 28 | """ 29 | Some methods of query the usage system hardware resources. 30 | The following is meaning of some identifier names: 31 | 32 | proc_*: query resources of specified process 33 | pid: process id, None means current process 34 | index: serial number of cpu or gpu, start from 0 35 | """ 36 | 37 | @classmethod 38 | def cpu_count(cls, logical=True): 39 | return psutil.cpu_count(logical) 40 | 41 | @classmethod 42 | def cpu_percent(cls, index=None): 43 | """ 44 | return the average used percent if index is None or less than 0 45 | """ 46 | if index is None or index < 0: 47 | return psutil.cpu_percent() 48 | else: 49 | return psutil.cpu_percent(percpu=True)[index] 50 | 51 | @classmethod 52 | def mem_total(cls): 53 | mem = psutil.virtual_memory() 54 | return mem.total 55 | 56 | @classmethod 57 | def mem_used(cls): 58 | mem = psutil.virtual_memory() 59 | return mem.used 60 | 61 | @classmethod 62 | def mem_available(cls): 63 | mem = psutil.virtual_memory() 64 | return mem.available 65 | 66 | @classmethod 67 | def mem_free(cls): 68 | mem = psutil.virtual_memory() 69 | return mem.free 70 | 71 | @classmethod 72 | def gpu_count(cls): 73 | try: 74 | return pynvml.nvmlDeviceGetCount() 75 | except: 76 | return 0 77 | 78 | @classmethod 79 | def gpu_mem_total(cls, index): 80 | mem_info = cls.__gpu_memory_info(index) 81 | return mem_info.total if mem_info is not None else 0 82 | 83 | @classmethod 84 | def gpu_mem_used(cls, index): 85 | mem_info = cls.__gpu_memory_info(index) 86 | return mem_info.used if mem_info is not None else 0 87 | 88 | @classmethod 89 | def gpu_mem_free(cls, index): 90 | mem_info = cls.__gpu_memory_info(index) 91 | return mem_info.free if mem_info is not None else 0 92 | 93 | @classmethod 94 | def gpu_utilization_rate(cls, index): 95 | utilization = cls.__gpu_utilization(index) 96 | return utilization.gpu / 100 if utilization is not None else 0 97 | 98 | @classmethod 99 | def proc_cpu_percent(cls, pid=None): 100 | pid = pid or os.getpid() 101 | return psutil.Process(pid).cpu_percent(interval=0.05) 102 | 103 | @classmethod 104 | def proc_mem_used(cls, pid=None): 105 | pid = pid or os.getpid() 106 | return psutil.Process(pid).memory_info().rss 107 | 108 | @classmethod 109 | def proc_gpu_mem_used(cls, index, pid=None): 110 | pid = pid or os.getpid() 111 | proc = cls.__gpu_process(index, pid) 112 | return proc.usedGpuMemory if proc is not None else 0 113 | 114 | @classmethod 115 | def __gpu_memory_info(cls, index): 116 | try: 117 | return pynvml.nvmlDeviceGetMemoryInfo(cls.__gpu_handle(index)) 118 | except: 119 | return None 120 | 121 | @classmethod 122 | def __gpu_utilization(cls, index): 123 | try: 124 | return pynvml.nvmlDeviceGetUtilizationRates(cls.__gpu_handle(index)) 125 | except: 126 | return None 127 | 128 | @classmethod 129 | def __gpu_process(cls, index, pid): 130 | try: 131 | processes = pynvml.nvmlDeviceGetComputeRunningProcesses(cls.__gpu_handle(index)) 132 | for p in processes: 133 | if p.pid == pid: 134 | return p 135 | return None 136 | except: 137 | return None 138 | 139 | @classmethod 140 | def __gpu_handle(cls, index): 141 | if not _NVML_INITIALIZED: 142 | raise pynvml.NVMLError(pynvml.NVML_ERROR_UNINITIALIZED) 143 | return pynvml.nvmlDeviceGetHandleByIndex(index) 144 | -------------------------------------------------------------------------------- /gfl/utils/zip_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | from io import BytesIO 17 | from pathlib import PurePath 18 | from typing import NoReturn, Union 19 | from zipfile import ZipFile, ZIP_DEFLATED 20 | 21 | 22 | class ZipUtils(object): 23 | 24 | @classmethod 25 | def compress_package(cls, package): 26 | """ 27 | 28 | :param package: 29 | :return: 30 | """ 31 | path = package.__file__ 32 | if path.endswith("__init__.py"): 33 | path = os.path.dirname(path) 34 | return cls.get_compress_data(path) 35 | 36 | @classmethod 37 | def get_compress_data(cls, src_paths: Union[str, list], basename=None) -> bytes: 38 | """ 39 | 40 | :param src_paths: 41 | :param basename: 42 | :return: 43 | """ 44 | zip_file = BytesIO() 45 | cls.compress(src_paths, zip_file, basename) 46 | zip_file.seek(0) 47 | data = zip_file.read() 48 | zip_file.close() 49 | return data 50 | 51 | @classmethod 52 | def extract_data(cls, data: bytes, dst_path: str) -> NoReturn: 53 | """ 54 | 55 | :param data: 56 | :param dst_path: 57 | :return: 58 | """ 59 | zip_file = BytesIO() 60 | zip_file.write(data) 61 | zip_file.seek(0) 62 | cls.extract(zip_file, dst_path) 63 | zip_file.close() 64 | 65 | @classmethod 66 | def compress(cls, src_paths: Union[str, list], dst_zip_file, basename=None) -> NoReturn: 67 | """ 68 | Compress the file that scr_paths point to into ZIP format and save it in dst_zip_file. 69 | 70 | :param src_paths: A compressed file or folder directory. Accepts multiple files or folders as a list. 71 | :param dst_zip_file: A file-like object or file path that stores compressed data 72 | :param basename: The root directory of the list of zip files. 73 | """ 74 | if type(src_paths) not in [list, tuple, set]: 75 | src_paths = (src_paths, ) 76 | if basename is None: 77 | basename = cls.__detect_basename(src_paths) 78 | if isinstance(dst_zip_file, str) and os.path.isdir(dst_zip_file): 79 | zip_filename = basename + ".zip" 80 | dst_zip_file = PurePath(dst_zip_file, zip_filename).as_posix() 81 | zip_file = ZipFile(dst_zip_file, "w", ZIP_DEFLATED) 82 | for p in src_paths: 83 | cls.__add_file(zip_file, basename, p) 84 | zip_file.close() 85 | 86 | @classmethod 87 | def extract(cls, src_zip_file, dst_path: str) -> NoReturn: 88 | """ 89 | Unzip the zip file to the specified directory. 90 | 91 | :param src_zip_file: The file-like object or file path to be extracted 92 | :param dst_path: The directory that the zip file is unzipped to. 93 | """ 94 | zip_file = ZipFile(src_zip_file, "r", ZIP_DEFLATED) 95 | zip_file.extractall(dst_path) 96 | zip_file.close() 97 | 98 | @classmethod 99 | def __add_file(cls, zip_file: ZipFile, basename, source): 100 | if not os.path.isdir(source): 101 | zip_file.write(source, basename) 102 | return 103 | for filename in os.listdir(source): 104 | new_source = PurePath(source, filename).as_posix() 105 | new_basename = PurePath(basename, filename).as_posix() 106 | if os.path.isdir(new_source): 107 | cls.__add_file(zip_file, new_basename, new_source) 108 | else: 109 | zip_file.write(new_source, new_basename) 110 | 111 | @classmethod 112 | def __detect_basename(cls, src_paths): 113 | if len(src_paths) == 1: 114 | return os.path.basename(src_paths[0]) 115 | else: 116 | parent_dirname = os.path.dirname(src_paths[0]) 117 | for p in src_paths[1:]: 118 | if parent_dirname != os.path.dirname(p): 119 | return "" 120 | return os.path.basename(parent_dirname) 121 | -------------------------------------------------------------------------------- /gfl/version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | grpcio 2 | Flask 3 | PyYAML 4 | daemoniker 5 | eciespy 6 | grpcio 7 | ipython 8 | protobuf 9 | psutil 10 | pynvml 11 | requests_toolbelt 12 | sqlalchemy 13 | web3 14 | zcommons==0.2.1 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import setuptools 16 | 17 | with open("README.md", "r", encoding="utf-8") as f: 18 | long_description = f.read() 19 | 20 | setuptools.setup( 21 | name="gfl_p", 22 | version="0.2.0", 23 | author="malanore", 24 | author_email="malanore.z@gmail.com", 25 | description="A Galaxy Federated Learning Framework", 26 | long_description=long_description, 27 | long_description_content_type="text/markdown", 28 | url="https://github.com/GalaxyLearning/GFL", 29 | project_urls={ 30 | "Bug Tracker": "https://github.com/GalaxyLearning/GFL/issues" 31 | }, 32 | classifiers=[ 33 | "Intended Audience :: Developers", 34 | "Programming Language :: Python :: 3", 35 | "License :: OSI Approved :: MIT License", 36 | "Operating System :: OS Independent", 37 | ], 38 | package_dir={"": "src"}, 39 | package_data={"": ["resources/*"]}, 40 | packages=setuptools.find_packages(where="src"), 41 | python_requires=">=3.6", 42 | install_requires=[ 43 | 44 | ], 45 | extras_requires={ 46 | "pytorch": [ 47 | "torch>=1.4.0", 48 | "torchvision>=0.5.0" 49 | ] 50 | }, 51 | entry_points={ 52 | "console_scripts": [ 53 | "gfl_p=gfl_p.shell.ipython:startup" 54 | ] 55 | } 56 | ) 57 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/fl_job/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import torch.nn as nn 16 | import torch.nn.functional as F 17 | 18 | 19 | class Net(nn.Module): 20 | 21 | def __init__(self, input_dim=2, output_dim=2): 22 | super(Net, self).__init__() 23 | self.fc_1 = nn.Linear(input_dim, 128) 24 | self.fc_2 = nn.Linear(output_dim, 128) 25 | 26 | def forward(self, x): 27 | x = F.relu(self.fc_1(x)) 28 | return self.fc_2(x) 29 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GalaxyLearning/GFL/007174933f4bb2cb35dd864758a9c4a02a3f46a5/tests/test.py -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import unittest 17 | import uuid 18 | 19 | import zcommons as zc 20 | 21 | from gfl.data import constants, meta, pramas 22 | from gfl.core.db import init_sqlite, DB 23 | from gfl.core.node import GflNode 24 | 25 | 26 | class DBTest(unittest.TestCase): 27 | 28 | def setUp(self) -> None: 29 | resource_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources") 30 | os.makedirs(resource_path, exist_ok=True) 31 | sqlite_filepath = os.path.join(resource_path, "test.sqlite") 32 | if os.path.exists(sqlite_filepath): 33 | os.remove(sqlite_filepath) 34 | init_sqlite(sqlite_filepath) 35 | self.db = DB(sqlite_filepath) 36 | 37 | def test_node(self): 38 | node = GflNode.new_node() 39 | self.db.add_node(node.address, node.pub_key) 40 | self.assertEqual(self.db.get_pub_key(node.address), node.pub_key) 41 | 42 | def test_job(self): 43 | job_id, owner, create_time, dataset_ids = self._add_job() 44 | self.db.update_job(job_id, constants.JobStatus.RUNNING.value) 45 | job = self.db.get_job(job_id) 46 | self.assertEqual(job.job_id, job_id) 47 | self.assertEqual(job.owner, owner) 48 | self.assertEqual(job.create_time, create_time) 49 | self.assertListEqual(list(job.dataset_ids), dataset_ids) 50 | 51 | def test_dataset(self): 52 | node = GflNode.new_node() 53 | dataset_id = str(uuid.uuid4()) 54 | owner = node.address 55 | create_time = zc.time.time_ms() 56 | status = constants.DatasetStatus.NEW.value 57 | type = constants.DatasetType.IMAGE.value 58 | size = zc.units.BinaryUnits.B.convert_from(10, zc.units.BinaryUnits.MiB) 59 | request_cnt = 0 60 | used_cnt = 0 61 | self.db.add_dataset(meta.DatasetMeta(id=dataset_id, 62 | owner=owner, 63 | create_time=create_time, 64 | status=status, 65 | type=type, 66 | size=size, 67 | request_cnt=request_cnt, 68 | used_cnt=used_cnt)) 69 | self.assertTrue(self.db.update_dataset(dataset_id, inc_request_cnt=32, inc_used_cnt=12)) 70 | dataset = self.db.get_dataset(dataset_id) 71 | self.assertEqual(dataset.owner, owner) 72 | self.assertEqual(dataset.status, constants.DatasetStatus.HOT.value) 73 | self.assertEqual(dataset.request_cnt, 32) 74 | self.assertEqual(dataset.used_cnt, 12) 75 | 76 | def test_job_trace(self): 77 | job_id, _, _, _ = self._add_job() 78 | job_trace = self.db.get_job_trace(job_id) 79 | self.assertEqual(job_trace.job_id, job_id) 80 | self.assertEqual(job_trace.ready_time, 0) 81 | 82 | timepoint = zc.time.time_ms() 83 | self.assertTrue(self.db.update_job_trace(job_id, end_timepoint=timepoint, inc_ready_time=10, inc_comm_time=20)) 84 | job_trace = self.db.get_job_trace(job_id) 85 | self.assertEqual(job_trace.end_timepoint, timepoint) 86 | self.assertEqual(job_trace.comm_time, 20) 87 | self.assertEqual(job_trace.ready_time, 10) 88 | 89 | def test_dataset_trace(self): 90 | job_id, _, _, dataset_ids = self._add_job() 91 | self.db.update_dataset_trace(dataset_ids[0], job_id, confirmed=True, score=78) 92 | d_t_1 = self.db.get_dataset_trace(dataset_ids[0], job_id)[0] 93 | self.assertTrue(d_t_1.confirmed) 94 | self.assertEqual(round(1000000 * d_t_1.score), 78000000) 95 | 96 | def test_params(self): 97 | node = GflNode.new_node() 98 | job_id = str(uuid.uuid4()) 99 | params = pramas.ModelParams(job_id=job_id, 100 | node_address=node.address, 101 | step=3, 102 | path="ipfs://", 103 | loss=1.09, 104 | metric_name="acc", 105 | metric_value=0.89, 106 | score=34.5, 107 | is_aggregate=True) 108 | self.db.add_params(params) 109 | self.db.update_params(job_id=job_id, node_address=node.address, dataset_id=None, step=3, is_aggregate=True, 110 | loss=0.98, score=54) 111 | p = self.db.get_params(job_id=job_id, node_address=node.address, dataset_id=None, step=3, is_aggregate=True) 112 | self.assertEqual(round(1000000 * p.loss), 980000) 113 | self.assertEqual(round(1000000 * p.score), 54000000) 114 | 115 | def _add_job(self): 116 | node = GflNode.new_node() 117 | job_id = str(uuid.uuid4()) 118 | owner = node.address 119 | create_time = zc.time.time_ms() 120 | dataset_ids = [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())] 121 | self.db.add_job(meta.JobMeta(id=job_id, owner=owner, create_time=create_time, 122 | status=constants.JobStatus.NEW.value, dataset_ids=dataset_ids)) 123 | return job_id, owner, create_time, dataset_ids 124 | 125 | 126 | if __name__ == '__main__': 127 | unittest.main() 128 | -------------------------------------------------------------------------------- /tests/test_db2.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | import unittest 18 | import uuid 19 | 20 | import zcommons as zc 21 | 22 | from gfl.data import constants 23 | from gfl.core.db import init_sqlite, DB 24 | from gfl.core.node import GflNode 25 | from gfl.data import meta, pramas 26 | 27 | class DBTest(unittest.TestCase): 28 | 29 | def setUp(self) -> None: 30 | sqlite_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "test.sqlite") 31 | if os.path.exists(sqlite_filepath): 32 | os.remove(sqlite_filepath) 33 | init_sqlite(sqlite_filepath) 34 | self.db = DB(sqlite_filepath) 35 | 36 | def test_various_node(self): 37 | """ 38 | 39 | : To test adding various nodes 40 | """ 41 | for i in range(10000): 42 | node = GflNode.new_node() 43 | self.db.add_node(node.address, node.pub_key) 44 | self.assertEqual(self.db.get_pub_key(node.address), node.pub_key) 45 | 46 | def test_job(self): 47 | """ 48 | 49 | : To test adding various jobs 50 | """ 51 | for i in range(10000): 52 | job_id, owner, create_time, dataset_ids = self._add_job() 53 | self.db.update_job(job_id, constants.JobStatus.RUNNING.value) 54 | job = self.db.get_job(job_id) 55 | self.assertEqual(job.job_id, job_id) 56 | self.assertEqual(job.owner, owner) 57 | self.assertEqual(job.create_time, create_time) 58 | self.assertListEqual(list(job.dataset_ids), dataset_ids) 59 | 60 | def test_dataset_huge_property(self): 61 | """ 62 | 63 | : To test data with huge properties and multiple changes 64 | """ 65 | node = GflNode.new_node() 66 | dataset_id = str(uuid.uuid4()) 67 | owner = node.address 68 | create_time = zc.time.time_ms() 69 | status = constants.DatasetStatus.NEW.value 70 | type = constants.DatasetType.IMAGE.value 71 | size = zc.units.BinaryUnits.B.convert_from(10**18, zc.units.BinaryUnits.MiB) 72 | request_cnt = int(10e9 - 10) 73 | used_cnt = 10**9-10 74 | self.db.add_dataset(meta.DatasetMeta(id=dataset_id, 75 | owner=owner, 76 | create_time=create_time, 77 | status=status, 78 | type=type, 79 | size=size, 80 | request_cnt=request_cnt, 81 | used_cnt=used_cnt)) 82 | for i in range(10000): 83 | self.assertTrue(self.db.update_dataset(dataset_id, inc_request_cnt=2, inc_used_cnt=3)) 84 | dataset = self.db.get_dataset(dataset_id) 85 | self.assertEqual(dataset.owner, owner) 86 | self.assertEqual(dataset.status, constants.DatasetStatus.HOT.value) 87 | self.assertEqual(dataset.request_cnt, 10 ** 9 - 10 + (i + 1) *2) 88 | self.assertEqual(dataset.used_cnt, 10 ** 9 + (i + 1) * 3) 89 | 90 | def test_job_trace_multi_change(self): 91 | """ 92 | 93 | : To test job trace with multiple changes 94 | """ 95 | job_id, _, _, _ = self._add_job() 96 | job_trace = self.db.get_job_trace(job_id) 97 | self.assertEqual(job_trace.job_id, job_id) 98 | self.assertEqual(job_trace.ready_time, 0) 99 | 100 | for i in range(10000): 101 | timepoint = zc.time.time_ms() 102 | self.assertTrue(self.db.update_job_trace(job_id, end_timepoint=timepoint, inc_ready_time=10, inc_comm_time=20)) 103 | job_trace = self.db.get_job_trace(job_id) 104 | self.assertEqual(job_trace.end_timepoint, timepoint) 105 | self.assertEqual(job_trace.comm_time, (i + 1) * 10) 106 | self.assertEqual(job_trace.ready_time, (i + 1) * 20) 107 | 108 | def test_dataset_trace_multi_change(self): 109 | """ 110 | 111 | : To test dataset trace with multiple changes 112 | """ 113 | job_id, _, _, dataset_ids = self._add_job() 114 | for i in range(1000): 115 | self.db.update_dataset_trace(dataset_ids[0], job_id, confirmed=True, score=i) 116 | d_t_1 = self.db.get_dataset_trace(dataset_ids[0], job_id)[0] 117 | self.assertTrue(d_t_1.confirmed) 118 | self.assertEqual(round(1000000 * d_t_1.score), i * 1000000) 119 | 120 | def test_params(self): 121 | """ 122 | 123 | : To test Model Params with multiple changes 124 | """ 125 | node = GflNode.new_node() 126 | job_id = str(uuid.uuid4()) 127 | params = pramas.ModelParams(job_id=job_id, 128 | node_address=node.address, 129 | step=3, 130 | path="ipfs://", 131 | loss=1.09, 132 | metric_name="acc", 133 | metric_value=0.89, 134 | score=34.5, 135 | is_aggregate=True) 136 | self.db.add_params(params) 137 | for i in range(1000): 138 | self.db.update_params(job_id=job_id, node_address=node.address, dataset_id=None, step=3, is_aggregate=True, 139 | loss=0.98, score=i) 140 | p = self.db.get_params(job_id=job_id, node_address=node.address, dataset_id=None, step=3, is_aggregate=True) 141 | self.assertEqual(round(1000000 * p.loss), 980000) 142 | self.assertEqual(round(1000000 * p.score), i * 1000000) 143 | 144 | def _add_job(self): 145 | node = GflNode.new_node() 146 | job_id = str(uuid.uuid4()) 147 | owner = node.address 148 | create_time = zc.time.time_ms() 149 | dataset_ids = [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())] 150 | self.db.add_job(meta.JobMeta(id=job_id, owner=owner, create_time=create_time, 151 | status=constants.JobStatus.NEW.value, dataset_ids=dataset_ids)) 152 | self.assertWarning() 153 | return job_id, owner, create_time, dataset_ids 154 | 155 | def test_dumplicate_job(self): 156 | """ 157 | 158 | : To test adding dumplicate jobs into a database 159 | """ 160 | node = GflNode.new_node() 161 | job_id = str(uuid.uuid4()) 162 | owner = node.address 163 | create_time = zc.time.time_ms() 164 | dataset_ids = [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())] 165 | self.db.add_job(meta.JobMeta(id=job_id, owner=owner, create_time=create_time, 166 | status=constants.JobStatus.NEW.value, dataset_ids=dataset_ids)) 167 | self.db.add_job(meta.JobMeta(id=job_id, owner=owner, create_time=create_time, 168 | status=constants.JobStatus.NEW.value, dataset_ids=dataset_ids)) 169 | self.assertWarning() 170 | 171 | 172 | 173 | if __name__ == '__main__': 174 | unittest.main() -------------------------------------------------------------------------------- /tests/test_server_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The GFL Authors. All Rights Reserved. 2 | # # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import unittest 17 | import uuid 18 | 19 | import zcommons as zc 20 | 21 | from gfl.core.db import init_sqlite 22 | from gfl.core.fs import FS 23 | from gfl.core.node import GflNode 24 | from gfl.data import * 25 | from gfl.runtime.config import GflConfig 26 | from gfl.runtime.manager.server_manager import ServerManager 27 | from gfl.utils import ZipUtils 28 | 29 | import tests.fl_job as fl_job 30 | 31 | 32 | class ServerManagerTest(unittest.TestCase): 33 | 34 | def setUp(self) -> None: 35 | home = os.path.join(os.path.dirname(os.path.abspath(__file__)), "server-manager-home") 36 | fs = FS(home) 37 | node = GflNode.global_instance() 38 | config = GflConfig() 39 | # init 40 | fs.init(overwrite=True) 41 | node.save(fs.path.key_file()) 42 | config.save(fs.path.config_file()) 43 | init_sqlite(fs.path.sqlite_file()) 44 | 45 | self.node = node 46 | self.server_manager = ServerManager(fs, node, config) 47 | 48 | def test_push_job(self): 49 | job_data = JobData( 50 | JobMeta( 51 | id=str(uuid.uuid4()), 52 | owner=self.node.address, 53 | create_time=zc.time.time_ms(), 54 | content="" 55 | ), 56 | JobConfig( 57 | trainer="gfl.abc.FLTrainer", 58 | aggregator="gfl.abc.FLAggregator" 59 | ), 60 | TrainConfig( 61 | model="Net", 62 | optimizer="SGD", 63 | criterion="CrossEntropyLoss" 64 | ), 65 | AggregateConfig( 66 | global_epoch=50 67 | ) 68 | ) 69 | package_data = ZipUtils.compress_package(fl_job) 70 | self.server_manager.push_job(job_data, package_data) 71 | --------------------------------------------------------------------------------