├── .github ├── ISSUE_TEMPLATE │ ├── 00-bug-report.md │ └── 00-feature-request.md └── workflows │ └── doc.yml ├── .gitignore ├── .gitmodules ├── .readthedocs.yml ├── LICENSE ├── README.md ├── README.zh.md ├── RELEASE.md ├── bin ├── init_env.sh └── service.sh ├── conf ├── casbin_model.conf ├── job_default_config.yaml ├── permission_casbin_model.conf ├── pulsar_route_table.yaml ├── rabbitmq_route_table.yaml └── service_conf.yaml ├── doc ├── bfia_access.md ├── bfia_access.zh.md ├── build │ ├── build.py │ └── build_swagger_server.py ├── data_access.md ├── data_access.zh.md ├── fate_access.md ├── fate_access.zh.md ├── fate_flow.md ├── fate_flow.zh.md ├── images │ ├── container_load.png │ ├── fate_arch.png │ ├── fate_deploy_directory.png │ ├── fate_flow_api.png │ ├── fate_flow_arch.png │ ├── fate_flow_authorization.png │ ├── fate_flow_component_dsl.png │ ├── fate_flow_component_registry.png │ ├── fate_flow_dag.png │ ├── fate_flow_detector.png │ ├── fate_flow_dsl.png │ ├── fate_flow_inputoutput.png │ ├── fate_flow_major_feature.png │ ├── fate_flow_model_storage.png │ ├── fate_flow_pipelined_model.png │ ├── fate_flow_scheduling_arch.png │ ├── federated_learning_pipeline.png │ ├── federationml_schedule.png │ ├── flow_cluster.png │ ├── log.png │ ├── muti_protocol.png │ ├── open_flow.png │ ├── open_flow │ │ ├── pipeline.png │ │ ├── pipeline_unionpay_lr.png │ │ ├── pipeline_unionpay_sbt.png │ │ ├── scheduler.png │ │ └── upload_data.png │ ├── pull.png │ ├── push.png │ ├── schedule_for_callback.png │ ├── schedule_for_component.png │ ├── schedule_for_poll.png │ ├── scheduler.png │ └── upload_data.png ├── index.md ├── index.zh.md ├── job_scheduling.md ├── job_scheduling.zh.md ├── mkdocs │ ├── README.md │ ├── assets │ │ └── animations │ │ │ └── ml.json │ ├── css │ │ ├── custom.css │ │ ├── extra.css │ │ ├── landing.css │ │ └── termynal.css │ ├── docker │ │ ├── Dockerfile │ │ └── README.md │ ├── js │ │ ├── custom.js │ │ ├── lottie-player.js │ │ └── termynal.js │ ├── requirements.txt │ └── theme │ │ ├── README.md │ │ └── overrides │ │ ├── home.html │ │ └── home.zh.html ├── provider_register.md ├── provider_register.zh.md ├── quick_start.md ├── quick_start.zh.md ├── swagger │ ├── index.md │ └── swagger.yaml ├── system_conf.md └── system_conf.zh.md ├── examples ├── bfia │ ├── fate │ │ ├── component_define │ │ │ ├── lr算法描述.json │ │ │ ├── psi算法描述.json │ │ │ ├── sbt算法描述.json │ │ │ └── transformer算法描述.json │ │ ├── job │ │ │ ├── dataframe_transformer.yaml │ │ │ ├── psi_lr.yaml │ │ │ └── psi_sbt.yaml │ │ ├── pipeline │ │ │ ├── test_dataframe_transformer.py │ │ │ ├── test_lr.py │ │ │ └── test_sbt.py │ │ ├── register │ │ │ └── fate_components.json │ │ └── upload │ │ │ ├── upload_guest.py │ │ │ └── upload_host.py │ └── unionpay │ │ ├── component_define │ │ ├── lr算法描述.json │ │ ├── psi算法描述.json │ │ └── sbt算法描述.json │ │ ├── job │ │ ├── psi_lr.yaml │ │ └── psi_sbt.yaml │ │ ├── pipeline │ │ ├── test_unionpay_lr.py │ │ └── test_unionpay_sbt.py │ │ └── register │ │ └── unionpay_components.json ├── data │ ├── breast_hetero_guest.csv │ └── breast_hetero_host.csv ├── lr │ └── train_lr.yaml ├── model │ ├── export.json │ └── import.json ├── permission │ ├── delete.json │ └── grant.json ├── provider │ ├── register.json │ └── register_image.json ├── transformer │ ├── transformer_guest.json │ └── transformer_host.json └── upload │ ├── upload_guest.json │ └── upload_host.json ├── fateflow.env ├── mkdocs.yml ├── proto ├── generate_proto_buffer.sh └── rollsite │ ├── basic-meta.proto │ └── proxy.proto └── python ├── __init__.py ├── fate_flow ├── __init__.py ├── _info.py ├── adapter │ ├── __init__.py │ ├── _controller.py │ ├── _loader.py │ └── bfia │ │ ├── __init__.py │ │ ├── apps │ │ ├── __init__.py │ │ ├── interconn_app.py │ │ └── platform_app.py │ │ ├── bridge │ │ ├── __init__.py │ │ └── job.py │ │ ├── conf │ │ └── route_table.yaml │ │ ├── container │ │ ├── __init__.py │ │ ├── entrypoint │ │ │ ├── __init__.py │ │ │ ├── cli.py │ │ │ └── runner.py │ │ └── wraps │ │ │ ├── __init__.py │ │ │ └── wraps.py │ │ ├── db │ │ ├── __init__.py │ │ └── output.py │ │ ├── engine_storage │ │ ├── __init__.py │ │ ├── client.py │ │ ├── consts.py │ │ ├── meta.py │ │ ├── session.py │ │ └── table.py │ │ ├── examples │ │ ├── job │ │ │ ├── fate │ │ │ │ ├── fate_components.json │ │ │ │ ├── fate_psi_lr.json │ │ │ │ ├── fate_psi_sbt.json │ │ │ │ ├── lr算法描述.json │ │ │ │ ├── psi算法描述.json │ │ │ │ └── sbt算法描述.json │ │ │ └── unionpay │ │ │ │ ├── bfia_psi_lr.json │ │ │ │ ├── bfia_psi_sbt.json │ │ │ │ └── unionpay_components.json │ │ └── test │ │ │ ├── data │ │ │ ├── guest │ │ │ │ ├── data_0 │ │ │ │ └── metadata │ │ │ └── host │ │ │ │ ├── data_0 │ │ │ │ └── metadata │ │ │ ├── test_job.py │ │ │ ├── test_task.py │ │ │ └── upload │ │ │ ├── upload_guest.py │ │ │ └── upload_host.py │ │ ├── runtime_config.py │ │ ├── scheduler │ │ ├── __init__.py │ │ ├── detector.py │ │ └── scheduler.py │ │ ├── settings.py │ │ ├── translator │ │ ├── __init__.py │ │ ├── component_spec.py │ │ └── dsl_translator.py │ │ ├── utils │ │ ├── api_utils.py │ │ ├── entity │ │ │ ├── __init__.py │ │ │ ├── code.py │ │ │ └── status.py │ │ └── spec │ │ │ ├── artifact.py │ │ │ ├── job.py │ │ │ └── task.py │ │ └── wheels │ │ ├── __init__.py │ │ ├── federated.py │ │ ├── job.py │ │ ├── output.py │ │ ├── parser.py │ │ ├── saver.py │ │ └── task.py ├── apps │ ├── __init__.py │ ├── client │ │ ├── __init__.py │ │ ├── client_app.py │ │ ├── data_app.py │ │ ├── job_app.py │ │ ├── log_app.py │ │ ├── model_app.py │ │ ├── output_app.py │ │ ├── permission_app.py │ │ ├── provider_app.py │ │ ├── server_app.py │ │ ├── site_app.py │ │ └── table_app.py │ ├── desc.py │ ├── desc_zh.py │ ├── partner │ │ ├── __init__.py │ │ └── partner_app.py │ ├── scheduler │ │ ├── __init__.py │ │ └── scheduler_app.py │ └── worker │ │ ├── __init__.py │ │ └── worker_app.py ├── commands │ ├── __init__.py │ ├── server_cli.py │ ├── service.py │ └── service.sh ├── components │ ├── __init__.py │ ├── __main__.py │ ├── components │ │ ├── __init__.py │ │ ├── download.py │ │ └── upload.py │ ├── cpn.py │ ├── define │ │ ├── download.yaml │ │ └── upload.yaml │ └── entrypoint │ │ ├── __init__.py │ │ └── cli.py ├── controller │ ├── __init__.py │ ├── federated.py │ ├── job.py │ ├── parser.py │ ├── permission.py │ └── task.py ├── db │ ├── __init__.py │ ├── base_models.py │ ├── casbin_models.py │ ├── db_models.py │ ├── permission_models.py │ ├── schedule_models.py │ └── storage_models.py ├── engine │ ├── __init__.py │ ├── backend │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── _eggroll.py │ │ ├── _eggroll_deepspeed.py │ │ ├── _session.py │ │ └── _spark.py │ ├── devices │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── container.py │ │ ├── deepspeed.py │ │ └── local.py │ ├── relation_ship.py │ └── storage │ │ ├── __init__.py │ │ ├── _abc.py │ │ ├── _partitioner.py │ │ ├── _session.py │ │ ├── _table.py │ │ ├── _types.py │ │ ├── eggroll │ │ ├── __init__.py │ │ ├── _session.py │ │ └── _table.py │ │ ├── file │ │ ├── __init__.py │ │ ├── _session.py │ │ └── _table.py │ │ ├── hdfs │ │ ├── __init__.py │ │ ├── _session.py │ │ └── _table.py │ │ ├── serdes │ │ ├── __init__.py │ │ ├── _integer_serdes.py │ │ ├── _serdes_base.py │ │ └── _unrestricted_serdes.py │ │ └── standalone │ │ ├── __init__.py │ │ ├── _session.py │ │ ├── _standalone.py │ │ └── _table.py ├── entity │ ├── __init__.py │ ├── _base.py │ ├── code │ │ ├── __init__.py │ │ ├── _api.py │ │ ├── _process.py │ │ └── _schedule.py │ ├── spec │ │ ├── __init__.py │ │ ├── dag │ │ │ ├── __init__.py │ │ │ ├── _artifact.py │ │ │ ├── _component.py │ │ │ ├── _computing.py │ │ │ ├── _device.py │ │ │ ├── _federation.py │ │ │ ├── _job.py │ │ │ ├── _logger.py │ │ │ ├── _mlmd.py │ │ │ ├── _output.py │ │ │ ├── _party.py │ │ │ └── _task.py │ │ └── flow │ │ │ ├── __init__.py │ │ │ ├── _model.py │ │ │ ├── _protocol.py │ │ │ ├── _provider.py │ │ │ ├── _scheduler.py │ │ │ └── _storage.py │ └── types │ │ ├── __init__.py │ │ ├── _address.py │ │ ├── _api.py │ │ ├── _artificats.py │ │ ├── _command.py │ │ ├── _engine.py │ │ ├── _federation.py │ │ ├── _input.py │ │ ├── _instance.py │ │ ├── _output.py │ │ ├── _permission.py │ │ ├── _provider.py │ │ ├── _status.py │ │ └── _work.py ├── errors │ ├── __init__.py │ ├── server_error.py │ └── zookeeper_error.py ├── fate_flow_server.py ├── hook │ ├── __init__.py │ ├── common │ │ ├── __init__.py │ │ └── parameters.py │ └── flow │ │ ├── __init__.py │ │ ├── client_authentication.py │ │ ├── permission.py │ │ └── site_authentication.py ├── hub │ ├── __init__.py │ ├── components_wraps │ │ ├── __init__.py │ │ └── fate │ │ │ ├── __init__.py │ │ │ └── _wraps.py │ ├── database │ │ ├── __init__.py │ │ ├── mysql.py │ │ └── sqlite.py │ ├── encrypt │ │ ├── __init__.py │ │ └── password_encrypt.py │ ├── flow_hub.py │ └── provider │ │ ├── __init__.py │ │ ├── docker.py │ │ └── local.py ├── manager │ ├── __init__.py │ ├── components │ │ ├── __init__.py │ │ ├── base.py │ │ ├── component_manager.py │ │ └── download.py │ ├── container │ │ ├── __init__.py │ │ ├── docker_manager.py │ │ ├── k8s_conf_template.yaml │ │ ├── k8s_manager.py │ │ └── k8s_template.yaml │ ├── operation │ │ ├── __init__.py │ │ ├── base_saver.py │ │ └── job_saver.py │ ├── outputs │ │ ├── __init__.py │ │ ├── data.py │ │ ├── log.py │ │ ├── metric.py │ │ └── model │ │ │ ├── __init__.py │ │ │ ├── engine │ │ │ ├── __init__.py │ │ │ ├── _mysql.py │ │ │ └── _tencent_cos.py │ │ │ ├── handel │ │ │ ├── __init__.py │ │ │ ├── _base.py │ │ │ ├── _file.py │ │ │ ├── _mysql.py │ │ │ └── _tencent_cos.py │ │ │ ├── model_manager.py │ │ │ └── model_meta.py │ ├── permission │ │ └── __init__.py │ ├── pipeline │ │ ├── __init__.py │ │ └── pipeline.py │ ├── service │ │ ├── __init__.py │ │ ├── app_manager.py │ │ ├── config_manager.py │ │ ├── provider_manager.py │ │ ├── resource_manager.py │ │ ├── service_manager.py │ │ └── worker_manager.py │ └── worker │ │ ├── __init__.py │ │ ├── deepspeed_download_model.py │ │ ├── fate_ds_executor.py │ │ ├── fate_executor.py │ │ └── fate_flow_executor.py ├── proto │ ├── __init__.py │ └── rollsite │ │ ├── __init__.py │ │ ├── basic_meta_pb2.py │ │ ├── proxy_pb2.py │ │ └── proxy_pb2_grpc.py ├── runtime │ ├── __init__.py │ ├── component_provider.py │ ├── env.py │ ├── job_default_config.py │ ├── reload_config_base.py │ ├── runtime_config.py │ └── system_settings.py ├── scheduler │ ├── __init__.py │ ├── detector.py │ └── scheduler.py ├── settings.py └── utils │ ├── __init__.py │ ├── api_utils.py │ ├── base_utils.py │ ├── conf_utils.py │ ├── cron.py │ ├── db_utils.py │ ├── engine_utils.py │ ├── file_utils.py │ ├── grpc_utils.py │ ├── io_utils.py │ ├── job_utils.py │ ├── log.py │ ├── log_utils.py │ ├── object_utils.py │ ├── password_utils.py │ ├── permission_utils.py │ ├── process_utils.py │ ├── requests_utils.py │ ├── schedule_utils.py │ ├── version.py │ ├── wraps_utils.py │ └── xthread.py ├── ofx ├── __init__.py └── api │ ├── __init__.py │ ├── client.py │ ├── entity.py │ ├── models │ ├── __init__.py │ ├── bfia │ │ ├── __init__.py │ │ ├── federated.py │ │ ├── resource.py │ │ ├── scheduler.py │ │ └── worker.py │ └── fate_flow │ │ ├── __init__.py │ │ ├── federated.py │ │ ├── resource.py │ │ ├── scheduler.py │ │ └── worker.py │ ├── proto │ ├── __init__.py │ └── rollsite │ │ ├── __init__.py │ │ ├── basic_meta_pb2.py │ │ ├── proxy_pb2.py │ │ └── proxy_pb2_grpc.py │ └── utils │ ├── __init__.py │ └── grpc_utils.py ├── requirements-container.txt ├── requirements-docker.txt ├── requirements-eggroll-client.txt ├── requirements-eggroll.txt ├── requirements-fate.txt ├── requirements-flow.txt ├── requirements-pulsar.txt ├── requirements-rabbitmq.txt ├── requirements-spark.txt ├── requirements.txt └── setup.py /.github/ISSUE_TEMPLATE/00-bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | name: Bug Report 4 | about: Use this template for reporting a bug 5 | labels: 'type:bug' 6 | 7 | --- 8 | 9 | **System information** 10 | 11 | - Have I written custom code (yes/no): 12 | - OS Platform and Distribution (e.g., Linux Ubuntu 16.04): 13 | - FATE Flow version (use command: python fate_flow_server.py --version): 14 | - Python version (use command: python --version): 15 | 16 | **Describe the current behavior** 17 | 18 | **Describe the expected behavior** 19 | 20 | **Other info / logs** Include any logs or source code that would be helpful to 21 | diagnose the problem. If including tracebacks, please include the full 22 | traceback. Large logs and files should be attached. 23 | 24 | - fateflow/logs/$job_id/fate_flow_schedule.log: scheduling log for a job 25 | - fateflow/logs/$job_id/* : all logs for a job 26 | - fateflow/logs/fate_flow/fate_flow_stat.log: a log of server stat 27 | - fateflow/logs/fate_flow/fate_flow_schedule.log: the starting scheduling log for all jobs 28 | - fateflow/logs/fate_flow/fate_flow_detect.log: the starting detecting log for all jobs 29 | 30 | **Contributing** 31 | 32 | - Do you want to contribute a PR? (yes/no): 33 | - Briefly describe your candidate solution(if contributing): 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/00-feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Use this template for raising a feature request 4 | labels: 'type:feature' 5 | 6 | --- 7 | 8 | **System information** 9 | 10 | - FATE Flow version (use command: python fate_flow_server.py --version): 11 | - Python version (use command: python --version): 12 | - Are you willing to contribute it (yes/no): 13 | 14 | **Describe the feature and the current behavior/state.** 15 | 16 | **Will this change the current api? How?** 17 | 18 | **Who will benefit with this feature?** 19 | 20 | **Any Other info.** 21 | -------------------------------------------------------------------------------- /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: generate doc 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | - 'develop-[0-9]+.[0-9]+.[0-9]+' 8 | - 'build-doc-[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]+' 9 | 10 | schedule: 11 | - cron: '0 8 * * *' 12 | 13 | workflow_dispatch: {} 14 | 15 | concurrency: 16 | group: doc_generator_${{ github.ref_name }} 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | doc_generator: 21 | name: generate doc on branch ${{ github.ref_name }} 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: check out the repo 25 | uses: actions/checkout@v2 26 | 27 | - name: fetch gh-pages 28 | continue-on-error: true 29 | run: git fetch origin gh-pages --depth=1 30 | 31 | - name: configure a git user 32 | run: | 33 | git config user.name github-actions[bot] 34 | git config user.email 41898282+github-actions[bot]@users.noreply.github.com 35 | 36 | - name: install python packages 37 | run: pip install -Ur doc/mkdocs/requirements.txt 38 | 39 | - name: build doc via mike 40 | shell: bash 41 | run: | 42 | VERSION='${{ github.ref_name }}' 43 | [ "$VERSION" == main ] && { VERSION=latest; ALIAS='main master'; } 44 | VERSION="${VERSION#develop-}" 45 | VERSION="${VERSION#build-doc-}" 46 | 47 | mike deploy --push --update-aliases "$VERSION" $ALIAS 48 | mike set-default --push latest 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # common file patterns 2 | .DS_STORE 3 | .idea 4 | *.iml 5 | *.pyc 6 | __pycache__ 7 | *.jar 8 | *.class 9 | .project 10 | *.prefs 11 | _build 12 | venv 13 | 14 | # excluded paths 15 | /data/ 16 | /model/ 17 | /logs/ 18 | /jobs/ 19 | /audit/ 20 | /localfs/ 21 | .vscode/* 22 | /temp/ 23 | /tmp 24 | /worker/ 25 | *.db 26 | *.db-journal 27 | *.whl 28 | /conf/local.*.yaml 29 | /cluster-deploy/FATE_install_* 30 | /python/component_plugins/ 31 | 32 | # doc 33 | /site/ 34 | 35 | /python/fate_flow/data 36 | /python/fate_flow/model 37 | /python/fate_flow/logs 38 | /python/fate_flow/jobs 39 | /python/fate_flow/localfs 40 | /python/fate_flow/*.env 41 | /python/fate_flow/conf 42 | /python/build 43 | /python/dist 44 | /python/*.egg-info -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/.gitmodules -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | mkdocs: 4 | configuration: mkdocs.yml 5 | fail_on_warning: false 6 | 7 | formats: all 8 | 9 | python: 10 | version: 3.7 11 | install: 12 | - requirements: doc/mkdocs/requirements.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CodeStyle](https://img.shields.io/badge/Check%20Style-Google-brightgreen)](https://checkstyle.sourceforge.io/google_style.html) [![Style](https://img.shields.io/badge/Check%20Style-Black-black)](https://checkstyle.sourceforge.io/google_style.html) 2 | 3 | [中文](./README.zh.md) 4 | 5 | FATE Flow is a multi-party federated task security scheduling platform for federated learning end-to-end pipeline 6 | 7 | - [Shared-State Scheduling Architecture](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/41684.pdf) 8 | - Secure Multi-Party Communication Across Data Centers 9 | 10 | Providing production-level service capabilities: 11 | 12 | - Data Access 13 | - Component Registry 14 | - Federated Job&Task Scheduling 15 | - Multi-Party Resource Coordination 16 | - Data Flow Tracking 17 | - Real-Time Job Monitoring 18 | - Multi-Party Federated Model Registry 19 | - Multi-Party Cooperation Authority Management 20 | - High Availability 21 | - CLI, REST API, Python API 22 | 23 | ## Deployment 24 | 25 | Please refer to [FATE](https://github.com/FederatedAI/FATE) 26 | 27 | ## Documentation 28 | 29 | The official FATE Flow documentation is here [https://federatedai.github.io/FATE-Flow/latest/](https://federatedai.github.io/FATE-Flow/latest/) 30 | 31 | ## License 32 | [Apache License 2.0](LICENSE) 33 | -------------------------------------------------------------------------------- /README.zh.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CodeStyle](https://img.shields.io/badge/Check%20Style-Google-brightgreen)](https://checkstyle.sourceforge.io/google_style.html) [![Style](https://img.shields.io/badge/Check%20Style-Black-black)](https://checkstyle.sourceforge.io/google_style.html) 2 | 3 | [English](./README.md) 4 | 5 | FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调度平台, 基于: 6 | 7 | - 共享状态调度架构 8 | - 跨数据中心的多方安全通信 9 | 10 | 提供生产级服务能力: 11 | 12 | - 数据接入 13 | - 任务组件注册中心 14 | - 联合作业&任务调度 15 | - 多方资源协调 16 | - 数据流动追踪 17 | - 作业实时监测 18 | - 联合模型注册中心 19 | - 多方合作权限管理 20 | - 系统高可用 21 | - CLI、REST API、Python API 22 | 23 | 24 | ## 部署 25 | 26 | 请查阅[FATE](https://github.com/FederatedAI/FATE) 27 | 28 | ## 文档 29 | 30 | FATE Flow官方文档在这里[https://federatedai.github.io/FATE-Flow/latest/zh/](https://federatedai.github.io/FATE-Flow/latest/zh/) 31 | 32 | ## License 33 | [Apache License 2.0](LICENSE) 34 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | ## Release 2.2.0 2 | ### Major Features and Improvements 3 | * Upgrade from Python 3.8 to Python 3.10 4 | 5 | ## Release 2.1.0 6 | ### Major Features and Improvements 7 | Improved the display issue of output data. 8 | Enhanced the PyPI package: configuration files have been relocated to the user's home directory, and the relative paths for uploading data are based on the user's home directory. 9 | Supported running FATE algorithms in Spark on YARN client mode. 10 | 11 | ### Bug-Fix 12 | Fixed an issue where failed tasks could not be retried. 13 | Fixed an issue where the system couldn't run when the task cores exceeded the system total cores. 14 | 15 | ## Release 2.0.0 16 | ### Major Features and Improvements 17 | * Adapted to new scalable and standardized federated DSL IR 18 | * Built an interconnected scheduling layer framework, supported the BFIA protocol 19 | * Optimized process scheduling, with scheduling separated and customizable, and added priority scheduling 20 | * Optimized algorithm component scheduling，support container-level algorithm loading， enhancing support for cross-platform heterogeneous scenarios 21 | * Optimized multi-version algorithm component registration, supporting registration for mode of components 22 | * Federated DSL IR extension enhancement: supports multi-party asymmetric scheduling 23 | * Optimized client authentication logic, supporting permission management for multiple clients 24 | * Optimized RESTful interface, making parameter fields and types, return fields, and status codes clearer 25 | * Added OFX(Open Flow Exchange) module: encapsulated scheduling client to allow cross-platform scheduling 26 | * Supported the new communication engine OSX, while remaining compatible with all engines from FATE Flow 1.x 27 | * Decoupled the System Layer and the Algorithm Layer, with system configuration moved from the FATE repository to the Flow repository 28 | * Published FATE Flow package to PyPI and added service-level CLI for service management 29 | * Migrated major functionality from FATE Flow 1.x 30 | -------------------------------------------------------------------------------- /bin/init_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Copyright 2019 The FATE Authors. All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../;cd ../;pwd) 20 | export FATE_PROJECT_BASE=$fate_project_base 21 | export PYTHONPATH= 22 | export SPARK_HOME= 23 | venv= 24 | 25 | source ${venv}/bin/activate 26 | 27 | -------------------------------------------------------------------------------- /conf/casbin_model.conf: -------------------------------------------------------------------------------- 1 | [request_definition] 2 | r = sub, obj, act 3 | 4 | [policy_definition] 5 | p = sub, obj, act 6 | 7 | [role_definition] 8 | g = _, _ 9 | 10 | [policy_effect] 11 | e = some(where (p.eft == allow)) 12 | 13 | [matchers] 14 | m = g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act -------------------------------------------------------------------------------- /conf/job_default_config.yaml: -------------------------------------------------------------------------------- 1 | # resource 2 | job_cores: 4 3 | computing_partitions: 8 4 | task_run: 5 | spark: 6 | num-executors: 2 7 | executor-cores: 2 8 | eggroll: 9 | cores: 4 10 | standalone: 11 | cores: 4 12 | task_timeout: 259200 # s 13 | remote_request_timeout: 30000 # ms 14 | federated_command_trys: 3 15 | auto_retries: 0 16 | sync_type: callback # poll or callback 17 | task_device: 18 | type: CPU 19 | -------------------------------------------------------------------------------- /conf/permission_casbin_model.conf: -------------------------------------------------------------------------------- 1 | [request_definition] 2 | r = party_id, type, value 3 | 4 | [policy_definition] 5 | p = party_id, type, value 6 | 7 | [policy_effect] 8 | e = some(where (p.eft == allow)) 9 | 10 | [matchers] 11 | m = r.party_id == p.party_id && r.type == p.type && r.value == p.value -------------------------------------------------------------------------------- /conf/pulsar_route_table.yaml: -------------------------------------------------------------------------------- 1 | 9999: 2 | # host can be a domain like 9999.fate.org 3 | host: 127.0.0.1 4 | port: 6650 5 | sslPort: 6651 6 | # set proxy address for this pulsar cluster 7 | proxy: "" 8 | 9 | 10000: 10 | # host can be a domain like 10000.fate.org 11 | host: 127.0.0.1 12 | port: 6650 13 | sslPort: 6651 14 | proxy: "" 15 | 16 | default: 17 | # compose host and proxy for party that does not exist in route table 18 | # in this example, the host for party 8888 will be 8888.fate.org 19 | proxy: "proxy.fate.org:443" 20 | domain: "fate.org" 21 | brokerPort: 6650 22 | brokerSslPort: 6651 -------------------------------------------------------------------------------- /conf/rabbitmq_route_table.yaml: -------------------------------------------------------------------------------- 1 | 9999: 2 | host: 192.168.0.4 3 | port: 5672 4 | 10000: 5 | host: 192.168.0.3 6 | port: 5672 7 | -------------------------------------------------------------------------------- /doc/build/build.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os.path 3 | import subprocess 4 | import sys 5 | import threading 6 | 7 | import requests 8 | 9 | 10 | def run_script(script_path, *args): 11 | result = subprocess.run(['python', script_path, *args]) 12 | return result.stderr 13 | 14 | 15 | if __name__ == '__main__': 16 | base_dir = os.path.dirname(__file__) 17 | build_path = os.path.join(base_dir, 'build_swagger_server.py') 18 | 19 | thread = threading.Thread(target=run_script, args=(build_path,)) 20 | thread.start() 21 | # 22 | thread.join() 23 | build_path = os.path.join(base_dir, 'swagger_server.py') 24 | port = "50000" 25 | server = threading.Thread(target=run_script, args=(build_path, port)) 26 | 27 | result = server.start() 28 | 29 | import time 30 | time.sleep(3) 31 | data = requests.get(url=f"http://127.0.0.1:{port}/swagger.json").text 32 | data = json.loads(data) 33 | swagger_file = os.path.join(os.path.dirname(base_dir), "swagger", "swagger.json") 34 | os.makedirs(os.path.dirname(swagger_file), exist_ok=True) 35 | with open(swagger_file, "w") as fw: 36 | json.dump(data, fw, indent=4) 37 | print("build success!") 38 | sys.exit() 39 | -------------------------------------------------------------------------------- /doc/fate_flow.zh.md: -------------------------------------------------------------------------------- 1 | # 整体设计 2 | 3 | ## 1. 设计架构图 4 | ![](./images/open_flow.png) 5 | - 应用层接口：供如fate-board、fate-client等上层使用 6 | - 互联互通层接口：分为调度器接口和参与方接口，调度器接口用于接收如创建、停止等调度命令并下发给参与方，参与方接口用于各参与方接收如创建、运行、停止等命令并执行 7 | - 底座接口： 用于接收算法容器上报的状态等 8 | - 调度器：联邦调度逻辑，解析DSL依赖及运行相关的作业及任务 9 | - 算法容器：是算法运行的环境，FATE Flow支持算法运行在本地进程、算法容器中，其运行方式类似。 10 | - 平台资源池： 抽象计算、通信、存储API 11 | 12 | 13 | ## 2. 整体架构 14 | 15 | ### 2.1 FATE整体架构 16 | 17 | ![](./images/fate_arch.png) 18 | 19 | ### 2.2 FATE Flow功能架构 20 | 21 | ![](./images/fate_flow_arch.png) 22 | 23 | ### 2.3 FATE Flow集群架构 24 | 25 | ![](./images/flow_cluster.png) 26 | 27 | ## 3. 调度架构 28 | ### 3.1 基于共享状态的调度架构 29 | 30 | - 剥离状态(资源、作业)与管理器(调度器、资源管理器) 31 | - 资源状态与作业状态持久化存于MySQL，全局共享，提供可靠事务性操作 32 | - 提高管理服务的高可用与扩展性 33 | - 作业可介入，支持实现如重启、重跑、并行控制、资源隔离等 34 | 35 | ![](./images/fate_flow_scheduling_arch.png) 36 | 37 | ### 3.2 状态驱动调度 38 | 39 | - 南北向状态上报/查询 40 | - 东西向多方任务状态计算联邦任务状态 41 | - 上下游任务状态计算作业作态 42 | #### 3.2.1 callback回调模式 43 | 调度器创建作业和任务，由各参与方主动回调作业或任务的状态 44 | 45 | ![](./images/schedule_for_callback.png) 46 | #### 3.2.2 poll轮询模式 47 | 调度器不仅需创建作业和任务，在调度过程中会轮询参与方的作业或任务的状态 48 | 49 | ![](./images/schedule_for_poll.png) 50 | 51 | ### 3.4 算法组件调度 52 | - 前处理: 数据、模型、算法参数等输入处理 53 | - 组件运行: 算法组件逻辑 54 | - 后处理: 算法组件输出内容处理 55 | ![](./images/schedule_for_component.png) 56 | 57 | 58 | ## 4. 多方资源协调 59 | 60 | - 每个引擎总资源大小通过配置文件配置，后续实现系统对接 61 | - 总资源大小中的cores表示每个计算节点cpu核数 62 | - FATEFlow server启动时从配置文件读取资源大小配置，并注册更新到数据库 63 | - 以Job维度申请资源，Job Conf提交时生效 64 | 65 | 66 | ## 5. 作业实时监测 67 | 68 | - 工作进程存活性检测 69 | - 作业超时检测 70 | - 资源回收检测 71 | - 基础引擎会话超时检测 72 | 73 | ![](./images/fate_flow_detector.png) 74 | 75 | ## 6. [任务组件中心](./provider_register.zh.md) 76 | 77 | ![](./images/fate_flow_component_registry.png) 78 | 79 | ## 7. [数据接入](./data_access.zh.md) 80 | 81 | ![](./images/upload_data.png) 82 | -------------------------------------------------------------------------------- /doc/images/container_load.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/container_load.png -------------------------------------------------------------------------------- /doc/images/fate_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_arch.png -------------------------------------------------------------------------------- /doc/images/fate_deploy_directory.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_deploy_directory.png -------------------------------------------------------------------------------- /doc/images/fate_flow_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_api.png -------------------------------------------------------------------------------- /doc/images/fate_flow_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_arch.png -------------------------------------------------------------------------------- /doc/images/fate_flow_authorization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_authorization.png -------------------------------------------------------------------------------- /doc/images/fate_flow_component_dsl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_component_dsl.png -------------------------------------------------------------------------------- /doc/images/fate_flow_component_registry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_component_registry.png -------------------------------------------------------------------------------- /doc/images/fate_flow_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_dag.png -------------------------------------------------------------------------------- /doc/images/fate_flow_detector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_detector.png -------------------------------------------------------------------------------- /doc/images/fate_flow_dsl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_dsl.png -------------------------------------------------------------------------------- /doc/images/fate_flow_inputoutput.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_inputoutput.png -------------------------------------------------------------------------------- /doc/images/fate_flow_major_feature.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_major_feature.png -------------------------------------------------------------------------------- /doc/images/fate_flow_model_storage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_model_storage.png -------------------------------------------------------------------------------- /doc/images/fate_flow_pipelined_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_pipelined_model.png -------------------------------------------------------------------------------- /doc/images/fate_flow_scheduling_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/fate_flow_scheduling_arch.png -------------------------------------------------------------------------------- /doc/images/federated_learning_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/federated_learning_pipeline.png -------------------------------------------------------------------------------- /doc/images/federationml_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/federationml_schedule.png -------------------------------------------------------------------------------- /doc/images/flow_cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/flow_cluster.png -------------------------------------------------------------------------------- /doc/images/log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/log.png -------------------------------------------------------------------------------- /doc/images/muti_protocol.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/muti_protocol.png -------------------------------------------------------------------------------- /doc/images/open_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/open_flow.png -------------------------------------------------------------------------------- /doc/images/open_flow/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/open_flow/pipeline.png -------------------------------------------------------------------------------- /doc/images/open_flow/pipeline_unionpay_lr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/open_flow/pipeline_unionpay_lr.png -------------------------------------------------------------------------------- /doc/images/open_flow/pipeline_unionpay_sbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/open_flow/pipeline_unionpay_sbt.png -------------------------------------------------------------------------------- /doc/images/open_flow/scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/open_flow/scheduler.png -------------------------------------------------------------------------------- /doc/images/open_flow/upload_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/open_flow/upload_data.png -------------------------------------------------------------------------------- /doc/images/pull.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/pull.png -------------------------------------------------------------------------------- /doc/images/push.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/push.png -------------------------------------------------------------------------------- /doc/images/schedule_for_callback.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/schedule_for_callback.png -------------------------------------------------------------------------------- /doc/images/schedule_for_component.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/schedule_for_component.png -------------------------------------------------------------------------------- /doc/images/schedule_for_poll.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/schedule_for_poll.png -------------------------------------------------------------------------------- /doc/images/scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/scheduler.png -------------------------------------------------------------------------------- /doc/images/upload_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/doc/images/upload_data.png -------------------------------------------------------------------------------- /doc/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | template: overrides/home.html 3 | title: Secure, Privacy-preserving Machine Learning Multi-Party Schduling System 4 | --- 5 | -------------------------------------------------------------------------------- /doc/index.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | template: overrides/home.zh.html 3 | title: 安全，隐私保护的机器学习多方调度系统 4 | --- 5 | -------------------------------------------------------------------------------- /doc/mkdocs/README.md: -------------------------------------------------------------------------------- 1 | # Build 2 | 3 | ## use docker 4 | 5 | At repo root, execute 6 | 7 | ```sh 8 | docker run --rm -it -p 8000:8000 -v ${PWD}:/docs sagewei0/mkdocs 9 | ``` 10 | 11 | to serve docs in http://localhost:8000 12 | 13 | or 14 | 15 | ```sh 16 | docker run --rm -it -p 8000:8000 -v ${PWD}:/docs sagewei0/mkdocs build 17 | ``` 18 | 19 | to build docs to `site` folder. 20 | 21 | ## manually 22 | 23 | [`mkdocs-material`](https://pypi.org/project/mkdocs-material/) and servel plugins are needed to build this docs 24 | 25 | Fisrt, create an python virtual environment 26 | 27 | ```sh 28 | python3 -m venv "fatedocs" 29 | source fatedocs/bin/activate 30 | pip install -U pip 31 | ``` 32 | And then install requirements 33 | 34 | ```sh 35 | pip install -r doc/mkdocs/requirements.txt 36 | ``` 37 | 38 | Now, use 39 | 40 | ```sh 41 | mkdocs serve 42 | ``` 43 | 44 | at repo root to serve docs or 45 | 46 | use 47 | 48 | ```sh 49 | mkdocs build 50 | ``` 51 | 52 | at repo root to build docs to folder `site` 53 | 54 | 55 | # Develop guide 56 | 57 | We use [mkdocs-material](https://squidfunk.github.io/mkdocs-material/) to build our docs. 58 | Servel markdown extensions are really useful to write pretty documents such as 59 | [admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/) and 60 | [content-tabs](https://squidfunk.github.io/mkdocs-material/reference/content-tabs/). 61 | 62 | Servel plugins are introdused to makes mkdocs-material much powerful: 63 | 64 | 65 | - [mkdocstrings](https://mkdocstrings.github.io/usage/) 66 | automatic documentation from sources code. We mostly use this to automatic generate 67 | `params api` for `federatedml`. 68 | 69 | - [awesome-pages](https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin) 70 | for powerful nav rule 71 | 72 | - [i18n](https://ultrabug.github.io/mkdocs-static-i18n/) 73 | for multi-languege support 74 | 75 | - [mkdocs-jupyter](https://github.com/danielfrg/mkdocs-jupyter) 76 | for jupyter format support 77 | 78 | - [mkdocs-simple-hooks](https://github.com/aklajnert/mkdocs-simple-hooks) 79 | for simple plugin-in -------------------------------------------------------------------------------- /doc/mkdocs/css/custom.css: -------------------------------------------------------------------------------- 1 | /* Indentation. */ 2 | div.doc-contents:not(.first) { 3 | padding-left: 25px; 4 | border-left: 4px solid rgba(230, 230, 230); 5 | margin-bottom: 80px; 6 | } 7 | 8 | /* Don't capitalize names. */ 9 | h5.doc-heading { 10 | text-transform: none !important; 11 | } 12 | 13 | /* Don't use vertical space on hidden ToC entries. */ 14 | h6.hidden-toc { 15 | margin: 0 !important; 16 | position: relative; 17 | top: -70px; 18 | } 19 | 20 | h6.hidden-toc::before { 21 | margin-top: 0 !important; 22 | padding-top: 0 !important; 23 | } 24 | 25 | /* Don't show permalink of hidden ToC entries. */ 26 | h6.hidden-toc a.headerlink { 27 | display: none; 28 | } 29 | 30 | /* Avoid breaking parameters name, etc. in table cells. */ 31 | td code { 32 | word-break: normal !important; 33 | } 34 | 35 | /* For pieces of Markdown rendered in table cells. */ 36 | td p { 37 | margin-top: 0 !important; 38 | margin-bottom: 0 !important; 39 | } 40 | -------------------------------------------------------------------------------- /doc/mkdocs/css/extra.css: -------------------------------------------------------------------------------- 1 | /* Remove default title on the page */ 2 | .md-content__inner h1:first-child { 3 | display: none; 4 | } 5 | 6 | /* Adjust to 2px to align with the title */ 7 | .md-logo { 8 | padding-top: 6px; 9 | } 10 | 11 | .btn { 12 | border: none; 13 | padding: 14px 28px; 14 | cursor: pointer; 15 | display: inline-block; 16 | 17 | background: #009688; 18 | color: white; 19 | } 20 | 21 | .btn:hover { 22 | background: #00bfa5; 23 | color: white; 24 | } 25 | 26 | .center { 27 | display: block; 28 | margin-left: auto; 29 | margin-right: auto; 30 | } 31 | 32 | .text-center { 33 | text-align: center; 34 | } 35 | -------------------------------------------------------------------------------- /doc/mkdocs/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.2-alpine3.13 2 | 3 | # Environment variables 4 | ENV PACKAGES=/usr/local/lib/python3.9/site-packages 5 | ENV PYTHONDONTWRITEBYTECODE=1 6 | 7 | # Set build directory 8 | WORKDIR /tmp 9 | 10 | COPY requirements.txt . 11 | 12 | RUN set -e ;\ 13 | apk upgrade --update-cache -a ;\ 14 | apk add --no-cache libstdc++ libffi-dev ;\ 15 | apk add --no-cache --virtual .build gcc g++ musl-dev python3-dev cargo openssl-dev git;\ 16 | pip install --no-cache-dir -r requirements.txt 17 | 18 | # clean 19 | RUN apk del .build ;\ 20 | rm -rf /tmp/* /root/.cache 21 | 22 | # Set working directory 23 | WORKDIR /docs 24 | 25 | # Expose MkDocs development server port 26 | EXPOSE 8000 27 | 28 | ENV PYTHONPATH=$PYTHONPATH:/docs/python 29 | # Start development server by default 30 | ENTRYPOINT ["mkdocs"] 31 | CMD ["serve", "--dev-addr=0.0.0.0:8000"] -------------------------------------------------------------------------------- /doc/mkdocs/docker/README.md: -------------------------------------------------------------------------------- 1 | # Image for build FATE's documents 2 | 3 | This image is modified from [mkdocs-meterial](https://squidfunk.github.io/mkdocs-material/) with some plugins embeded. 4 | 5 | Usage 6 | 7 | Mount the folder where your mkdocs.yml resides as a volume into /docs: 8 | 9 | - Start development server on http://localhost:8000 10 | 11 | ```console 12 | docker run --rm -it -p 8000:8000 -v ${PWD}:/docs sagewei0/mkdocs 13 | ``` 14 | 15 | - Build documentation 16 | 17 | ```console 18 | docker run --rm -it -v ${PWD}:/docs sagewei/mkdocs build 19 | ``` 20 | 21 | - Deploy documentation to GitHub Pages 22 | 23 | ```console 24 | docker run --rm -it -v ~/.ssh:/root/.ssh -v ${PWD}:/docs sagewei0/mkdocs gh-deploy 25 | ``` 26 | -------------------------------------------------------------------------------- /doc/mkdocs/requirements.txt: -------------------------------------------------------------------------------- 1 | mike 2 | mkdocs-material 3 | python-markdown-math 4 | mkdocs-awesome-pages-plugin 5 | mkdocs-render-swagger-plugin 6 | mkdocs-static-i18n 7 | git+https://github.com/jarviszeng-zjc/markdown-include-snippet.git@develop-0.1.0#egg=markdown-include-snippet 8 | -------------------------------------------------------------------------------- /doc/mkdocs/theme/README.md: -------------------------------------------------------------------------------- 1 | Mostly copied from https://github.com/cirruslabs/cirrus-ci-docs/tree/master/theme 2 | -------------------------------------------------------------------------------- /doc/swagger/index.md: -------------------------------------------------------------------------------- 1 | ## Swagger API 2 | 3 | !!swagger swagger.yaml!! 4 | -------------------------------------------------------------------------------- /examples/bfia/fate/job/dataframe_transformer.yaml: -------------------------------------------------------------------------------- 1 | dag: 2 | initiator: 3 | - "guest" 4 | - "JG0100001100000010" 5 | conf: 6 | extra: 7 | initiator: 8 | role: guest 9 | party_id: JG0100001100000010 10 | parties: 11 | - party_id: 12 | - JG0100001100000010 13 | role: guest 14 | stage: default 15 | tasks: 16 | transformer_0: 17 | parameters: 18 | name: breast_hetero_guest 19 | namespace: experiment 20 | site_name: null 21 | conf: 22 | provider: "fate" 23 | version: "2.0.0" 24 | inputs: 25 | data: 26 | table: 27 | data_warehouse: 28 | dataset_id: upload#guest 29 | outputs: 30 | data: 31 | dataframe_output: 32 | output_artifact_key_alias: dataframe_output 33 | output_artifact_type_alias: dataset 34 | metric: 35 | metric: 36 | output_artifact_key_alias: metric 37 | output_artifact_type_alias: report 38 | component_ref: dataframe_transformer 39 | parties: 40 | - party_id: 41 | - JG0100001100000010 42 | role: guest 43 | schema_version: 2.0.0 44 | kind: bfia 45 | -------------------------------------------------------------------------------- /examples/bfia/fate/pipeline/test_dataframe_transformer.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from fate_client.pipeline import FateFlowPipeline 17 | from fate_client.pipeline.components.fate import DataFrameTransformer 18 | from fate_client.pipeline.interface.channel import DataWarehouseChannel 19 | 20 | 21 | def main(): 22 | guest_party_id = "JG0100001100000010" 23 | 24 | pipeline = FateFlowPipeline().set_parties(guest=guest_party_id) 25 | transformer_0 = DataFrameTransformer( 26 | "transformer_0", 27 | namespace="test", 28 | name="guest", 29 | table=DataWarehouseChannel( 30 | dataset_id="upload#guest" 31 | ) 32 | ) 33 | pipeline.set_site_role("guest") 34 | pipeline.set_site_party_id(guest_party_id) 35 | 36 | pipeline.add_tasks([transformer_0]) 37 | pipeline.protocol_kind = "bfia" 38 | pipeline.conf.set( 39 | "extra", 40 | dict(initiator={'party_id': 'JG0100001100000010', 'role': 'guest'}) 41 | ) 42 | pipeline.compile() 43 | # print(pipeline.get_dag()) 44 | pipeline.fit() 45 | 46 | 47 | if __name__ == "__main__": 48 | main() -------------------------------------------------------------------------------- /examples/bfia/fate/upload/upload_guest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | 4 | from fate_flow.adapter.bfia.container.wraps.wraps import DataIo 5 | from fate_flow.components.components.upload import Upload, UploadParam 6 | from fate_flow.entity.spec.dag import Metadata 7 | 8 | 9 | def upload_data(s3_address, namespace, name, file, meta, head=True, partitions=16, extend_sid=True, storage_engine="standalone"): 10 | upload_object = Upload() 11 | params = { 12 | 'name': name, 13 | 'namespace': namespace, 14 | 'file': file, 15 | 'storage_engine': storage_engine, 16 | 'head': head, 17 | 'partitions': partitions, 18 | 'extend_sid': extend_sid, 19 | 'meta': meta 20 | } 21 | params = UploadParam(**params) 22 | 23 | with tempfile.TemporaryDirectory() as data_home: 24 | os.environ["STANDALONE_DATA_HOME"] = data_home 25 | data_meta = upload_object.run(params).get("data_meta") 26 | 27 | metadata = Metadata(metadata=dict(options=dict(partitions=partitions), schema=data_meta)) 28 | data_path = os.path.join(data_home, namespace, name) 29 | engine = DataIo(s3_address) 30 | engine.upload_to_s3(data_path, name=name, namespace=namespace, metadata=metadata.dict()) 31 | 32 | 33 | if __name__ == "__main__": 34 | s3_address = "s3://127.0.0.1:9000?username=admin&password=12345678" 35 | namespace = "upload" 36 | name = "guest" 37 | file = 'examples/data/breast_hetero_guest.csv' 38 | 39 | meta = { 40 | "delimiter": ",", 41 | "label_name": "y", 42 | "match_id_name": "id" 43 | } 44 | upload_data(s3_address=s3_address, namespace=namespace, name=name, file=file, meta=meta) 45 | -------------------------------------------------------------------------------- /examples/bfia/fate/upload/upload_host.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | 4 | from fate_flow.adapter.bfia.container.wraps.wraps import DataIo 5 | from fate_flow.components.components.upload import Upload, UploadParam 6 | from fate_flow.entity.spec.dag import Metadata 7 | 8 | 9 | def upload_data(s3_address, namespace, name, file, meta, head=True, partitions=16, extend_sid=True, storage_engine="standalone"): 10 | upload_object = Upload() 11 | params = { 12 | 'name': name, 13 | 'namespace': namespace, 14 | 'file': file, 15 | 'storage_engine': storage_engine, 16 | 'head': head, 17 | 'partitions': partitions, 18 | 'extend_sid': extend_sid, 19 | 'meta': meta 20 | } 21 | params = UploadParam(**params) 22 | 23 | with tempfile.TemporaryDirectory() as data_home: 24 | os.environ["STANDALONE_DATA_HOME"] = data_home 25 | data_meta = upload_object.run(params).get("data_meta") 26 | 27 | metadata = Metadata(metadata=dict(options=dict(partitions=partitions), schema=data_meta)) 28 | data_path = os.path.join(data_home, namespace, name) 29 | engine = DataIo(s3_address) 30 | engine.upload_to_s3(data_path, name=name, namespace=namespace, metadata=metadata.dict()) 31 | 32 | 33 | if __name__ == "__main__": 34 | s3_address = "s3://127.0.0.1:9000?username=admin&password=12345678" 35 | namespace = "upload" 36 | name = "host" 37 | file = 'examples/data/breast_hetero_host.csv' 38 | 39 | meta = { 40 | "delimiter": ",", 41 | "match_id_name": "id" 42 | } 43 | upload_data(s3_address=s3_address, namespace=namespace, name=name, file=file, meta=meta) 44 | -------------------------------------------------------------------------------- /examples/model/export.json: -------------------------------------------------------------------------------- 1 | { 2 | "model_id": "202307251326130289470", 3 | "model_version": "0", 4 | "role": "guest", 5 | "party_id": "9999", 6 | "path": "xxx/dir" 7 | } -------------------------------------------------------------------------------- /examples/model/import.json: -------------------------------------------------------------------------------- 1 | { 2 | "model_id": "xxx", 3 | "model_version": "xxx", 4 | "path": "xxx" 5 | } -------------------------------------------------------------------------------- /examples/permission/delete.json: -------------------------------------------------------------------------------- 1 | { 2 | "party_id": "9999", 3 | "component": "reader", 4 | "dataset": [{ 5 | "name": "xxx", "namespace": "xxx" 6 | }] 7 | } -------------------------------------------------------------------------------- /examples/permission/grant.json: -------------------------------------------------------------------------------- 1 | { 2 | "party_id": "9999", 3 | "component": "reader", 4 | "dataset": [{ 5 | "name": "xxx", "namespace": "xxx" 6 | }] 7 | } -------------------------------------------------------------------------------- /examples/provider/register.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fate", 3 | "device": "local", 4 | "version": "2.0.0", 5 | "metadata": { 6 | "path": "xxx", 7 | "venv": "xxx" 8 | } 9 | } -------------------------------------------------------------------------------- /examples/provider/register_image.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fate", 3 | "device": "docker", 4 | "version": "2.1.0", 5 | "metadata": { 6 | "base_url": "", 7 | "image": "federatedai/fate:2.1.0" 8 | } 9 | } -------------------------------------------------------------------------------- /examples/transformer/transformer_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "experiment", 3 | "name": "breast_homo_guest", 4 | "data_warehouse": { 5 | "name": "7e1441e4-9413-11ee-9bb0-16b977118319", 6 | "namespace": "upload" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /examples/transformer/transformer_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "experiment", 3 | "name": "breast_homo_host", 4 | "data_warehouse": { 5 | "name": "9fa4905c-9413-11ee-9bb0-16b977118319", 6 | "namespace": "upload" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /examples/upload/upload_guest.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_hetero_guest.csv", 3 | "head": true, 4 | "partitions": 16, 5 | "extend_sid": true, 6 | "meta": { 7 | "delimiter": ",", 8 | "label_name": "y", 9 | "match_id_name": "id" 10 | }, 11 | "namespace": "experiment", 12 | "name": "breast_hetero_guest" 13 | } 14 | -------------------------------------------------------------------------------- /examples/upload/upload_host.json: -------------------------------------------------------------------------------- 1 | { 2 | "file": "examples/data/breast_hetero_host.csv", 3 | "head": true, 4 | "partitions": 16, 5 | "extend_sid": true, 6 | "meta": { 7 | "delimiter": ",", 8 | "match_id_name": "id" 9 | }, 10 | "namespace": "experiment", 11 | "name": "breast_hetero_host" 12 | } 13 | -------------------------------------------------------------------------------- /fateflow.env: -------------------------------------------------------------------------------- 1 | FATE=2.1.0 2 | FATE_FLOW=2.1.0 3 | PYTHON=3.8 -------------------------------------------------------------------------------- /python/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from ._info import __provider__, __version__ 16 | -------------------------------------------------------------------------------- /python/fate_flow/_info.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | __version__ = "2.2.0" 16 | __provider__ = "fate_flow" 17 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/__init__.py: -------------------------------------------------------------------------------- 1 | from ._loader import init_adapter, load_adapter_apps 2 | from ._controller import AdapterJobController 3 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/_controller.py: -------------------------------------------------------------------------------- 1 | from fate_flow.adapter._loader import load_bridge_module 2 | from fate_flow.entity.spec.flow import SubmitJobInput, SubmitJobOutput, QueryJobInput, QueryJobOutput, StopJobInput, \ 3 | StopJobOutput, QueryTaskOutput, QueryTaskInput 4 | 5 | 6 | class AdapterJobController(object): 7 | def __init__(self, protocol_name): 8 | packages = load_bridge_module(protocol_name=protocol_name) 9 | self.controller_adapter = getattr(packages, "JobController") 10 | 11 | def create_job(self, submit_job_input: SubmitJobInput) -> SubmitJobOutput: 12 | return self.controller_adapter.create_job(submit_job_input) 13 | 14 | def query_job(self, query_job_input: QueryJobInput) -> QueryJobOutput: 15 | return self.controller_adapter.query_job(query_job_input) 16 | 17 | def stop_job(self, stop_job_input: StopJobInput) -> StopJobOutput: 18 | return self.controller_adapter.stop_job(stop_job_input) 19 | 20 | def query_task(self, query_task_input: QueryTaskInput) -> QueryTaskOutput: 21 | return self.controller_adapter.query_task(query_task_input) 22 | 23 | def rerun_job(self): 24 | pass 25 | 26 | @staticmethod 27 | def query_output_data(): 28 | return {} 29 | 30 | @staticmethod 31 | def query_output_model(): 32 | return {} 33 | 34 | @staticmethod 35 | def query_output_metric(): 36 | return {} 37 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["apps"] 2 | 3 | from fate_flow.adapter.bfia.runtime_config import BfiaRuntimeConfig 4 | 5 | BfiaRuntimeConfig.init() 6 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/apps/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "v1" 2 | 3 | 4 | def before_request(): 5 | # 签名、认证 6 | pass 7 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/bridge/__init__.py: -------------------------------------------------------------------------------- 1 | from .job import JobController 2 | 3 | __all__ = ["JobController"] 4 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/conf/route_table.yaml: -------------------------------------------------------------------------------- 1 | JG0100001100000000: 2 | host: 127.0.0.1 3 | port: 9380 4 | JG0100001100000010: 5 | host: 127.0.0.1 6 | port: 9380 7 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/container/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/container/entrypoint/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/container/entrypoint/runner.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | class Submit: 18 | @staticmethod 19 | def run(): 20 | import click 21 | from fate_flow.adapter.bfia.container.entrypoint.cli import component 22 | 23 | cli = click.Group() 24 | cli.add_command(component) 25 | cli(prog_name="python -m fate_flow.adapter.bfia.container.entrypoint") 26 | 27 | 28 | if __name__ == "__main__": 29 | Submit.run() 30 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/container/wraps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/adapter/bfia/container/wraps/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/db/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ["ComponentOutput"] 2 | 3 | from fate_flow.adapter.bfia.db.output import ComponentOutput 4 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/db/output.py: -------------------------------------------------------------------------------- 1 | from peewee import CompositeKey, CharField, TextField, BigIntegerField 2 | 3 | from fate_flow.db import DataBaseModel, JSONField 4 | 5 | 6 | class ComponentOutput(DataBaseModel): 7 | f_job_id = CharField(max_length=25, index=True) 8 | f_role = CharField(max_length=50, index=True) 9 | f_node_id = CharField(max_length=50, index=True) 10 | f_task_name = CharField(max_length=50) 11 | f_component = CharField(max_length=50) 12 | f_task_id = CharField(max_length=100) 13 | f_type = CharField(max_length=20) 14 | f_key = CharField(max_length=20) 15 | f_engine = JSONField() 16 | f_address = JSONField() 17 | 18 | class Meta: 19 | db_table = "t_bfia_component_output" 20 | primary_key = CompositeKey('f_job_id', 'f_role', 'f_node_id', 'f_task_id', 'f_type', 'f_key') 21 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/engine_storage/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from urllib.parse import urlparse 3 | 4 | from .session import S3Session 5 | 6 | 7 | def inti_session(): 8 | """创建客户端对象""" 9 | 10 | storage_str = os.getenv('system.storage') 11 | assert storage_str is not None, 'system.storage 配置为空' 12 | 13 | storage_str = urlparse(storage_str) 14 | storage_query = {i.split('=')[0]: i.split('=')[1] for i in storage_str.query.split('&')} 15 | username = storage_query.get('username') 16 | password = storage_query.get('password') 17 | url = storage_str.netloc 18 | service_type = storage_str.scheme 19 | 20 | assert url is not None, 'system.storage 配置无法解析url' 21 | assert username is not None, 'system.storage 配置无法解析username' 22 | assert password is not None, 'system.storage 配置无法解析password' 23 | assert service_type is not None, 'system.storage 配置无法解析service_type' 24 | 25 | if service_type == 's3': 26 | url = f"http://{url}" 27 | return S3Session(url=url, username=username, password=password) 28 | else: 29 | assert Exception(f'不支持相应配置, storage_service_type: {service_type}') 30 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/engine_storage/consts.py: -------------------------------------------------------------------------------- 1 | S3_BUCKET_NAME = 'storage' 2 | S3_META_KEY = '{namespace}/{name}/metadata' 3 | S3_OBJECT_KEY = '{namespace}/{name}/data_{partition}' 4 | S3_SAVE_PATH = '/{namespace}/{name}' 5 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/examples/test/data/guest/data_0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/adapter/bfia/examples/test/data/guest/data_0 -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/examples/test/data/host/data_0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/adapter/bfia/examples/test/data/host/data_0 -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/examples/test/test_job.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | 5 | base_url = "http://127.0.0.1:9380" 6 | 7 | 8 | def register_components(): 9 | uri = "/v2/provider/register" 10 | config_path = "../job/unionpay/unionpay_components.json" 11 | body = json.load(open(config_path, "r")) 12 | resp = requests.post(base_url+uri, json=body) 13 | print(resp.text) 14 | 15 | 16 | register_components() 17 | 18 | 19 | def submit_job(): 20 | uri = "/v1/platform/schedule/job/create_all" 21 | config_path = "../job/unionpay/bfia_psi_sbt.json" 22 | body = json.load(open(config_path, "r")) 23 | resp = requests.post(base_url+uri, json=body) 24 | print(resp.text) 25 | 26 | 27 | def start_job(job_id): 28 | uri = "/v1/interconn/schedule/job/start" 29 | resp = requests.post(base_url+uri, json={"job_id": job_id}) 30 | print(resp.text) 31 | 32 | 33 | def stop_job(job_id): 34 | uri = "/v1/interconn/schedule/job/stop_all" 35 | resp = requests.post(base_url+uri, json={"job_id": job_id}) 36 | print(resp.text) 37 | 38 | 39 | submit_job() 40 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/examples/test/test_task.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | 5 | base_url = "http://127.0.0.1:9380" 6 | 7 | 8 | def start_task(job_id, task_id, task_name): 9 | uri = "/v1/interconn/schedule/task/start" 10 | resp = requests.post(base_url+uri, json={"job_id": job_id, "task_id": task_id, "task_name": task_name}) 11 | print(resp.text) 12 | 13 | 14 | def stop_job(job_id): 15 | uri = "/v1/platform/schedule/job/stop_all" 16 | resp = requests.post(base_url+uri, json={"job_id": job_id}) 17 | print(resp.text) 18 | 19 | 20 | # submit_job() 21 | # start_task("202310161542273925260", "202310161422165657200_wzh3", "intersect_rsa_1") 22 | # stop_job("202310161542273925260") 23 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/examples/test/upload/upload_guest.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from fate_flow.adapter.bfia import engine_storage 4 | 5 | url = "http://127.0.0.1:9000" 6 | username = "admin" 7 | password = "12345678" 8 | 9 | session = engine_storage.session.S3Session(url=url, username=username, password=password) 10 | 11 | namespace = "test_data" 12 | name = "guest" 13 | fp = open("../data/guest/metadata") 14 | metadata = yaml.safe_load(fp) 15 | print(metadata) 16 | 17 | table = session.create_table(name=name, namespace=namespace, metadata=metadata.get("metadata"), column_info=[]) 18 | table.upload("../data/guest/data_0") 19 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/examples/test/upload/upload_host.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from fate_flow.adapter.bfia import engine_storage 4 | 5 | url = "http://127.0.0.1:9000" 6 | username = "admin" 7 | password = "12345678" 8 | 9 | session = engine_storage.session.S3Session(url=url, username=username, password=password) 10 | 11 | namespace = "test_data" 12 | name = "host" 13 | fp = open("../data/host/metadata") 14 | metadata = yaml.safe_load(fp) 15 | print(metadata) 16 | 17 | table = session.create_table(name=name, namespace=namespace, column_info=[], metadata=metadata.get("metadata")) 18 | table.upload("../data/host/data_0") 19 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/runtime_config.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from fate_flow.runtime.system_settings import HOST, HTTP_PORT 4 | from fate_flow.adapter.bfia.settings import LOCAL_SITE_ID as PARTY_ID 5 | from fate_flow.settings import HTTP_REQUEST_TIMEOUT 6 | from fate_flow.utils.file_utils import load_yaml_conf 7 | from ofx.api.client import BfiaSchedulerApi 8 | 9 | 10 | class BfiaRuntimeConfig(object): 11 | ROUTE_TABLE: dict = {} 12 | SCHEDULE_CLIENT: BfiaSchedulerApi = None 13 | 14 | @classmethod 15 | def load_route_table_from_file(cls): 16 | name = "route_table.yaml" 17 | route_table_path = os.path.join(os.path.dirname(__file__), "conf", name) 18 | cls.ROUTE_TABLE.update(load_yaml_conf(route_table_path)) 19 | 20 | @classmethod 21 | def set_schedule_client(cls, schedule_client): 22 | cls.SCHEDULE_CLIENT = schedule_client 23 | 24 | @classmethod 25 | def init(cls): 26 | # init route table 27 | cls.load_route_table_from_file() 28 | 29 | # init schedule client 30 | cls.set_schedule_client( 31 | BfiaSchedulerApi( 32 | host=HOST, 33 | port=HTTP_PORT, 34 | timeout=HTTP_REQUEST_TIMEOUT, 35 | route_table=cls.ROUTE_TABLE, 36 | self_node_id=PARTY_ID 37 | ) 38 | ) 39 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from .detector import BfiaDetector 17 | from .scheduler import BfiaScheduler 18 | 19 | __all__ = [BfiaScheduler, BfiaDetector] 20 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/settings.py: -------------------------------------------------------------------------------- 1 | from fate_flow.runtime.system_settings import PARTY_ID 2 | 3 | LOCAL_SITE_ID = PARTY_ID 4 | STORAGE_NAME = "s3" 5 | STORAGE_ADDRESS = "s3://127.0.0.1:9000?username=admin&password=12345678" 6 | TRANSPORT = "127.0.0.1:9377" 7 | SESSION_ID = "session_{}" 8 | TOKEN = "session_{}" 9 | FATE_CONTAINER_HOME = "/data/projects/fate/fate_flow" 10 | CONTAINER_LOG_PATH = f"{FATE_CONTAINER_HOME}/logs" 11 | CALLBACK_ADDRESS = "http://127.0.0.1:9380" 12 | CALLBACK = f"{CALLBACK_ADDRESS}/v1/platform/schedule/task/callback" 13 | 14 | 15 | VOLUME = { 16 | # "/data/projects/fate/fate_flow/logs": { 17 | # 'bind': "/data/projects/fate/fate_flow/logs", 18 | # 'mode': 'rw' 19 | # } 20 | } 21 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/translator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/translator/component_spec.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from pydantic import BaseModel 16 | from typing import Optional, Dict, List, Literal 17 | 18 | 19 | class DataSpec(BaseModel): 20 | name: str 21 | description: str 22 | category: str 23 | dataFormat: List[str] 24 | 25 | 26 | class BFIAComponentSpec(BaseModel): 27 | componentName: str 28 | title: str 29 | provider: str 30 | version: str 31 | description: str 32 | roleList: List[Literal["guest", "host", "arbiter", "local"]] 33 | desVersion: str 34 | storageEngine: List[str] 35 | inputParam: Optional[List[Dict]] 36 | inputData: Optional[List[DataSpec]] 37 | outputData: Optional[List[DataSpec]] 38 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/api_utils.py: -------------------------------------------------------------------------------- 1 | from flask import jsonify 2 | 3 | from fate_flow.adapter.bfia.utils.entity.code import ReturnCode 4 | from fate_flow.utils.api_utils import API 5 | 6 | 7 | class BfiaAPI(API): 8 | class Output: 9 | @staticmethod 10 | def json(code=ReturnCode.SUCCESS, msg='success', data=None, **kwargs): 11 | result_dict = { 12 | "code": code, 13 | "msg": msg, 14 | "data": data, 15 | } 16 | 17 | response = {} 18 | for key, value in result_dict.items(): 19 | if value is not None: 20 | response[key] = value 21 | # extra resp 22 | for key, value in kwargs.items(): 23 | response[key] = value 24 | return jsonify(response) -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/entity/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/adapter/bfia/utils/entity/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/entity/code.py: -------------------------------------------------------------------------------- 1 | class ReturnCode: 2 | SUCCESS = 0 3 | FAILED = 1 4 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/entity/status.py: -------------------------------------------------------------------------------- 1 | from fate_flow.entity.types import BaseStateTransitionRule, BaseStatus 2 | 3 | 4 | class StatusSet(BaseStatus): 5 | PENDING = "PENDING" 6 | READY = 'READY' 7 | RUNNING = "RUNNING" 8 | FINISHED = "FINISHED" 9 | REJECTED = "REJECTED" 10 | SUCCESS = "SUCCESS" 11 | FAILED = "FAILED" 12 | 13 | @classmethod 14 | def get_level(cls, status): 15 | return dict(zip(cls.status_list(), range(len(cls.status_list())))).get(status, None) 16 | 17 | 18 | class JobStatus(BaseStatus): 19 | PENDING = StatusSet.PENDING 20 | READY = StatusSet.READY 21 | REJECTED = StatusSet.REJECTED 22 | RUNNING = StatusSet.RUNNING 23 | FINISHED = StatusSet.FINISHED 24 | 25 | class StateTransitionRule(BaseStateTransitionRule): 26 | RULES = { 27 | StatusSet.PENDING: [StatusSet.READY, StatusSet.REJECTED], 28 | StatusSet.READY: [StatusSet.RUNNING, StatusSet.FINISHED], 29 | StatusSet.RUNNING: [StatusSet.FINISHED], 30 | StatusSet.FINISHED: [] 31 | } 32 | 33 | 34 | class TaskStatus(BaseStatus): 35 | PENDING = StatusSet.PENDING 36 | READY = StatusSet.READY 37 | RUNNING = StatusSet.RUNNING 38 | SUCCESS = StatusSet.SUCCESS 39 | FAILED = StatusSet.FAILED 40 | 41 | class StateTransitionRule(BaseStateTransitionRule): 42 | RULES = { 43 | StatusSet.PENDING: [StatusSet.READY, StatusSet.RUNNING, StatusSet.SUCCESS, StatusSet.FAILED], 44 | StatusSet.READY: [StatusSet.RUNNING, StatusSet.FAILED, StatusSet.SUCCESS], 45 | StatusSet.RUNNING: [StatusSet.SUCCESS, StatusSet.FAILED], 46 | StatusSet.FAILED: [], 47 | StatusSet.SUCCESS: [], 48 | } 49 | 50 | 51 | class EndStatus(BaseStatus): 52 | FAILED = StatusSet.FAILED 53 | FINISHED = StatusSet.FINISHED 54 | SUCCESS = StatusSet.SUCCESS 55 | 56 | 57 | class InterruptStatus(BaseStatus): 58 | FAILED = StatusSet.FAILED 59 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/spec/artifact.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class S3Address(BaseModel): 7 | url: str 8 | 9 | 10 | class Engine(BaseModel): 11 | name: str = "s3" 12 | address: S3Address 13 | 14 | 15 | class ArtifactAddress(BaseModel): 16 | name: str 17 | namespace: str 18 | 19 | 20 | class Artifact(BaseModel): 21 | input: Dict[str, ArtifactAddress] 22 | output: Dict[str, ArtifactAddress] 23 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/spec/job.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, List 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | class InitiatorSpec(BaseModel): 7 | role: str 8 | node_id: str 9 | 10 | 11 | class RoleSpec(BaseModel): 12 | guest: Optional[List[str]] 13 | host: Optional[List[str]] 14 | arbiter: Optional[List[str]] 15 | 16 | 17 | class JobCommonSpec(BaseModel): 18 | sync_type: Optional[str] = "poll" 19 | 20 | 21 | class JobParamsSpec(BaseModel): 22 | common: Optional[JobCommonSpec] 23 | guest: Optional[Dict] 24 | host: Optional[Dict] 25 | arbiter: Optional[Dict] 26 | 27 | 28 | class TaskParamsSpec(BaseModel): 29 | common: Optional[Dict] 30 | guest: Optional[Dict] 31 | host: Optional[Dict] 32 | arbiter: Optional[Dict] 33 | 34 | 35 | class ConfSpec(BaseModel): 36 | initiator: InitiatorSpec 37 | role: RoleSpec 38 | job_params: JobParamsSpec 39 | task_params: TaskParamsSpec 40 | version: str 41 | 42 | 43 | class DataSpec(BaseModel): 44 | key: str 45 | type: str 46 | 47 | 48 | class DagComponentSpec(BaseModel): 49 | name: str 50 | componentName: str 51 | provider: str 52 | version: str 53 | input: Optional[List[DataSpec]] = [] 54 | output: Optional[List[DataSpec]] = [] 55 | 56 | 57 | class DagSpec(BaseModel): 58 | components: List[DagComponentSpec] 59 | version: str 60 | 61 | 62 | class BFIADagSpec(BaseModel): 63 | config: ConfSpec 64 | dag: DagSpec 65 | flow_id: Optional[str] 66 | old_job_id: Optional[str] 67 | 68 | 69 | class DagSchemaSpec(BaseModel): 70 | dag: BFIADagSpec 71 | schema_version: str 72 | kind: str = "bfia" 73 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/utils/spec/task.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict 2 | 3 | from pydantic import BaseModel 4 | 5 | from fate_flow.adapter.bfia.utils.spec.artifact import ArtifactAddress 6 | 7 | 8 | class RuntimeConf(BaseModel): 9 | name: str 10 | parameter: Dict = {} 11 | input: Optional[Dict[str, ArtifactAddress]] 12 | output: Optional[Dict[str, ArtifactAddress]] 13 | 14 | 15 | class RuntimeComponent(BaseModel): 16 | component: RuntimeConf 17 | 18 | 19 | class LogPath(BaseModel): 20 | path: str 21 | 22 | 23 | class Config(BaseModel): 24 | task_id: str 25 | trace_id: Optional[str] 26 | session_id: str = "" 27 | token: str = "" 28 | inst_id: Dict 29 | node_id: Dict 30 | log: Optional[LogPath] 31 | self_role: str 32 | 33 | 34 | class SystemConf(BaseModel): 35 | storage: str 36 | transport: str 37 | callback: str 38 | 39 | 40 | class TaskRuntimeEnv(BaseModel): 41 | runtime: RuntimeComponent 42 | config: Config 43 | system: SystemConf 44 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/wheels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/adapter/bfia/wheels/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/wheels/output.py: -------------------------------------------------------------------------------- 1 | from fate_flow.adapter.bfia.db import ComponentOutput 2 | from fate_flow.db.base_models import BaseModelOperate 3 | from fate_flow.utils.wraps_utils import filter_parameters 4 | 5 | 6 | class OutputMeta(BaseModelOperate): 7 | @classmethod 8 | def save(cls, **meta_info): 9 | cls._create_entity(ComponentOutput, meta_info) 10 | 11 | @classmethod 12 | @filter_parameters() 13 | def query(cls, **kwargs): 14 | return cls._query(ComponentOutput, **kwargs) 15 | 16 | @classmethod 17 | @filter_parameters() 18 | def delete(cls, **kwargs): 19 | return cls._delete(ComponentOutput, **kwargs) 20 | -------------------------------------------------------------------------------- /python/fate_flow/adapter/bfia/wheels/saver.py: -------------------------------------------------------------------------------- 1 | from fate_flow.adapter.bfia.utils.entity.status import TaskStatus, JobStatus, EndStatus 2 | from fate_flow.db import ScheduleJob, Task 3 | from fate_flow.entity.types import PROTOCOL 4 | from fate_flow.manager.operation.job_saver import JobSaver, ScheduleJobSaver 5 | 6 | 7 | class BfiaJobSaver(JobSaver): 8 | @classmethod 9 | def check_task_status(cls, old_status, dest_status): 10 | return TaskStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=dest_status) 11 | 12 | @classmethod 13 | def check_job_status(cls, old_status, dest_status): 14 | return JobStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=dest_status) 15 | 16 | @classmethod 17 | def end_status_contains(cls, status): 18 | return EndStatus.contains(status) 19 | 20 | @classmethod 21 | def query_task(cls, only_latest=True, reverse=None, order_by=None, protocol=PROTOCOL.BFIA, **kwargs): 22 | return cls._query_task( 23 | Task, only_latest=only_latest, reverse=reverse, order_by=order_by, protocol=protocol, **kwargs 24 | ) 25 | 26 | 27 | class BfiaScheduleJobSaver(ScheduleJobSaver, BfiaJobSaver): 28 | @classmethod 29 | def query_job(cls, reverse=None, order_by=None, protocol=PROTOCOL.BFIA, **kwargs): 30 | return cls._query_job(ScheduleJob, reverse, order_by, protocol=protocol, **kwargs) 31 | -------------------------------------------------------------------------------- /python/fate_flow/apps/client/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. -------------------------------------------------------------------------------- /python/fate_flow/apps/client/site_app.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.entity.code import ReturnCode 17 | from fate_flow.errors.server_error import IsStandalone 18 | from fate_flow.runtime.system_settings import PARTY_ID, IS_STANDALONE 19 | from fate_flow.utils.api_utils import API 20 | 21 | 22 | @manager.route('/info/query', methods=['GET']) 23 | def query_site_info(): 24 | if not IS_STANDALONE: 25 | return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data={"party_id": PARTY_ID}) 26 | else: 27 | return API.Output.fate_flow_exception(IsStandalone()) 28 | -------------------------------------------------------------------------------- /python/fate_flow/apps/partner/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | # Define federated scheduling api 19 | -------------------------------------------------------------------------------- /python/fate_flow/apps/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/apps/worker/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/commands/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/components/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/components/__main__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | """ 18 | execute with python -m fate.components --config xxx 19 | """ 20 | 21 | if __name__ == "__main__": 22 | import click 23 | from fate_flow.components.entrypoint.cli import component 24 | 25 | cli = click.Group() 26 | cli.add_command(component) 27 | cli(prog_name="python -m fate_flow.components") 28 | -------------------------------------------------------------------------------- /python/fate_flow/components/components/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from .upload import upload 16 | from .download import download 17 | 18 | BUILDIN_COMPONENTS = [upload, download] 19 | -------------------------------------------------------------------------------- /python/fate_flow/components/cpn.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import inspect 16 | import logging 17 | from typing import Any 18 | 19 | from pydantic import BaseModel 20 | 21 | 22 | class Params(BaseModel): 23 | class TaskParams(BaseModel): 24 | job_id: str 25 | 26 | component_params: Any 27 | task_params: TaskParams 28 | 29 | 30 | class _Component: 31 | def __init__( 32 | self, 33 | name: str, 34 | callback 35 | ) -> None: 36 | self.name = name 37 | self.callback = callback 38 | 39 | def execute(self, config, outputs): 40 | return self.callback(config, outputs) 41 | 42 | 43 | def component(*args, **kwargs): 44 | def decorator(f): 45 | cpn_name = f.__name__.lower() 46 | if isinstance(f, _Component): 47 | raise TypeError("Attempted to convert a callback into a component twice.") 48 | cpn = _Component( 49 | name=cpn_name, 50 | callback=f 51 | ) 52 | cpn.__doc__ = f.__doc__ 53 | # cpn.validate_declare() 54 | return cpn 55 | return decorator 56 | -------------------------------------------------------------------------------- /python/fate_flow/components/define/download.yaml: -------------------------------------------------------------------------------- 1 | component: 2 | name: download 3 | description: '' 4 | provider: fate_flow 5 | version: 2.1.0 6 | labels: [] 7 | roles: 8 | - guest 9 | - host 10 | - local 11 | parameters: 12 | name: 13 | type: str 14 | default: 15 | optional: false 16 | description: '' 17 | type_meta: 18 | title: str 19 | type: string 20 | default: 21 | description: '' 22 | namespace: 23 | type: str 24 | default: 25 | optional: false 26 | description: '' 27 | type_meta: 28 | title: str 29 | type: string 30 | default: 31 | description: '' 32 | path: 33 | type: str 34 | default: 35 | optional: false 36 | description: '' 37 | type_meta: 38 | title: str 39 | type: string 40 | default: 41 | description: '' 42 | input_artifacts: 43 | data: {} 44 | model: {} 45 | output_artifacts: 46 | data: {} 47 | model: {} 48 | metric: 49 | metric: 50 | types: 51 | - json_metric 52 | optional: false 53 | stages: 54 | - default 55 | roles: 56 | - guest 57 | - host 58 | - local 59 | description: metric, invisible for user 60 | is_multi: false 61 | schema_version: v1 62 | -------------------------------------------------------------------------------- /python/fate_flow/components/entrypoint/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/controller/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/db/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.db.storage_models import * 17 | from fate_flow.db.schedule_models import * 18 | from fate_flow.db.db_models import * 19 | -------------------------------------------------------------------------------- /python/fate_flow/db/permission_models.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from peewee import CharField, CompositeKey 17 | 18 | from fate_flow.db import DataBaseModel 19 | 20 | 21 | class AppInfo(DataBaseModel): 22 | f_app_name = CharField(max_length=100, index=True) 23 | f_app_id = CharField(max_length=100, primary_key=True) 24 | f_app_token = CharField(max_length=100) 25 | f_app_type = CharField(max_length=20, index=True) 26 | 27 | class Meta: 28 | db_table = "t_app_info" 29 | 30 | 31 | class PartnerAppInfo(DataBaseModel): 32 | f_party_id = CharField(max_length=100, primary_key=True) 33 | f_app_id = CharField(max_length=100) 34 | f_app_token = CharField(max_length=100) 35 | 36 | class Meta: 37 | db_table = "t_partner_app_info" 38 | -------------------------------------------------------------------------------- /python/fate_flow/engine/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/engine/backend/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.engine.backend._session import build_backend 16 | 17 | __all__ = [ 18 | "build_backend", 19 | ] 20 | 21 | 22 | -------------------------------------------------------------------------------- /python/fate_flow/engine/backend/_eggroll.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import yaml 17 | 18 | from fate_flow.engine.backend._base import LocalEngine 19 | from fate_flow.entity.spec.dag import TaskConfigSpec 20 | from fate_flow.entity.types import WorkerName, ComputingEngine 21 | from fate_flow.manager.service.worker_manager import WorkerManager 22 | 23 | 24 | class EggrollEngine(LocalEngine): 25 | def run(self, task_info, run_parameters, engine_run, provider_name, output_path, conf_path, sync=False, **kwargs): 26 | parameters = TaskConfigSpec.parse_obj(run_parameters) 27 | if parameters.conf.computing.type == ComputingEngine.EGGROLL: 28 | # update eggroll options 29 | cores = engine_run.pop("task_cores_per_node", None) 30 | if cores: 31 | engine_run["eggroll.session.processors.per.node"] = cores 32 | parameters.conf.computing.metadata.options.update(engine_run) 33 | with open(conf_path, "w") as f: 34 | # update parameters 35 | yaml.dump(parameters.dict(), f) 36 | return WorkerManager.start_task_worker( 37 | worker_name=WorkerName.TASK_EXECUTE, 38 | task_info=task_info, 39 | common_cmd=self.generate_component_run_cmd(provider_name, conf_path, output_path, ), 40 | sync=sync, 41 | **kwargs 42 | ).returncode 43 | -------------------------------------------------------------------------------- /python/fate_flow/engine/backend/_session.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.engine.backend._eggroll import EggrollEngine 17 | from fate_flow.engine.backend._spark import SparkEngine 18 | from fate_flow.engine.backend._eggroll_deepspeed import Deepspeed 19 | from fate_flow.entity.types import ComputingEngine, LauncherType 20 | 21 | 22 | def build_backend(backend_name: str, launcher_name: str = LauncherType.DEFAULT): 23 | if backend_name in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: 24 | if launcher_name == LauncherType.DEEPSPEED: 25 | backend = Deepspeed() 26 | elif not launcher_name or launcher_name == LauncherType.DEFAULT: 27 | backend = EggrollEngine() 28 | else: 29 | raise ValueError(f'backend "{backend_name}" launcher {launcher_name} is not supported') 30 | elif backend_name == ComputingEngine.SPARK: 31 | backend = SparkEngine() 32 | else: 33 | raise ValueError(f'backend "{backend_name}" is not supported') 34 | return backend 35 | -------------------------------------------------------------------------------- /python/fate_flow/engine/devices/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.engine.devices._base import EngineABC 16 | from fate_flow.entity.types import ProviderDevice, LauncherType 17 | from fate_flow.manager.service.provider_manager import ProviderManager 18 | 19 | 20 | def build_engine(provider_name: str, launcher_name: str = LauncherType.DEFAULT): 21 | provider = ProviderManager.get_provider_by_provider_name(provider_name) 22 | 23 | if launcher_name == LauncherType.DEEPSPEED: 24 | if provider.device in {ProviderDevice.LOCAL}: 25 | from fate_flow.engine.devices.deepspeed import EggrollDeepspeedEngine 26 | engine_session = EggrollDeepspeedEngine(provider) 27 | else: 28 | raise ValueError(f'engine launcher {LauncherType.DEEPSPEED} device "{provider.device}" is not supported') 29 | 30 | else: 31 | if provider.device in {ProviderDevice.DOCKER, ProviderDevice.K8S}: 32 | from fate_flow.engine.devices.container import ContainerdEngine 33 | engine_session = ContainerdEngine(provider) 34 | 35 | elif provider.device in {ProviderDevice.LOCAL}: 36 | from fate_flow.engine.devices.local import LocalEngine 37 | engine_session = LocalEngine(provider) 38 | 39 | else: 40 | raise ValueError(f'engine device "{provider.device}" is not supported') 41 | 42 | return engine_session 43 | -------------------------------------------------------------------------------- /python/fate_flow/engine/devices/_base.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | import abc 19 | import typing 20 | 21 | from fate_flow.db.db_models import Task 22 | 23 | 24 | class EngineABC(metaclass=abc.ABCMeta): 25 | @abc.abstractmethod 26 | def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs) -> typing.Dict: 27 | ... 28 | 29 | @abc.abstractmethod 30 | def kill(self, task: Task): 31 | ... 32 | 33 | @abc.abstractmethod 34 | def is_alive(self, task: Task): 35 | ... 36 | 37 | @abc.abstractmethod 38 | def cleanup(self, task: Task): 39 | ... 40 | 41 | @abc.abstractmethod 42 | def download_output(self, task: Task): 43 | ... 44 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.engine.storage._types import EggRollStoreType, StorageEngine, StandaloneStoreType, DataType, StorageOrigin 16 | from fate_flow.engine.storage._table import StorageTableBase, StorageTableMeta 17 | from fate_flow.engine.storage._session import StorageSessionBase, Session 18 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/eggroll/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.engine.storage.eggroll._table import StorageTable 17 | from fate_flow.engine.storage.eggroll._session import StorageSession 18 | 19 | __all__ = ["StorageTable", "StorageSession"] 20 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/file/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.engine.storage.file._session import StorageSession 17 | from fate_flow.engine.storage.file._table import StorageTable 18 | 19 | __all__ = ["StorageTable", "StorageSession"] 20 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/file/_session.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.engine.storage import StorageSessionBase, StorageEngine 17 | from fate_flow.engine.storage.file._table import StorageTable 18 | from fate_flow.entity.types import AddressABC, FileAddress 19 | 20 | 21 | class StorageSession(StorageSessionBase): 22 | def __init__(self, session_id, options=None): 23 | super(StorageSession, self).__init__(session_id=session_id, engine=StorageEngine.FILE) 24 | 25 | def table(self, address: AddressABC, name, namespace, partitions, storage_type=None, options=None, **kwargs): 26 | if isinstance(address, FileAddress): 27 | return StorageTable(address=address, name=name, namespace=namespace, 28 | partitions=partitions) 29 | raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") 30 | 31 | def load( 32 | self, 33 | name, 34 | namespace, 35 | address, 36 | store_type, 37 | partitions, 38 | options=None, 39 | **kwargs, 40 | ): 41 | if isinstance(address, FileAddress): 42 | return StorageTable(address=address, name=name, namespace=namespace, 43 | partitions=partitions, options=options) 44 | raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") 45 | 46 | def cleanup(self, name, namespace): 47 | pass 48 | 49 | def stop(self): 50 | pass 51 | 52 | def kill(self): 53 | pass 54 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/hdfs/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.engine.storage.hdfs._table import StorageTable 17 | from fate_flow.engine.storage.hdfs._session import StorageSession 18 | 19 | __all__ = ["StorageTable", "StorageSession"] 20 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/serdes/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | def get_serdes_by_type(serdes_type: int): 19 | if serdes_type == 0: 20 | from ._unrestricted_serdes import get_unrestricted_serdes 21 | 22 | return get_unrestricted_serdes() 23 | elif serdes_type == 1: 24 | from ._integer_serdes import get_integer_serdes 25 | 26 | return get_integer_serdes() 27 | else: 28 | raise ValueError(f"serdes type `{serdes_type}` not supported") 29 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/serdes/_integer_serdes.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | def get_integer_serdes(): 19 | return IntegerSerdes() 20 | 21 | 22 | class IntegerSerdes: 23 | def __init__(self): 24 | ... 25 | 26 | def serialize(self, obj) -> bytes: 27 | return obj.to_bytes(8, "big") 28 | 29 | def deserialize(self, bytes) -> object: 30 | return int.from_bytes(bytes, "big") 31 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/serdes/_serdes_base.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from pickle import loads as p_loads 18 | from pickle import dumps as p_dumps 19 | from pickle import Pickler, Unpickler 20 | 21 | __all__ = ["p_dumps", "p_loads", "Pickler", "Unpickler"] -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/serdes/_unrestricted_serdes.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from ._serdes_base import p_dumps, p_loads 18 | 19 | 20 | def get_unrestricted_serdes(): 21 | return UnrestrictedSerdes 22 | 23 | 24 | class UnrestrictedSerdes: 25 | @staticmethod 26 | def serialize(obj) -> bytes: 27 | return p_dumps(obj) 28 | 29 | @staticmethod 30 | def deserialize(bytes) -> object: 31 | return p_loads(bytes) 32 | -------------------------------------------------------------------------------- /python/fate_flow/engine/storage/standalone/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.engine.storage.standalone._table import StorageTable 17 | from fate_flow.engine.storage.standalone._session import StorageSession 18 | 19 | __all__ = ["StorageTable", "StorageSession"] 20 | -------------------------------------------------------------------------------- /python/fate_flow/entity/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from ._base import BaseEntity, BaseModel, CustomEnum 18 | 19 | __all__ = ["BaseEntity", "BaseModel", "CustomEnum"] 20 | -------------------------------------------------------------------------------- /python/fate_flow/entity/_base.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from enum import Enum 17 | 18 | from pydantic import BaseModel as Base 19 | 20 | from fate_flow.utils.base_utils import BaseType 21 | 22 | 23 | class BaseEntity(BaseType): 24 | pass 25 | 26 | 27 | class BaseModel(Base): 28 | def to_dict(self): 29 | d = {} 30 | for k, v in self.__dict__.items(): 31 | d[k] = v 32 | return d 33 | 34 | def __str__(self): 35 | return str(self.to_dict()) 36 | 37 | 38 | class CustomEnum(Enum): 39 | @classmethod 40 | def valid(cls, value): 41 | try: 42 | cls(value) 43 | return True 44 | except: 45 | return False 46 | 47 | @classmethod 48 | def values(cls): 49 | return [member.value for member in cls.__members__.values()] 50 | 51 | @classmethod 52 | def names(cls): 53 | return [member.name for member in cls.__members__.values()] -------------------------------------------------------------------------------- /python/fate_flow/entity/code/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from ._schedule import * 17 | from ._api import * 18 | from ._process import * 19 | -------------------------------------------------------------------------------- /python/fate_flow/entity/code/_process.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from enum import IntEnum 17 | 18 | from fate_flow.entity import CustomEnum 19 | 20 | 21 | class KillProcessRetCode(IntEnum, CustomEnum): 22 | KILLED = 0 23 | NOT_FOUND = 1 24 | ERROR_PID = 2 25 | -------------------------------------------------------------------------------- /python/fate_flow/entity/code/_schedule.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | class SchedulingStatusCode(object): 17 | SUCCESS = 0 18 | NO_RESOURCE = 1 19 | PASS = 1 20 | NO_NEXT = 2 21 | HAVE_NEXT = 3 22 | FAILED = 4 23 | 24 | 25 | class FederatedSchedulingStatusCode(object): 26 | SUCCESS = 0 27 | FAILED = 1 28 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/dag/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.entity.spec.dag._output import ComponentOutputMeta, MetricData, OutputArtifactType, OutputArtifactSpec, \ 17 | OutputArtifacts, IOMeta 18 | from fate_flow.entity.spec.dag._party import PartySpec 19 | from fate_flow.entity.spec.dag._job import DAGSchema, DAGSpec, JobConfSpec, TaskConfSpec, TaskSpec, PartyTaskSpec, \ 20 | InheritConfSpec, PartyTaskRefSpec 21 | from fate_flow.entity.spec.dag._task import TaskConfigSpec, PreTaskConfigSpec, TaskRuntimeConfSpec, \ 22 | TaskCleanupConfigSpec 23 | from fate_flow.entity.spec.dag._artifact import RuntimeTaskOutputChannelSpec, DataWarehouseChannelSpec, \ 24 | ModelWarehouseChannelSpec, RuntimeInputArtifacts, FlowRuntimeInputArtifacts,\ 25 | ArtifactInputApplySpec, Metadata, RuntimeTaskOutputChannelSpec, InputArtifactSpec, \ 26 | ArtifactOutputApplySpec, ModelWarehouseChannelSpec, ArtifactOutputSpec, ArtifactSource 27 | from fate_flow.entity.spec.dag._component import ComponentSpec, ComponentIOArtifactsTypeSpec, ComponentSpecV1 28 | from fate_flow.entity.spec.dag._computing import EggrollComputingSpec, SparkComputingSpec, StandaloneComputingSpec 29 | from fate_flow.entity.spec.dag._federation import StandaloneFederationSpec, RollSiteFederationSpec, OSXFederationSpec, \ 30 | PulsarFederationSpec, RabbitMQFederationSpec 31 | from fate_flow.entity.spec.dag._logger import FlowLogger 32 | from fate_flow.entity.spec.dag._mlmd import MLMDSpec 33 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/dag/_computing.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from typing import Literal, TypeVar 16 | 17 | import pydantic 18 | from pydantic import typing 19 | 20 | 21 | class StandaloneComputingSpec(pydantic.BaseModel): 22 | class MetadataSpec(pydantic.BaseModel): 23 | computing_id: str 24 | options: dict = {} 25 | 26 | type: Literal["standalone"] 27 | metadata: MetadataSpec 28 | 29 | 30 | class EggrollComputingSpec(pydantic.BaseModel): 31 | class MetadataSpec(pydantic.BaseModel): 32 | computing_id: str 33 | host: typing.Optional[str] = None 34 | port: typing.Optional[int] = None 35 | config_options: typing.Optional[dict] = None 36 | config_properties_file: typing.Optional[str] = None 37 | options: dict = {} 38 | 39 | type: Literal["eggroll"] 40 | metadata: MetadataSpec 41 | 42 | 43 | class SparkComputingSpec(pydantic.BaseModel): 44 | class MetadataSpec(pydantic.BaseModel): 45 | computing_id: str 46 | options: dict = {} 47 | 48 | type: Literal["spark"] 49 | metadata: MetadataSpec 50 | 51 | 52 | class CustomComputingSpec(pydantic.BaseModel): 53 | type: Literal["custom"] 54 | metadata: dict 55 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/dag/_device.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from typing import Literal 16 | 17 | import pydantic 18 | from pydantic import typing 19 | 20 | 21 | class CPUSpec(pydantic.BaseModel): 22 | type: Literal["CPU"] 23 | metadata: dict = {} 24 | 25 | 26 | class GPUSpec(pydantic.BaseModel): 27 | type: Literal["GPU"] 28 | metadata: dict = {} 29 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/dag/_mlmd.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from typing import Optional, Any, Dict, Union 17 | 18 | import pydantic 19 | 20 | 21 | class FlowMLMDMetadata(pydantic.BaseModel): 22 | host: Optional[str] 23 | port: Optional[int] 24 | protocol: Optional[str] 25 | 26 | 27 | class MLMDSpec(pydantic.BaseModel): 28 | type: str 29 | metadata: Union[Dict[str, Any], FlowMLMDMetadata] 30 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/dag/_party.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The FATE Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | from typing import Union, Literal, List 16 | 17 | from pydantic import BaseModel 18 | 19 | 20 | class PartySpec(BaseModel): 21 | role: Union[Literal["guest", "host", "arbiter", "local"]] 22 | party_id: List[str] 23 | 24 | def tuple(self): 25 | return self.role, self.party_id 26 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/flow/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from ._model import MLModelSpec, Metadata 16 | from ._storage import FileStorageSpec, MysqlStorageSpec, TencentCosStorageSpec 17 | from ._provider import ProviderSpec, DockerProviderSpec, K8sProviderSpec, LocalProviderSpec 18 | from ._scheduler import SchedulerInfoSpec 19 | from ._protocol import SubmitJobInput, SubmitJobOutput, QueryJobInput, QueryJobOutput, StopJobInput, StopJobOutput, \ 20 | QueryTaskOutput, QueryTaskInput 21 | 22 | __all__ = ["MLModelSpec", "FileStorageSpec", "MysqlStorageSpec", "TencentCosStorageSpec", "ProviderSpec", 23 | "DockerProviderSpec", "K8sProviderSpec", "LocalProviderSpec", "SchedulerInfoSpec", "Metadata", 24 | "SubmitJobInput", "SubmitJobOutput", "QueryJobInput", "QueryJobOutput", "StopJobInput", "StopJobOutput", 25 | "QueryTaskInput", "QueryTaskOutput"] 26 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/flow/_model.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from datetime import datetime 16 | from typing import List, Optional, Dict, Any, Union 17 | 18 | import pydantic 19 | 20 | 21 | class MLModelComponentSpec(pydantic.BaseModel): 22 | name: str 23 | provider: str 24 | version: str 25 | metadata: dict 26 | 27 | 28 | class MLModelPartiesSpec(pydantic.BaseModel): 29 | guest: List[str] 30 | host: List[str] 31 | arbiter: List[str] 32 | 33 | 34 | class MLModelFederatedSpec(pydantic.BaseModel): 35 | 36 | task_id: str 37 | parties: MLModelPartiesSpec 38 | component: MLModelComponentSpec 39 | 40 | 41 | class MLModelModelSpec(pydantic.BaseModel): 42 | name: str 43 | created_time: datetime 44 | file_format: str 45 | metadata: dict 46 | 47 | 48 | class MLModelPartySpec(pydantic.BaseModel): 49 | 50 | party_task_id: str 51 | role: str 52 | partyid: str 53 | models: List[MLModelModelSpec] 54 | 55 | 56 | class MLModelSpec(pydantic.BaseModel): 57 | 58 | federated: MLModelFederatedSpec 59 | party: MLModelPartySpec 60 | 61 | 62 | class Metadata(pydantic.BaseModel): 63 | metadata: dict 64 | model_overview: MLModelSpec 65 | model_key: str 66 | index: Optional[Union[int, None]] = None 67 | source: Optional[Dict[str, Any]] = None 68 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/flow/_protocol.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, Any, List 2 | 3 | import pydantic 4 | 5 | from fate_flow.entity.spec.dag import DAGSchema 6 | 7 | 8 | class SubmitJobInput(pydantic.BaseModel): 9 | dag_schema: DAGSchema 10 | 11 | 12 | class SubmitJobOutput(pydantic.BaseModel): 13 | message: str = "success" 14 | code: int = 0 15 | job_id: str 16 | data: Optional[Dict[str, Any]] = {} 17 | 18 | 19 | class QueryJobInput(pydantic.BaseModel): 20 | jobs: List[Any] 21 | 22 | 23 | class QueryJobOutput(pydantic.BaseModel): 24 | jobs: List[Any] 25 | 26 | 27 | class StopJobInput(pydantic.BaseModel): 28 | job_id: str 29 | 30 | 31 | class StopJobOutput(pydantic.BaseModel): 32 | message: str = "success" 33 | code: int = 0 34 | 35 | 36 | class QueryTaskInput(pydantic.BaseModel): 37 | tasks: List[Any] 38 | 39 | 40 | class QueryTaskOutput(pydantic.BaseModel): 41 | tasks: List[Any] 42 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/flow/_provider.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | from typing import Optional, Union 17 | 18 | import pydantic 19 | 20 | from fate_flow.errors.server_error import FileNoFound 21 | 22 | 23 | class BaseProvider(pydantic.BaseModel): 24 | def __init__(self, check=False, **kwargs): 25 | super(BaseProvider, self).__init__(**kwargs) 26 | if check: 27 | self.check() 28 | 29 | def check(self): 30 | pass 31 | 32 | 33 | class LocalProviderSpec(BaseProvider): 34 | def check(self): 35 | if not os.path.exists(self.path): 36 | raise FileNoFound(path=self.path) 37 | if self.venv and not os.path.exists(self.venv): 38 | raise FileNoFound(venv=self.venv) 39 | 40 | path: str 41 | venv: Optional[str] 42 | 43 | 44 | class DockerProviderSpec(BaseProvider): 45 | base_url: str 46 | image: str 47 | 48 | 49 | class K8sProviderSpec(BaseProvider): 50 | image: str 51 | namespace: str 52 | config: Optional[dict] 53 | 54 | 55 | class ProviderSpec(BaseProvider): 56 | name: str 57 | version: str 58 | device: str 59 | metadata: Union[LocalProviderSpec, DockerProviderSpec, K8sProviderSpec] 60 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/flow/_scheduler.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from typing import Any, List, Union, Dict 16 | 17 | from pydantic import BaseModel 18 | 19 | from fate_flow.entity.spec.dag import PartySpec 20 | 21 | 22 | class SchedulerInfoSpec(BaseModel): 23 | dag: Dict[str, Any] 24 | parties: List[PartySpec] 25 | initiator_party_id: str 26 | scheduler_party_id: str 27 | federated_status_collect_type: str 28 | model_id: str 29 | model_version: Union[str, int] 30 | 31 | 32 | -------------------------------------------------------------------------------- /python/fate_flow/entity/spec/flow/_storage.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from typing import Union 16 | 17 | import pydantic 18 | 19 | 20 | class FileStorageSpec(pydantic.BaseModel): 21 | path: Union[str, None] 22 | 23 | 24 | class MysqlStorageSpec(pydantic.BaseModel): 25 | name: str 26 | user: str 27 | passwd: str 28 | host: str 29 | port: int 30 | max_connections: int 31 | stale_timeout: int 32 | 33 | 34 | class TencentCosStorageSpec(pydantic.BaseModel): 35 | Region: str 36 | SecretId: str 37 | SecretKey: str 38 | Bucket: str 39 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from ._work import * 17 | from ._address import * 18 | from ._engine import * 19 | from ._output import * 20 | from ._provider import * 21 | from ._status import * 22 | from ._federation import * 23 | from ._command import * 24 | from ._api import * 25 | from ._instance import * 26 | from ._permission import * 27 | from ._input import * 28 | from ._artificats import * 29 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_api.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | class AppType: 17 | SITE = "site" 18 | CLIENT = "client" 19 | 20 | 21 | class PROTOCOL: 22 | FATE_FLOW = "fate" 23 | BFIA = "bfia" 24 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_artificats.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from typing import List 17 | 18 | from typing_extensions import Protocol 19 | 20 | 21 | class ArtifactType(Protocol): 22 | type_name: str 23 | path_type: str 24 | uri_types: List[str] 25 | 26 | 27 | class DataframeArtifactType(ArtifactType): 28 | type_name = "dataframe" 29 | path_type = "distributed" 30 | uri_types = ["eggroll", "hdfs"] 31 | 32 | 33 | class TableArtifactType(ArtifactType): 34 | type_name = "table" 35 | path_type = "distributed" 36 | uri_types = ["eggroll", "hdfs"] 37 | 38 | 39 | class DataDirectoryArtifactType(ArtifactType): 40 | type_name = "data_directory" 41 | path_type = "directory" 42 | uri_types = ["file"] 43 | 44 | 45 | class ModelDirectoryArtifactType(ArtifactType): 46 | type_name = "model_directory" 47 | path_type = "directory" 48 | uri_types = ["file"] 49 | 50 | 51 | class JsonModelArtifactType(ArtifactType): 52 | type_name = "json_model" 53 | path_type = "file" 54 | uri_types = ["file"] 55 | 56 | 57 | class JsonMetricArtifactType(ArtifactType): 58 | type_name = "json_metric" 59 | path_type = "file" 60 | uri_types = ["file"] 61 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_command.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.entity import CustomEnum 17 | 18 | 19 | class ResourceOperation(CustomEnum): 20 | APPLY = "apply" 21 | RETURN = "return" 22 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_engine.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | class EngineType(object): 16 | COMPUTING = "computing" 17 | STORAGE = "storage" 18 | FEDERATION = "federation" 19 | 20 | 21 | class FederationEngine(object): 22 | ROLLSITE = "rollsite" 23 | RABBITMQ = "rabbitmq" 24 | STANDALONE = "standalone" 25 | PULSAR = "pulsar" 26 | OSX = "osx" 27 | 28 | 29 | class ComputingEngine(object): 30 | EGGROLL = "eggroll" 31 | SPARK = "spark" 32 | STANDALONE = "standalone" 33 | 34 | 35 | class StorageEngine(object): 36 | STANDALONE = "standalone" 37 | EGGROLL = "eggroll" 38 | HDFS = "hdfs" 39 | MYSQL = "mysql" 40 | SIMPLE = "simple" 41 | PATH = "path" 42 | HIVE = "hive" 43 | FILE = "file" 44 | API = "api" 45 | 46 | 47 | class CoordinationProxyService(object): 48 | ROLLSITE = "rollsite" 49 | NGINX = "nginx" 50 | FATEFLOW = "fateflow" 51 | FIREWORK = "firework" 52 | OSX = "osx" 53 | 54 | 55 | class FederatedCommunicationType(object): 56 | POLL = "poll" 57 | CALLBACK = "callback" 58 | 59 | 60 | class LauncherType(object): 61 | DEFAULT = "default" 62 | DEEPSPEED = "deepspeed" 63 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_federation.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | class CoordinationCommunicationProtocol(object): 17 | HTTP = "http" 18 | GRPC = "grpc" 19 | 20 | 21 | class OSXMode(object): 22 | STREAM = "stream" 23 | QUEUE = "queue" 24 | 25 | 26 | class FederatedMode(object): 27 | SINGLE = "SINGLE" 28 | MULTIPLE = "MULTIPLE" 29 | 30 | def is_single(self, value): 31 | return value == self.SINGLE 32 | 33 | def is_multiple(self, value): 34 | return value == self.MULTIPLE 35 | 36 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_input.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | class ArtifactSourceType(object): 18 | TASK_OUTPUT_ARTIFACT = "task_output_artifact" 19 | MODEL_WAREHOUSE = "model_warehouse" 20 | DATA_WAREHOUSE = "data_warehouse" 21 | 22 | 23 | class InputArtifactType(object): 24 | DATA = "data" 25 | MODEL = "model" 26 | 27 | @classmethod 28 | def types(cls): 29 | for _type in [cls.DATA, cls.MODEL]: 30 | yield _type -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_instance.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from .._base import BaseEntity 17 | 18 | 19 | class FlowInstance(BaseEntity): 20 | def __init__(self, **kwargs): 21 | self.instance_id = None, 22 | self.timestamp = None, 23 | self.version = None, 24 | self.host = None, 25 | self.grpc_port = None, 26 | self.http_port = None 27 | for k, v in kwargs.items(): 28 | if hasattr(self, k): 29 | setattr(self, k, v) 30 | 31 | def to_dict(self): 32 | d = {} 33 | for k, v in self.__dict__.items(): 34 | if v is None: 35 | continue 36 | d[k] = v 37 | return d 38 | 39 | @property 40 | def grpc_address(self): 41 | return f'{self.host}:{self.grpc_port}' 42 | 43 | @property 44 | def http_address(self): 45 | return f'{self.host}:{self.http_port}' 46 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_output.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from typing import Dict, List, Union, Any 16 | 17 | from fate_flow.entity import BaseModel 18 | 19 | 20 | class ModelStorageEngine(object): 21 | FILE = "file" 22 | MYSQL = "mysql" 23 | TENCENT_COS = "tencent_cos" 24 | 25 | 26 | class ModelFileFormat(object): 27 | JSON = "json" 28 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_provider.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | class ProviderDevice(object): 16 | LOCAL = "local" 17 | DOCKER = "docker" 18 | K8S = "k8s" 19 | 20 | 21 | class ProviderName(object): 22 | FATE = "fate" 23 | FATE_FLOW = "fate_flow" 24 | -------------------------------------------------------------------------------- /python/fate_flow/entity/types/_work.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.entity import CustomEnum 17 | 18 | 19 | class ProcessRole(CustomEnum): 20 | DRIVER = "driver" 21 | WORKER = "worker" 22 | 23 | 24 | class WorkerName(CustomEnum): 25 | TASK_SUBMIT = "submit" 26 | TASK_ENTRYPOINT = "task_entrypoint" 27 | TASK_EXECUTE = "task_execute" 28 | COMPONENT_DEFINE = "component_define" 29 | TASK_CLEAN = "task_clean" 30 | TASK_EXECUTE_CLEAN = "execute_clean" 31 | -------------------------------------------------------------------------------- /python/fate_flow/errors/__init__.py: -------------------------------------------------------------------------------- 1 | class FateFlowError(Exception): 2 | code = None 3 | message = 'Unknown Fate Flow Error' 4 | 5 | def __init__(self, message=None, **kwargs): 6 | self.code = self.code 7 | self.message = str(message) if message is not None else self.message 8 | suffix = "" 9 | if kwargs: 10 | for k, v in kwargs.items(): 11 | if v is not None and not callable(v): 12 | if suffix: 13 | suffix += "," 14 | suffix += f"{k}[{v}]" 15 | if suffix: 16 | self.message += f": {suffix}" 17 | super().__init__(self.code, self.message) 18 | -------------------------------------------------------------------------------- /python/fate_flow/errors/zookeeper_error.py: -------------------------------------------------------------------------------- 1 | from fate_flow.errors import FateFlowError 2 | 3 | __all__ = ['ServicesError', 'ServiceNotSupported', 'ZooKeeperNotConfigured', 4 | 'MissingZooKeeperUsernameOrPassword', 'ZooKeeperBackendError'] 5 | 6 | 7 | class ServicesError(FateFlowError): 8 | message = 'Unknown services error' 9 | 10 | 11 | class ServiceNotSupported(ServicesError): 12 | message = 'The service {service_name} is not supported' 13 | 14 | 15 | class ZooKeeperNotConfigured(ServicesError): 16 | message = 'ZooKeeper has not been configured' 17 | 18 | 19 | class MissingZooKeeperUsernameOrPassword(FateFlowError): 20 | message = 'Using ACL for ZooKeeper is enabled but username or password is not configured' 21 | 22 | 23 | class ZooKeeperBackendError(ServicesError): 24 | message = 'ZooKeeper backend error: {error_message}' 25 | -------------------------------------------------------------------------------- /python/fate_flow/hook/common/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # Copyright 2019 The FATE Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /python/fate_flow/hook/common/parameters.py: -------------------------------------------------------------------------------- 1 | from fate_flow.entity.code import ReturnCode 2 | 3 | 4 | class ParametersBase: 5 | def to_dict(self): 6 | d = {} 7 | for k, v in self.__dict__.items(): 8 | d[k] = v 9 | return d 10 | 11 | 12 | class AuthenticationParameters(ParametersBase): 13 | def __init__(self, path, method, headers, form, data, json, full_path): 14 | self.path = path 15 | self.method = method 16 | self.headers = headers 17 | self.form = form 18 | self.data = data 19 | self.json = json 20 | self.full_path = full_path 21 | 22 | 23 | class AuthenticationReturn(ParametersBase): 24 | def __init__(self, code=ReturnCode.Base.SUCCESS, message="success"): 25 | self.code = code 26 | self.message = message 27 | 28 | 29 | class SignatureParameters(ParametersBase): 30 | def __init__(self, party_id, body, initiator_party_id=""): 31 | self.party_id = party_id 32 | self.initiator_party_id = initiator_party_id 33 | self.body = body 34 | 35 | 36 | class SignatureReturn(ParametersBase): 37 | def __init__(self, code=ReturnCode.Base.SUCCESS, signature=None, message=""): 38 | self.code = code 39 | self.signature = signature 40 | self.message = message 41 | 42 | 43 | class PermissionCheckParameters(ParametersBase): 44 | def __init__(self, initiator_party_id, roles, component_list, dataset_list, dag_schema, component_parameters): 45 | self.party_id = initiator_party_id 46 | self.roles = roles 47 | self.component_list = component_list 48 | self.dataset_list = dataset_list 49 | self.dag_schema = dag_schema 50 | self.component_parameters = component_parameters 51 | 52 | 53 | class PermissionReturn(ParametersBase): 54 | def __init__(self, code=ReturnCode.Base.SUCCESS, message="success"): 55 | self.code = code 56 | self.message = message 57 | 58 | 59 | -------------------------------------------------------------------------------- /python/fate_flow/hook/flow/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/hook/flow/client_authentication.py: -------------------------------------------------------------------------------- 1 | from fate_flow.controller.permission import Authentication, PermissionController 2 | from fate_flow.entity.code import ReturnCode 3 | from fate_flow.errors.server_error import InvalidParameter 4 | from fate_flow.hook import HookManager 5 | from fate_flow.hook.common.parameters import AuthenticationReturn, AuthenticationParameters 6 | 7 | 8 | @HookManager.register_client_authentication_hook 9 | def authentication(parm: AuthenticationParameters) -> AuthenticationReturn: 10 | app_id = parm.headers.get("appId") 11 | user_name = parm.headers.get("userName") 12 | timestamp = parm.headers.get("Timestamp") 13 | nonce = parm.headers.get("Nonce") 14 | signature = parm.headers.get("Signature") 15 | check_parameters(app_id, user_name, timestamp, nonce, signature) 16 | if Authentication.md5_verify(app_id, timestamp, nonce, signature, user_name): 17 | if PermissionController.enforcer(app_id, parm.path, parm.method): 18 | return AuthenticationReturn(code=ReturnCode.Base.SUCCESS, message="success") 19 | else: 20 | return AuthenticationReturn(code=ReturnCode.API.AUTHENTICATION_FAILED, 21 | message="Authentication Failed") 22 | else: 23 | return AuthenticationReturn(code=ReturnCode.API.VERIFY_FAILED, message="varify failed!") 24 | 25 | 26 | def check_parameters(app_id, user_name, time_stamp, nonce, signature): 27 | if not app_id: 28 | raise InvalidParameter(name="appId") 29 | if not time_stamp or not isinstance(time_stamp, str): 30 | raise InvalidParameter(name="Timestamp") 31 | if not nonce or not isinstance(time_stamp, str) or len(nonce) != 4: 32 | raise InvalidParameter(name="Nonce") 33 | if not signature: 34 | raise InvalidParameter(name="Signature") 35 | -------------------------------------------------------------------------------- /python/fate_flow/hook/flow/permission.py: -------------------------------------------------------------------------------- 1 | from fate_flow.controller.permission import PermissionCheck 2 | from fate_flow.entity.code import ReturnCode 3 | from fate_flow.hook import HookManager 4 | from fate_flow.hook.common.parameters import PermissionCheckParameters, PermissionReturn 5 | from fate_flow.runtime.system_settings import LOCAL_PARTY_ID, PARTY_ID 6 | 7 | 8 | @HookManager.register_permission_check_hook 9 | def permission(parm: PermissionCheckParameters) -> PermissionReturn: 10 | if parm.party_id == LOCAL_PARTY_ID or parm.party_id == PARTY_ID: 11 | return PermissionReturn() 12 | 13 | checker = PermissionCheck(**parm.to_dict()) 14 | component_result = checker.check_component() 15 | 16 | if component_result.code != ReturnCode.Base.SUCCESS: 17 | return component_result 18 | 19 | dataset_result = checker.check_dataset() 20 | if dataset_result.code != ReturnCode.Base.SUCCESS: 21 | return dataset_result 22 | return PermissionReturn() 23 | -------------------------------------------------------------------------------- /python/fate_flow/hub/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/hub/components_wraps/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import abc 16 | from abc import ABCMeta 17 | 18 | 19 | class WrapsABC(metaclass=ABCMeta): 20 | @abc.abstractmethod 21 | def run(self): 22 | ... 23 | 24 | @abc.abstractmethod 25 | def cleanup(self): 26 | ... 27 | -------------------------------------------------------------------------------- /python/fate_flow/hub/components_wraps/fate/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.hub.components_wraps.fate._wraps import FlowWraps 16 | 17 | __all__ = ["FlowWraps"] 18 | -------------------------------------------------------------------------------- /python/fate_flow/hub/database/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/hub/database/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/hub/database/mysql.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from playhouse.pool import PooledMySQLDatabase 17 | 18 | from fate_flow.utils.password_utils import decrypt_database_config 19 | 20 | 21 | def get_database_connection(config, decrypt_key): 22 | database_config = config.copy() 23 | db_name = database_config.pop("name") 24 | decrypt_database_config(database_config, decrypt_key=decrypt_key) 25 | return PooledMySQLDatabase(db_name, **database_config) 26 | -------------------------------------------------------------------------------- /python/fate_flow/hub/database/sqlite.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from peewee import Insert 17 | 18 | from fate_flow.runtime.system_settings import SQLITE_PATH 19 | 20 | 21 | def get_database_connection(config, decrypt_key): 22 | Insert.on_conflict = lambda self, *args, **kwargs: self.on_conflict_replace() 23 | from playhouse.apsw_ext import APSWDatabase 24 | path = config.get("path") 25 | if not path: 26 | path = SQLITE_PATH 27 | return APSWDatabase(path) 28 | -------------------------------------------------------------------------------- /python/fate_flow/hub/encrypt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/hub/encrypt/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/hub/encrypt/password_encrypt.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from Crypto import Random 4 | from Crypto.PublicKey import RSA 5 | from Crypto.Cipher import PKCS1_v1_5 as PKCS1_cipher 6 | 7 | 8 | def rsa_key_generate(): 9 | random_generator = Random.new().read 10 | rsa = RSA.generate(2048, random_generator) 11 | private_pem = rsa.exportKey().decode() 12 | public_pem = rsa.publickey().exportKey().decode() 13 | with open('private_key.pem', "w") as f: 14 | f.write(private_pem) 15 | with open('public_key.pem', "w") as f: 16 | f.write(public_pem) 17 | return private_pem, public_pem 18 | 19 | 20 | def encrypt_data(public_key, msg): 21 | cipher = PKCS1_cipher.new(RSA.importKey(public_key)) 22 | encrypt_text = base64.b64encode(cipher.encrypt(bytes(msg.encode("utf8")))) 23 | return encrypt_text.decode('utf-8') 24 | 25 | 26 | def pwdecrypt(private_key, encrypt_msg): 27 | try: 28 | cipher = PKCS1_cipher.new(RSA.importKey(private_key)) 29 | back_text = cipher.decrypt(base64.b64decode(encrypt_msg), 0) 30 | return back_text.decode('utf-8') 31 | except Exception as e: 32 | raise RuntimeError(f"passwd decrypt failed: {e}") 33 | 34 | 35 | def test_encrypt_decrypt(): 36 | msg = "fate" 37 | private_key, public_key = rsa_key_generate() 38 | encrypt_text = encrypt_data(public_key, msg) 39 | print(encrypt_text) 40 | decrypt_text = pwdecrypt(private_key, encrypt_text) 41 | print(msg == decrypt_text) 42 | -------------------------------------------------------------------------------- /python/fate_flow/hub/provider/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import abc 16 | from typing import List, Dict 17 | 18 | 19 | class EntrypointABC: 20 | @property 21 | @abc.abstractmethod 22 | def component_list(self) -> List: 23 | ... 24 | 25 | @property 26 | @abc.abstractmethod 27 | def component_description(self) -> Dict: 28 | ... 29 | -------------------------------------------------------------------------------- /python/fate_flow/hub/provider/docker.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import re 16 | from abc import ABC 17 | 18 | from fate_flow.hub.provider import EntrypointABC 19 | from fate_flow.manager.container.docker_manager import DockerManager 20 | 21 | 22 | class DockerEntrypoint(EntrypointABC, ABC): 23 | def __init__(self, provider): 24 | self.provider = provider 25 | self.manager = DockerManager(provider) 26 | 27 | @property 28 | def component_list(self): 29 | return self.component_dict.keys() 30 | 31 | @property 32 | def component_dict(self): 33 | labels = self.manager.get_labels() 34 | _dict = {} 35 | pattern = r'^component\.\d*\.name$' 36 | for key, cpn_name in labels.items(): 37 | if re.match(pattern, key): 38 | _k = key.rstrip(".name") 39 | _dict[key] = cpn_name 40 | return _dict 41 | 42 | @property 43 | def component_description(self): 44 | return {} 45 | -------------------------------------------------------------------------------- /python/fate_flow/hub/provider/local.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import sys 16 | 17 | from fate_flow.hub.provider import EntrypointABC 18 | 19 | 20 | class LocalFateEntrypoint(EntrypointABC): 21 | def __init__(self, provider): 22 | self.provider = provider 23 | 24 | @property 25 | def component_list(self): 26 | if self.provider.python_path and self.provider.python_path not in sys.path: 27 | sys.path.append(self.provider.python_path) 28 | from fate.components.core import list_components 29 | # {'buildin': [], 'thirdparty': []} 30 | components = list_components() 31 | _list = components.get('buildin', []) 32 | _list.extend(components.get("thirdparty", [])) 33 | return _list 34 | 35 | @property 36 | def component_description(self): 37 | return {} 38 | 39 | 40 | class FATEFLowEntrypoint(EntrypointABC): 41 | def __init__(self, provider): 42 | self.provider = provider 43 | 44 | @property 45 | def component_list(self): 46 | from fate_flow.components.components import BUILDIN_COMPONENTS 47 | return [component.name for component in BUILDIN_COMPONENTS] 48 | 49 | @property 50 | def component_description(self): 51 | return {} 52 | -------------------------------------------------------------------------------- /python/fate_flow/manager/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/manager/components/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /python/fate_flow/manager/components/download.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os 17 | 18 | from fate_flow.engine import storage 19 | from fate_flow.manager.outputs.data import DataManager 20 | 21 | 22 | class Param(object): 23 | def to_dict(self): 24 | d = {} 25 | for k, v in self.__dict__.items(): 26 | if v is None: 27 | continue 28 | d[k] = v 29 | return d 30 | 31 | 32 | class DownloadParam(Param): 33 | def __init__( 34 | self, 35 | dir_name="", 36 | namespace="", 37 | name="" 38 | ): 39 | self.dir_name = dir_name 40 | self.namespace = namespace 41 | self.name = name 42 | 43 | 44 | class Download: 45 | def __init__(self): 46 | self.table = None 47 | self.schema = {} 48 | 49 | def run(self, parameters: DownloadParam, job_id=""): 50 | data_table_meta = storage.StorageTableMeta( 51 | name=parameters.name, 52 | namespace=parameters.namespace 53 | ) 54 | DataManager.send_table( 55 | output_tables_meta={"table": data_table_meta}, 56 | download_dir=os.path.abspath(parameters.dir_name) 57 | ) 58 | 59 | -------------------------------------------------------------------------------- /python/fate_flow/manager/container/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/manager/container/k8s_conf_template.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: '' 5 | namespace: fate-9999 6 | data: 7 | service_conf.yaml: | 8 | -------------------------------------------------------------------------------- /python/fate_flow/manager/container/k8s_template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: '' 5 | namespace: fate-9999 6 | spec: 7 | backoffLimit: 0 8 | template: 9 | spec: 10 | restartPolicy: Never 11 | containers: 12 | - name: '' 13 | image: '' 14 | imagePullPolicy: IfNotPresent 15 | command: [] 16 | args: [] 17 | env: [] 18 | volumeMounts: 19 | - mountPath: /data/projects/fate/eggroll/conf/ 20 | name: service-conf 21 | - mountPath: /data/projects/fate/fate_flow/logs/ 22 | name: local-path 23 | subPath: logs 24 | - mountPath: /data/projects/fate/fate_flow/data/ 25 | name: local-path 26 | subPath: data 27 | volumes: 28 | - name: service-conf 29 | configMap: 30 | name: service-conf 31 | - name: local-path 32 | hostPath: 33 | path: /data/projects/fate/fate_flow 34 | -------------------------------------------------------------------------------- /python/fate_flow/manager/operation/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | -------------------------------------------------------------------------------- /python/fate_flow/manager/outputs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/manager/outputs/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/manager/outputs/model/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.manager.outputs.model.model_manager import PipelinedModel 16 | from fate_flow.manager.outputs.model.model_meta import ModelMeta 17 | 18 | __all__ = ["PipelinedModel", "ModelMeta"] 19 | -------------------------------------------------------------------------------- /python/fate_flow/manager/outputs/model/engine/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.manager.outputs.model.engine._mysql import MysqlModelStorage 16 | from fate_flow.manager.outputs.model.engine._tencent_cos import TencentCosStorage 17 | 18 | __all__ = ["MysqlModelStorage", "TencentCosStorage"] 19 | -------------------------------------------------------------------------------- /python/fate_flow/manager/outputs/model/handel/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.manager.outputs.model.handel._base import IOHandle 16 | from fate_flow.manager.outputs.model.handel._file import FileHandle 17 | from fate_flow.manager.outputs.model.handel._mysql import MysqlHandel 18 | from fate_flow.manager.outputs.model.handel._tencent_cos import TencentCosHandel 19 | 20 | __all__ = ["IOHandle", "FileHandle", "MysqlHandel", "TencentCosHandel"] 21 | -------------------------------------------------------------------------------- /python/fate_flow/manager/outputs/model/model_meta.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.db.base_models import BaseModelOperate 16 | from fate_flow.db.db_models import PipelineModelMeta 17 | from fate_flow.utils.wraps_utils import filter_parameters 18 | 19 | 20 | class ModelMeta(BaseModelOperate): 21 | @classmethod 22 | def save(cls, **meta_info): 23 | cls._create_entity(PipelineModelMeta, meta_info) 24 | 25 | @classmethod 26 | @filter_parameters() 27 | def query(cls, **kwargs): 28 | return cls._query(PipelineModelMeta, **kwargs) 29 | 30 | @classmethod 31 | @filter_parameters() 32 | def delete(cls, **kwargs): 33 | return cls._delete(PipelineModelMeta, **kwargs) 34 | -------------------------------------------------------------------------------- /python/fate_flow/manager/permission/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/manager/permission/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/manager/pipeline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/manager/pipeline/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/manager/service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FederatedAI/FATE-Flow/0f8daa372ac07ce15a10956f2058ae3fd7846903/python/fate_flow/manager/service/__init__.py -------------------------------------------------------------------------------- /python/fate_flow/manager/service/config_manager.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.manager.service.resource_manager import ResourceManager 17 | from fate_flow.manager.service.service_manager import ServerRegistry 18 | from fate_flow.runtime.job_default_config import JobDefaultConfig 19 | 20 | 21 | class ConfigManager: 22 | @classmethod 23 | def load(cls): 24 | JobDefaultConfig.load() 25 | ResourceManager.initialize() 26 | ServerRegistry.load() 27 | -------------------------------------------------------------------------------- /python/fate_flow/manager/worker/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/manager/worker/fate_ds_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import json 17 | import os 18 | import sys 19 | 20 | 21 | class FateSubmit: 22 | @staticmethod 23 | def run(): 24 | import runpy 25 | runpy.run_module(mod_name='fate.components', run_name='__main__') 26 | 27 | 28 | if __name__ == "__main__": 29 | # replace deepspeed rank env 30 | print(os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_LOGS_DIR")) 31 | print(os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_MODELS_DIR")) 32 | print(os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_RESULT_DIR")) 33 | 34 | result_index = sys.argv.index("--execution-final-meta-path") + 1 35 | result_path = os.environ.get(sys.argv[result_index], "") 36 | 37 | env_name_index = sys.argv.index("--env-name") + 1 38 | env_key = sys.argv[env_name_index] 39 | sys.argv[result_index] = sys.argv[result_index].replace( 40 | os.environ.get("DEEPSPEED_RESULT_PLACEHOLDER"), 41 | os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_RESULT_DIR") 42 | ) 43 | 44 | env_str = os.environ.get(sys.argv[env_name_index], "") 45 | env_str = env_str.replace(os.environ.get("DEEPSPEED_LOGS_DIR_PLACEHOLDER"), os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_LOGS_DIR")) 46 | env_str = env_str.replace(os.environ.get("DEEPSPEED_MODEL_DIR_PLACEHOLDER"), os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_MODELS_DIR")) 47 | print(json.loads(env_str)) 48 | os.environ[env_key] = env_str 49 | FateSubmit.run() 50 | -------------------------------------------------------------------------------- /python/fate_flow/manager/worker/fate_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | class FateSubmit: 19 | @staticmethod 20 | def run(): 21 | import runpy 22 | runpy.run_module(mod_name='fate.components', run_name='__main__') 23 | 24 | 25 | if __name__ == "__main__": 26 | FateSubmit.run() 27 | -------------------------------------------------------------------------------- /python/fate_flow/manager/worker/fate_flow_executor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import argparse 17 | import sys 18 | 19 | from fate_flow.entity import BaseEntity 20 | from fate_flow.utils.log import getLogger 21 | 22 | 23 | class FateFlowSubmit: 24 | @staticmethod 25 | def run(): 26 | import runpy 27 | runpy.run_module(mod_name='fate_flow.components', run_name='__main__') 28 | 29 | 30 | if __name__ == "__main__": 31 | FateFlowSubmit.run() 32 | -------------------------------------------------------------------------------- /python/fate_flow/proto/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/proto/rollsite/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | import sys 17 | 18 | _pb_path = os.path.abspath(os.path.join(__file__, os.path.pardir)) 19 | if _pb_path not in sys.path: 20 | sys.path.append(_pb_path) -------------------------------------------------------------------------------- /python/fate_flow/runtime/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/runtime/env.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import fate_flow 3 | 4 | 5 | def is_in_virtualenv(): 6 | try: 7 | module_path = fate_flow.__file__ 8 | return sys.prefix in module_path 9 | except ImportError: 10 | return False 11 | -------------------------------------------------------------------------------- /python/fate_flow/runtime/job_default_config.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | from fate_flow.runtime.system_settings import FATE_FLOW_JOB_DEFAULT_CONFIG_PATH 17 | from .reload_config_base import ReloadConfigBase 18 | from ..utils import file_utils 19 | from ..utils.log import getLogger 20 | 21 | stat_logger = getLogger() 22 | 23 | 24 | class JobDefaultConfig(ReloadConfigBase): 25 | job_cores = None 26 | computing_partitions = None 27 | task_run = None 28 | remote_request_timeout = None 29 | federated_command_trys = None 30 | job_timeout = None 31 | auto_retries = None 32 | sync_type = None 33 | 34 | task_logger = None 35 | task_device = None 36 | task_timeout = None 37 | 38 | @classmethod 39 | def load(cls): 40 | conf = file_utils.load_yaml_conf(FATE_FLOW_JOB_DEFAULT_CONFIG_PATH) 41 | if not isinstance(conf, dict): 42 | raise ValueError("invalid config file") 43 | 44 | for k, v in conf.items(): 45 | if hasattr(cls, k): 46 | setattr(cls, k, v) 47 | else: 48 | stat_logger.warning(f"job default config not supported {k}") 49 | 50 | return cls.get_all() 51 | -------------------------------------------------------------------------------- /python/fate_flow/runtime/reload_config_base.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | class ReloadConfigBase: 17 | @classmethod 18 | def get_all(cls): 19 | configs = {} 20 | for k, v in cls.__dict__.items(): 21 | if not callable(getattr(cls, k)) and not k.startswith("__") and not k.startswith("_"): 22 | configs[k] = v 23 | return configs 24 | 25 | @classmethod 26 | def get(cls, config_name): 27 | return getattr(cls, config_name) if hasattr(cls, config_name) else None 28 | -------------------------------------------------------------------------------- /python/fate_flow/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from fate_flow.scheduler.scheduler import SchedulerABC 16 | from fate_flow.runtime.runtime_config import RuntimeConfig 17 | from fate_flow.runtime.system_settings import HOST, HTTP_PORT, PROXY_PROTOCOL, API_VERSION, HTTP_REQUEST_TIMEOUT 18 | from fate_flow.utils.api_utils import get_federated_proxy_address, generate_headers 19 | from ofx.api.client import FlowSchedulerApi 20 | 21 | 22 | def init_scheduler(): 23 | remote_host, remote_port, remote_protocol, grpc_channel = get_federated_proxy_address() 24 | 25 | protocol = remote_protocol if remote_protocol else PROXY_PROTOCOL 26 | 27 | # schedule client 28 | RuntimeConfig.set_schedule_client( 29 | FlowSchedulerApi( 30 | host=HOST, 31 | port=HTTP_PORT, 32 | api_version=API_VERSION, 33 | timeout=HTTP_REQUEST_TIMEOUT, 34 | remote_protocol=protocol, 35 | remote_host=remote_host, 36 | remote_port=remote_port, 37 | grpc_channel=grpc_channel, 38 | callback=generate_headers) 39 | ) 40 | 41 | -------------------------------------------------------------------------------- /python/fate_flow/settings.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # GRPC 17 | GRPC_SERVER_MAX_WORKERS = None # default: (os.cpu_count() or 1) * 5 18 | 19 | # Request 20 | HTTP_REQUEST_TIMEOUT = 10 # s 21 | REMOTE_REQUEST_TIMEOUT = 30000 # ms 22 | 23 | LOG_DIR = "" 24 | DATA_DIR = "" 25 | MODEL_DIR = "" 26 | JOB_DIR = "" 27 | DEFAULT_FATE_DIR = "" 28 | DEFAULT_SERVER_CONF_PATH = "" 29 | UPLOAD_DATA_HOME = "" 30 | 31 | # sqlite 32 | SQLITE_FILE_DIR = "" 33 | SQLITE_FILE_NAME = "fate_flow_sqlite.db" 34 | 35 | 36 | # Client Manager 37 | APP_TOKEN_LENGTH = 16 38 | ADMIN_ID = "admin" 39 | ADMIN_KEY = "fate_flow_admin" 40 | -------------------------------------------------------------------------------- /python/fate_flow/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/fate_flow/utils/conf_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os 17 | 18 | from .file_utils import load_yaml_conf, get_fate_flow_directory 19 | from ..settings import DEFAULT_SERVER_CONF_PATH 20 | 21 | SERVICE_CONF = "service_conf.yaml" 22 | TRANSFER_CONF = "transfer_conf.yaml" 23 | 24 | 25 | def conf_realpath(conf_name): 26 | conf_path = DEFAULT_SERVER_CONF_PATH or os.path.join(get_fate_flow_directory(), "conf") 27 | return f"{conf_path}/{conf_name}" 28 | 29 | 30 | def get_base_config(key, default=None, conf_name=SERVICE_CONF) -> dict: 31 | local_config = {} 32 | local_path = conf_realpath(f"local.{conf_name}") 33 | 34 | if os.path.exists(local_path): 35 | local_config = load_yaml_conf(local_path) 36 | if not isinstance(local_config, dict): 37 | raise ValueError(f'Invalid config file: "{local_path}".') 38 | 39 | if key is not None and key in local_config: 40 | return local_config[key] 41 | 42 | config_path = conf_realpath(conf_name) 43 | config = load_yaml_conf(config_path) 44 | 45 | if not isinstance(config, dict): 46 | raise ValueError(f'Invalid config file: "{config_path}".') 47 | 48 | config.update(local_config) 49 | return config.get(key, default) if key is not None else config 50 | -------------------------------------------------------------------------------- /python/fate_flow/utils/db_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | def get_dynamic_db_model(base, job_id): 16 | return type(base.model(table_index=get_dynamic_tracking_table_index(job_id=job_id))) 17 | 18 | 19 | def get_dynamic_tracking_table_index(job_id): 20 | return job_id[:8] -------------------------------------------------------------------------------- /python/fate_flow/utils/object_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import importlib 17 | 18 | 19 | def from_dict_hook(in_dict: dict): 20 | if "type" in in_dict and "data" in in_dict: 21 | if in_dict["module"] is None: 22 | return in_dict["data"] 23 | else: 24 | return getattr(importlib.import_module(in_dict["module"]), in_dict["type"])(**in_dict["data"]) 25 | else: 26 | return in_dict 27 | -------------------------------------------------------------------------------- /python/fate_flow/utils/password_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os.path 17 | from importlib import import_module 18 | 19 | from fate_flow.runtime.system_settings import ENCRYPT_CONF 20 | from fate_flow.utils.conf_utils import conf_realpath 21 | 22 | 23 | def decrypt_database_config(database, passwd_key="passwd", decrypt_key=""): 24 | database[passwd_key] = decrypt_password(database[passwd_key], key=decrypt_key) 25 | return database 26 | 27 | 28 | def decrypt_password(password, key=""): 29 | if not ENCRYPT_CONF or not key or key not in ENCRYPT_CONF: 30 | return password 31 | encrypt_module = ENCRYPT_CONF.get(key).get("module", "") 32 | private_path = ENCRYPT_CONF.get(key).get("private_path", "") 33 | if not encrypt_module: 34 | raise ValueError(f"module is {encrypt_module}") 35 | if not private_path: 36 | raise ValueError(f"private_path is {private_path}") 37 | if not os.path.isabs(private_path): 38 | private_path = conf_realpath(private_path) 39 | with open(private_path) as f: 40 | private_key = f.read() 41 | module_func = encrypt_module.split("#") 42 | encrypt_func = getattr(import_module(module_func[0]), module_func[1]) 43 | return encrypt_func(private_key, password) 44 | -------------------------------------------------------------------------------- /python/fate_flow/utils/requests_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2021 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import functools 17 | import json 18 | import requests 19 | 20 | from fate_flow.utils.base_utils import CustomJSONEncoder 21 | 22 | requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder) 23 | 24 | 25 | def request(**kwargs): 26 | sess = requests.Session() 27 | stream = kwargs.pop('stream', sess.stream) 28 | timeout = kwargs.pop('timeout', None) 29 | if kwargs.get('headers'): 30 | kwargs['headers'] = {k.replace('_', '-').upper(): v for k, v in kwargs.get('headers', {}).items()} 31 | prepped = requests.Request(**kwargs).prepare() 32 | return sess.send(prepped, stream=stream, timeout=timeout) 33 | -------------------------------------------------------------------------------- /python/fate_flow/utils/version.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | import os 17 | 18 | import dotenv 19 | import typing 20 | 21 | from fate_flow.runtime.system_settings import VERSION_FILE_PATH 22 | 23 | 24 | def get_versions() -> typing.Mapping[str, typing.Any]: 25 | return dotenv.dotenv_values( 26 | dotenv_path=VERSION_FILE_PATH 27 | ) 28 | 29 | 30 | def get_flow_version() -> typing.Optional[str]: 31 | return get_versions().get("FATE_FLOW") 32 | 33 | 34 | def get_default_fate_version() -> typing.Optional[str]: 35 | return get_versions().get("FATE") 36 | -------------------------------------------------------------------------------- /python/ofx/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /python/ofx/api/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/ofx/api/entity.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from typing import Optional, List 16 | 17 | import pydantic 18 | from pydantic import BaseModel 19 | 20 | OSX_EXCHANGE = "" 21 | 22 | 23 | class PROTOCOL: 24 | FATE_FLOW = "fate" 25 | BFIA = "bfia" 26 | 27 | class RoleSpec(BaseModel): 28 | guest: Optional[List[str]] 29 | host: Optional[List[str]] 30 | arbiter: Optional[List[str]] 31 | local: Optional[List[str]] 32 | 33 | 34 | class BFIAHttpHeadersSpec(pydantic.BaseModel): 35 | x_auth_sign: Optional[str] 36 | x_node_id: Optional[str] 37 | x_nonce: Optional[str] 38 | x_trace_id: Optional[str] 39 | x_timestamp: Optional[str] 40 | 41 | 42 | class BFIAHeadersSpec(pydantic.BaseModel): 43 | x_ptp_version: Optional[str] 44 | x_ptp_provider_code: Optional[str] 45 | x_ptp_trace_id: Optional[str] 46 | x_ptp_token: Optional[str] 47 | x_ptp_uri: Optional[str] 48 | x_ptp_from_node_id: Optional[str] 49 | x_ptp_from_inst_id: Optional[str] 50 | x_ptp_target_node_id: Optional[str] 51 | x_ptp_target_inst_id: Optional[str] 52 | x_ptp_session_id: Optional[str] 53 | x_ptp_topic: Optional[str] 54 | -------------------------------------------------------------------------------- /python/ofx/api/models/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from ofx.api.models import bfia, fate_flow 16 | 17 | __all__ = [bfia, fate_flow] 18 | -------------------------------------------------------------------------------- /python/ofx/api/models/bfia/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/ofx/api/models/bfia/federated.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from .resource import BaseAPI 16 | 17 | 18 | class Federated(BaseAPI): 19 | def create_job(self, node_list, command_body): 20 | return self.job_command( 21 | node_list=node_list, 22 | command_body=command_body, 23 | parallel=False, 24 | endpoint="/v1/interconn/schedule/job/create" 25 | ) 26 | 27 | def start_job(self, node_list, command_body): 28 | return self.job_command( 29 | node_list=node_list, 30 | command_body=command_body, 31 | parallel=False, 32 | endpoint="/v1/interconn/schedule/job/start" 33 | ) 34 | 35 | def stop_job(self, node_list, command_body): 36 | return self.job_command( 37 | node_list=node_list, 38 | command_body=command_body, 39 | parallel=False, 40 | endpoint="/v1/interconn/schedule/job/stop" 41 | ) 42 | 43 | def start_task(self, node_list, command_body): 44 | return self.job_command( 45 | node_list=node_list, 46 | command_body=command_body, 47 | parallel=False, 48 | endpoint="/v1/interconn/schedule/task/start" 49 | ) 50 | 51 | def poll_task(self, node_list, command_body): 52 | return self.job_command( 53 | node_list=node_list, 54 | command_body=command_body, 55 | parallel=False, 56 | endpoint="/v1/interconn/schedule/task/poll" 57 | ) 58 | -------------------------------------------------------------------------------- /python/ofx/api/models/bfia/scheduler.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from .resource import BaseAPI 16 | 17 | 18 | class Scheduler(BaseAPI): 19 | def create_job(self, party_id, command_body): 20 | return self.scheduler_command( 21 | endpoint="/v1/interconn/schedule/job/create_all", 22 | node_id=party_id, 23 | command_body=command_body 24 | ) 25 | 26 | def audit_confirm(self, party_id, command_body): 27 | return self.scheduler_command( 28 | endpoint="/v1/interconn/schedule/job/audit_confirm", 29 | node_id=party_id, 30 | command_body=command_body 31 | ) 32 | 33 | def stop_job(self, party_id, command_body): 34 | return self.scheduler_command( 35 | endpoint="/v1/interconn/schedule/job/stop_all", 36 | node_id=party_id, 37 | command_body=command_body 38 | ) 39 | 40 | def report_task(self, party_id, command_body): 41 | return self.scheduler_command( 42 | endpoint="/v1/interconn/schedule/task/callback", 43 | node_id=party_id, 44 | command_body=command_body 45 | ) 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /python/ofx/api/models/bfia/worker.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from .resource import BaseAPI 16 | 17 | 18 | class Worker(BaseAPI): 19 | def report_task_status(self, status, task_id, role): 20 | return self.client.post( 21 | endpoint="/v1/platform/schedule/task/callback", 22 | json={ 23 | "status": status, 24 | "task_id": task_id, 25 | "role": role 26 | }) 27 | -------------------------------------------------------------------------------- /python/ofx/api/models/fate_flow/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/ofx/api/models/fate_flow/scheduler.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from .resource import BaseAPI 16 | 17 | 18 | class Scheduler(BaseAPI): 19 | def create_job(self, party_id, initiator_party_id, command_body): 20 | return self.scheduler_command(command="job/create", 21 | party_id=party_id, 22 | initiator_party_id=initiator_party_id, 23 | command_body=command_body 24 | ) 25 | 26 | def stop_job(self, party_id, command_body): 27 | return self.scheduler_command(command="job/stop", 28 | party_id=party_id, 29 | command_body=command_body 30 | ) 31 | 32 | def rerun_job(self, party_id, command_body): 33 | return self.scheduler_command(command="job/rerun", 34 | party_id=party_id, 35 | command_body=command_body 36 | ) 37 | 38 | def report_task(self, party_id, command_body): 39 | return self.scheduler_command(command="task/report", 40 | party_id=party_id, 41 | command_body=command_body 42 | ) 43 | -------------------------------------------------------------------------------- /python/ofx/api/proto/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/ofx/api/proto/rollsite/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | import sys 17 | 18 | _pb_path = os.path.abspath(os.path.join(__file__, os.path.pardir)) 19 | if _pb_path not in sys.path: 20 | sys.path.append(_pb_path) -------------------------------------------------------------------------------- /python/ofx/api/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 The FATE Authors. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /python/requirements-container.txt: -------------------------------------------------------------------------------- 1 | docker 2 | kubernetes -------------------------------------------------------------------------------- /python/requirements-docker.txt: -------------------------------------------------------------------------------- 1 | # fate flow 2 | -r requirements-flow.txt 3 | 4 | # fate 5 | -r requirements-fate.txt 6 | 7 | # container 8 | -r requirements-container.txt 9 | 10 | # eggroll-client 11 | -r requirements-eggroll-client.txt 12 | -------------------------------------------------------------------------------- /python/requirements-eggroll-client.txt: -------------------------------------------------------------------------------- 1 | eggroll -------------------------------------------------------------------------------- /python/requirements-eggroll.txt: -------------------------------------------------------------------------------- 1 | opentelemetry-api 2 | opentelemetry-sdk 3 | protobuf==4.24.4 4 | grpcio==1.62.1 5 | grpcio-tools==1.62.1 6 | cloudpickle==2.1.0 7 | psutil>=5.7.0 8 | pynvml==11.5.0 9 | -------------------------------------------------------------------------------- /python/requirements-fate.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://download.pytorch.org/whl/cpu 2 | 3 | lmdb==1.4.1 4 | torch==2.3.1+cpu 5 | fate_utils 6 | pydantic==1.10.12 7 | cloudpickle==2.1.0 8 | click 9 | ruamel-yaml==0.16 10 | numpy<2.0.0 11 | pandas 12 | transformers==4.37.2 13 | accelerate==0.27.2 14 | beautifultable 15 | requests<2.26.0 16 | scikit-learn==1.4.1.post1 17 | omegaconf 18 | rich==13.7.1 19 | opentelemetry-api 20 | opentelemetry-sdk 21 | opentelemetry-exporter-otlp-proto-grpc 22 | mmh3==3.0.0 23 | protobuf==4.24.4 24 | grpcio==1.62.1 25 | safetensors==0.4.1 26 | scipy 27 | -------------------------------------------------------------------------------- /python/requirements-flow.txt: -------------------------------------------------------------------------------- 1 | pip>=21 2 | apsw>=3.10 3 | Flask==2.2.5 4 | grpcio==1.62.1 5 | grpcio-tools==1.62.1 6 | requests<2.26.0 7 | urllib3==1.26.18 8 | ruamel-yaml==0.16 9 | cachetools==3.0.0 10 | filelock==3.3.1 11 | pydantic==1.10.12 12 | webargs 13 | peewee==3.17.1 14 | python-dotenv==0.13.0 15 | pyyaml 16 | networkx 17 | psutil>=5.7.0 18 | casbin_peewee_adapter 19 | casbin 20 | pymysql 21 | kazoo 22 | shortuuid 23 | cos-python-sdk-v5==1.9.27 24 | typing-extensions==4.8.0 25 | boto3 26 | pyarrow==15.0.1 27 | -------------------------------------------------------------------------------- /python/requirements-pulsar.txt: -------------------------------------------------------------------------------- 1 | pulsar-client==3.4.0 2 | -------------------------------------------------------------------------------- /python/requirements-rabbitmq.txt: -------------------------------------------------------------------------------- 1 | pika==1.2.1 -------------------------------------------------------------------------------- /python/requirements-spark.txt: -------------------------------------------------------------------------------- 1 | pyspark -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | # fate flow 2 | -r requirements-flow.txt 3 | 4 | # container 5 | -r requirements-container.txt 6 | 7 | # eggroll 8 | -r requirements-eggroll.txt 9 | 10 | # rabbitmq 11 | -r requirements-rabbitmq.txt 12 | 13 | # pulsar 14 | -r requirements-pulsar.txt 15 | 16 | # spark 17 | -r requirements-spark.txt 18 | --------------------------------------------------------------------------------