├── pyproject.toml ├── docs ├── MVCS.jpeg ├── PCPArch.jpg ├── XORoutput.jpg ├── ShardingDesignFB.jpg ├── AggregateOneParty.jpg ├── AggregateTwoParties.jpg ├── ShardingDesignClient.jpg ├── MVCS.md └── FBPCPComponents.md ├── lint_requirements.txt ├── .pyre_configuration ├── fbpcp ├── __init__.py ├── cloud │ └── __init__.py ├── error │ ├── __init__.py │ ├── mapper │ │ ├── __init__.py │ │ ├── gcp.py │ │ ├── aws.py │ │ └── k8s.py │ └── pcp.py ├── util │ ├── __init__.py │ ├── typing.py │ ├── reflect.py │ ├── arg_builder.py │ ├── yaml.py │ ├── gcspath.py │ └── s3path.py ├── decorator │ ├── __init__.py │ ├── error_handler.py │ └── metrics.py ├── entity │ ├── __init__.py │ ├── cloud_provider.py │ ├── file_information.py │ ├── container_permission.py │ ├── log_event.py │ ├── subnet.py │ ├── container_metadata.py │ ├── secret.py │ ├── pce.py │ ├── container_insight.py │ ├── cloud_cost.py │ ├── pce_compute.py │ ├── container_definition.py │ ├── firewall_ruleset.py │ ├── insight.py │ ├── vpc_instance.py │ ├── cluster_instance.py │ ├── pce_network.py │ ├── policy_statement.py │ ├── route_table.py │ ├── vpc_peering.py │ ├── policy_settings_config.py │ ├── container_instance.py │ └── container_type.py ├── gateway │ ├── __init__.py │ ├── gcp.py │ ├── aws.py │ ├── cloudwatch.py │ ├── kms.py │ ├── costexplorer.py │ └── secrets_manager.py ├── mapper │ └── __init__.py ├── metrics │ ├── __init__.py │ ├── getter.py │ └── emitter.py └── service │ ├── __init__.py │ ├── pce.py │ ├── insights.py │ ├── billing.py │ ├── log.py │ ├── policy_validation.py │ ├── secrets_manager.py │ ├── log_cloudwatch.py │ ├── billing_aws.py │ ├── storage.py │ ├── secrets_manager_aws.py │ ├── policy_validation_aws.py │ └── pce_aws.py ├── pce ├── __init__.py ├── entity │ ├── __init__.py │ ├── log_group_aws.py │ ├── mpc_roles.py │ └── iam_role.py ├── gateway │ ├── __init__.py │ ├── sts.py │ ├── tests │ │ ├── test_sts.py │ │ └── test_logs_aws.py │ ├── ecs.py │ ├── logs_aws.py │ ├── iam.py │ └── ec2.py ├── mapper │ ├── __init__.py │ └── aws.py ├── validator │ ├── __init__.py │ ├── message_templates │ │ ├── __init__.py │ │ ├── resource_names.py │ │ ├── pce_standard_constants.py │ │ ├── validator_step_names.py │ │ └── warning_message_templates.py │ └── __main__.py └── README.md ├── onedocker ├── __init__.py ├── common │ ├── __init__.py │ ├── env.py │ └── util.py ├── entity │ ├── __init__.py │ ├── package_info.py │ ├── measurement.py │ ├── opawdl_workflow.py │ ├── opawdl_state_instance.py │ ├── metadata.py │ ├── attestation_document.py │ ├── opawdl_state.py │ ├── opawdl_workflow_instance.py │ └── exit_code.py ├── mapper │ ├── __init__.py │ └── aws.py ├── script │ ├── __init__.py │ ├── cli │ │ ├── __init__.py │ │ └── __main__.py │ ├── runner │ │ ├── __init__.py │ │ └── __main__.py │ └── config │ │ └── cli_config_template.yml ├── util │ ├── __init__.py │ ├── opawdl_parser.py │ └── service_builder.py ├── gateway │ ├── __init__.py │ └── repository_service.py ├── repository │ ├── __init__.py │ ├── opawdl_workflow_instance_repository.py │ ├── opawdl_workflow_instance_repository_local.py │ └── onedocker_package.py ├── service │ ├── __init__.py │ ├── attestation.py │ ├── certificate.py │ ├── measurement.py │ ├── attestation_factory.py │ ├── metadata.py │ └── attestation_pc.py └── tests │ ├── common │ └── test_util.py │ ├── entity │ ├── test_opawdl_state.py │ ├── test_opawdl_state_instance.py │ ├── test_opawdl_workflow.py │ └── test_opawdl_workflow_instance.py │ ├── mapper │ └── test_aws.py │ ├── service │ ├── test_measurement.py │ ├── test_opawdl_driver.py │ └── test_attestation_factory.py │ └── util │ ├── test_service_builder.py │ └── test_opawdl_parser.py ├── tests ├── util │ ├── test_reflect.py │ ├── test_typing.py │ ├── test_arg_builder.py │ ├── test_gcspath.py │ ├── test_s3path.py │ ├── test_yaml.py │ └── test_aws.py ├── error │ └── mapper │ │ ├── test_gcp.py │ │ ├── test_k8s.py │ │ └── test_aws.py ├── service │ ├── test_storage.py │ ├── test_log_cloudwatch.py │ └── test_policy_validation_aws.py ├── gateway │ ├── test_cloudwatch.py │ └── test_kms.py ├── entity │ ├── test_container_insight.py │ └── test_certificate_request.py └── decorator │ ├── test_metrics.py │ └── test_error_handler.py ├── .github └── workflows │ ├── tests.yml │ ├── lint.yml │ ├── pyre.yml │ ├── publish_project.yml │ └── integ_e2e_tests.yml ├── scripts ├── run-python-tests.sh └── compare_package_version.py ├── LICENSE ├── setup.py ├── CONTRIBUTING.md ├── .flake8 ├── CODE_OF_CONDUCT.md └── README.md /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.usort] 2 | first_party_detection = false 3 | -------------------------------------------------------------------------------- /docs/MVCS.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/MVCS.jpeg -------------------------------------------------------------------------------- /docs/PCPArch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/PCPArch.jpg -------------------------------------------------------------------------------- /docs/XORoutput.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/XORoutput.jpg -------------------------------------------------------------------------------- /docs/ShardingDesignFB.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/ShardingDesignFB.jpg -------------------------------------------------------------------------------- /docs/AggregateOneParty.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/AggregateOneParty.jpg -------------------------------------------------------------------------------- /docs/AggregateTwoParties.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/AggregateTwoParties.jpg -------------------------------------------------------------------------------- /docs/ShardingDesignClient.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/fbpcp/HEAD/docs/ShardingDesignClient.jpg -------------------------------------------------------------------------------- /lint_requirements.txt: -------------------------------------------------------------------------------- 1 | black==24.3.0 2 | ufmt==2.5.1 3 | usort==1.0.8 4 | libcst==1.1.0 5 | flake8==3.9.0 6 | flake8-bugbear==21.3.2 7 | -------------------------------------------------------------------------------- /.pyre_configuration: -------------------------------------------------------------------------------- 1 | { 2 | "site_package_search_strategy": "all", 3 | "source_directories": [ 4 | "fbpcp", 5 | "onedocker", 6 | "pce" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /fbpcp/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/cloud/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/error/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/entity/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/mapper/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/decorator/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/entity/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/mapper/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/service/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/common/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/entity/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/mapper/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/script/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/validator/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /fbpcp/error/mapper/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/repository/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/script/cli/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/service/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /onedocker/script/runner/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/validator/message_templates/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | -------------------------------------------------------------------------------- /pce/entity/log_group_aws.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from dataclasses import dataclass 9 | 10 | 11 | @dataclass 12 | class LogGroup: 13 | log_group_name: str 14 | -------------------------------------------------------------------------------- /pce/entity/mpc_roles.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from enum import Enum 9 | 10 | 11 | class MPCRoles(Enum): 12 | PUBLISHER = "PUBLISHER" 13 | PARTNER = "PARTNER" 14 | -------------------------------------------------------------------------------- /pce/validator/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | 10 | from pce.validator.validator import main 11 | 12 | if __name__ == "__main__": 13 | main() 14 | -------------------------------------------------------------------------------- /onedocker/script/cli/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | 10 | from onedocker.script.cli.onedocker_cli import main 11 | 12 | if __name__ == "__main__": 13 | main() 14 | -------------------------------------------------------------------------------- /onedocker/script/runner/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | 10 | from onedocker.script.runner.onedocker_runner import main 11 | 12 | if __name__ == "__main__": 13 | main() 14 | -------------------------------------------------------------------------------- /fbpcp/entity/cloud_provider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from enum import Enum 9 | 10 | 11 | class CloudProvider(Enum): 12 | AWS = "AWS" 13 | GCP = "GCP" 14 | AZURE = "AZURE" 15 | -------------------------------------------------------------------------------- /fbpcp/entity/file_information.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | 10 | 11 | @dataclass 12 | class FileInfo: 13 | file_name: str 14 | last_modified: str 15 | file_size: int 16 | -------------------------------------------------------------------------------- /onedocker/entity/package_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | 11 | 12 | @dataclass 13 | class PackageInfo: 14 | package_name: str 15 | version: str 16 | last_modified: str 17 | package_size: int 18 | -------------------------------------------------------------------------------- /fbpcp/entity/container_permission.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | 11 | from dataclasses_json import dataclass_json 12 | 13 | 14 | @dataclass_json 15 | @dataclass 16 | class ContainerPermissionConfig: 17 | role_id: str 18 | -------------------------------------------------------------------------------- /fbpcp/entity/log_event.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | 11 | from dataclasses_json import dataclass_json 12 | 13 | 14 | @dataclass_json 15 | @dataclass 16 | class LogEvent: 17 | timestamp: int 18 | message: str 19 | -------------------------------------------------------------------------------- /fbpcp/entity/subnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass, field 9 | from typing import Dict 10 | 11 | 12 | @dataclass 13 | class Subnet: 14 | id: str 15 | availability_zone: str 16 | tags: Dict[str, str] = field(default_factory=dict) 17 | -------------------------------------------------------------------------------- /fbpcp/service/pce.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | 11 | from fbpcp.entity.pce import PCE 12 | 13 | 14 | class PCEService(abc.ABC): 15 | @abc.abstractmethod 16 | def get_pce( 17 | self, 18 | pce_id: str, 19 | ) -> PCE: 20 | pass 21 | -------------------------------------------------------------------------------- /fbpcp/service/insights.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | import abc 9 | 10 | 11 | class InsightsService(abc.ABC): 12 | @abc.abstractmethod 13 | async def emit_async(self, message: str) -> None: 14 | pass 15 | 16 | @abc.abstractmethod 17 | def emit(self, message: str) -> None: 18 | pass 19 | -------------------------------------------------------------------------------- /fbpcp/error/pcp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | 10 | class PcpError(Exception): 11 | pass 12 | 13 | 14 | class InvalidParameterError(PcpError): 15 | pass 16 | 17 | 18 | class ThrottlingError(PcpError): 19 | pass 20 | 21 | 22 | class LimitExceededError(PcpError): 23 | pass 24 | -------------------------------------------------------------------------------- /fbpcp/entity/container_metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | 11 | from dataclasses_json import dataclass_json 12 | 13 | 14 | @dataclass_json 15 | @dataclass 16 | class ContainerStoppedMetadata: 17 | stopped_at: str 18 | stop_code: str 19 | stopped_reason: str 20 | -------------------------------------------------------------------------------- /fbpcp/entity/secret.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from typing import Dict 11 | 12 | 13 | @dataclass 14 | class StringSecret: 15 | id: str 16 | name: str 17 | value: str 18 | create_date: str 19 | tags: Dict[str, str] = field(default_factory=dict) 20 | -------------------------------------------------------------------------------- /fbpcp/util/typing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Type, TypeVar 10 | 11 | 12 | T = TypeVar("T") 13 | V = TypeVar("V") 14 | 15 | 16 | def checked_cast(typ: Type[T], val: V) -> T: 17 | if not isinstance(val, typ): 18 | raise ValueError(f"Value was not of type {type!r}:\n{val!r}") 19 | return val 20 | -------------------------------------------------------------------------------- /fbpcp/entity/pce.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | 10 | from fbpcp.entity.pce_compute import PCECompute 11 | from fbpcp.entity.pce_network import PCENetwork 12 | 13 | 14 | @dataclass 15 | class PCE: 16 | pce_id: str 17 | region: str 18 | pce_network: PCENetwork 19 | pce_compute: PCECompute 20 | -------------------------------------------------------------------------------- /pce/entity/iam_role.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from dataclasses import dataclass 9 | from typing import Any, Dict 10 | 11 | 12 | RoleId = str 13 | RoleName = str 14 | PolicyName = str 15 | PolicyContents = Dict[str, Any] 16 | 17 | 18 | @dataclass 19 | class IAMRole: 20 | role_id: RoleId 21 | attached_policy_contents: Dict[PolicyName, PolicyContents] 22 | -------------------------------------------------------------------------------- /fbpcp/util/reflect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from importlib import import_module 10 | from typing import Any 11 | 12 | 13 | # pyre-ignore 14 | def get_class(class_path: str) -> Any: 15 | module_name, class_name = class_path.rsplit(".", 1) 16 | module = import_module(module_name) 17 | 18 | return getattr(module, class_name) 19 | -------------------------------------------------------------------------------- /fbpcp/entity/container_insight.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | 9 | from dataclasses import dataclass 10 | from typing import Optional 11 | 12 | from fbpcp.entity.insight import Insight 13 | 14 | 15 | @dataclass 16 | class ContainerInsight(Insight): 17 | time: float 18 | cluster_name: str 19 | instance_id: str 20 | status: str 21 | exit_code: Optional[int] = None 22 | -------------------------------------------------------------------------------- /fbpcp/metrics/getter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | 11 | from fbpcp.metrics.emitter import MetricsEmitter 12 | 13 | 14 | class MetricsGetter(abc.ABC): 15 | @abc.abstractmethod 16 | def has_metrics(self) -> bool: 17 | pass 18 | 19 | @abc.abstractmethod 20 | def get_metrics(self) -> MetricsEmitter: 21 | pass 22 | -------------------------------------------------------------------------------- /fbpcp/util/arg_builder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | 10 | from shlex import quote 11 | 12 | 13 | def build_cmd_args( 14 | **kwargs: object, 15 | ) -> str: 16 | return " ".join( 17 | [ 18 | f"--{key}={quote(str(value))}" 19 | for key, value in kwargs.items() 20 | if value is not None 21 | ] 22 | ) 23 | -------------------------------------------------------------------------------- /onedocker/service/attestation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | from typing import Dict 11 | 12 | from onedocker.entity.attestation_document import AttestationPolicy 13 | 14 | 15 | class AttestationService(abc.ABC): 16 | @abc.abstractmethod 17 | def validate(self, policy: AttestationPolicy, measurements: Dict[str, str]) -> bool: 18 | pass 19 | -------------------------------------------------------------------------------- /fbpcp/entity/cloud_cost.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | from decimal import Decimal 10 | from typing import List 11 | 12 | 13 | @dataclass 14 | class CloudCostItem: 15 | service: str 16 | cost_amount: Decimal 17 | 18 | 19 | @dataclass 20 | class CloudCost: 21 | total_cost_amount: Decimal 22 | details: List[CloudCostItem] 23 | -------------------------------------------------------------------------------- /tests/util/test_reflect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.util.reflect import get_class 12 | from fbpcp.util.s3path import S3Path 13 | 14 | TEST_CLASS_PATH = "fbpcp.util.s3path.S3Path" 15 | 16 | 17 | class TestReflect(unittest.TestCase): 18 | def test_get_class(self): 19 | self.assertEqual(S3Path, get_class(TEST_CLASS_PATH)) 20 | -------------------------------------------------------------------------------- /pce/validator/message_templates/resource_names.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | # patternlint-disable f-string-may-be-missing-leading-f 10 | from enum import Enum 11 | 12 | 13 | class ResourceNamePlural(Enum): 14 | VPC = "VPCs" 15 | ROUTE_TABLE = "Route Tables" 16 | VPC_PEERING = "VPC Peerings" 17 | CLUSTER = "ECS Clusters" 18 | CONTAINER_DEFINITION = "Container Definitions" 19 | -------------------------------------------------------------------------------- /fbpcp/service/billing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | from datetime import date 11 | from typing import Optional 12 | 13 | from fbpcp.entity.cloud_cost import CloudCost 14 | 15 | 16 | class BillingService(abc.ABC): 17 | @abc.abstractmethod 18 | def get_cost( 19 | self, start_date: date, end_date: date, region: Optional[str] = None 20 | ) -> CloudCost: 21 | pass 22 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | workflow_dispatch: 10 | 11 | jobs: 12 | tests: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.8' 20 | 21 | - name: Install Package 22 | run: | 23 | python3 -m pip install --upgrade pip 24 | python3 -m pip install . 25 | - name: Run Tests 26 | run: | 27 | ./scripts/run-python-tests.sh 28 | -------------------------------------------------------------------------------- /fbpcp/entity/pce_compute.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | from typing import Optional 10 | 11 | from fbpcp.entity.cluster_instance import Cluster 12 | from fbpcp.entity.container_definition import ContainerDefinition 13 | 14 | 15 | @dataclass 16 | class PCECompute: 17 | region: str 18 | cluster: Optional[Cluster] 19 | container_definition: Optional[ContainerDefinition] 20 | -------------------------------------------------------------------------------- /fbpcp/metrics/emitter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | 11 | 12 | class MetricsEmitter(abc.ABC): 13 | @abc.abstractmethod 14 | def count( 15 | self, 16 | name: str, 17 | value: int, 18 | ) -> None: 19 | pass 20 | 21 | @abc.abstractmethod 22 | def gauge( 23 | self, 24 | name: str, 25 | value: int, 26 | ) -> None: 27 | pass 28 | -------------------------------------------------------------------------------- /fbpcp/util/yaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from pathlib import Path 10 | from typing import Any, Dict 11 | 12 | import yaml 13 | 14 | 15 | def load(file_path: Path) -> Dict[str, Any]: 16 | with open(file_path) as stream: 17 | return yaml.safe_load(stream) 18 | 19 | 20 | def dump(data: Dict[str, Any], file_path: Path) -> None: 21 | with open(file_path, "w") as f: 22 | return yaml.dump(data, f) 23 | -------------------------------------------------------------------------------- /onedocker/tests/common/test_util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import subprocess 10 | import unittest 11 | 12 | from onedocker.common.util import run_cmd 13 | 14 | 15 | class TestUtil(unittest.TestCase): 16 | def test_run_cmd(self) -> None: 17 | self.assertEqual(0, run_cmd("cat", 1)) 18 | 19 | def test_run_cmd_with_timeout(self) -> None: 20 | self.assertRaises(subprocess.TimeoutExpired, run_cmd, "vi", 1) 21 | -------------------------------------------------------------------------------- /fbpcp/entity/container_definition.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from typing import Dict, List 11 | 12 | 13 | @dataclass 14 | class ContainerDefinition: 15 | id: str 16 | image: str 17 | cpu: int 18 | memory: int 19 | entry_point: List[str] 20 | environment: Dict[str, str] 21 | task_role_id: str 22 | tags: Dict[str, str] = field(default_factory=lambda: {}) 23 | -------------------------------------------------------------------------------- /onedocker/service/certificate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | import abc 9 | 10 | from fbpcp.entity.certificate_request import CertificateRequest 11 | 12 | 13 | class CertificateService(abc.ABC): 14 | @abc.abstractmethod 15 | def __init__(self, cert_request: CertificateRequest, exe_path: str) -> None: 16 | pass 17 | 18 | @abc.abstractmethod 19 | def generate_certificate( 20 | self, 21 | ) -> str: 22 | pass 23 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | workflow_dispatch: 10 | 11 | jobs: 12 | lint: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.8' 20 | 21 | - name: Install Dependencies 22 | run: | 23 | python3 -m pip install --upgrade pip 24 | python3 -m pip install -r lint_requirements.txt 25 | - name: Run ufmt 26 | run: | 27 | ufmt check . 28 | - name: Run Flake 29 | run: | 30 | flake8 31 | -------------------------------------------------------------------------------- /fbpcp/entity/firewall_ruleset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass, field 9 | from typing import Dict, List 10 | 11 | 12 | @dataclass 13 | class FirewallRule: 14 | from_port: int 15 | to_port: int 16 | ip_protocol: str 17 | cidr: str 18 | 19 | 20 | @dataclass 21 | class FirewallRuleset: 22 | id: str 23 | vpc_id: str 24 | ingress: List[FirewallRule] 25 | egress: List[FirewallRule] 26 | tags: Dict[str, str] = field(default_factory=dict) 27 | -------------------------------------------------------------------------------- /scripts/run-python-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | set -e 8 | 9 | SCRIPTS_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 10 | echo "${SCRIPTS_DIRECTORY}" 11 | cd "${SCRIPTS_DIRECTORY}/.." 12 | 13 | files=$(find tests onedocker/tests pce/gateway/tests pce/validator/tests "${SCRIPTS_DIRECTORY}" -name '*.py') 14 | echo "${files}" 15 | if [[ -z "${files}" ]]; then 16 | echo 'No test files found, exiting.' 17 | exit 1 18 | fi 19 | 20 | echo " Running all tests:" 21 | echo "${files}" | xargs python3 -m unittest -v 22 | -------------------------------------------------------------------------------- /fbpcp/service/log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | from typing import List 11 | 12 | from fbpcp.entity.container_instance import ContainerInstance 13 | from fbpcp.entity.log_event import LogEvent 14 | 15 | 16 | class LogService(abc.ABC): 17 | @abc.abstractmethod 18 | def fetch(self, log_path: str, start_time: int) -> List[LogEvent]: 19 | pass 20 | 21 | @abc.abstractmethod 22 | def get_log_path(self, container_instance: ContainerInstance) -> str: 23 | pass 24 | -------------------------------------------------------------------------------- /tests/util/test_typing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.util.typing import checked_cast 12 | 13 | TEST_STR = "test" 14 | TEST_NUM = 123 15 | 16 | 17 | class TestTyping(unittest.TestCase): 18 | def test_checked_cast(self): 19 | error = f"Value was not of type {type!r}:\n{TEST_STR!r}" 20 | with self.assertRaisesRegex(ValueError, error): 21 | checked_cast(int, TEST_STR) 22 | 23 | self.assertEqual(checked_cast(int, TEST_NUM), TEST_NUM) 24 | -------------------------------------------------------------------------------- /fbpcp/entity/insight.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | import json 9 | from dataclasses import dataclass 10 | 11 | from dataclasses_json import dataclass_json, DataClassJsonMixin, Undefined 12 | 13 | 14 | @dataclass_json(undefined=Undefined.EXCLUDE) 15 | @dataclass 16 | class Insight(DataClassJsonMixin): 17 | def convert_to_str_with_class_name(self) -> str: 18 | class_name = {"class_name": self.__class__.__name__} 19 | insight = self.to_dict() 20 | insight.update(class_name) 21 | return json.dumps(insight) 22 | -------------------------------------------------------------------------------- /onedocker/entity/measurement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | 10 | from enum import Enum 11 | 12 | from dataclasses_json import dataclass_json 13 | 14 | 15 | class MeasurementType(Enum): 16 | sha256 = "sha256" 17 | sha512 = "sha512" 18 | 19 | @classmethod 20 | def has_member(cls, name: str) -> bool: 21 | return name in cls.__members__ 22 | 23 | 24 | @dataclass_json 25 | @dataclass 26 | class Measurement: 27 | key: MeasurementType 28 | value: str 29 | -------------------------------------------------------------------------------- /fbpcp/entity/vpc_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from enum import Enum 11 | from typing import Dict 12 | 13 | from dataclasses_json import dataclass_json 14 | 15 | 16 | class VpcState(Enum): 17 | UNKNOWN = "UNKNOWN" 18 | PENDING = "PENDING" 19 | AVAILABLE = "AVAILABLE" 20 | 21 | 22 | @dataclass_json 23 | @dataclass 24 | class Vpc: 25 | vpc_id: str 26 | cidr: str 27 | state: VpcState = VpcState.UNKNOWN 28 | tags: Dict[str, str] = field(default_factory=dict) 29 | -------------------------------------------------------------------------------- /.github/workflows/pyre.yml: -------------------------------------------------------------------------------- 1 | name: pyre 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | workflow_dispatch: 10 | 11 | jobs: 12 | pyre: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.8' 20 | 21 | - name: Install Dependencies 22 | run: | 23 | python3 -m pip install --upgrade pip 24 | python3 -m venv env 25 | source env/bin/activate 26 | python3 -m pip install . 27 | python3 -m pip install pyre-check==0.9.19 28 | - name: Run Pyre 29 | run: | 30 | source env/bin/activate 31 | pyre check 32 | -------------------------------------------------------------------------------- /onedocker/entity/opawdl_workflow.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from typing import Dict 11 | 12 | from dataclasses_json import config, DataClassJsonMixin 13 | 14 | from onedocker.entity.opawdl_state import OPAWDLState 15 | 16 | 17 | @dataclass 18 | class OPAWDLWorkflow(DataClassJsonMixin): 19 | starts_at: str = field(metadata=config(field_name="StartAt")) 20 | states: Dict[str, OPAWDLState] = field(metadata=config(field_name="States")) 21 | 22 | def __str__(self) -> str: 23 | return self.to_json() 24 | -------------------------------------------------------------------------------- /fbpcp/gateway/gcp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import json 10 | import logging 11 | from typing import Any, Dict, Optional 12 | 13 | 14 | class GCPGateway: 15 | def __init__( 16 | self, 17 | credentials_json: Optional[str] = None, 18 | config: Optional[Dict[str, Any]] = None, 19 | ) -> None: 20 | self.logger: logging.Logger = logging.getLogger(__name__) 21 | self.config: Dict[str, Any] = config or {} 22 | 23 | if credentials_json is not None: 24 | self.config["credentials_json"] = json.loads(credentials_json) 25 | -------------------------------------------------------------------------------- /fbpcp/error/mapper/gcp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from fbpcp.error.pcp import InvalidParameterError, PcpError, ThrottlingError 10 | from google.cloud.exceptions import GoogleCloudError 11 | 12 | 13 | # reference: https://gcloud.readthedocs.io/en/latest/_modules/google/cloud/exceptions.html 14 | def map_gcp_error(error: GoogleCloudError) -> PcpError: 15 | code = error.code 16 | message = error.message 17 | if code == 429: 18 | return ThrottlingError(message) 19 | if code == 400: 20 | return InvalidParameterError(message) 21 | return PcpError(message) 22 | -------------------------------------------------------------------------------- /onedocker/common/env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | # This is the repository path that OneDocker downloads binaries from 10 | ONEDOCKER_REPOSITORY_PATH = "ONEDOCKER_REPOSITORY_PATH" 11 | 12 | # This is the repository path that OneDocker downloads binary checksums from 13 | ONEDOCKER_CHECKSUM_REPOSITORY_PATH = "ONEDOCKER_CHECKSUM_REPOSITORY_PATH" 14 | 15 | # This is the local path that the binaries reside 16 | ONEDOCKER_EXE_PATH = "ONEDOCKER_EXE_PATH" 17 | 18 | # This is the type of checksum we want to compare when running program 19 | ONEDOCKER_CHECKSUM_TYPE = "ONEDOCKER_CHECKSUM_TYPE" 20 | -------------------------------------------------------------------------------- /tests/util/test_arg_builder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.util.arg_builder import build_cmd_args 12 | 13 | 14 | class TestArgBuilder(unittest.TestCase): 15 | def test_build_cmd_args(self) -> None: 16 | expected_cmd_args = ( 17 | "--arg1=value1 --arg2=value2 --arg3='--k1=v1 --k2=v2' --arg4=0" 18 | ) 19 | self.assertEqual( 20 | expected_cmd_args, 21 | build_cmd_args( 22 | arg1="value1", arg2="value2", arg3="--k1=v1 --k2=v2", arg4=0, arg5=None 23 | ), 24 | ) 25 | -------------------------------------------------------------------------------- /fbpcp/entity/cluster_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from enum import Enum 11 | from typing import Dict 12 | 13 | from dataclasses_json import dataclass_json 14 | 15 | 16 | class ClusterStatus(Enum): 17 | ACTIVE = "ACTIVE" 18 | INACTIVE = "INACTIVE" 19 | UNKNOWN = "UNKNOWN" 20 | 21 | 22 | @dataclass_json 23 | @dataclass 24 | class Cluster: 25 | cluster_arn: str 26 | cluster_name: str 27 | pending_tasks: int 28 | running_tasks: int 29 | status: ClusterStatus = ClusterStatus.UNKNOWN 30 | tags: Dict[str, str] = field(default_factory=dict) 31 | -------------------------------------------------------------------------------- /onedocker/entity/opawdl_state_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | 10 | from dataclasses import dataclass 11 | from enum import Enum 12 | 13 | from dataclasses_json import DataClassJsonMixin 14 | from onedocker.entity.opawdl_state import OPAWDLState 15 | 16 | 17 | class Status(Enum): 18 | CREATED = "CREATED" 19 | STARTED = "STARTED" 20 | FAILED = "FAILED" 21 | COMPLETED = "COMPLETED" 22 | 23 | 24 | @dataclass 25 | class OPAWDLStateInstance(DataClassJsonMixin): 26 | opawdl_state: OPAWDLState 27 | status: Status = Status.CREATED 28 | 29 | def __str__(self) -> str: 30 | return self.to_json() 31 | -------------------------------------------------------------------------------- /onedocker/entity/metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from typing import Any, Dict 11 | 12 | from onedocker.entity.measurement import MeasurementType 13 | 14 | 15 | @dataclass 16 | class PackageMetadata: 17 | package_name: str 18 | version: str 19 | measurements: Dict[MeasurementType, str] = field(default_factory=dict) 20 | 21 | def to_dict(self) -> Dict[str, Any]: 22 | return { 23 | "package_name": self.package_name, 24 | "version": self.version, 25 | "measurements": {k.value: v for k, v in self.measurements.items()}, 26 | } 27 | -------------------------------------------------------------------------------- /fbpcp/entity/pce_network.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | from typing import List, Optional 10 | 11 | from fbpcp.entity.firewall_ruleset import FirewallRuleset 12 | from fbpcp.entity.route_table import RouteTable 13 | from fbpcp.entity.subnet import Subnet 14 | from fbpcp.entity.vpc_instance import Vpc 15 | from fbpcp.entity.vpc_peering import VpcPeering 16 | 17 | 18 | @dataclass 19 | class PCENetwork: 20 | region: str 21 | vpc: Optional[Vpc] 22 | subnets: List[Subnet] 23 | route_table: Optional[RouteTable] 24 | vpc_peering: Optional[VpcPeering] 25 | firewall_rulesets: List[FirewallRuleset] 26 | -------------------------------------------------------------------------------- /fbpcp/service/policy_validation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | import abc 9 | from typing import List 10 | 11 | from fbpcp.entity.policy_settings_config import PolicySettingsConfig 12 | from fbpcp.entity.policy_statement import PolicyStatement 13 | 14 | 15 | class PolicyValidationService: 16 | @abc.abstractmethod 17 | def is_bucket_policy_statements_valid( 18 | self, 19 | bucket: str, 20 | bucket_statements: List[PolicyStatement], 21 | policy_settings: List[PolicySettingsConfig], 22 | ) -> bool: 23 | """Returns whether the bucket's policy is valid according to policy_settings""" 24 | pass 25 | -------------------------------------------------------------------------------- /onedocker/mapper/aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from typing import Any, Dict 9 | 10 | from onedocker.entity.measurement import MeasurementType 11 | 12 | from onedocker.entity.metadata import PackageMetadata 13 | 14 | 15 | def map_dynamodbitem_to_packagemetadata( 16 | dynamodb_item: Dict[str, Any] 17 | ) -> PackageMetadata: 18 | measurements = { 19 | MeasurementType(key): value 20 | for key, value in dynamodb_item.get("measurements", {}).items() 21 | } 22 | return PackageMetadata( 23 | package_name=dynamodb_item.get("package_name"), 24 | version=dynamodb_item.get("version"), 25 | measurements=measurements, 26 | ) 27 | -------------------------------------------------------------------------------- /tests/error/mapper/test_gcp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import unittest 10 | 11 | from fbpcp.error.mapper.gcp import map_gcp_error 12 | from fbpcp.error.pcp import PcpError, ThrottlingError 13 | from google.cloud.exceptions import GoogleCloudError 14 | 15 | 16 | class TestMapGCPError(unittest.TestCase): 17 | def test_pcs_error(self) -> None: 18 | err = GoogleCloudError("Test") 19 | err = map_gcp_error(err) 20 | 21 | self.assertIsInstance(err, PcpError) 22 | 23 | def test_throttling_error(self) -> None: 24 | err = GoogleCloudError("Test") 25 | err.code = 429 26 | err = map_gcp_error(err) 27 | 28 | self.assertIsInstance(err, ThrottlingError) 29 | -------------------------------------------------------------------------------- /onedocker/entity/attestation_document.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | from enum import Enum 11 | from typing import Dict, Optional 12 | 13 | from dataclasses_json import DataClassJsonMixin 14 | 15 | 16 | class PolicyName(str, Enum): 17 | BINARY_MATCH = "BINARY_MATCH" 18 | 19 | 20 | @dataclass 21 | class PolicyParams(DataClassJsonMixin): 22 | package_name: Optional[str] 23 | version: Optional[str] 24 | 25 | 26 | @dataclass 27 | class AttestationPolicy(DataClassJsonMixin): 28 | policy_name: PolicyName 29 | params: PolicyParams 30 | 31 | 32 | @dataclass 33 | class AttestationDocument(DataClassJsonMixin): 34 | policy: AttestationPolicy 35 | measurements: Dict[str, str] 36 | -------------------------------------------------------------------------------- /onedocker/gateway/repository_service.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Dict 10 | 11 | from fbpcp.entity.cloud_provider import CloudProvider 12 | from onedocker.repository.onedocker_repository_service import OneDockerRepositoryService 13 | 14 | from onedocker.util.service_builder import build_repository_service 15 | 16 | 17 | class RepositoryServiceGateway: 18 | def __init__(self) -> None: 19 | self.repository_service: OneDockerRepositoryService = build_repository_service( 20 | cloud_provider=CloudProvider.AWS 21 | ) 22 | 23 | def get_measurements(self, package_name: str, version: str) -> Dict[str, str]: 24 | return self.repository_service.get_package_measurements(package_name, version) 25 | -------------------------------------------------------------------------------- /onedocker/entity/opawdl_state.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass, field 10 | from typing import List, Optional 11 | 12 | from dataclasses_json import config, DataClassJsonMixin 13 | 14 | 15 | @dataclass 16 | class OPAWDLState(DataClassJsonMixin): 17 | plugin_name: str = field(metadata=config(field_name="PluginName")) 18 | cmd_args_list: List[str] = field(metadata=config(field_name="CmdArgsList")) 19 | timeout: Optional[int] = field(metadata=config(field_name="Timeout"), default=None) 20 | next_: Optional[str] = field(metadata=config(field_name="Next"), default=None) 21 | is_end: Optional[bool] = field(metadata=config(field_name="IsEnd"), default=True) 22 | 23 | def __str__(self) -> str: 24 | return self.to_json() 25 | -------------------------------------------------------------------------------- /pce/gateway/sts.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from typing import Any, Dict, Optional 9 | 10 | import boto3 11 | from fbpcp.gateway.aws import AWSGateway 12 | 13 | 14 | class STSGateway(AWSGateway): 15 | def __init__( 16 | self, 17 | region: str, 18 | access_key_id: Optional[str] = None, 19 | access_key_data: Optional[str] = None, 20 | config: Optional[Dict[str, Any]] = None, 21 | ) -> None: 22 | super().__init__(region, access_key_id, access_key_data, config) 23 | 24 | # pyre-ignore 25 | self.client = boto3.client("sts", region_name=self.region, **self.config) 26 | 27 | def get_caller_arn( 28 | self, 29 | ) -> str: 30 | response = self.client.get_caller_identity() 31 | 32 | return response["Arn"] 33 | -------------------------------------------------------------------------------- /onedocker/repository/opawdl_workflow_instance_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | 11 | from onedocker.entity.opawdl_workflow_instance import OPAWDLWorkflowInstance 12 | 13 | 14 | class OPAWDLWorkflowInstanceRepository(abc.ABC): 15 | @abc.abstractmethod 16 | def create(self, instance: OPAWDLWorkflowInstance) -> None: 17 | pass 18 | 19 | @abc.abstractmethod 20 | def get(self, instance_id: str) -> OPAWDLWorkflowInstance: 21 | pass 22 | 23 | @abc.abstractmethod 24 | def update(self, instance: OPAWDLWorkflowInstance) -> None: 25 | pass 26 | 27 | @abc.abstractmethod 28 | def delete(self, instance_id: str) -> None: 29 | pass 30 | 31 | @abc.abstractmethod 32 | def exist(self, instance_id: str) -> bool: 33 | pass 34 | -------------------------------------------------------------------------------- /fbpcp/entity/policy_statement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass 9 | from typing import List 10 | 11 | 12 | @dataclass 13 | class PolicyStatement: 14 | """AWS Policy Statement: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_statement.html""" 15 | 16 | effect: str 17 | principals: List[str] 18 | actions: List[str] 19 | resources: List[str] 20 | sid: str = "" 21 | 22 | 23 | @dataclass 24 | class PublicAccessBlockConfig: 25 | """AWS PublicAccessBlockConfiguration: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-publicaccessblockconfiguration.html""" 26 | 27 | block_public_acls: bool 28 | ignore_public_acls: bool 29 | block_public_policy: bool 30 | restrict_public_buckets: bool 31 | -------------------------------------------------------------------------------- /fbpcp/entity/route_table.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass, field 9 | from enum import Enum 10 | from typing import Dict, List 11 | 12 | 13 | class RouteTargetType(Enum): 14 | OTHER = "OTHER" 15 | INTERNET = "INTERNET" 16 | VPC_PEERING = "VPC_PEERING" 17 | 18 | 19 | class RouteState(Enum): 20 | UNKNOWN = "UNKNOWN" 21 | ACTIVE = "ACTIVE" 22 | 23 | 24 | @dataclass 25 | class RouteTarget: 26 | route_target_id: str 27 | route_target_type: RouteTargetType 28 | 29 | 30 | @dataclass 31 | class Route: 32 | destination_cidr_block: str 33 | route_target: RouteTarget 34 | state: RouteState 35 | 36 | 37 | @dataclass 38 | class RouteTable: 39 | id: str 40 | routes: List[Route] 41 | vpc_id: str 42 | tags: Dict[str, str] = field(default_factory=dict) 43 | -------------------------------------------------------------------------------- /tests/service/test_storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.service.storage import PathType, StorageService 12 | 13 | 14 | class TestStorageService(unittest.TestCase): 15 | def test_path_type_s3(self) -> None: 16 | type_ = StorageService.path_type( 17 | "https://bucket-name.s3.Region.amazonaws.com/key-name" 18 | ) 19 | self.assertEqual(type_, PathType.S3) 20 | 21 | def test_path_type_gcs(self) -> None: 22 | type_ = StorageService.path_type( 23 | "https://storage.cloud.google.com/bucket-name/key-name" 24 | ) 25 | self.assertEqual(type_, PathType.GCS) 26 | 27 | def test_path_type_local(self) -> None: 28 | type_ = StorageService.path_type("/usr/file") 29 | self.assertEqual(type_, PathType.Local) 30 | -------------------------------------------------------------------------------- /fbpcp/entity/vpc_peering.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from dataclasses import dataclass, field 9 | from enum import Enum 10 | from typing import Dict, Optional 11 | 12 | 13 | class VpcPeeringState(Enum): 14 | PENDING_ACCEPTANCE = "PENDING_ACCEPTANCE" 15 | ACTIVE = "ACTIVE" 16 | INACTIVE = "INACTIVE" 17 | PROVISIONING = "PROVISIONING" 18 | INITIATING = "INITIATING" 19 | 20 | 21 | class VpcPeeringRole(Enum): 22 | REQUESTER = "REQUESTER" 23 | ACCEPTER = "ACCEPTER" 24 | 25 | 26 | @dataclass 27 | class VpcPeering: 28 | id: str 29 | status: VpcPeeringState 30 | role: VpcPeeringRole 31 | requester_vpc_id: str 32 | accepter_vpc_id: str 33 | requester_vpc_cidr: Optional[str] = None 34 | accepter_vpc_cidr: Optional[str] = None 35 | tags: Dict[str, str] = field(default_factory=dict) 36 | -------------------------------------------------------------------------------- /onedocker/util/opawdl_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from onedocker.entity.opawdl_workflow import OPAWDLWorkflow 10 | 11 | 12 | class OPAWDLParser: 13 | def parse_json_str_to_workflow( 14 | self, 15 | input_str: str, 16 | ) -> OPAWDLWorkflow: 17 | workflow = OPAWDLWorkflow.from_json(input_str) 18 | 19 | # Validate end state 20 | has_end = 0 21 | for state in workflow.states.values(): 22 | if state.is_end: 23 | has_end += 1 24 | if has_end == 0: 25 | raise Exception("Input workflow string does not have an ending state.") 26 | elif has_end > 1: 27 | raise Exception( 28 | f"Input workflow string has multiple({has_end}) ending states." 29 | ) 30 | 31 | return workflow 32 | -------------------------------------------------------------------------------- /fbpcp/entity/policy_settings_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | from dataclasses import dataclass 8 | from enum import Enum 9 | from typing import List 10 | 11 | 12 | class Effect(str, Enum): 13 | ALLOW = "Allow" 14 | DENY = "Deny" 15 | 16 | 17 | @dataclass 18 | class PolicySettingsConfig: 19 | """This is a config struct used by Repository Monitor to set constraints for bucket policy. 20 | Args: 21 | exist: Wether the specified permission exists in the bucket's policy. 22 | effect: The specified effect. 23 | principal: The specified principal. Use normal string or "re()" for regex matching. 24 | actions: List of specified actions. 25 | resources: List of resources. 26 | """ 27 | 28 | exist: bool 29 | effect: Effect 30 | principal: str 31 | actions: List[str] 32 | resources: List[str] 33 | -------------------------------------------------------------------------------- /pce/validator/message_templates/pce_standard_constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | # patternlint-disable f-string-may-be-missing-leading-f 10 | 11 | from typing import TypeVar 12 | 13 | 14 | FIREWALL_RULE_INITIAL_PORT = 5000 15 | FIREWALL_RULE_FINAL_PORT = 15500 16 | 17 | 18 | AvailabilityZone = TypeVar("AvailabilityZone", str, bytes) 19 | 20 | 21 | CONTAINER_CPU = 4096 22 | CONTAINER_MEMORY = 30720 23 | CONTAINER_IMAGE = "539290649537.dkr.ecr.us-west-2.amazonaws.com/one-docker-prod:latest" 24 | TASK_POLICY = { 25 | "Version": "2012-10-17", 26 | "Statement": [ 27 | {"Effect": "Allow", "Action": ["s3:*", "s3-object-lambda:*"], "Resource": "*"} 28 | ], 29 | } 30 | 31 | IGW_ROUTE_TARGET_PREFIX: str = "igw-" 32 | IGW_ROUTE_DESTINATION_CIDR_BLOCK: str = "0.0.0.0/0" 33 | 34 | DEFAULT_VPC_CIDR = "10.0.0.0/16" 35 | DEFAULT_PARTNER_VPC_CIDR = "10.1.0.0/16" 36 | -------------------------------------------------------------------------------- /fbpcp/util/gcspath.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import re 10 | from typing import Tuple 11 | 12 | 13 | class GCSPath: 14 | bucket: str 15 | key: str 16 | 17 | def __init__(self, fileURL: str) -> None: 18 | self.bucket, self.key = self._get_bucket_key(fileURL) 19 | 20 | def __eq__(self, other: "GCSPath") -> bool: 21 | return self.bucket == other.bucket and self.key == other.key 22 | 23 | # virtual host style url 24 | # https://storage.cloud.google.com// 25 | def _get_bucket_key(self, fileURL: str) -> Tuple[str, str]: 26 | match = re.search("^https?://storage.cloud.google.com/(.*)$", fileURL) 27 | if not match: 28 | raise ValueError(f"Could not parse {fileURL} as an GCSPath") 29 | bucket, *rest = match.group(1).split("/") 30 | key = "/".join(rest) 31 | return (bucket, key) 32 | -------------------------------------------------------------------------------- /onedocker/entity/opawdl_workflow_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | from enum import Enum 11 | from typing import List 12 | 13 | from dataclasses_json import DataClassJsonMixin 14 | from onedocker.entity.opawdl_state_instance import OPAWDLStateInstance 15 | from onedocker.entity.opawdl_workflow import OPAWDLWorkflow 16 | 17 | 18 | class Status(Enum): 19 | CREATED = "CREATED" 20 | STARTED = "STARTED" 21 | FAILED = "FAILED" 22 | COMPLETED = "COMPLETED" 23 | 24 | 25 | @dataclass 26 | class OPAWDLWorkflowInstance(DataClassJsonMixin): 27 | instance_id: str 28 | opawdl_workflow: OPAWDLWorkflow 29 | state_instances: List[OPAWDLStateInstance] 30 | status: Status = Status.CREATED 31 | 32 | def get_instance_id(self) -> str: 33 | return self.instance_id 34 | 35 | def __str__(self) -> str: 36 | return self.to_json() 37 | -------------------------------------------------------------------------------- /fbpcp/error/mapper/aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from botocore.exceptions import ClientError 10 | from fbpcp.error.pcp import ( 11 | InvalidParameterError, 12 | LimitExceededError, 13 | PcpError, 14 | ThrottlingError, 15 | ) 16 | 17 | 18 | # reference: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/error-handling.html 19 | def map_aws_error(error: ClientError) -> PcpError: 20 | code = error.response["Error"]["Code"] 21 | response_metadata = error.response["ResponseMetadata"] 22 | message = f"{error}\n\n Details: {response_metadata}\n" 23 | 24 | if code == "InvalidParameterException": 25 | return InvalidParameterError(message) 26 | 27 | if code == "ThrottlingException": 28 | return ThrottlingError(message) 29 | 30 | if code == "LimitExceededException": 31 | return LimitExceededError(message) 32 | 33 | return PcpError(message) 34 | -------------------------------------------------------------------------------- /fbpcp/error/mapper/k8s.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from fbpcp.error.pcp import InvalidParameterError, PcpError, ThrottlingError 10 | from kubernetes.client.exceptions import ( 11 | ApiException, 12 | ApiTypeError, 13 | ApiValueError, 14 | OpenApiException, 15 | ) 16 | 17 | 18 | # reference: https://github.com/kubernetes-client/python/blob/d3de7a85a63fa6bec6518d1cc75dc5e9458b9bbc/kubernetes/client/exceptions.py 19 | def map_k8s_error(error: OpenApiException) -> PcpError: 20 | message = str(error) 21 | if isinstance(error, (ApiValueError, ApiTypeError)): 22 | return InvalidParameterError(message) 23 | elif isinstance(error, ApiException): 24 | code = error.status 25 | if code == 429: 26 | return ThrottlingError(message) 27 | if code == 400 or code == 404: 28 | return InvalidParameterError(message) 29 | 30 | return PcpError(message) 31 | -------------------------------------------------------------------------------- /tests/util/test_gcspath.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.util.gcspath import GCSPath 12 | 13 | 14 | class TestGCSPath(unittest.TestCase): 15 | def test_gcspath_no_subfolder(self): 16 | test_gcspath = GCSPath("https://storage.cloud.google.com/bucket-name/key-name") 17 | self.assertEqual(test_gcspath.bucket, "bucket-name") 18 | self.assertEqual(test_gcspath.key, "key-name") 19 | 20 | def test_gcspath_with_subfoler(self): 21 | test_gcspath = GCSPath( 22 | "https://storage.cloud.google.com/bucket-name/subfolder/key" 23 | ) 24 | self.assertEqual(test_gcspath.bucket, "bucket-name") 25 | self.assertEqual(test_gcspath.key, "subfolder/key") 26 | 27 | def test_gcspath_invalid_fileURL(self): 28 | test_url = "an invalid fileURL" 29 | with self.assertRaises(ValueError): 30 | GCSPath(test_url) 31 | -------------------------------------------------------------------------------- /tests/gateway/test_cloudwatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | from unittest.mock import MagicMock, patch 11 | 12 | from fbpcp.entity.log_event import LogEvent 13 | from fbpcp.gateway.cloudwatch import CloudWatchGateway 14 | 15 | 16 | class TestCloudWatchGateway(unittest.TestCase): 17 | REGION = "us-west-1" 18 | GROUP_NAME = "test-group-name" 19 | STREAM_NAME = "test-stream-name" 20 | 21 | @patch("boto3.client") 22 | def test_get_log_events(self, BotoClient): 23 | gw = CloudWatchGateway(self.REGION) 24 | mocked_log = {"events": [{"timestamp": 1, "message": "log"}]} 25 | gw.client = BotoClient() 26 | gw.client.get_log_events = MagicMock(return_value=mocked_log) 27 | returned_log = gw.get_log_events(self.GROUP_NAME, self.STREAM_NAME) 28 | gw.client.get_log_events.assert_called() 29 | self.assertEqual([LogEvent(1, "log")], returned_log) 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Meta Platforms, Inc. and affiliates. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /fbpcp/gateway/aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import logging 10 | from typing import Any, Dict, Optional 11 | 12 | 13 | class AWSGateway: 14 | def __init__( 15 | self, 16 | region: Optional[str] = None, 17 | access_key_id: Optional[str] = None, 18 | access_key_data: Optional[str] = None, 19 | config: Optional[Dict[str, Any]] = None, 20 | session_token: Optional[str] = None, 21 | ) -> None: 22 | self.logger: logging.Logger = logging.getLogger(__name__) 23 | self.region = region 24 | self.config: Dict[str, Any] = config or {} 25 | 26 | if access_key_id is not None: 27 | self.config["aws_access_key_id"] = access_key_id 28 | 29 | if access_key_data is not None: 30 | self.config["aws_secret_access_key"] = access_key_data 31 | 32 | if session_token is not None: 33 | self.config["aws_session_token"] = session_token 34 | -------------------------------------------------------------------------------- /tests/service/test_log_cloudwatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | from unittest.mock import MagicMock, patch 11 | 12 | from fbpcp.service.log_cloudwatch import CloudWatchLogService 13 | 14 | REGION = "us-west-1" 15 | LOG_GROUP = "test-group-name" 16 | LOG_PATH = "test-log-path" 17 | 18 | 19 | class TestCloudWatchLogService(unittest.TestCase): 20 | @patch("fbpcp.gateway.cloudwatch.CloudWatchGateway") 21 | def test_fetch(self, MockCloudWatchGateway): 22 | log_service = CloudWatchLogService(LOG_GROUP, REGION) 23 | mocked_log = {"test-events": [{"test-event-name": "test-event-data"}]} 24 | log_service.cloudwatch_gateway = MockCloudWatchGateway() 25 | log_service.cloudwatch_gateway.fetch = MagicMock(return_value=mocked_log) 26 | returned_log = log_service.cloudwatch_gateway.fetch(LOG_PATH) 27 | log_service.cloudwatch_gateway.fetch.assert_called() 28 | self.assertEqual(mocked_log, returned_log) 29 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from setuptools import find_packages, setup 10 | 11 | install_requires = [ 12 | "boto3==1.18.57", 13 | "urllib3==1.26.19", 14 | "dataclasses-json==0.5.2", 15 | "pyyaml==5.4.1", 16 | "tqdm==4.66.3", 17 | "google-cloud-storage==1.30.0", 18 | "docopt==0.6.2", 19 | "schema==0.7.5", 20 | "psutil==5.8.0", 21 | "click==7.1.2", 22 | "kubernetes==12.0.1", 23 | ] 24 | 25 | with open("README.md", encoding="utf-8") as f: 26 | long_description = f.read() 27 | 28 | setup( 29 | name="fbpcp", 30 | version="0.6.4", 31 | description="Facebook Private Computation Platform", 32 | author="Facebook", 33 | author_email="researchtool-help@fb.com", 34 | url="https://github.com/facebookresearch/fbpcp", 35 | install_requires=install_requires, 36 | packages=find_packages(), 37 | long_description_content_type="text/markdown", 38 | long_description=long_description, 39 | python_requires=">=3.8", 40 | ) 41 | -------------------------------------------------------------------------------- /pce/mapper/aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Any, Dict, Optional 10 | 11 | from pce.entity.iam_role import IAMRole, PolicyContents, PolicyName, RoleId 12 | 13 | 14 | def map_attachedrolepolicies_to_rolepolicies( 15 | role_id: RoleId, 16 | attached_role_policies: Dict[PolicyName, PolicyContents], 17 | ) -> Optional[IAMRole]: 18 | return IAMRole(role_id, attached_role_policies) if attached_role_policies else None 19 | 20 | 21 | def map_ecstaskdefinition_to_awslogsgroupname( 22 | task_definition: Dict[str, Any], 23 | ) -> Optional[str]: 24 | try: 25 | container_definition = task_definition["containerDefinitions"][0] 26 | log_configuration = container_definition["logConfiguration"] 27 | log_group_name = log_configuration["options"]["awslogs-group"] 28 | return log_group_name 29 | # Key error indicates the log group does not exist, error will be returned in error message templates 30 | except KeyError: 31 | return None 32 | -------------------------------------------------------------------------------- /onedocker/entity/exit_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from enum import IntEnum 10 | 11 | 12 | class ExitCode(IntEnum): 13 | """Custom exit codes for OneDocker containers. 14 | 15 | Unix uses error codes above 125 specially for failures, so they should be avoided for program errors. 16 | 17 | The meaning of the codes is approximately as follows: 18 | SUCCESS -- The program succeeded. 19 | EXE_ERROR -- The executable itself exited with non-zero code. 20 | ERROR -- Catchall for general errors. 21 | SERVICE_UNAVAILABLE -- A service is unavailable. This can occur if a support program or file is unavailable. 22 | TIMEOUT -- The program timed out. 23 | SIGINT -- The program received SIGINT signal. 24 | 25 | For other exit codes please refer to https://man.freebsd.org/cgi/man.cgi?sysexits. 26 | """ 27 | 28 | SUCCESS = 0 29 | EXE_ERROR = 1 30 | ERROR = 64 31 | SERVICE_UNAVAILABLE = 69 32 | TIMEOUT = 124 33 | SIGINT = 130 34 | -------------------------------------------------------------------------------- /fbpcp/entity/container_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | from enum import Enum 11 | from typing import Optional 12 | 13 | from dataclasses_json import dataclass_json 14 | from fbpcp.entity.container_metadata import ContainerStoppedMetadata 15 | from fbpcp.entity.container_permission import ContainerPermissionConfig 16 | 17 | 18 | class ContainerInstanceStatus(Enum): 19 | UNKNOWN = "UNKNOWN" 20 | STARTED = "STARTED" 21 | COMPLETED = "COMPLETED" 22 | FAILED = "FAILED" 23 | 24 | 25 | @dataclass_json 26 | @dataclass 27 | class ContainerInstance: 28 | instance_id: str 29 | ip_address: Optional[str] = None 30 | status: ContainerInstanceStatus = ContainerInstanceStatus.UNKNOWN 31 | cpu: Optional[int] = None # Number of vCPU 32 | memory: Optional[int] = None # Memory in GB 33 | exit_code: Optional[int] = None 34 | permission: Optional[ContainerPermissionConfig] = None 35 | stopped_metadata: Optional[ContainerStoppedMetadata] = None 36 | -------------------------------------------------------------------------------- /onedocker/tests/entity/test_opawdl_state.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | 12 | from onedocker.entity.opawdl_state import OPAWDLState 13 | 14 | 15 | class TestOPAWDLState(unittest.TestCase): 16 | def setUp(self) -> None: 17 | self.test_plugin_name = "test_plugin" 18 | self.test_cmd_args_list = ["-a=b", "-c=d"] 19 | self.test_opawdl_state = OPAWDLState( 20 | plugin_name=self.test_plugin_name, cmd_args_list=self.test_cmd_args_list 21 | ) 22 | 23 | def test__str__(self) -> None: 24 | # Arrange 25 | expected_opawdl_state_json_str = json.dumps( 26 | { 27 | "PluginName": self.test_plugin_name, 28 | "CmdArgsList": self.test_cmd_args_list, 29 | "Timeout": None, 30 | "Next": None, 31 | "IsEnd": True, 32 | } 33 | ) 34 | # Act and Assert 35 | self.assertEqual(str(self.test_opawdl_state), expected_opawdl_state_json_str) 36 | -------------------------------------------------------------------------------- /tests/util/test_s3path.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.util.s3path import S3Path 12 | 13 | 14 | class TestS3Path(unittest.TestCase): 15 | def test_s3path_no_subfolder(self) -> None: 16 | test_s3path = S3Path("https://bucket-name.s3.Region.amazonaws.com/key-name") 17 | self.assertEqual(test_s3path.region, "Region") 18 | self.assertEqual(test_s3path.bucket, "bucket-name") 19 | self.assertEqual(test_s3path.key, "key-name") 20 | 21 | def test_s3path_with_subfoler(self) -> None: 22 | test_s3path = S3Path( 23 | "https://bucket-name.s3.Region.amazonaws.com/subfolder/key" 24 | ) 25 | self.assertEqual(test_s3path.region, "Region") 26 | self.assertEqual(test_s3path.bucket, "bucket-name") 27 | self.assertEqual(test_s3path.key, "subfolder/key") 28 | 29 | def test_s3path_invalid_fileURL(self) -> None: 30 | test_url = "an invalid fileURL" 31 | with self.assertRaises(ValueError): 32 | S3Path(test_url) 33 | -------------------------------------------------------------------------------- /onedocker/service/measurement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import hashlib 10 | from typing import Dict, List 11 | 12 | from onedocker.entity.measurement import MeasurementType 13 | 14 | 15 | class MeasurementService: 16 | def _get_content_bytes(self, file_path: str) -> bytes: 17 | with open(file_path, "rb") as file: 18 | content_bytes = file.read() 19 | return content_bytes 20 | 21 | def _generate_measurement( 22 | self, 23 | content_bytes: bytes, 24 | measurement_type: MeasurementType, 25 | ) -> str: 26 | hash_function = hashlib.new(measurement_type.value, content_bytes) 27 | return hash_function.hexdigest() 28 | 29 | def generate_measurements( 30 | self, measurement_types: List[MeasurementType], file_path: str 31 | ) -> Dict[MeasurementType, str]: 32 | content_bytes = self._get_content_bytes(file_path) 33 | measurements = { 34 | t: self._generate_measurement(content_bytes, t) for t in measurement_types 35 | } 36 | return measurements 37 | -------------------------------------------------------------------------------- /fbpcp/util/s3path.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import re 10 | from typing import Tuple 11 | 12 | 13 | class S3Path: 14 | region: str 15 | bucket: str 16 | key: str 17 | 18 | def __init__(self, fileURL: str) -> None: 19 | self.region, self.bucket, self.key = self._get_region_bucket_key(fileURL) 20 | 21 | def __eq__(self, other: "S3Path") -> bool: 22 | return ( 23 | self.region == other.region 24 | and self.bucket == other.bucket 25 | and self.key == other.key 26 | ) 27 | 28 | # virtual host style url 29 | # https://bucket-name.s3.Region.amazonaws.com/key-name 30 | def _get_region_bucket_key(self, fileURL: str) -> Tuple[str, str, str]: 31 | match = re.search("^https?:/([^.]+).s3.([^.]+).amazonaws.com/(.*)$", fileURL) 32 | if not match: 33 | raise ValueError(f"Could not parse {fileURL} as an S3Path") 34 | bucket, region, key = ( 35 | match.group(1).strip("/"), 36 | match.group(2), 37 | match.group(3).strip("/"), 38 | ) 39 | return (region, bucket, key) 40 | -------------------------------------------------------------------------------- /onedocker/script/config/cli_config_template.yml: -------------------------------------------------------------------------------- 1 | onedocker-cli: 2 | dependency: 3 | StorageService: 4 | class: classpath.classname #TODO: change this to actual class name that derived from abstract class: fbpcp.service.storage.StorageService 5 | constructor: 6 | attribute_name: value #TODO: change this to actual construction attribute name and value 7 | ContainerService: 8 | class: classpath.classname #TODO: change this to actual class name that derived from abstract class: fbpcp.service.container.ContainerService 9 | constructor: 10 | attribute_name: value #TODO: change this to actual construction attribute name and value 11 | LogService: 12 | class: classpath.classname #TODO: change this to actual class name that derived from abstract class: fbpcp.service.log.LogService 13 | constructor: 14 | attribute_name: value #TODO: change this to actual construction attribute name and value 15 | setting: 16 | repository_path: repositorypath/ #TODO: change this to the path to your repository, example: https://PATH.s3.REGION.amazonaws.com/ 17 | checksum_repository_path: checksumrepositorypath/ #TODO: [OPTIONAL] change this to the path to your repository, example: https://PATH.s3.REGION.amazonaws.com/ 18 | task_definition: taskdefinition #TODO: change this to your task_definition 19 | -------------------------------------------------------------------------------- /pce/gateway/tests/test_sts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from unittest import TestCase 10 | from unittest.mock import MagicMock, patch 11 | 12 | from pce.gateway.sts import STSGateway 13 | 14 | 15 | class TestSTSGateway(TestCase): 16 | REGION = "us-west-2" 17 | TEST_AWS_ACCOUNT_ID = "123456789012" 18 | 19 | def setUp(self) -> None: 20 | self.aws_sts = MagicMock() 21 | with patch("boto3.client") as mock_client: 22 | mock_client.return_value = self.aws_sts 23 | self.sts = STSGateway(self.REGION) 24 | 25 | def test_get_caller_arn(self) -> None: 26 | # Arrange 27 | client_return_response = { 28 | "UserId": "userID", 29 | "Account": "123456789012", 30 | "Arn": "foo", 31 | } 32 | 33 | self.aws_sts.get_caller_identity = MagicMock( 34 | return_value=client_return_response 35 | ) 36 | 37 | expected_arn = "foo" 38 | 39 | # Act 40 | arn = self.sts.get_caller_arn() 41 | 42 | # Assert 43 | self.assertEqual(expected_arn, arn) 44 | self.aws_sts.get_caller_identity.assert_called() 45 | -------------------------------------------------------------------------------- /onedocker/tests/mapper/test_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from onedocker.entity.measurement import MeasurementType 12 | from onedocker.entity.metadata import PackageMetadata 13 | from onedocker.mapper.aws import map_dynamodbitem_to_packagemetadata 14 | 15 | 16 | class TestAWSMapper(unittest.TestCase): 17 | def test_map_dynamodbitem_to_packagemetadata(self): 18 | # Arrange 19 | test_package_name = "PA" 20 | test_package_version = "0.0.1" 21 | test_sha256_measurement = "123" 22 | test_dynamodb_item = { 23 | "package_name": test_package_name, 24 | "version": test_package_version, 25 | "measurements": {"sha256": test_sha256_measurement}, 26 | } 27 | expect_res = PackageMetadata( 28 | package_name=test_package_name, 29 | version=test_package_version, 30 | measurements={MeasurementType.sha256: test_sha256_measurement}, 31 | ) 32 | 33 | # Act 34 | res = map_dynamodbitem_to_packagemetadata(dynamodb_item=test_dynamodb_item) 35 | 36 | # Assert 37 | self.assertEqual(expect_res, res) 38 | -------------------------------------------------------------------------------- /fbpcp/decorator/error_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import functools 10 | from typing import Callable 11 | 12 | from botocore.exceptions import ClientError 13 | from fbpcp.error.mapper.aws import map_aws_error 14 | from fbpcp.error.mapper.gcp import map_gcp_error 15 | from fbpcp.error.mapper.k8s import map_k8s_error 16 | from fbpcp.error.pcp import PcpError 17 | from google.cloud.exceptions import GoogleCloudError 18 | from kubernetes.client.exceptions import OpenApiException 19 | 20 | 21 | def error_handler(f: Callable) -> Callable: 22 | @functools.wraps(f) 23 | def wrapper(*args, **kwargs): 24 | try: 25 | return f(*args, **kwargs) 26 | except PcpError as err: 27 | raise err from None 28 | # AWS Error 29 | except ClientError as err: 30 | raise map_aws_error(err) from None 31 | # GCP Error 32 | except GoogleCloudError as err: 33 | raise map_gcp_error(err) from None 34 | except OpenApiException as err: 35 | raise map_k8s_error(err) from None 36 | except Exception as err: 37 | raise PcpError(err) from None 38 | 39 | return wrapper 40 | -------------------------------------------------------------------------------- /onedocker/common/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import os 10 | import signal 11 | import subprocess 12 | from types import FrameType 13 | from typing import Optional 14 | 15 | 16 | def run_cmd(cmd: str, timeout: Optional[int]) -> int: 17 | # The handler dealing signal SIGINT, which could be Ctrl + C from user's terminal 18 | def _handler(signum: int, frame: Optional[FrameType]) -> None: 19 | raise InterruptedError 20 | 21 | signal.signal(signal.SIGINT, _handler) 22 | """ 23 | If start_new_session is true the setsid() system call will be made in the 24 | child process prior to the execution of the subprocess, which makes sure 25 | every process in the same process group can be killed by OS if timeout occurs. 26 | note: setsid() will set the pgid to its pid. 27 | """ 28 | with subprocess.Popen(cmd, shell=True, start_new_session=True) as proc: 29 | try: 30 | proc.communicate(timeout=timeout) 31 | except (subprocess.TimeoutExpired, InterruptedError) as e: 32 | proc.terminate() 33 | os.killpg(proc.pid, signal.SIGTERM) 34 | raise e 35 | 36 | return proc.wait() 37 | -------------------------------------------------------------------------------- /tests/util/test_yaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | from unittest.mock import mock_open, patch 12 | 13 | from fbpcp.util.yaml import dump, load 14 | 15 | TEST_FILENAME = "TEST_FILE" 16 | TEST_DICT = { 17 | "test_dict": [ 18 | {"test_key_1": "test_value_1"}, 19 | {"test_key_1": "test_value_2"}, 20 | ] 21 | } 22 | 23 | 24 | class TestYaml(unittest.TestCase): 25 | data = json.dumps(TEST_DICT) 26 | 27 | @patch("builtins.open", new_callable=mock_open, read_data=data) 28 | def test_load(self, mock_file): 29 | self.assertEqual(open(TEST_FILENAME).read(), self.data) 30 | load_data = load(TEST_FILENAME) 31 | self.assertEqual(load_data, TEST_DICT) 32 | mock_file.assert_called_with(TEST_FILENAME) 33 | 34 | @patch("builtins.open") 35 | @patch("yaml.dump") 36 | def test_dump(self, mock_dump, mock_open): 37 | mock_dump.return_value = None 38 | stream = mock_open().__enter__.return_value 39 | self.assertIsNone(dump(TEST_DICT, TEST_FILENAME)) 40 | mock_open.assert_called_with(TEST_FILENAME, "w") 41 | mock_dump.assert_called_with(TEST_DICT, stream) 42 | -------------------------------------------------------------------------------- /fbpcp/service/secrets_manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | from typing import Dict, Optional 11 | 12 | from fbpcp.entity.secret import StringSecret 13 | 14 | 15 | class SecretsManagerService(abc.ABC): 16 | @abc.abstractmethod 17 | def create_secret( 18 | self, secret_name: str, secret_value: str, tags: Optional[Dict[str, str]] = None 19 | ) -> str: 20 | pass 21 | 22 | @abc.abstractmethod 23 | def get_secret( 24 | self, 25 | secret_id: str, 26 | ) -> StringSecret: 27 | pass 28 | 29 | @abc.abstractmethod 30 | async def create_secret_async( 31 | self, secret_name: str, secret_value: str, tags: Optional[Dict[str, str]] = None 32 | ) -> str: 33 | pass 34 | 35 | @abc.abstractmethod 36 | async def get_secret_async( 37 | self, 38 | secret_id: str, 39 | ) -> StringSecret: 40 | pass 41 | 42 | @abc.abstractmethod 43 | def delete_secret( 44 | self, 45 | secret_id: str, 46 | ) -> None: 47 | pass 48 | 49 | @abc.abstractmethod 50 | async def delete_secret_async( 51 | self, 52 | secret_id: str, 53 | ) -> None: 54 | pass 55 | -------------------------------------------------------------------------------- /onedocker/service/attestation_factory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from onedocker.entity.attestation_document import ( 10 | AttestationDocument, 11 | AttestationPolicy, 12 | PolicyName, 13 | ) 14 | from onedocker.service.attestation import AttestationService 15 | from onedocker.service.attestation_pc import PCAttestationService 16 | 17 | 18 | class AttestationFactoryService: 19 | def validate(self, attestation_document: str) -> bool: 20 | try: 21 | document = AttestationDocument.from_json(attestation_document) 22 | except Exception as e: 23 | raise ValueError( 24 | f"Failed to parse attestation document json, reason: {e}" 25 | ) from e 26 | attestation_svc = self._get_attestation_service(document.policy) 27 | return attestation_svc.validate(document.policy, document.measurements) 28 | 29 | def _get_attestation_service(self, policy: AttestationPolicy) -> AttestationService: 30 | if policy.policy_name == PolicyName.BINARY_MATCH: 31 | return PCAttestationService() 32 | else: 33 | raise NotImplementedError( 34 | f"Policy name {policy.policy_name} is NOT supported for now." 35 | ) 36 | -------------------------------------------------------------------------------- /fbpcp/service/log_cloudwatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Any, Dict, List, Optional 10 | 11 | from fbpcp.entity.container_instance import ContainerInstance 12 | from fbpcp.entity.log_event import LogEvent 13 | from fbpcp.gateway.cloudwatch import CloudWatchGateway 14 | from fbpcp.service.log import LogService 15 | 16 | 17 | class CloudWatchLogService(LogService): 18 | def __init__( 19 | self, 20 | log_group: str, 21 | region: str = "us-west-1", 22 | access_key_id: Optional[str] = None, 23 | access_key_data: Optional[str] = None, 24 | config: Optional[Dict[str, Any]] = None, 25 | ) -> None: 26 | self.cloudwatch_gateway = CloudWatchGateway( 27 | region, access_key_id, access_key_data, config 28 | ) 29 | self.log_group = log_group 30 | 31 | def fetch(self, log_path: str, start_time: int = 0) -> List[LogEvent]: 32 | """Fetch logs""" 33 | return self.cloudwatch_gateway.get_log_events( 34 | self.log_group, log_path, start_time 35 | ) 36 | 37 | def get_log_path(self, container_instance: ContainerInstance) -> str: 38 | return self.log_group[1:] + "/" + container_instance.instance_id.split("/")[-1] 39 | -------------------------------------------------------------------------------- /fbpcp/service/billing_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from datetime import date 10 | from typing import Any, Dict, Optional 11 | 12 | from fbpcp.entity.cloud_cost import CloudCost 13 | from fbpcp.gateway.costexplorer import CostExplorerGateway 14 | from fbpcp.service.billing import BillingService 15 | 16 | 17 | class AWSBillingService(BillingService): 18 | def __init__( 19 | self, 20 | access_key_id: Optional[str] = None, 21 | access_key_data: Optional[str] = None, 22 | config: Optional[Dict[str, Any]] = None, 23 | ) -> None: 24 | self.ce_gateway = CostExplorerGateway(access_key_id, access_key_data, config) 25 | 26 | def get_cost( 27 | self, start_date: date, end_date: date, region: Optional[str] = None 28 | ) -> CloudCost: 29 | """Get cost between start_date and end_date 30 | Keyword arguments: 31 | start_date: start date for cost 32 | end_date: end date for cost 33 | region: region name as optional filter for cost. 34 | """ 35 | date_format = "%Y-%m-%d" 36 | return self.ce_gateway.get_cost( 37 | start_date.strftime(date_format), 38 | end_date.strftime(date_format), 39 | region, 40 | ) 41 | -------------------------------------------------------------------------------- /fbpcp/entity/container_type.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from dataclasses import dataclass 10 | from enum import Enum 11 | from typing import Dict 12 | 13 | from fbpcp.entity.cloud_provider import CloudProvider 14 | from fbpcp.error.pcp import InvalidParameterError 15 | 16 | 17 | class ContainerType(Enum): 18 | SMALL = "SMALL" 19 | MEDIUM = "MEDIUM" 20 | LARGE = "LARGE" 21 | 22 | 23 | CONTAINER_TYPES: Dict[CloudProvider, Dict[ContainerType, Dict[str, int]]] = { 24 | CloudProvider.AWS: { 25 | ContainerType.SMALL: {"cpu": 1, "memory": 8}, 26 | ContainerType.MEDIUM: {"cpu": 4, "memory": 30}, 27 | ContainerType.LARGE: {"cpu": 16, "memory": 120}, 28 | } 29 | } 30 | 31 | 32 | @dataclass 33 | class ContainerTypeConfig: 34 | cpu: int # Number of vCPU 35 | memory: int # Memory in GB 36 | 37 | @classmethod 38 | def get_config( 39 | cls, cloud_provider: CloudProvider, container_type: ContainerType 40 | ) -> "ContainerTypeConfig": 41 | if cloud_provider != CloudProvider.AWS: 42 | raise InvalidParameterError( 43 | f"Cloud provider {cloud_provider} is not supported." 44 | ) 45 | return cls(**CONTAINER_TYPES[cloud_provider][container_type]) 46 | -------------------------------------------------------------------------------- /pce/gateway/ecs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from typing import Any, Dict, Optional 9 | 10 | import boto3 11 | import botocore 12 | from fbpcp.gateway.aws import AWSGateway 13 | from fbpcp.util.aws import split_container_definition 14 | from pce.mapper.aws import map_ecstaskdefinition_to_awslogsgroupname 15 | 16 | 17 | class ECSGateway(AWSGateway): 18 | def __init__( 19 | self, 20 | region: str, 21 | access_key_id: Optional[str] = None, 22 | access_key_data: Optional[str] = None, 23 | config: Optional[Dict[str, Any]] = None, 24 | ) -> None: 25 | super().__init__(region, access_key_id, access_key_data, config) 26 | 27 | self.client: botocore.client.BaseClient = boto3.client( 28 | "ecs", region_name=self.region, **self.config 29 | ) 30 | 31 | def extract_log_group_name(self, container_definition_id: str) -> Optional[str]: 32 | task_definition_arn = split_container_definition(container_definition_id)[0] 33 | 34 | response = self.client.describe_task_definition( 35 | taskDefinition=task_definition_arn 36 | ) 37 | log_group_name = map_ecstaskdefinition_to_awslogsgroupname( 38 | response["taskDefinition"] 39 | ) 40 | return log_group_name 41 | -------------------------------------------------------------------------------- /docs/MVCS.md: -------------------------------------------------------------------------------- 1 | # Why MVCS 2 | MVC was designed for single web application. In this type of app, backend usually exposes only one kind of endpoint which is web/http. In modern SOA or micro service architecture, it becomes more complex. One service may have to support different kinds of endpoints, such as web, mobile, and api (rest/thrift). For each set of endpoints, we need to implement a set of controllers. Code could be duplicated and difficult to manage and maintain. That’s why a new abstraction layer named Service was introduced and MVCS design pattern was defined. 3 | 4 | # What is MVCS 5 | MVCS defines the following components. 6 | 7 | Figure 1: MVCS 8 | 9 | * Handler: Exposes external endpoints, e.g. XController, GraphAPI, thrift etc. Handlers will address request related issues, like parameter validation, response generation, authentication, authorization, rate limiting, etc. It should not handle any business logic. (e.g. [GraphAPI design principles](https://developers.facebook.com/docs/graph-api/)). 10 | * Repository: Encapsulates database operations. 11 | * Gateway: Encapsulates interface of dependent services. 12 | * Mapper: Deals with data transformation between components, such as thrift object to entity and vice versa. 13 | * Entity: Represents business objects. 14 | * Service: Holds all business logic and exposes internal APIs to handlers or internal components within the same code base. There could be multiple services defined and implemented in this layer. 15 | -------------------------------------------------------------------------------- /fbpcp/gateway/cloudwatch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Any, Dict, List, Optional 10 | 11 | import boto3 12 | from botocore.client import BaseClient 13 | from fbpcp.decorator.error_handler import error_handler 14 | from fbpcp.entity.log_event import LogEvent 15 | from fbpcp.gateway.aws import AWSGateway 16 | 17 | 18 | class CloudWatchGateway(AWSGateway): 19 | def __init__( 20 | self, 21 | region: str, 22 | access_key_id: Optional[str] = None, 23 | access_key_data: Optional[str] = None, 24 | config: Optional[Dict[str, Any]] = None, 25 | ) -> None: 26 | super().__init__(region, access_key_id, access_key_data, config) 27 | self.client: BaseClient = boto3.client( 28 | "logs", region_name=self.region, **self.config 29 | ) 30 | 31 | @error_handler 32 | def get_log_events( 33 | self, 34 | log_group: str, 35 | log_stream: str, 36 | start_time: int = 0, 37 | ) -> List[LogEvent]: 38 | events = self.client.get_log_events( 39 | logGroupName=log_group, 40 | logStreamName=log_stream, 41 | startTime=start_time, 42 | )["events"] 43 | 44 | return [LogEvent(event["timestamp"], event["message"]) for event in events] 45 | -------------------------------------------------------------------------------- /onedocker/tests/service/test_measurement.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-unsafe 7 | 8 | import hashlib 9 | import unittest 10 | from unittest.mock import mock_open, patch 11 | 12 | from onedocker.entity.measurement import MeasurementType 13 | 14 | from onedocker.service.measurement import MeasurementService 15 | 16 | 17 | class TestMeasurementService(unittest.TestCase): 18 | def setUp(self) -> None: 19 | self.measurement_svc = MeasurementService() 20 | 21 | def test_generate_measurements(self): 22 | # Arrange 23 | test_binary = b"123" 24 | test_path = "test_path" 25 | test_measurement_types = [MeasurementType.sha256, MeasurementType.sha512] 26 | expect_hash_sha256 = hashlib.sha256(test_binary).hexdigest() 27 | expect_hash_sha512 = hashlib.sha512(test_binary).hexdigest() 28 | expect_res = { 29 | MeasurementType.sha256: expect_hash_sha256, 30 | MeasurementType.sha512: expect_hash_sha512, 31 | } 32 | open_mock = mock_open(read_data=test_binary) 33 | 34 | # Act 35 | with patch("builtins.open", open_mock, create=True): 36 | result = self.measurement_svc.generate_measurements( 37 | measurement_types=test_measurement_types, file_path=test_path 38 | ) 39 | 40 | # Assert 41 | self.assertEqual(expect_res, result) 42 | -------------------------------------------------------------------------------- /pce/gateway/logs_aws.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from typing import Any, Dict, Optional 9 | 10 | import boto3 11 | import botocore 12 | from fbpcp.gateway.aws import AWSGateway 13 | from pce.entity.log_group_aws import LogGroup 14 | 15 | 16 | class LogsGateway(AWSGateway): 17 | def __init__( 18 | self, 19 | region: str, 20 | access_key_id: Optional[str] = None, 21 | access_key_data: Optional[str] = None, 22 | config: Optional[Dict[str, Any]] = None, 23 | ) -> None: 24 | super().__init__(region, access_key_id, access_key_data, config) 25 | 26 | self.client: botocore.client.BaseClient = boto3.client( 27 | "logs", region_name=self.region, **self.config 28 | ) 29 | 30 | def describe_log_group(self, log_group_name: str) -> Optional[LogGroup]: 31 | response = self.client.describe_log_groups(logGroupNamePrefix=log_group_name) 32 | """ 33 | Only 1 log group name will be expected in this case 34 | Though this API returns up to 50 matches and a nextToken is required for pagination, the first match will be the exact log_group_name 35 | """ 36 | for group in response["logGroups"]: 37 | if group["logGroupName"] == log_group_name: 38 | log_group = LogGroup(log_group_name=group["logGroupName"]) 39 | return log_group 40 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to FBPCP 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Our Development Process 6 | ... (in particular how this is synced with internal changes to the project) 7 | 8 | ## Pull Requests 9 | We actively welcome your pull requests. 10 | 11 | 1. Fork the repo and create your branch from `master`. 12 | 2. If you've added code that should be tested, add tests. 13 | 3. If you've changed APIs, update the documentation. 14 | 4. Ensure the test suite passes. 15 | 5. Make sure your code lints. 16 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 17 | 18 | ## Contributor License Agreement ("CLA") 19 | In order to accept your pull request, we need you to submit a CLA. You only need 20 | to do this once to work on any of Facebook's open source projects. 21 | 22 | Complete your CLA here: 23 | 24 | ## Issues 25 | We use GitHub issues to track public bugs. Please ensure your description is 26 | clear and has sufficient instructions to be able to reproduce the issue. 27 | 28 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 29 | disclosure of security bugs. In those cases, please go through the process 30 | outlined on that page and do not file a public issue. 31 | 32 | ## Coding Style 33 | * 2 spaces for indentation rather than tabs 34 | * 80 character line length 35 | * ... 36 | 37 | ## License 38 | By contributing to FBPCS, you agree that your contributions will be licensed 39 | under the LICENSE file in the root directory of this source tree. 40 | -------------------------------------------------------------------------------- /tests/error/mapper/test_k8s.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.error.mapper.k8s import map_k8s_error 12 | from fbpcp.error.pcp import InvalidParameterError, PcpError, ThrottlingError 13 | from kubernetes.client.exceptions import ApiException, ApiTypeError, ApiValueError 14 | 15 | 16 | class TestMapK8SError(unittest.TestCase): 17 | def test_invalid_error(self): 18 | src_errs = [ 19 | ApiValueError( 20 | "Missing the required parameter `namespace` when calling `create_namespaced_job`" 21 | ), 22 | ApiTypeError( 23 | "Got an unexpected keyword argument 'unexpected_arg' to method get_api_group" 24 | ), 25 | ApiException(status=400, reason="Bad Request"), 26 | ] 27 | 28 | for src_err in src_errs: 29 | dst_err = map_k8s_error(src_err) 30 | self.assertIsInstance(dst_err, InvalidParameterError) 31 | 32 | def test_throttling_error(self): 33 | src_err = ApiException(status=429, reason="Too Many Requests") 34 | dst_err = map_k8s_error(src_err) 35 | 36 | self.assertIsInstance(dst_err, ThrottlingError) 37 | 38 | def test_default_pce_error(self): 39 | src_err = ApiException(status=503, reason="Service Unavailable") 40 | dst_err = map_k8s_error(src_err) 41 | 42 | self.assertIsInstance(dst_err, PcpError) 43 | -------------------------------------------------------------------------------- /onedocker/tests/entity/test_opawdl_state_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | 12 | from onedocker.entity.opawdl_state import OPAWDLState 13 | from onedocker.entity.opawdl_state_instance import OPAWDLStateInstance, Status 14 | 15 | 16 | class TestOPAWDLStateInstance(unittest.TestCase): 17 | def setUp(self) -> None: 18 | self.test_plugin_name = "test_plugin" 19 | self.test_cmd_args_list = ["-a=b", "-c=d"] 20 | self.test_opawdl_state = OPAWDLState( 21 | plugin_name=self.test_plugin_name, cmd_args_list=self.test_cmd_args_list 22 | ) 23 | self.test_opawdl_state_instance = OPAWDLStateInstance( 24 | opawdl_state=self.test_opawdl_state, status=Status.COMPLETED 25 | ) 26 | 27 | def test__str__(self) -> None: 28 | # Arrange 29 | expected_opawdl_state_instance_json_str = json.dumps( 30 | { 31 | "opawdl_state": { 32 | "PluginName": self.test_plugin_name, 33 | "CmdArgsList": self.test_cmd_args_list, 34 | "Timeout": None, 35 | "Next": None, 36 | "IsEnd": True, 37 | }, 38 | "status": "COMPLETED", 39 | } 40 | ) 41 | # Act and Assert 42 | self.assertEqual( 43 | str(self.test_opawdl_state_instance), 44 | expected_opawdl_state_instance_json_str, 45 | ) 46 | -------------------------------------------------------------------------------- /onedocker/tests/util/test_service_builder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | from unittest.mock import MagicMock, patch 11 | 12 | from fbpcp.entity.cloud_provider import CloudProvider 13 | from onedocker.util.service_builder import ( 14 | build_repository_service, 15 | METADATA_TABLE_KEY_NAME, 16 | METADATA_TABLES, 17 | ) 18 | 19 | TEST_CLOUD_PROVIDER: CloudProvider = CloudProvider.AWS 20 | TEST_ENV: str = "STAGING" 21 | 22 | 23 | class TestServiceBuilder(unittest.TestCase): 24 | @patch("onedocker.util.service_builder.MetadataService") 25 | @patch("onedocker.util.service_builder.OneDockerRepositoryService") 26 | @patch("onedocker.util.service_builder.S3StorageService") 27 | def test_build_repository_service( 28 | self, MockS3StorageService, MockRepositoryService, MockMetadataService 29 | ) -> None: 30 | # Arrange 31 | expected_repo_svc = MagicMock() 32 | MockRepositoryService.return_value = expected_repo_svc 33 | # Act 34 | repo_svc = build_repository_service(TEST_CLOUD_PROVIDER) 35 | # Assert 36 | MockS3StorageService.assert_called_once_with( 37 | "us-west-2", unsigned_enabled=False 38 | ) 39 | MockMetadataService.assert_called_once_with( 40 | region="us-west-2", 41 | table_name=METADATA_TABLES[TEST_ENV], 42 | key_name=METADATA_TABLE_KEY_NAME, 43 | ) 44 | self.assertEqual(repo_svc, expected_repo_svc) 45 | -------------------------------------------------------------------------------- /tests/entity/test_container_insight.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | 12 | from fbpcp.entity.container_insight import ContainerInsight 13 | 14 | 15 | class TestContainerInsight(unittest.TestCase): 16 | def setUp(self) -> None: 17 | self.test_time = 1674196156.00 18 | self.test_cluster_name = "pci-tests" 19 | self.test_instance_id = "a4e3bc43995f473697f7ede38699fcbe" 20 | self.test_status = "COMPLETED" 21 | self.test_exit_code = 1 22 | self.class_name = "ContainerInsight" 23 | 24 | self.test_container_insight = ContainerInsight( 25 | time=self.test_time, 26 | cluster_name=self.test_cluster_name, 27 | instance_id=self.test_instance_id, 28 | status=self.test_status, 29 | exit_code=self.test_exit_code, 30 | ) 31 | 32 | def test_convert_to_str_with_class_name(self) -> None: 33 | # Arrange 34 | expected_str = json.dumps( 35 | { 36 | "time": self.test_time, 37 | "cluster_name": self.test_cluster_name, 38 | "instance_id": self.test_instance_id, 39 | "status": self.test_status, 40 | "exit_code": self.test_exit_code, 41 | "class_name": self.class_name, 42 | } 43 | ) 44 | # Act 45 | actual_str = self.test_container_insight.convert_to_str_with_class_name() 46 | 47 | # Assert 48 | self.assertEqual(actual_str, expected_str) 49 | -------------------------------------------------------------------------------- /onedocker/tests/entity/test_opawdl_workflow.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | 12 | from onedocker.entity.opawdl_state import OPAWDLState 13 | from onedocker.entity.opawdl_workflow import OPAWDLWorkflow 14 | 15 | 16 | class TestOPAWDLWorkflow(unittest.TestCase): 17 | def setUp(self) -> None: 18 | self.test_plugin_name = "test_plugin" 19 | self.test_cmd_args_list = ["-a=b", "-c=d"] 20 | self.test_opawdl_state = OPAWDLState( 21 | plugin_name=self.test_plugin_name, cmd_args_list=self.test_cmd_args_list 22 | ) 23 | self.test_state_name = "state_1" 24 | self.test_opawdl_workflow = OPAWDLWorkflow( 25 | starts_at=self.test_state_name, 26 | states={self.test_state_name: self.test_opawdl_state}, 27 | ) 28 | 29 | def test__str__(self) -> None: 30 | # Arrange 31 | expected_opawdl_workflow_json_str = json.dumps( 32 | { 33 | "StartAt": self.test_state_name, 34 | "States": { 35 | self.test_state_name: { 36 | "PluginName": self.test_plugin_name, 37 | "CmdArgsList": self.test_cmd_args_list, 38 | "Timeout": None, 39 | "Next": None, 40 | "IsEnd": True, 41 | }, 42 | }, 43 | } 44 | ) 45 | # Act and Assert 46 | self.assertEqual( 47 | str(self.test_opawdl_workflow), expected_opawdl_workflow_json_str 48 | ) 49 | -------------------------------------------------------------------------------- /tests/error/mapper/test_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from botocore.exceptions import ClientError 12 | from fbpcp.error.mapper.aws import map_aws_error 13 | from fbpcp.error.pcp import PcpError, ThrottlingError 14 | 15 | 16 | class TestMapAwsError(unittest.TestCase): 17 | def test_pcs_error(self): 18 | request_id = "76f3e69f-0d46-436f-803a-5e88956b7308" 19 | err = ClientError( 20 | { 21 | "Error": { 22 | "Code": "Exception", 23 | "Message": "test", 24 | }, 25 | "ResponseMetadata": { 26 | "RequestId": request_id, 27 | "HTTPStatusCode": 400, 28 | }, 29 | }, 30 | "test", 31 | ) 32 | err = map_aws_error(err) 33 | 34 | self.assertIsInstance(err, PcpError) 35 | self.assertIn(request_id, str(err)) 36 | 37 | def test_throttling_error(self): 38 | request_id = "3e65a825-6a43-4261-b8a4-953972aa7065" 39 | err = ClientError( 40 | { 41 | "Error": { 42 | "Code": "ThrottlingException", 43 | "Message": "test", 44 | }, 45 | "ResponseMetadata": { 46 | "RequestId": request_id, 47 | "HTTPStatusCode": 400, 48 | }, 49 | }, 50 | "test", 51 | ) 52 | err = map_aws_error(err) 53 | 54 | self.assertIsInstance(err, ThrottlingError) 55 | self.assertIn(request_id, str(err)) 56 | -------------------------------------------------------------------------------- /onedocker/util/service_builder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Dict 10 | 11 | from fbpcp.entity.cloud_provider import CloudProvider 12 | from fbpcp.service.storage_s3 import S3StorageService 13 | from fbpcp.util.s3path import S3Path 14 | from onedocker.repository.onedocker_repository_service import OneDockerRepositoryService 15 | from onedocker.service.metadata import MetadataService 16 | 17 | PROD = "PROD" 18 | STAGING = "STAGING" 19 | REPOSITORY_PATHS: Dict[str, str] = { 20 | PROD: "https://one-docker-repository-prod.s3.us-west-2.amazonaws.com/", 21 | STAGING: "https://onedocker-repository-test.s3.us-west-2.amazonaws.com/", 22 | } 23 | 24 | METADATA_TABLE_KEY_NAME = "primary_key" 25 | METADATA_TABLES: Dict[str, str] = {STAGING: "metadata_test"} 26 | 27 | 28 | def build_repository_service( 29 | cloud_provider: CloudProvider, env: str = "STAGING", unsigned_enabled: bool = False 30 | ) -> OneDockerRepositoryService: 31 | if cloud_provider != CloudProvider.AWS: 32 | raise NotImplementedError( 33 | "Only AWS is supported for building Repository Service for now." 34 | ) 35 | repository_path = REPOSITORY_PATHS[env] 36 | region = S3Path(repository_path).region 37 | storage_svc = S3StorageService( 38 | region, 39 | unsigned_enabled=unsigned_enabled, 40 | ) 41 | metadata_svc = MetadataService( 42 | region=region, table_name=METADATA_TABLES[env], key_name=METADATA_TABLE_KEY_NAME 43 | ) 44 | return OneDockerRepositoryService( 45 | storage_svc=storage_svc, 46 | package_repository_path=repository_path, 47 | metadata_svc=metadata_svc, 48 | ) 49 | -------------------------------------------------------------------------------- /pce/validator/message_templates/validator_step_names.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | # patternlint-disable f-string-may-be-missing-leading-f 10 | from enum import Enum 11 | from typing import List 12 | 13 | 14 | class ValidationStepNames(Enum): 15 | """ 16 | Enumerates the names of validation steps, each step corresponding to a 17 | method of pce.validator.ValidationSuite called validate_{code_name} 18 | """ 19 | 20 | VPC_CIDR = ("VPC CIDR", "vpc_cidr") 21 | VPC_PEERING = ("VPC peering", "vpc_peering") 22 | FIREWALL = ("Firewall", "firewall") 23 | ROUTE_TABLE = ("Route table", "route_table") 24 | SUBNETS = ("Subnets", "subnets") 25 | CLUSTER_DEFINITION = ("Cluster definition", "cluster_definition") 26 | IAM_ROLES = ("IAM roles", "iam_roles") 27 | LOG_GROUP = ("Log group", "log_group") 28 | 29 | def __init__(self, formatted_name: str, code_name: str) -> None: 30 | """ 31 | set the Enum member values under better attribute names for 32 | easier access later eg ValidationStepNames.VPC_CIDR.code_name 33 | """ 34 | self.formatted_name = formatted_name 35 | self.code_name = code_name 36 | 37 | @staticmethod 38 | def code_names() -> List[str]: 39 | return [step.code_name for step in ValidationStepNames] 40 | 41 | @staticmethod 42 | def from_code_name(code_name: str) -> "ValidationStepNames": 43 | for step_name in ValidationStepNames: 44 | if step_name.code_name == code_name: 45 | return step_name 46 | raise ValueError( 47 | f"No ValidationStepName member exists with code_name={code_name}" 48 | ) 49 | -------------------------------------------------------------------------------- /onedocker/service/metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Any, Dict, Optional 10 | 11 | from onedocker.entity.metadata import PackageMetadata 12 | from onedocker.gateway.dynamodb import DynamoDBGateway 13 | from onedocker.mapper.aws import map_dynamodbitem_to_packagemetadata 14 | 15 | 16 | class MetadataService: 17 | def __init__( 18 | self, 19 | region: str, 20 | table_name: str, 21 | key_name: str, 22 | access_key_id: Optional[str] = None, 23 | access_key_data: Optional[str] = None, 24 | config: Optional[Dict[str, Any]] = None, 25 | ) -> None: 26 | self.table_name = table_name 27 | self.key_name = key_name 28 | self.dynamodb_gateway = DynamoDBGateway( 29 | region, access_key_id, access_key_data, config 30 | ) 31 | 32 | def _build_key(self, package_name: str, version: str) -> str: 33 | return f"{package_name}#{version}" 34 | 35 | def get_medadata(self, package_name: str, version: str) -> PackageMetadata: 36 | dynamodb_item = self.dynamodb_gateway.get_item( 37 | table_name=self.table_name, 38 | key_name=self.key_name, 39 | key_value=self._build_key(package_name=package_name, version=version), 40 | ) 41 | 42 | return map_dynamodbitem_to_packagemetadata(dynamodb_item=dynamodb_item) 43 | 44 | def put_metadata(self, metadata: PackageMetadata) -> None: 45 | md_dict = metadata.to_dict() 46 | md_dict.update( 47 | {self.key_name: self._build_key(metadata.package_name, metadata.version)} 48 | ) 49 | self.dynamodb_gateway.put_item(table_name=self.table_name, item=md_dict) 50 | -------------------------------------------------------------------------------- /.github/workflows/publish_project.yml: -------------------------------------------------------------------------------- 1 | name: Build and Publish Python Distributions to PyPI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | paths: 7 | - 'setup.py' 8 | jobs: 9 | check-package-version: 10 | name: Check Package Version 11 | runs-on: ubuntu-latest 12 | outputs: 13 | version_check: ${{ steps.version_check.outputs.version_check }} 14 | steps: 15 | - uses: actions/checkout@master 16 | - name: Set up Python 3.8 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: 3.8 20 | - name: Install packages 21 | run: >- 22 | python3 -m 23 | pip install 24 | requests packaging 25 | - name: Package version check 26 | run: >- 27 | python3 ./scripts/compare_package_version.py 28 | - name: Get version check output 29 | id: version_check 30 | run: >- 31 | echo "version_check=$(python3 ./scripts/compare_package_version.py | tail -1)" >> $GITHUB_OUTPUT 32 | 33 | build-n-publish: 34 | name: Build and publish distributions to PyPI 35 | runs-on: ubuntu-latest 36 | needs: check-package-version 37 | if : needs.check-package-version.outputs.version_check == 'higher' 38 | steps: 39 | - uses: actions/checkout@master 40 | - name: Set up Python 3.8 41 | uses: actions/setup-python@v4 42 | with: 43 | python-version: 3.8 44 | - name: Install pypa/build 45 | run: >- 46 | python3 -m 47 | pip install 48 | build 49 | --user 50 | - name: Build a binary wheel and a source tarball 51 | run: >- 52 | python -m 53 | build 54 | --sdist 55 | --wheel 56 | --outdir dist/ 57 | . 58 | - name: Publish distribution 📦 to PyPI 59 | uses: pypa/gh-action-pypi-publish@release/v1 60 | with: 61 | user: __token__ 62 | password: ${{ secrets.PYPI_API_TOKEN }} 63 | -------------------------------------------------------------------------------- /scripts/compare_package_version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # Usage: this script is to run from Github Actions Workflow 7 | import re 8 | import sys 9 | 10 | import requests 11 | from packaging.version import parse as version_parse 12 | 13 | 14 | package = "fbpcp" 15 | setup_file_path = "./setup.py" 16 | 17 | 18 | def get_setup_version(file_path: str) -> str: 19 | setup_text = open(file_path).read().strip() 20 | version = re.search("version=['\"]([^'\"]*)['\"]", setup_text) 21 | 22 | return version.group(1) 23 | 24 | 25 | def get_pypi_version(package: str) -> str: 26 | try: 27 | pypi_package_url = f"https://pypi.python.org/pypi/{package}/json" 28 | response = requests.get(pypi_package_url) 29 | except requests.exceptions.RequestException as e: 30 | print(f"{str(e)}") 31 | sys.exit(1) 32 | if response is not None: 33 | response_json = response.json() 34 | version = response_json["info"]["version"] 35 | 36 | return version 37 | 38 | 39 | def main() -> None: 40 | setup_version = get_setup_version(setup_file_path) 41 | pypi_version = get_pypi_version(package) 42 | if version_parse(setup_version) > version_parse(pypi_version): 43 | print(f"setup.py {setup_version} is higher than Pypi version {pypi_version}") 44 | print("higher") 45 | elif version_parse(setup_version) == version_parse(pypi_version): 46 | print(f"setup.py {setup_version} is equal to Pypi version {pypi_version}") 47 | print("equal") 48 | else: 49 | print( 50 | f"Error: setup.py {setup_version} is lower than to Pypi version {pypi_version}, this is not exepected." 51 | ) 52 | # when exit code is 1, the workflow will stop and raise an error 53 | sys.exit(1) 54 | 55 | 56 | if __name__ == "__main__": 57 | main() 58 | -------------------------------------------------------------------------------- /.github/workflows/integ_e2e_tests.yml: -------------------------------------------------------------------------------- 1 | name: Integration and E2E tests 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | pce_id: 7 | description: PCE ID 8 | required: false 9 | type: string 10 | default: "test-pci-partner" 11 | 12 | workflow_call: 13 | inputs: 14 | pce_id: 15 | description: PCE ID 16 | required: false 17 | type: string 18 | default: "test-pci-partner" 19 | 20 | 21 | jobs: 22 | pce_validator_tests: 23 | name: PCE validator E2E test 24 | runs-on: ubuntu-latest 25 | timeout-minutes: 15 26 | permissions: 27 | id-token: write 28 | contents: read 29 | steps: 30 | - uses: actions/checkout@v3 31 | 32 | - name: Set up Python 3.8 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: 3.8 36 | 37 | - name: Install Python packages 38 | run: | 39 | python3 -m pip install --upgrade pip 40 | python3 -m pip install . 41 | 42 | - name: Configure AWS credentials 43 | uses: aws-actions/configure-aws-credentials@v2 44 | with: 45 | role-to-assume: ${{ secrets.AWS_PCE_VALIDATOR_ROLE_TO_ASSUME }} 46 | aws-region: us-west-1 47 | role-duration-seconds: 1200 48 | 49 | - name: PCE validator runner 50 | id: runner 51 | run: python3 ./pce/validator/validator.py --region="us-west-1" --pce-id=${{ inputs.pce_id }} 52 | 53 | onedocker_runner_tests: 54 | name: Onedocker Runner tests 55 | runs-on: ubuntu-latest 56 | timeout-minutes: 15 57 | steps: 58 | - uses: actions/checkout@v3 59 | 60 | - name: Set up Python 3.8 61 | uses: actions/setup-python@v4 62 | with: 63 | python-version: 3.8 64 | 65 | - name: Install Python packages 66 | run: | 67 | python3 -m pip install --upgrade pip 68 | python3 -m pip install . 69 | 70 | - name: Ondocker runner 71 | id: onedocker_runner 72 | run: python3.8 -m onedocker.script.runner ls --version=latest --repository_path=local --exe_path="/usr/bin/" 73 | -------------------------------------------------------------------------------- /pce/gateway/iam.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from re import sub 9 | from typing import Any, Dict, Optional 10 | 11 | import boto3 12 | from fbpcp.gateway.aws import AWSGateway 13 | from pce.entity.iam_role import IAMRole, RoleId, RoleName 14 | from pce.mapper.aws import map_attachedrolepolicies_to_rolepolicies 15 | 16 | 17 | class IAMGateway(AWSGateway): 18 | def __init__( 19 | self, 20 | region: str, 21 | access_key_id: Optional[str] = None, 22 | access_key_data: Optional[str] = None, 23 | config: Optional[Dict[str, Any]] = None, 24 | ) -> None: 25 | super().__init__(region, access_key_id, access_key_data, config) 26 | 27 | # pyre-ignore 28 | self.client = boto3.client("iam", region_name=self.region, **self.config) 29 | 30 | @classmethod 31 | def _role_id_to_name(cls, role_id: RoleId) -> RoleName: 32 | return sub(r".*?:role/", "", role_id) 33 | 34 | def get_policies_for_role( 35 | self, 36 | role_id: RoleId, 37 | ) -> Optional[IAMRole]: 38 | return map_attachedrolepolicies_to_rolepolicies( 39 | role_id, 40 | { 41 | policy["PolicyName"]: 42 | # Retrieving the policy document requires retrieving the version id for each policy arn, hence `get_policy` has to be called for each `get_policy_version` invocation 43 | self.client.get_policy_version( 44 | PolicyArn=policy["PolicyArn"], 45 | VersionId=self.client.get_policy(PolicyArn=policy["PolicyArn"])[ 46 | "Policy" 47 | ]["DefaultVersionId"], 48 | )["PolicyVersion"]["Document"] 49 | for policy_result in self.client.get_paginator( 50 | "list_attached_role_policies" 51 | ).paginate(RoleName=self._role_id_to_name(role_id)) 52 | for policy in policy_result["AttachedPolicies"] 53 | }, 54 | ) 55 | -------------------------------------------------------------------------------- /fbpcp/service/storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import abc 10 | import re 11 | from enum import Enum 12 | from typing import List 13 | 14 | from fbpcp.entity.file_information import FileInfo 15 | from fbpcp.entity.policy_statement import PolicyStatement, PublicAccessBlockConfig 16 | 17 | 18 | class PathType(Enum): 19 | Local = 1 20 | S3 = 2 21 | GCS = 3 22 | 23 | 24 | class StorageService(abc.ABC): 25 | @abc.abstractmethod 26 | def read(self, filename: str) -> str: 27 | pass 28 | 29 | @abc.abstractmethod 30 | def write(self, filename: str, data: str) -> None: 31 | pass 32 | 33 | @abc.abstractmethod 34 | def copy(self, source: str, destination: str) -> None: 35 | pass 36 | 37 | @abc.abstractmethod 38 | def file_exists(self, filename: str) -> bool: 39 | pass 40 | 41 | @staticmethod 42 | def path_type(filename: str) -> PathType: 43 | s3_match = re.search( 44 | "^https?:/([^.]+).s3.([^.]+).amazonaws.com/(.*)$", filename 45 | ) 46 | if s3_match: 47 | return PathType.S3 48 | 49 | gcs_match = re.search("^https?://storage.cloud.google.com/(.*)$", filename) 50 | if gcs_match: 51 | return PathType.GCS 52 | 53 | return PathType.Local 54 | 55 | @abc.abstractmethod 56 | def get_file_size(self, filename: str) -> int: 57 | pass 58 | 59 | @abc.abstractmethod 60 | def get_file_info(self, filename: str) -> FileInfo: 61 | pass 62 | 63 | @abc.abstractmethod 64 | def list_folders(self, filename: str) -> List[str]: 65 | pass 66 | 67 | @abc.abstractmethod 68 | def get_bucket_policy_statements(self, bucket: str) -> List[PolicyStatement]: 69 | pass 70 | 71 | @abc.abstractmethod 72 | def get_bucket_public_access_block(self, bucket: str) -> PublicAccessBlockConfig: 73 | pass 74 | 75 | @abc.abstractmethod 76 | def list_files(self, dirPath: str) -> List[str]: 77 | pass 78 | -------------------------------------------------------------------------------- /onedocker/repository/opawdl_workflow_instance_repository_local.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from pathlib import Path 10 | 11 | from onedocker.entity.opawdl_workflow_instance import OPAWDLWorkflowInstance 12 | from onedocker.repository.opawdl_workflow_instance_repository import ( 13 | OPAWDLWorkflowInstanceRepository, 14 | ) 15 | 16 | 17 | class LocalOPAWDLWorkflowInstanceRepository(OPAWDLWorkflowInstanceRepository): 18 | def __init__(self, base_dir: str) -> None: 19 | self.base_dir = Path(base_dir) 20 | 21 | def exist(self, instance_id: str) -> bool: 22 | return self.base_dir.joinpath(instance_id).exists() 23 | 24 | def create(self, instance: OPAWDLWorkflowInstance) -> None: 25 | if self.exist(instance.get_instance_id()): 26 | raise Exception( 27 | f"Fail to create the workflow instance: {instance.get_instance_id()} already exists." 28 | ) 29 | 30 | path = self.base_dir.joinpath(instance.get_instance_id()) 31 | with open(path, "w") as f: 32 | f.write(str(instance)) 33 | 34 | def get(self, instance_id: str) -> OPAWDLWorkflowInstance: 35 | if not self.exist(instance_id): 36 | raise Exception(f"{instance_id} does NOT exist") 37 | 38 | path = self.base_dir.joinpath(instance_id) 39 | with open(path, "r") as f: 40 | return OPAWDLWorkflowInstance.from_json((f.read().strip())) 41 | 42 | def update(self, instance: OPAWDLWorkflowInstance) -> None: 43 | if not self.exist(instance.get_instance_id()): 44 | raise Exception(f"{instance.get_instance_id()} does not exist") 45 | 46 | path = self.base_dir.joinpath(instance.get_instance_id()) 47 | with open(path, "w") as f: 48 | f.write(str(instance)) 49 | 50 | def delete(self, instance_id: str) -> None: 51 | if not self.exist(instance_id): 52 | raise Exception(f"{instance_id} does not exist") 53 | 54 | self.base_dir.joinpath(instance_id).unlink() 55 | -------------------------------------------------------------------------------- /tests/gateway/test_kms.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env fbpython 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import base64 10 | import base64 as b64 11 | import unittest 12 | from unittest.mock import MagicMock, patch 13 | 14 | from fbpcp.gateway.kms import KMSGateway 15 | 16 | 17 | class TestKMSGateway(unittest.TestCase): 18 | REGION = "us-west-2" 19 | TEST_ACCESS_KEY_ID = "test-access-key-id" 20 | TEST_ACCESS_KEY_DATA = "test-access-key-data" 21 | 22 | @patch("boto3.client") 23 | def setUp(self, BotoClient) -> None: 24 | self.kms = KMSGateway( 25 | region=self.REGION, 26 | access_key_id=self.TEST_ACCESS_KEY_ID, 27 | access_key_data=self.TEST_ACCESS_KEY_DATA, 28 | ) 29 | self.kms.client = BotoClient() 30 | 31 | def test_sign(self) -> None: 32 | # Arrange 33 | sign_args = { 34 | "key_id": "test_key_id", 35 | "message": "test_message", 36 | "message_type": "test_message_type", 37 | "grant_tokens": [], 38 | "signing_algorithm": "", 39 | } 40 | signed_message = "test_signed_message" 41 | self.kms.client.sign = MagicMock( 42 | return_value={"Signature": signed_message.encode()} 43 | ) 44 | 45 | # Act 46 | b64_signature = self.kms.sign(**sign_args) 47 | signature = b64.b64decode(b64_signature.encode()).decode() 48 | 49 | # Assert 50 | self.assertEqual(signature, signed_message) 51 | 52 | def test_verify(self) -> None: 53 | # Arrange 54 | verify_args = { 55 | "key_id": "test_key_id", 56 | "message": "test_message", 57 | "message_type": "test_message_type", 58 | "signature": "dGVzdF9tZXNzYWdl", 59 | "grant_tokens": [], 60 | "signing_algorithm": "", 61 | } 62 | self.kms.client.verify = MagicMock(return_value={"SignatureValid": True}) 63 | 64 | # Act 65 | verification = self.kms.verify(**verify_args) 66 | 67 | # Assert 68 | self.assertTrue(verification) 69 | -------------------------------------------------------------------------------- /fbpcp/gateway/kms.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from base64 import b64decode, b64encode 10 | from typing import Any, Dict, List, Optional 11 | 12 | import boto3 13 | from botocore.client import BaseClient 14 | from fbpcp.decorator.error_handler import error_handler 15 | from fbpcp.gateway.aws import AWSGateway 16 | 17 | 18 | class KMSGateway(AWSGateway): 19 | def __init__( 20 | self, 21 | region: str, 22 | access_key_id: Optional[str] = None, 23 | access_key_data: Optional[str] = None, 24 | config: Optional[Dict[str, Any]] = None, 25 | ) -> None: 26 | super().__init__(region, access_key_id, access_key_data, config) 27 | self.client: BaseClient = boto3.client( 28 | "kms", region_name=self.region, **self.config 29 | ) 30 | 31 | @error_handler 32 | def sign( 33 | self, 34 | key_id: str, 35 | message: str, 36 | message_type: str, 37 | grant_tokens: List[str], 38 | signing_algorithm: str, 39 | ) -> str: 40 | response = self.client.sign( 41 | KeyId=key_id, 42 | Message=message.encode(), 43 | MessageType=message_type, 44 | GrantTokens=grant_tokens, 45 | SigningAlgorithm=signing_algorithm, 46 | ) 47 | signature = b64encode(response["Signature"]).decode() 48 | return signature 49 | 50 | @error_handler 51 | def verify( 52 | self, 53 | key_id: str, 54 | message: str, 55 | message_type: str, 56 | signature: str, 57 | signing_algorithm: str, 58 | grant_tokens: List[str], 59 | ) -> bool: 60 | b64_signature = b64decode(signature.encode()) 61 | response = self.client.verify( 62 | KeyId=key_id, 63 | Message=message.encode(), 64 | MessageType=message_type, 65 | Signature=b64_signature, 66 | SigningAlgorithm=signing_algorithm, 67 | GrantTokens=grant_tokens, 68 | ) 69 | return response["SignatureValid"] 70 | -------------------------------------------------------------------------------- /pce/README.md: -------------------------------------------------------------------------------- 1 | # PCE (Private Computation Environment) 2 | PCE is the environment to run [Multi Party Computation games](https://en.wikipedia.org/wiki/Secure_multi-party_computation). This environment includes two major components: networking and compute. 3 | 4 | Networking: a virtual private cloud with subnets, routing table and firewalls setting. In order to keep the communication private, it is also required the two parties virtual clouds can communicate privately. 5 | Compute: a container service with sufficient CPU, memory and enough permissions to interact with other services 6 | 7 | ## PCE ID: 8 | PCE ID is the identifier for a PCE. It should be unique within a given Cloud account. 9 | 10 | # PCE Validator 11 | PCE Validator is to verify above PCE components are set up correctly. It will check all the resources for Networking and Compute are setup correctly and raise warning/error if there is a mis-configuration. Partner can use this tool to verify PCE environment before running a game. 12 | 13 | Currently, PCE Validator only supports AWS(Amazon Web Services). You should tag all your resource for the PCE with {"pce:pce-id": ""} in order to run validator. 14 | 15 | ## MPC Roles 16 | MPC(Multi Party Computation) Roles includes publisher and partner. If you donot pass in a role, it will use partner role by default. All advertisers need to use partner role, publisher role is for advertising provider. 17 | 18 | 19 | ## Installing PCE 20 | You need to install through fbpcp [README](https://github.com/facebookresearch/fbpcp/blob/main/README.md). 21 | 22 | ## PCE Validator Usage 23 | python3.8 -m pce.validator --region= --key-id= --key-data= --pce-id= --role= 24 | 25 | Example1: for resources tagged with {"pce:pce-id": "test-pce-tag-value"} 26 | python3.8 -m pce.validator --region=us-west-2 --key-id=AWS_ACCESS_KEY_ID --key-data=AWS_SECRET_ACCESS_KEY --pce-id="test-pce-tag-value" 27 | 28 | Example2: if your host has AWS environment variables setup, you can remove --key-id= and --key-data 29 | python3.8 -m pce.validator --region=us-west-2 --pce-id="test-pce-tag-value" 30 | 31 | Example3: you can optionally pass in the role, the output will stay the same 32 | python3.8 -m pce.validator --region=us-west-2 --pce-id="test-pce-tag-value" --role partner 33 | -------------------------------------------------------------------------------- /docs/FBPCPComponents.md: -------------------------------------------------------------------------------- 1 | ### Components: 2 | Facebook Private Computation Platform follows [MVCS(Model View Controller Service)](MVCS.md) design pattern. 3 | 4 | ### Repository 5 | Repository is responsible for encapsulating database-like operations. In our design, we have MPC instance repositories for both Amazon S3 and local storage. The end point service will call MPC service to create an MPC instance and all the files and information related to this instance will be stored on Amazon S3 or local storage, depending on which repository the end point service is using. 6 | 7 | ### Gateway: 8 | Gateway is responsible for encapsulating the interface of dependent services, which is AWS API in our design. Since we need to run tasks on ECS and store files on S3, it is required to call AWS API to do the operations and these api calls reside in the gateways. 9 | 10 | ### Mapper: 11 | Mapper deals with data transformation between components. Any response from AWS API calls should be mapped to the data we self defined. 12 | 13 | ### Entity: 14 | Entity represents business objects, in our case, the MPC Instance, Container Instance and Cluster Instance, etc. In our design: 15 | 16 | MPC Instance contains information about a MPC game. For example, MPC game name, ECS fargate containers running the tasks, etc. 17 | 18 | Container Instance contains information about a container on an ECS cluster. For example, the instance id, ip address and container status. 19 | 20 | ### Service: 21 | MPCService is the public interface that FBPCP provides. All other services are internal only so subject to changes. 22 | 23 | Service holds all business logic and exposes internal APIs to controllers or other services within the same code base. Besides MPC Sevice, MPC Game Service and OneDocker Service: 24 | 25 | * OneDockerService is a cloud agnostic, serverless container management service. Currently, it supports AWS ECS. 26 | 27 | * MPCGameService bridges MPCService and OneDockerService together. Given a MPC game and it's arguments, MPCGameService transforms them to OneDocker arguments. 28 | 29 | * ContainerService is a generic interface that each cloud may extend to implement a concrete container service. Currently, we support AWS ECS. 30 | 31 | * Storage Service provides APIs to do CRUD operations on a particular storage, such as local and S3. 32 | -------------------------------------------------------------------------------- /onedocker/service/attestation_pc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | 10 | import logging 11 | from typing import Dict 12 | 13 | from fbpcp.error.pcp import InvalidParameterError 14 | 15 | from onedocker.entity.attestation_document import AttestationPolicy, PolicyName 16 | from onedocker.gateway.repository_service import RepositoryServiceGateway 17 | from onedocker.service.attestation import AttestationService 18 | 19 | 20 | class PCAttestationService(AttestationService): 21 | def __init__(self) -> None: 22 | self.repository_service_gateway = RepositoryServiceGateway() 23 | self.logger: logging.Logger = logging.getLogger(__name__) 24 | 25 | def validate(self, policy: AttestationPolicy, measurements: Dict[str, str]) -> bool: 26 | if policy.policy_name != PolicyName.BINARY_MATCH: 27 | raise NotImplementedError("Only BINARY_MATCH policy is supported for now.") 28 | if policy.params.package_name is None or policy.params.version is None: 29 | raise InvalidParameterError( 30 | "Package name and version must be specified in policy to perform binary match." 31 | ) 32 | return self.binary_match( 33 | policy.params.package_name, policy.params.version, measurements 34 | ) 35 | 36 | def binary_match( 37 | self, package_name: str, version: str, measurements: Dict[str, str] 38 | ) -> bool: 39 | allowlist = self.repository_service_gateway.get_measurements( 40 | package_name, version 41 | ) 42 | for measurement_key, measurement_value in measurements.items(): 43 | if measurement_key not in allowlist: 44 | self.logger.error( 45 | f"Cannot find measurement with key '{measurement_key}' in allowlist." 46 | ) 47 | return False 48 | if allowlist[measurement_key] != measurement_value: 49 | self.logger.error( 50 | f"Measurement with key '{measurement_key}' does not match the allowed value." 51 | ) 52 | return False 53 | return True 54 | -------------------------------------------------------------------------------- /pce/validator/message_templates/warning_message_templates.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | # patternlint-disable f-string-may-be-missing-leading-f 10 | 11 | from enum import Enum 12 | 13 | from pce.validator.message_templates.pce_standard_constants import ( 14 | FIREWALL_RULE_FINAL_PORT, 15 | FIREWALL_RULE_INITIAL_PORT, 16 | ) 17 | 18 | # for scenarios should raise up warning messages are 19 | # 1) the current PCE setup is not a blocker for running study, however not fully follow the PCE set up standard. 20 | # 2) run time components are still in pending status 21 | 22 | 23 | class NetworkingValidationWarningDescriptionTemplate(Enum): 24 | NETWORKING_VPC_PEERING_PEERING_NOT_READY = ( 25 | "VPC Peering Connection request is pending acceptance." 26 | ) 27 | NETWORKING_FIREWALL_CIDR_EXCEED_EXPECTED_RANGE = f"Ingress cidr {{fr_vpc_id}}:{{fri_cidr}}:{{fri_from_port}}-{{fri_to_port}} exceeds the expected port range {FIREWALL_RULE_INITIAL_PORT}-{FIREWALL_RULE_FINAL_PORT}" 28 | NETWORKING_FIREWALL_FLAGGED_RULESETS = ( 29 | "These issues are not fatal but are worth noticing: {warning_reasons}" 30 | ) 31 | 32 | 33 | class ValidationWarningDescriptionTemplate(Enum): 34 | CLUSTER_DEFINITION_FLAGGED_VALUE = ( 35 | "{resource_name} value '{value}' is not expected, should be '{expected_value}'." 36 | ) 37 | CLUSTER_DEFINITION_FLAGGED_VALUES = ( 38 | "Container has outlier values which are non-fatal: {warning_reasons}" 39 | ) 40 | MORE_POLICIES_THAN_EXPECTED = ( 41 | "Policies {policy_names} attached to {role_id} are not expected." 42 | ) 43 | CLOUDWATCH_LOGS_NOT_CONFIGURED_IN_TASK_DEFINITION = ( 44 | "CloudWatch logs are not configured in ECS task definition." 45 | ) 46 | CLOUDWATCH_LOGS_NOT_FOUND = "CloudWatch logs are not found, please check if log group name {log_group_name_from_task} is correct or if this log group gets deleted " 47 | 48 | 49 | class ValidationWarningSolutionHintTemplate(Enum): 50 | VPC_PEERING_PEERING_NOT_READY = "Please work with owner of Acceptor VPC (VPC ID: {accepter_vpc_id}) to accept peering request." 51 | MORE_POLICIES_THAN_EXPECTED = ( 52 | "Consider removing additional policies to strengthen security." 53 | ) 54 | -------------------------------------------------------------------------------- /pce/gateway/tests/test_logs_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from unittest import TestCase 10 | from unittest.mock import MagicMock, patch 11 | 12 | from pce.entity.log_group_aws import LogGroup 13 | from pce.gateway.logs_aws import LogsGateway 14 | 15 | 16 | class TestLogsGateway(TestCase): 17 | REGION = "us-west-1" 18 | TEST_LOG_GROUP_NAME = "/ecs/test-log-group-name" 19 | TEST_LOG_GROUP_ARN = ( 20 | f"arn:aws:logs:{REGION}:123456789012:log-group:{TEST_LOG_GROUP_NAME}:*" 21 | ) 22 | TEST_ADDITIONAL_LOG_GROUP_NAME = "/ecs/test-log_group-name" 23 | TEST_ADDITIONAL_LOG_GROUP_ARN = ( 24 | f"arn:aws:logs:{REGION}:123456789012:log-group:{TEST_LOG_GROUP_NAME}:*" 25 | ) 26 | 27 | @patch("boto3.client") 28 | def setUp(self, mock_boto_client: MagicMock) -> None: 29 | self.aws_logs = MagicMock() 30 | mock_boto_client.return_value = self.aws_logs 31 | self.logs = LogsGateway(region=self.REGION) 32 | 33 | def test_describe_log_group(self) -> None: 34 | test_log_groups_response = { 35 | "logGroups": [ 36 | { 37 | "logGroupName": self.TEST_LOG_GROUP_NAME, 38 | "creationTime": 1640137658065, 39 | "metricFilterCount": 0, 40 | "arn": self.TEST_LOG_GROUP_ARN, 41 | "storedBytes": 0, 42 | }, 43 | { 44 | "logGroupName": self.TEST_ADDITIONAL_LOG_GROUP_NAME, 45 | "creationTime": 1633025558042, 46 | "metricFilterCount": 0, 47 | "arn": self.TEST_ADDITIONAL_LOG_GROUP_ARN, 48 | "storedBytes": 980, 49 | }, 50 | ] 51 | } 52 | self.aws_logs.describe_log_groups = MagicMock( 53 | return_value=test_log_groups_response 54 | ) 55 | exist_log_group = self.logs.describe_log_group( 56 | log_group_name=self.TEST_LOG_GROUP_NAME 57 | ) 58 | 59 | expected_log_group = LogGroup( 60 | log_group_name=self.TEST_LOG_GROUP_NAME, 61 | ) 62 | 63 | self.assertEqual(exist_log_group, expected_log_group) 64 | self.aws_logs.describe_log_group.assert_called 65 | -------------------------------------------------------------------------------- /tests/decorator/test_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | from unittest import IsolatedAsyncioTestCase 11 | from unittest.mock import ANY, patch 12 | 13 | from fbpcp.decorator.metrics import duration_time, error_counter, request_counter 14 | from fbpcp.metrics.getter import MetricsGetter 15 | 16 | METRICS_NAME = "test_metrics" 17 | 18 | 19 | class TestMetrics(MetricsGetter): 20 | def __init__(self, metrics): 21 | self.metrics = metrics 22 | 23 | def has_metrics(self): 24 | return True 25 | 26 | def get_metrics(self): 27 | return self.metrics 28 | 29 | @duration_time(METRICS_NAME) 30 | @request_counter(METRICS_NAME) 31 | def test_sync(self): 32 | pass 33 | 34 | @duration_time(METRICS_NAME) 35 | @request_counter(METRICS_NAME) 36 | async def test_async(self): 37 | pass 38 | 39 | @error_counter(METRICS_NAME) 40 | def test_error_sync(self): 41 | raise ValueError("test") 42 | 43 | @error_counter(METRICS_NAME) 44 | async def test_error_async(self): 45 | raise ValueError("test") 46 | 47 | 48 | class TestMetricsDecoratorSync(unittest.TestCase): 49 | @patch("fbpcp.metrics.emitter.MetricsEmitter") 50 | def test_sync(self, MockMetricsEmitter): 51 | metrics = MockMetricsEmitter() 52 | test_metrics = TestMetrics(metrics) 53 | test_metrics.test_sync() 54 | metrics.count.assert_called_with(METRICS_NAME, 1) 55 | metrics.gauge.assert_called_with(METRICS_NAME, ANY) 56 | with self.assertRaises(ValueError): 57 | test_metrics.test_error_sync() 58 | metrics.count.assert_called_with(METRICS_NAME, 1) 59 | 60 | 61 | class TestMetricsDecoratorAsync(IsolatedAsyncioTestCase): 62 | @patch("fbpcp.metrics.emitter.MetricsEmitter") 63 | async def test_async(self, MockMetricsEmitter): 64 | metrics = MockMetricsEmitter() 65 | test_metrics = TestMetrics(metrics) 66 | await test_metrics.test_async() 67 | metrics.count.assert_called_with(METRICS_NAME, 1) 68 | metrics.gauge.assert_called_with(METRICS_NAME, ANY) 69 | with self.assertRaises(ValueError): 70 | await test_metrics.test_error_async() 71 | metrics.count.assert_called_with(METRICS_NAME, 1) 72 | -------------------------------------------------------------------------------- /fbpcp/service/secrets_manager_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import asyncio 10 | 11 | from typing import Any, Dict, Optional 12 | 13 | from fbpcp.entity.secret import StringSecret 14 | 15 | from fbpcp.gateway.secrets_manager import AWSSecretsManagerGateway 16 | from fbpcp.service.secrets_manager import SecretsManagerService 17 | 18 | 19 | class AWSSecretsManagerService(SecretsManagerService): 20 | def __init__( 21 | self, 22 | region: str, 23 | access_key_id: Optional[str] = None, 24 | access_key_data: Optional[str] = None, 25 | config: Optional[Dict[str, Any]] = None, 26 | ) -> None: 27 | self.secret_gateway = AWSSecretsManagerGateway( 28 | region, access_key_id, access_key_data, config 29 | ) 30 | 31 | def create_secret( 32 | self, 33 | secret_name: str, 34 | secret_value: str, 35 | tags: Optional[Dict[str, str]] = None, 36 | ) -> str: 37 | secret_id = self.secret_gateway.create_secret( 38 | secret_name=secret_name, secret_value=secret_value, tags=tags 39 | ) 40 | 41 | return secret_id 42 | 43 | def get_secret( 44 | self, 45 | secret_id: str, 46 | ) -> StringSecret: 47 | # secret id can be ARN or secret name 48 | secret = self.secret_gateway.get_secret(secret_id=secret_id) 49 | 50 | return secret 51 | 52 | async def create_secret_async( 53 | self, 54 | secret_name: str, 55 | secret_value: str, 56 | tags: Optional[Dict[str, str]] = None, 57 | ) -> str: 58 | loop = asyncio.get_running_loop() 59 | result = await loop.run_in_executor( 60 | None, self.create_secret, secret_name, secret_value, tags 61 | ) 62 | return result 63 | 64 | async def get_secret_async(self, secret_id: str) -> StringSecret: 65 | loop = asyncio.get_running_loop() 66 | result = await loop.run_in_executor(None, self.get_secret, secret_id) 67 | return result 68 | 69 | def delete_secret(self, secret_id: str) -> None: 70 | self.secret_gateway.delete_secret(secret_id=secret_id) 71 | 72 | async def delete_secret_async(self, secret_id: str) -> None: 73 | loop = asyncio.get_running_loop() 74 | await loop.run_in_executor(None, self.delete_secret, secret_id) 75 | -------------------------------------------------------------------------------- /onedocker/tests/service/test_opawdl_driver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | import uuid 11 | from unittest.mock import MagicMock, mock_open, patch 12 | 13 | from onedocker.entity.opawdl_state import OPAWDLState 14 | from onedocker.entity.opawdl_workflow import OPAWDLWorkflow 15 | from onedocker.service.opawdl_driver import OPAWDLDriver 16 | 17 | 18 | class TestOPAWDLDriver(unittest.TestCase): 19 | @patch( 20 | "onedocker.repository.opawdl_workflow_instance_repository_local.LocalOPAWDLWorkflowInstanceRepository" 21 | ) 22 | def setUp(self, MockLocalOPAWDLWorkflowRepository): 23 | self.test_plugin_name = "test_plugin" 24 | self.test_cmd_args_list = ["-a=b", "-c=d"] 25 | self.test_opawdl_state = OPAWDLState( 26 | plugin_name=self.test_plugin_name, cmd_args_list=self.test_cmd_args_list 27 | ) 28 | self.test_state_name = "state_1" 29 | self.test_opawdl_workflow = OPAWDLWorkflow( 30 | starts_at=self.test_state_name, 31 | states={self.test_state_name: self.test_opawdl_state}, 32 | ) 33 | self.test_opawdl_workflow_path = "test_workflow.json" 34 | self.test_workflow_instance_id = str(uuid.uuid4()) 35 | self.mock_opawdl_repo = MockLocalOPAWDLWorkflowRepository("test_repo_path") 36 | 37 | @patch("onedocker.util.opawdl_parser.OPAWDLParser.parse_json_str_to_workflow") 38 | @patch("onedocker.service.opawdl_driver.subprocess.run") 39 | def test_run_workflow( 40 | self, mock_subprocess_run, mock_parse_json_str_to_workflow 41 | ) -> None: 42 | # Arrange 43 | self.mock_opawdl_repo.create.return_value = MagicMock() 44 | self.mock_opawdl_repo.update.return_value = MagicMock() 45 | mock_parse_json_str_to_workflow.return_value = self.test_opawdl_workflow 46 | expected_executed_cmd = ( 47 | self.test_plugin_name + " " + " ".join(self.test_cmd_args_list) 48 | ) 49 | 50 | # Act 51 | with patch("builtins.open", new_callable=mock_open()) as open_mock: 52 | driver = OPAWDLDriver( 53 | instance_id=self.test_workflow_instance_id, 54 | workflow_path=self.test_opawdl_workflow_path, 55 | repo=self.mock_opawdl_repo, 56 | ) 57 | driver.run_workflow() 58 | 59 | # Assert 60 | open_mock.assert_called_once_with(self.test_opawdl_workflow_path, "r") 61 | mock_subprocess_run.assert_called_once_with(expected_executed_cmd, shell=True) 62 | -------------------------------------------------------------------------------- /onedocker/tests/util/test_opawdl_parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from onedocker.entity.opawdl_state import OPAWDLState 12 | from onedocker.entity.opawdl_workflow import OPAWDLWorkflow 13 | from onedocker.util.opawdl_parser import OPAWDLParser 14 | 15 | 16 | class TestOPAWDLParser(unittest.TestCase): 17 | def setUp(self): 18 | self.parser = OPAWDLParser() 19 | self.test_plugin_name = "test_plugin" 20 | self.test_cmd_args_list = ["-a=b", "-c=d"] 21 | self.test_opawdl_state = OPAWDLState( 22 | plugin_name=self.test_plugin_name, cmd_args_list=self.test_cmd_args_list 23 | ) 24 | self.test_state_name = "state_1" 25 | self.test_opawdl_workflow = OPAWDLWorkflow( 26 | starts_at=self.test_state_name, 27 | states={self.test_state_name: self.test_opawdl_state}, 28 | ) 29 | 30 | def test_parse_json_str_to_workflow(self): 31 | # Arrange 32 | valid_workflow = OPAWDLWorkflow( 33 | starts_at=self.test_state_name, 34 | states={self.test_state_name: self.test_opawdl_state}, 35 | ) 36 | # Act 37 | result = self.parser.parse_json_str_to_workflow(str(valid_workflow)) 38 | # Assert 39 | self.assertEqual(result, valid_workflow) 40 | 41 | def test_parse_json_str_to_workflow_no_end(self): 42 | # Arrange 43 | test_state_no_end = self.test_opawdl_state 44 | test_state_no_end.is_end = False 45 | test_workflow_no_end = OPAWDLWorkflow( 46 | starts_at=self.test_state_name, 47 | states={self.test_state_name: test_state_no_end}, 48 | ) 49 | # Act and Assert 50 | with self.assertRaisesRegex( 51 | Exception, 52 | "Input workflow string does not have an ending state.", 53 | ): 54 | self.parser.parse_json_str_to_workflow(str(test_workflow_no_end)) 55 | 56 | def test_parse_json_str_to_workflow_multiple_ends(self): 57 | # Arrange 58 | test_workflow_multiple_ends = OPAWDLWorkflow( 59 | starts_at=self.test_state_name, 60 | states={ 61 | self.test_state_name: self.test_opawdl_state, 62 | "state_2": self.test_opawdl_state, 63 | }, 64 | ) 65 | # Act and Assert 66 | with self.assertRaisesRegex( 67 | Exception, 68 | "Input workflow string has multiple", 69 | ): 70 | self.parser.parse_json_str_to_workflow(str(test_workflow_multiple_ends)) 71 | -------------------------------------------------------------------------------- /onedocker/repository/onedocker_package.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | from typing import List 9 | 10 | from fbpcp.service.storage import StorageService 11 | from onedocker.entity.package_info import PackageInfo 12 | 13 | 14 | class OneDockerPackageRepository: 15 | def __init__(self, storage_svc: StorageService, repository_path: str) -> None: 16 | self.storage_svc = storage_svc 17 | self.repository_path = repository_path 18 | 19 | def _build_package_path(self, package_name: str, version: str) -> str: 20 | return f"{self.repository_path}{package_name}/{version}/{package_name.split('/')[-1]}" 21 | 22 | def _build_archive_path(self, package_name: str, version: str) -> str: 23 | return f"{self.repository_path}archived/{package_name}/{version}/{package_name.split('/')[-1]}" 24 | 25 | def upload(self, package_name: str, version: str, source: str) -> None: 26 | package_path = self._build_package_path(package_name, version) 27 | self.storage_svc.copy(source, package_path) 28 | 29 | def download(self, package_name: str, version: str, destination: str) -> None: 30 | package_path = self._build_package_path(package_name, version) 31 | self.storage_svc.copy(package_path, destination) 32 | 33 | def get_package_versions( 34 | self, 35 | package_name: str, 36 | ) -> List[str]: 37 | package_parent_path = f"{self.repository_path}{package_name}/" 38 | return self.storage_svc.list_folders(package_parent_path) 39 | 40 | def get_package_info(self, package_name: str, version: str) -> PackageInfo: 41 | package_path = self._build_package_path(package_name, version) 42 | 43 | if not self.storage_svc.file_exists(package_path): 44 | raise ValueError( 45 | f"Package {package_name}, version {version} not found in repository" 46 | ) 47 | 48 | file_info = self.storage_svc.get_file_info(package_path) 49 | return PackageInfo( 50 | package_name=package_name, 51 | version=version, 52 | last_modified=file_info.last_modified, 53 | package_size=file_info.file_size, 54 | ) 55 | 56 | def archive_package(self, package_name: str, version: str) -> None: 57 | current_path = self._build_package_path(package_name, version) 58 | if not self.storage_svc.file_exists(current_path): 59 | raise FileNotFoundError( 60 | f"Cant find the package to be archived for package {package_name}, version {version}" 61 | ) 62 | archive_path = self._build_archive_path(package_name, version) 63 | self.storage_svc.copy(current_path, archive_path) 64 | -------------------------------------------------------------------------------- /tests/decorator/test_error_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from botocore.exceptions import ClientError 12 | from fbpcp.decorator.error_handler import error_handler 13 | from fbpcp.error.pcp import LimitExceededError, PcpError, ThrottlingError 14 | from google.cloud.exceptions import TooManyRequests 15 | 16 | 17 | class TestErrorHandler(unittest.TestCase): 18 | def test_pcs_error(self): 19 | @error_handler 20 | def foo(): 21 | raise ValueError("just a test") 22 | 23 | self.assertRaises(PcpError, foo) 24 | 25 | def test_throttling_error(self): 26 | @error_handler 27 | def foo(): 28 | err = ClientError( 29 | { 30 | "Error": { 31 | "Code": "ThrottlingException", 32 | "Message": "test", 33 | }, 34 | "ResponseMetadata": { 35 | "RequestId": "test_id", 36 | "HTTPStatusCode": 400, 37 | }, 38 | }, 39 | "test", 40 | ) 41 | 42 | raise err 43 | 44 | self.assertRaises(ThrottlingError, foo) 45 | 46 | def test_wrapped_function_args(self): 47 | @error_handler 48 | def foo(**kwargs): 49 | raise ValueError("just a test f") 50 | 51 | error_msgs = { 52 | "error_type1": "error_msg1", 53 | "error_type2": "error_msg2", 54 | } 55 | self.assertRaises(PcpError, foo, error_msgs) 56 | 57 | def test_wrapped_function_kwargs(self): 58 | @error_handler 59 | def foo(*args): 60 | raise ValueError("just a test") 61 | 62 | self.assertRaises(PcpError, foo, "error1", "error2") 63 | 64 | def test_gcp_throttling_error(self): 65 | @error_handler 66 | def foo(): 67 | # Exception mapping a 429 Too Many Requests response 68 | err = TooManyRequests("test") 69 | raise err 70 | 71 | self.assertRaises(ThrottlingError, foo) 72 | 73 | def test_limit_exceeded_error(self): 74 | @error_handler 75 | def foo(): 76 | err = ClientError( 77 | { 78 | "Error": { 79 | "Code": "LimitExceededException", 80 | "Message": "test", 81 | }, 82 | "ResponseMetadata": { 83 | "RequestId": "test_id123", 84 | "HTTPStatusCode": 400, 85 | }, 86 | }, 87 | "test", 88 | ) 89 | 90 | raise err 91 | 92 | self.assertRaises(LimitExceededError, foo) 93 | -------------------------------------------------------------------------------- /fbpcp/service/policy_validation_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | import logging 9 | import re 10 | from typing import List 11 | 12 | from fbpcp.entity.policy_settings_config import PolicySettingsConfig 13 | from fbpcp.entity.policy_statement import PolicyStatement 14 | from fbpcp.service.policy_validation import PolicyValidationService 15 | 16 | 17 | class AWSPolicyValidationService(PolicyValidationService): 18 | def __init__(self) -> None: 19 | self.logger: logging.Logger = logging.getLogger(__name__) 20 | 21 | def _principal_match(self, principal_settings: str, principal: str) -> bool: 22 | """Returns whether the principal matches the principal_settings. 23 | Args: 24 | principal_settings: Principal settings in the PolicySettingsConfig. 25 | This can either be a normal string, 26 | or in format "re()" to indicate regex matching. 27 | principal: The principal string. 28 | """ 29 | match = re.fullmatch(r"re\((.*?)\)", principal_settings) 30 | if match: 31 | principal_regex = match.group(1) 32 | return bool(re.fullmatch(principal_regex, principal)) 33 | return principal_settings == principal 34 | 35 | def _policy_exists_in_statements( 36 | self, 37 | statements: List[PolicyStatement], 38 | effect: str, 39 | principal: str, 40 | actions: List[str], 41 | resources: List[str], 42 | ) -> bool: 43 | for statement in statements: 44 | contain_principal = any( 45 | self._principal_match(principal, stmt_principal) 46 | for stmt_principal in statement.principals 47 | ) 48 | if ( 49 | contain_principal 50 | and effect == statement.effect 51 | and set(actions).issubset(statement.actions) 52 | and set(resources).issubset(statement.resources) 53 | ): 54 | return True 55 | return False 56 | 57 | def is_bucket_policy_statements_valid( 58 | self, 59 | bucket: str, 60 | bucket_statements: List[PolicyStatement], 61 | policy_settings: List[PolicySettingsConfig], 62 | ) -> bool: 63 | for rule in policy_settings: 64 | if rule.exist != self._policy_exists_in_statements( 65 | bucket_statements, 66 | rule.effect, 67 | rule.principal, 68 | rule.actions, 69 | rule.resources, 70 | ): 71 | self.logger.error( 72 | "The policy of bucket %s does not satisfy the following policy settings: %s.", 73 | bucket, 74 | str(rule), 75 | ) 76 | return False 77 | return True 78 | -------------------------------------------------------------------------------- /onedocker/tests/service/test_attestation_factory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | from typing import Dict 11 | from unittest.mock import MagicMock, patch 12 | 13 | from onedocker.entity.attestation_document import ( 14 | AttestationDocument, 15 | AttestationPolicy, 16 | PolicyName, 17 | PolicyParams, 18 | ) 19 | 20 | from onedocker.service.attestation_factory import AttestationFactoryService 21 | 22 | TEST_PACKAGE: str = "echo" 23 | TEST_VERSION: str = "1.0" 24 | TEST_PARAMS: PolicyParams = PolicyParams( 25 | package_name=TEST_PACKAGE, version=TEST_VERSION 26 | ) 27 | TEST_POLICY: AttestationPolicy = AttestationPolicy( 28 | policy_name=PolicyName.BINARY_MATCH, params=TEST_PARAMS 29 | ) 30 | TEST_MEASUREMENTS: Dict[str, str] = {"MD5": "md5-hash"} 31 | 32 | 33 | class TestAttestationFactory(unittest.TestCase): 34 | def setUp(self) -> None: 35 | self.attestation_factory_service = AttestationFactoryService() 36 | 37 | @patch( 38 | "onedocker.service.attestation_factory.AttestationFactoryService._get_attestation_service" 39 | ) 40 | def test_validate(self, mock_get_attestation_service) -> None: 41 | # Arrange 42 | mock_get_attestation_service.return_value.validate = MagicMock( 43 | return_value=True 44 | ) 45 | test_document = AttestationDocument( 46 | policy=TEST_POLICY, measurements=TEST_MEASUREMENTS 47 | ) 48 | # Act 49 | result = self.attestation_factory_service.validate(test_document.to_json()) 50 | # Assert 51 | mock_get_attestation_service.assert_called_once_with(test_document.policy) 52 | self.assertEqual(result, True) 53 | 54 | def test_validate_invalid_document(self) -> None: 55 | # Arrange 56 | test_document = "invalid-document" 57 | # Act and Assert 58 | with self.assertRaises(ValueError): 59 | self.attestation_factory_service.validate(test_document) 60 | 61 | @patch("onedocker.service.attestation_factory.PCAttestationService") 62 | def test_get_attestation_service(self, MockPCAttestationService) -> None: 63 | # Arrange 64 | expected_service = MagicMock() 65 | MockPCAttestationService.return_value = expected_service 66 | # Act 67 | service = self.attestation_factory_service._get_attestation_service(TEST_POLICY) 68 | # Assert 69 | self.assertEqual(service, expected_service) 70 | 71 | def test_get_attestation_service_unsupported_policy(self) -> None: 72 | # Arrange 73 | test_policy = AttestationPolicy( 74 | # pyre-ignore 75 | policy_name="unsupported-policy", 76 | params=TEST_PARAMS, 77 | ) 78 | # Act and Assert 79 | with self.assertRaises(NotImplementedError): 80 | self.attestation_factory_service._get_attestation_service(test_policy) 81 | -------------------------------------------------------------------------------- /fbpcp/gateway/costexplorer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from typing import Any, Dict, Optional 10 | 11 | import boto3 12 | from fbpcp.decorator.error_handler import error_handler 13 | from fbpcp.entity.cloud_cost import CloudCost 14 | from fbpcp.gateway.aws import AWSGateway 15 | from fbpcp.mapper.aws import map_cecost_to_cloud_cost 16 | 17 | COST_GRANULARITY = "DAILY" 18 | 19 | 20 | class CostExplorerGateway(AWSGateway): 21 | def __init__( 22 | self, 23 | access_key_id: Optional[str] = None, 24 | access_key_data: Optional[str] = None, 25 | config: Optional[Dict[str, Any]] = None, 26 | ) -> None: 27 | super().__init__( 28 | access_key_id=access_key_id, access_key_data=access_key_data, config=config 29 | ) 30 | # pyre-ignore 31 | self.client = boto3.client("ce", **self.config) 32 | 33 | @error_handler 34 | def get_cost( 35 | self, 36 | start_date: str, 37 | end_date: str, 38 | region: Optional[str] = None, 39 | ) -> CloudCost: 40 | """ 41 | Get cost between start_date and end_date from CostExplorer API using get_cost_and_usage() 42 | get_cost_and_usage() referece: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce.html#CostExplorer.Client.get_cost_and_usage 43 | :param start_date: start date for cost, required format "yyyy-mm-dd" (e.g "2020-12-01") 44 | :param end_date: end date for cost, required format "yyyy-mm-dd" (e.g "2020-12-01") 45 | :param region: region name as additional filter for cost. 46 | :return: CloudCost object that has the total cost and a list of CloudCostItem objects group by region and service. Unit of cost_amount is USD 47 | """ 48 | 49 | page_token = None 50 | results_by_time = [] 51 | while True: 52 | kwargs = {"NextPageToken": page_token} if page_token else {} 53 | kwargs.update( 54 | { 55 | "Filter": { 56 | "Dimensions": { 57 | "Key": "REGION", 58 | "Values": [region], 59 | } 60 | } 61 | } 62 | if region 63 | else {} 64 | ) 65 | client_response = self.client.get_cost_and_usage( 66 | TimePeriod={"Start": start_date, "End": end_date}, 67 | Granularity=COST_GRANULARITY, 68 | Metrics=["UnblendedCost"], 69 | GroupBy=[ 70 | {"Type": "DIMENSION", "Key": "SERVICE"}, 71 | ], 72 | **kwargs, 73 | ) 74 | results_by_time.extend(client_response.get("ResultsByTime")) 75 | page_token = client_response.get("NextPageToken") 76 | if not page_token: 77 | break 78 | 79 | return map_cecost_to_cloud_cost(results_by_time) 80 | -------------------------------------------------------------------------------- /fbpcp/decorator/metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | """Metrics Decorators 10 | 11 | The decorators in this file are designed to deocrate classes that implement MetricsGetter. 12 | """ 13 | 14 | import asyncio 15 | import functools 16 | import time 17 | from typing import Callable 18 | 19 | from fbpcp.metrics.getter import MetricsGetter 20 | 21 | 22 | def request_counter(metrics_name: str) -> Callable: 23 | def wrap(f: Callable): 24 | @functools.wraps(f) 25 | def wrapper_sync(self: MetricsGetter, *args, **kwargs): 26 | if self.has_metrics(): 27 | self.get_metrics().count(metrics_name, 1) 28 | return f(self, *args, **kwargs) 29 | 30 | @functools.wraps(f) 31 | async def wrapper_async(self: MetricsGetter, *args, **kwargs): 32 | if self.has_metrics(): 33 | self.get_metrics().count(metrics_name, 1) 34 | return await f(self, *args, **kwargs) 35 | 36 | return wrapper_async if asyncio.iscoroutinefunction(f) else wrapper_sync 37 | 38 | return wrap 39 | 40 | 41 | def error_counter(metrics_name: str) -> Callable: 42 | def wrap(f: Callable): 43 | @functools.wraps(f) 44 | def wrapper_sync(self: MetricsGetter, *args, **kwargs): 45 | try: 46 | return f(self, *args, **kwargs) 47 | except Exception as err: 48 | if self.has_metrics(): 49 | self.get_metrics().count(metrics_name, 1) 50 | raise err 51 | 52 | @functools.wraps(f) 53 | async def wrapper_async(self: MetricsGetter, *args, **kwargs): 54 | try: 55 | return await f(self, *args, **kwargs) 56 | except Exception as err: 57 | if self.has_metrics(): 58 | self.get_metrics().count(metrics_name, 1) 59 | raise err 60 | 61 | return wrapper_async if asyncio.iscoroutinefunction(f) else wrapper_sync 62 | 63 | return wrap 64 | 65 | 66 | def duration_time(metrics_name: str) -> Callable: 67 | def wrap(f: Callable): 68 | @functools.wraps(f) 69 | def wrapper_sync(self: MetricsGetter, *args, **kwargs): 70 | start = time.perf_counter_ns() 71 | res = f(self, *args, **kwargs) 72 | end = time.perf_counter_ns() 73 | 74 | if self.has_metrics(): 75 | self.get_metrics().gauge(metrics_name, int((end - start) / 1e6)) 76 | 77 | return res 78 | 79 | @functools.wraps(f) 80 | async def wrapper_async(self: MetricsGetter, *args, **kwargs): 81 | start = time.perf_counter_ns() 82 | res = await f(self, *args, **kwargs) 83 | end = time.perf_counter_ns() 84 | 85 | if self.has_metrics(): 86 | self.get_metrics().gauge(metrics_name, int((end - start) / 1e6)) 87 | 88 | return res 89 | 90 | return wrapper_async if asyncio.iscoroutinefunction(f) else wrapper_sync 91 | 92 | return wrap 93 | -------------------------------------------------------------------------------- /tests/util/test_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.error.pcp import InvalidParameterError 12 | 13 | from fbpcp.util.aws import ( 14 | convert_dict_to_list, 15 | convert_list_to_dict, 16 | is_container_definition_valid, 17 | prepare_tags, 18 | split_container_definition, 19 | ) 20 | 21 | TEST_DICT = {"k1": "v1", "k2": "v2"} 22 | TEST_LIST = [{"Name": "k1", "Value": "v1"}, {"Name": "k2", "Value": "v2"}] 23 | 24 | 25 | class TestAWSUtil(unittest.TestCase): 26 | def test_convert_dict_to_list(self): 27 | expected_list = [ 28 | {"Name": "k1", "Values": ["v1"]}, 29 | {"Name": "k2", "Values": ["v2"]}, 30 | ] 31 | self.assertEqual( 32 | expected_list, convert_dict_to_list(TEST_DICT, "Name", "Values") 33 | ) 34 | self.assertEqual([], convert_dict_to_list({}, "Name", "Values")) 35 | 36 | def test_convert_list_dict(self): 37 | self.assertEqual(TEST_DICT, convert_list_to_dict(TEST_LIST, "Name", "Value")) 38 | 39 | def test_prepare_tags(self): 40 | expected_tags = {"tag:k1": "v1", "tag:k2": "v2"} 41 | self.assertEqual(expected_tags, prepare_tags(TEST_DICT)) 42 | 43 | def test_is_container_definition_valid_true(self): 44 | # Arrange 45 | valid_container_definitions = [ 46 | "pl-task-fake-business:2#pl-container-fake-business", 47 | "arn:aws:ecs:us-west-2:123456789012:task-definition/onedocker-task-shared-us-west-2:1#onedocker-container-shared-us-west-2", 48 | ] 49 | 50 | # Act and Assert 51 | for container_definition in valid_container_definitions: 52 | self.assertTrue(is_container_definition_valid(container_definition)) 53 | 54 | def test_is_container_definition_valid_false(self): 55 | # Arrange 56 | invalid_container_definitions = [ 57 | "pl-task-fake-business:2#", 58 | "pl-task-fake-business:2##pl-container-fake-business", 59 | "pl-task-fake-business#pl-container-fake-business", 60 | "pl-container-fake-business", 61 | ] 62 | 63 | # Act and Assert 64 | for container_definition in invalid_container_definitions: 65 | self.assertFalse(is_container_definition_valid(container_definition)) 66 | 67 | def test_split_container_definition_throw(self): 68 | # Arrange 69 | invalid_container_definition = "pl-task-fake-business:2#" 70 | 71 | # Act & Assert 72 | with self.assertRaises(InvalidParameterError): 73 | split_container_definition(invalid_container_definition) 74 | 75 | def test_split_container_definition(self): 76 | # Arrange 77 | valid_container_definition = ( 78 | "pl-task-fake-business:2#pl-container-fake-business" 79 | ) 80 | 81 | # Act & Assert 82 | self.assertEqual( 83 | ("pl-task-fake-business:2", "pl-container-fake-business"), 84 | split_container_definition(valid_container_definition), 85 | ) 86 | -------------------------------------------------------------------------------- /pce/gateway/ec2.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | 8 | from typing import Any, Dict, List, Optional 9 | 10 | import boto3 11 | from fbpcp.decorator.error_handler import error_handler 12 | from fbpcp.entity.vpc_peering import VpcPeering 13 | from fbpcp.gateway.aws import AWSGateway 14 | from fbpcp.mapper.aws import map_ec2vpcpeering_to_vpcpeering 15 | 16 | 17 | class EC2Gateway(AWSGateway): 18 | def __init__( 19 | self, 20 | region: str, 21 | access_key_id: Optional[str] = None, 22 | access_key_data: Optional[str] = None, 23 | config: Optional[Dict[str, Any]] = None, 24 | ) -> None: 25 | super().__init__(region, access_key_id, access_key_data, config) 26 | 27 | # pyre-ignore 28 | self.client = boto3.client("ec2", region_name=self.region, **self.config) 29 | 30 | def describe_availability_zones( 31 | self, 32 | tags: Optional[Dict[str, str]] = None, 33 | ) -> List[str]: 34 | response = self.client.describe_availability_zones() 35 | 36 | return [aws_az["ZoneName"] for aws_az in response["AvailabilityZones"]] 37 | 38 | def describe_vpc_peering_connections_with_accepter_vpc_id( 39 | self, 40 | vpc_id: str, 41 | ) -> Optional[List[VpcPeering]]: 42 | filters = [ 43 | {"Name": "accepter-vpc-info.vpc-id", "Values": [vpc_id]}, 44 | ] 45 | 46 | response = self.client.describe_vpc_peering_connections(Filters=filters) 47 | return ( 48 | [ 49 | map_ec2vpcpeering_to_vpcpeering(vpc_conn, vpc_id) 50 | for vpc_conn in response["VpcPeeringConnections"] 51 | ] 52 | if response["VpcPeeringConnections"] 53 | else None 54 | ) 55 | 56 | @error_handler 57 | def accept_vpc_peering_connection( 58 | self, vpc_peering_connection_id: str, vpc_id: str 59 | ) -> Optional[VpcPeering]: 60 | 61 | response = self.client.accept_vpc_peering_connection( 62 | VpcPeeringConnectionId=vpc_peering_connection_id 63 | ) 64 | 65 | return ( 66 | map_ec2vpcpeering_to_vpcpeering(response["VpcPeeringConnection"], vpc_id) 67 | if response["VpcPeeringConnection"] 68 | else None 69 | ) 70 | 71 | @error_handler 72 | def create_route( 73 | self, route_table_id: str, vpc_peering_connection_id: str, dest_cidr: str 74 | ) -> bool: 75 | 76 | response = self.client.create_route( 77 | RouteTableId=route_table_id, 78 | DestinationCidrBlock=dest_cidr, 79 | VpcPeeringConnectionId=vpc_peering_connection_id, 80 | ) 81 | 82 | return response["Return"] 83 | 84 | @error_handler 85 | def replace_route( 86 | self, route_table_id: str, vpc_peering_connection_id: str, dest_cidr: str 87 | ) -> None: 88 | 89 | self.client.replace_route( 90 | RouteTableId=route_table_id, 91 | DestinationCidrBlock=dest_cidr, 92 | VpcPeeringConnectionId=vpc_peering_connection_id, 93 | ) 94 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | select = B,C,E,F,P,W,B9 3 | max-line-length = 80 4 | max-complexity = 12 5 | # Main Explanation Docs: https://github.com/grantmcconnaughey/Flake8Rules 6 | ignore = 7 | # Black conflicts and overlaps. 8 | # Found in https://github.com/psf/black/issues/429 9 | B950, # Line too long. (Use `arc lint`'s LINEWRAP instead) 10 | C901, # The function is too complex 11 | E111, # Indentation is not a multiple of four. 12 | E115, # Expected an indented block (comment). 13 | E117, # Over-indented. 14 | E121, # Continuation line under-indented for hanging indent. 15 | E122, # Continuation line missing indentation or outdented. 16 | E123, # Closing bracket does not match indentation of opening bracket's line. 17 | E124, # Closing bracket does not match visual indentation. 18 | E125, # Continuation line with same indent as next logical line. 19 | E126, # Continuation line over-indented for hanging indent. 20 | E127, # Continuation line over-indented for visual indent. 21 | E128, # Continuation line under-indented for visual indent. 22 | E129, # Visually indented line with same indent as next logical line. 23 | E131, # Continuation line unaligned for hanging indent. 24 | E201, # Whitespace after '('. 25 | E202, # Whitespace before ')'. 26 | E203, # Whitespace before ':'. 27 | E221, # Multiple spaces before operator. 28 | E222, # Multiple spaces after operator. 29 | E225, # Missing whitespace around operator. 30 | E226, # Missing whitespace around arithmetic operator. 31 | E227, # Missing whitespace around bitwise or shift operator. 32 | E231, # Missing whitespace after ',', ';', or ':'. 33 | E241, # Multiple spaces after ','. 34 | E251, # Unexpected spaces around keyword / parameter equals. 35 | E252, # Missing whitespace around parameter equals. 36 | E261, # At least two spaces before inline comment. 37 | E262, # Inline comment should start with '# '. 38 | E265, # Block comment should start with '# '. 39 | E271, # Multiple spaces after keyword. 40 | E272, # Multiple spaces before keyword. 41 | E301, # Expected 1 blank line, found 0. 42 | E302, # Expected 2 blank lines, found 0. 43 | E303, # Too many blank lines (3). 44 | E305, # Expected 2 blank lines after end of function or class. 45 | E306, # Expected 1 blank line before a nested definition. 46 | E501, # Line too long (82 > 79 characters). 47 | E502, # The backslash is redundant between brackets. 48 | E701, # Multiple statements on one line (colon). 49 | E702, # Multiple statements on one line (semicolon). 50 | E703, # Statement ends with a semicolon. 51 | E704, # Multiple statements on one line (def). 52 | W291, # Trailing whitespace. 53 | W292, # No newline at end of file. 54 | W293, # Blank line contains whitespace. 55 | W391, # Blank line at end of file. 56 | W504, # Line break occurred after a binary operator. 57 | 58 | # Too opinionated. 59 | E265, # Block comment should start with '# '. 60 | E266, # Too many leading '#' for block comment. 61 | E402, # Module level import not at top of file. (Use cases like demandimport https://fburl.com/demandimport require statements before imports) 62 | E722, # Do not use bare except, specify exception instead. (Duplicate of B001) 63 | P207, # (Duplicate of B003) 64 | P208, # (Duplicate of C403) 65 | W503 # Line break occurred before a binary operator. 66 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Open Source Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | Using welcoming and inclusive language 12 | Being respectful of differing viewpoints and experiences 13 | Gracefully accepting constructive criticism 14 | Focusing on what is best for the community 15 | Showing empathy towards other community members 16 | Examples of unacceptable behavior by participants include: 17 | 18 | The use of sexualized language or imagery and unwelcome sexual attention or advances 19 | Trolling, insulting/derogatory comments, and personal or political attacks 20 | Public or private harassment 21 | Publishing others’ private information, such as a physical or electronic address, without explicit permission 22 | Other conduct which could reasonably be considered inappropriate in a professional setting 23 | 24 | ## Our Responsibilities 25 | 26 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 27 | 28 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 29 | 30 | ## Scope 31 | 32 | This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 33 | 34 | ## Enforcement 35 | 36 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at opensource-conduct@fb.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 37 | 38 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. 39 | 40 | ## Attribution 41 | 42 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 43 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 44 | 45 | [homepage]: https://www.contributor-covenant.org 46 | -------------------------------------------------------------------------------- /fbpcp/gateway/secrets_manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | from functools import reduce 10 | from typing import Any, Dict, List, Optional 11 | 12 | import boto3 13 | from botocore.client import BaseClient 14 | from fbpcp.decorator.error_handler import error_handler 15 | from fbpcp.entity.secret import StringSecret 16 | from fbpcp.gateway.aws import AWSGateway 17 | from fbpcp.util.aws import convert_list_to_dict 18 | 19 | 20 | class AWSSecretsManagerGateway(AWSGateway): 21 | def __init__( 22 | self, 23 | region: str, 24 | access_key_id: Optional[str] = None, 25 | access_key_data: Optional[str] = None, 26 | config: Optional[Dict[str, Any]] = None, 27 | ) -> None: 28 | super().__init__(region, access_key_id, access_key_data, config) 29 | self.client: BaseClient = boto3.client( 30 | "secretsmanager", region_name=self.region, **self.config 31 | ) 32 | 33 | @error_handler 34 | def create_secret( 35 | self, 36 | secret_name: str, 37 | secret_value: str, 38 | tags: Optional[Dict[str, str]] = None, 39 | ) -> str: 40 | """ 41 | Returns the id (ARN) of the created secret 42 | """ 43 | tags_dict = [] 44 | if tags: 45 | tags_dict = self._generate_tags_dict(tags) 46 | 47 | response = self.client.create_secret( 48 | Name=secret_name, SecretString=secret_value, Tags=tags_dict 49 | ) 50 | return response["ARN"] 51 | 52 | @error_handler 53 | def get_secret( 54 | self, 55 | secret_id: str, 56 | ) -> StringSecret: 57 | # Get secret value. 58 | # It retrieves the current version of the secret. 59 | val_response = self.client.get_secret_value(SecretId=secret_id) 60 | # Get secret details. E.g tags 61 | descr_response = self.client.describe_secret(SecretId=secret_id) 62 | 63 | return self._convert_resp_to_secret(descr_response, val_response) 64 | 65 | @error_handler 66 | def delete_secret( 67 | self, 68 | secret_id: str, 69 | ) -> None: 70 | # Delete secret. 71 | self.client.delete_secret(SecretId=secret_id) 72 | 73 | def _convert_resp_to_secret( 74 | self, descr_resp: Dict[str, Any], val_resp: Dict[str, Any] 75 | ) -> StringSecret: 76 | """ 77 | Encapsulate the responses into a Secret object 78 | """ 79 | id = descr_resp["ARN"] 80 | name = descr_resp["Name"] 81 | value = val_resp.get("SecretString") 82 | create_date = descr_resp.get("CreatedDate") 83 | tags = convert_list_to_dict(descr_resp.get("Tags"), "Key", "Value") 84 | 85 | return StringSecret( 86 | id=id, name=name, value=value, create_date=create_date, tags=tags 87 | ) 88 | 89 | def _generate_tags_dict(self, tags: Dict[str, str]) -> List[Dict[str, str]]: 90 | # Input tag format {"tag1": "v1", "tag2", "v2", ...} 91 | # AWS required format [{"Key": "tag1", "Value": "v1"}, ...} 92 | new_dict = reduce( 93 | lambda x, y: [*x, {"Key": y, "Value": tags[y]}], tags.keys(), [] 94 | ) 95 | 96 | return new_dict 97 | -------------------------------------------------------------------------------- /tests/service/test_policy_validation_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import unittest 10 | 11 | from fbpcp.entity.policy_settings_config import Effect, PolicySettingsConfig 12 | from fbpcp.entity.policy_statement import PolicyStatement 13 | from fbpcp.service.policy_validation_aws import AWSPolicyValidationService 14 | 15 | TEST_ACTION_1 = "s3:GetObject" 16 | TEST_ACTION_2 = "s3:ListBucket" 17 | TEST_BUCKET = "onedocker-repo-test" 18 | TEST_RESOURCE_1 = f"arn:aws:s3:::{TEST_BUCKET}/*" 19 | TEST_RESOURCE_2 = f"arn:aws:s3:::{TEST_BUCKET}" 20 | TEST_STATEMENT = [ 21 | PolicyStatement( 22 | effect="Allow", 23 | principals=["*"], 24 | actions=[TEST_ACTION_1], 25 | resources=[TEST_RESOURCE_1], 26 | ), 27 | PolicyStatement( 28 | effect="Allow", 29 | principals=["arn:aws:iam::account-id:root"], 30 | actions=[TEST_ACTION_2], 31 | resources=[TEST_RESOURCE_2], 32 | ), 33 | ] 34 | 35 | 36 | class TestAWSPolicyValidationService(unittest.TestCase): 37 | def setUp(self): 38 | self.checker = AWSPolicyValidationService() 39 | 40 | def test_is_bucket_policy_statements_valid(self): 41 | # Arrange 42 | policy_settings = [ 43 | # Test normal principal 44 | PolicySettingsConfig( 45 | exist=True, 46 | effect=Effect.ALLOW.value, 47 | principal="*", 48 | actions=[TEST_ACTION_1], 49 | resources=[TEST_RESOURCE_1], 50 | ), 51 | # Test regex principal 52 | PolicySettingsConfig( 53 | exist=True, 54 | effect=Effect.ALLOW.value, 55 | principal="re(arn:aws:iam::account-id:.*)", 56 | actions=[TEST_ACTION_2], 57 | resources=[TEST_RESOURCE_2], 58 | ), 59 | # Test not exist policy 60 | PolicySettingsConfig( 61 | exist=False, 62 | effect=Effect.ALLOW.value, 63 | principal="re(.*)", 64 | actions=["not-exist-action"], 65 | resources=["non-existing-resource"], 66 | ), 67 | ] 68 | # Act 69 | result = self.checker.is_bucket_policy_statements_valid( 70 | TEST_BUCKET, TEST_STATEMENT, policy_settings 71 | ) 72 | # Assert 73 | self.assertTrue(result) 74 | 75 | def test_is_bucket_policy_statements_invalid(self): 76 | # Arrange 77 | policy_settings = [ 78 | PolicySettingsConfig( 79 | exist=True, 80 | effect=Effect.DENY.value, 81 | principal="*", 82 | actions=[TEST_ACTION_1], 83 | resources=[TEST_RESOURCE_1], 84 | ), 85 | PolicySettingsConfig( 86 | exist=True, 87 | effect=Effect.ALLOW.value, 88 | principal="re(arn:aws:iam::account-id:.*)", 89 | actions=[TEST_ACTION_2], 90 | resources=[TEST_RESOURCE_1], 91 | ), 92 | ] 93 | # Act 94 | result = self.checker.is_bucket_policy_statements_valid( 95 | TEST_BUCKET, TEST_STATEMENT, policy_settings 96 | ) 97 | # Assert 98 | self.assertFalse(result) 99 | -------------------------------------------------------------------------------- /onedocker/tests/entity/test_opawdl_workflow_instance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | 12 | import onedocker.entity.opawdl_state_instance as state_instance 13 | import onedocker.entity.opawdl_workflow_instance as workflow_instance 14 | 15 | from onedocker.entity.opawdl_state import OPAWDLState 16 | from onedocker.entity.opawdl_workflow import OPAWDLWorkflow 17 | 18 | 19 | class TestOPAWDLWorkflowInstance(unittest.TestCase): 20 | def setUp(self) -> None: 21 | self.test_plugin_name = "test_plugin" 22 | self.test_cmd_args_list = ["-a=b", "-c=d"] 23 | self.test_opawdl_state = OPAWDLState( 24 | plugin_name=self.test_plugin_name, cmd_args_list=self.test_cmd_args_list 25 | ) 26 | self.test_opawdl_state_instance = state_instance.OPAWDLStateInstance( 27 | opawdl_state=self.test_opawdl_state, 28 | status=state_instance.Status.COMPLETED, 29 | ) 30 | self.test_state_name = "state_1" 31 | self.test_opawdl_workflow = OPAWDLWorkflow( 32 | starts_at=self.test_state_name, 33 | states={self.test_state_name: self.test_opawdl_state}, 34 | ) 35 | self.test_instance_id = "abcd" 36 | self.test_opawdl_workflow_instance = workflow_instance.OPAWDLWorkflowInstance( 37 | instance_id=self.test_instance_id, 38 | opawdl_workflow=self.test_opawdl_workflow, 39 | state_instances=[self.test_opawdl_state_instance], 40 | status=workflow_instance.Status.STARTED, 41 | ) 42 | 43 | def test_get_instance_id(self) -> None: 44 | # Act and Assert 45 | self.assertEqual( 46 | self.test_opawdl_workflow_instance.get_instance_id(), self.test_instance_id 47 | ) 48 | 49 | def test__str__(self) -> None: 50 | # Arrange 51 | expected_opawdl_workflow_instance_json_str = json.dumps( 52 | { 53 | "instance_id": self.test_instance_id, 54 | "opawdl_workflow": { 55 | "StartAt": self.test_state_name, 56 | "States": { 57 | self.test_state_name: { 58 | "PluginName": self.test_plugin_name, 59 | "CmdArgsList": self.test_cmd_args_list, 60 | "Timeout": None, 61 | "Next": None, 62 | "IsEnd": True, 63 | }, 64 | }, 65 | }, 66 | "state_instances": [ 67 | { 68 | "opawdl_state": { 69 | "PluginName": self.test_plugin_name, 70 | "CmdArgsList": self.test_cmd_args_list, 71 | "Timeout": None, 72 | "Next": None, 73 | "IsEnd": True, 74 | }, 75 | "status": "COMPLETED", 76 | } 77 | ], 78 | "status": "STARTED", 79 | } 80 | ) 81 | # Act and Assert 82 | self.assertEqual( 83 | str(self.test_opawdl_workflow_instance), 84 | expected_opawdl_workflow_instance_json_str, 85 | ) 86 | -------------------------------------------------------------------------------- /fbpcp/service/pce_aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-strict 8 | 9 | import logging 10 | from typing import Any, Dict, Optional 11 | 12 | from fbpcp.entity.pce import PCE 13 | from fbpcp.entity.pce_compute import PCECompute 14 | from fbpcp.entity.pce_network import PCENetwork 15 | from fbpcp.gateway.ec2 import EC2Gateway 16 | from fbpcp.gateway.ecs import ECSGateway 17 | from fbpcp.service.pce import PCEService 18 | 19 | 20 | PCE_ID_KEY = "pce:pce-id" 21 | SHARED_TASK_DEFINITION_PREFIX = "onedocker-task-shared-" 22 | 23 | 24 | class AWSPCEService(PCEService): 25 | def __init__( 26 | self, 27 | region: str, 28 | access_key_id: Optional[str] = None, 29 | access_key_data: Optional[str] = None, 30 | config: Optional[Dict[str, Any]] = None, 31 | ) -> None: 32 | self.logger: logging.Logger = logging.getLogger(__name__) 33 | self.region = region 34 | self.ec2_gateway = EC2Gateway(region, access_key_id, access_key_data, config) 35 | self.ecs_gateway = ECSGateway(region, access_key_id, access_key_data, config) 36 | 37 | def get_pce( 38 | self, 39 | pce_id: str, 40 | ) -> PCE: 41 | pce_network = self._get_network(pce_id) 42 | pce_compute = self._get_compute(pce_id) 43 | return PCE(pce_id, self.region, pce_network, pce_compute) 44 | 45 | def _get_network(self, pce_id: str) -> PCENetwork: 46 | 47 | tags = {PCE_ID_KEY: pce_id} 48 | vpcs = self.ec2_gateway.describe_vpcs(tags=tags) 49 | vpc = vpcs[0] if vpcs else None 50 | subnets = self.ec2_gateway.describe_subnets(tags=tags) 51 | firewall_rulesets = self.ec2_gateway.describe_security_groups(tags=tags) 52 | route_tables = self.ec2_gateway.describe_route_tables(tags=tags) 53 | route_table = route_tables[0] if route_tables else None 54 | vpc_peering = None 55 | if vpc: 56 | vpc_peerings = self.ec2_gateway.describe_vpc_peerings( 57 | vpc_id=vpc.vpc_id, tags=tags 58 | ) 59 | vpc_peering = vpc_peerings[0] if vpc_peerings else None 60 | return PCENetwork( 61 | region=self.region, 62 | vpc=vpc, 63 | subnets=subnets, 64 | route_table=route_table, 65 | vpc_peering=vpc_peering, 66 | firewall_rulesets=firewall_rulesets, 67 | ) 68 | 69 | def _get_compute(self, pce_id: str) -> PCECompute: 70 | tags = {PCE_ID_KEY: pce_id} 71 | clusters = self.ecs_gateway.describe_clusters(tags=tags) 72 | cluster = clusters[0] if clusters else None 73 | container_definitions = self.ecs_gateway.describe_task_definitions(tags=tags) 74 | if container_definitions: 75 | container_definition = container_definitions[0] 76 | else: 77 | try: 78 | container_definition = self.ecs_gateway.describe_task_definition( 79 | SHARED_TASK_DEFINITION_PREFIX + self.region 80 | ) 81 | except Exception as err: 82 | # The PCE service will only be responsible for getting the resources 83 | # The caller will decide how to handle the case that PCE don't have the container definition 84 | container_definition = None 85 | self.logger.exception(err) 86 | return PCECompute(self.region, cluster, container_definition) 87 | -------------------------------------------------------------------------------- /tests/entity/test_certificate_request.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # This source code is licensed under the MIT license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # pyre-unsafe 8 | 9 | import json 10 | import unittest 11 | 12 | from fbpcp.entity.certificate_request import CertificateRequest, KeyAlgorithm 13 | from fbpcp.error.pcp import InvalidParameterError 14 | 15 | 16 | class TestCertificateRequest(unittest.TestCase): 17 | TEST_KEY_ALGORITHM = KeyAlgorithm.RSA 18 | TEST_KEY_SIZE = 4096 19 | TEST_PASSPHRASE = "test" 20 | TEST_ORGANIZATION_NAME = "Test Company" 21 | TEST_COUNTRY_NAME = "US" 22 | 23 | def test_create_instance(self): 24 | # Arrange 25 | expected = CertificateRequest( 26 | key_algorithm=self.TEST_KEY_ALGORITHM, 27 | key_size=self.TEST_KEY_SIZE, 28 | passphrase=self.TEST_PASSPHRASE, 29 | cert_folder=None, 30 | private_key_name=None, 31 | certificate_name=None, 32 | days_valid=None, 33 | country_name=self.TEST_COUNTRY_NAME, 34 | state_or_province_name=None, 35 | locality_name=None, 36 | organization_name=None, 37 | common_name=None, 38 | dns_name=None, 39 | ) 40 | test_cert_params = expected.convert_to_cert_params() 41 | 42 | # Act 43 | result = CertificateRequest.create_instance(test_cert_params) 44 | 45 | # Assert 46 | self.assertEqual(expected, result) 47 | 48 | def test_create_instance_format_exception(self): 49 | # Arrange 50 | wrong_cert_params = "'test':" 51 | 52 | # Act 53 | # Assert 54 | with self.assertRaises(InvalidParameterError): 55 | CertificateRequest.create_instance(wrong_cert_params) 56 | 57 | def test_create_instance_missing_parameter_exception(self): 58 | # Arrange 59 | wrong_cert_params = str( 60 | { 61 | "key_algorithm": self.TEST_KEY_ALGORITHM.value, 62 | "key_size": self.TEST_KEY_SIZE, 63 | "organization_name": self.TEST_ORGANIZATION_NAME, 64 | } 65 | ) 66 | 67 | # Act 68 | # Assert 69 | with self.assertRaises(InvalidParameterError): 70 | CertificateRequest.create_instance(wrong_cert_params) 71 | 72 | def test_convert_to_cert_params(self): 73 | # Arrange 74 | cert_request = CertificateRequest( 75 | key_algorithm=self.TEST_KEY_ALGORITHM, 76 | key_size=self.TEST_KEY_SIZE, 77 | passphrase=self.TEST_PASSPHRASE, 78 | cert_folder=None, 79 | private_key_name=None, 80 | certificate_name=None, 81 | days_valid=None, 82 | country_name=None, 83 | state_or_province_name=None, 84 | locality_name=None, 85 | organization_name=self.TEST_ORGANIZATION_NAME, 86 | common_name=None, 87 | dns_name=None, 88 | ) 89 | expected_cert_params = json.dumps( 90 | { 91 | "key_algorithm": self.TEST_KEY_ALGORITHM.value, 92 | "key_size": self.TEST_KEY_SIZE, 93 | "passphrase": self.TEST_PASSPHRASE, 94 | "organization_name": self.TEST_ORGANIZATION_NAME, 95 | } 96 | ) 97 | # Act 98 | test_cert_params = cert_request.convert_to_cert_params() 99 | 100 | # Assert 101 | self.assertEqual(test_cert_params, expected_cert_params) 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Maintenance Mode](https://img.shields.io/badge/maintenance%20mode-%E2%9C%94%EF%B8%8F-red) 2 | **THIS PROJECT IS CURRENTLY IN MAINTENANCE MODE. Only critical bug patches will be applied; no new features will be added.** 3 | 4 | --- 5 | 6 | # FBPCP (Facebook Private Computation Platform) 7 | [Secure multi-party computation](https://en.wikipedia.org/wiki/Secure_multi-party_computation) (also known as secure computation, multi-party computation (MPC), or privacy-preserving computation) is a subfield of cryptography with the goal of creating methods for parties to jointly compute a function over their inputs while keeping those inputs private. 8 | 9 | FBPCP (Facebook Private Computation Platform) is a secure, privacy safe and scalable architecture to deploy MPC (Multi Party Computation) applications in a distributed way on virtual private clouds. [FBPCF](https://github.com/facebookresearch/fbpcf) (Facebook Private Computation Framework) is for scaling MPC computation up via threading, while FBPCP is for scaling MPC computation out via [Private Scaling](https://github.com/facebookresearch/fbpcp/blob/main/docs/PrivateScaling.md) architecture. FBPCP consists of various services, interfaces that enable various private measurement solutions, e.g. [Private Lift](https://github.com/facebookresearch/fbpcf/blob/master/docs/PrivateLift.md). 10 | 11 | [Private Scaling](https://github.com/facebookresearch/fbpcp/blob/main/docs/PrivateScaling.md) resembles the map/reduce architecture and is secure against a semi-honest adversary who tries to learn the inputs of the computation. The goal is to secure the intermediate output of each shard to prevent potential privacy leak. 12 | 13 | ## Installation Requirements: 14 | ### Prerequisites for working on Ubuntu 18.04: 15 | * An AWS account (Access Key ID, Secret Access Key) to use AWS SDK (boto3 API) in FBPCP 16 | * python >= 3.8 17 | * python3-pip 18 | 19 | ## Installing prerequisites on Ubuntu 18.04: 20 | * python3.8 21 | ```sh 22 | sudo apt-get install -y python3.8 23 | ``` 24 | * python3-pip 25 | ```sh 26 | sudo apt-get install -y python3-pip 27 | ``` 28 | ## Installing fbpcp 29 | ```sh 30 | python3.8 -m pip install 'git+https://github.com/facebookresearch/fbpcp.git' 31 | # (add --user if you don't have permission) 32 | 33 | # Or, to install it from a local clone: 34 | git clone https://github.com/facebookresearch/fbpcp.git 35 | python3.8 -m pip install -e fbpcp 36 | # (add --user if you don't have permission) 37 | 38 | # Or, to install it from Pypi 39 | python3.8 -m pip install fbpcp 40 | ``` 41 | 42 | ## Upgrading fbpcp 43 | * To latest version in github main branch 44 | ```sh 45 | python3.8 -m pip uninstall fbpcp 46 | # uninstall fbpcp first 47 | 48 | python3.8 -m pip install 'git+https://github.com/facebookresearch/fbpcp.git' 49 | # (add --user if you don't have permission) 50 | # re-install fbpcp from github repository 51 | ``` 52 | 53 | * To latest version in Pypi 54 | ```sh 55 | python3.8 -m pip install fbpcp --upgrade 56 | ``` 57 | 58 | ## Architecture 59 | Figure 1: Architecture of FBPCP 60 | 61 | ### Services: 62 | 63 | * MPCService is the public interface that provides APIs to distribute a MPC application with large dataset to multiple MPC workers on cloud. 64 | 65 | 66 | ### [Other components](https://github.com/facebookresearch/fbpcp/blob/main/docs/FBPCPComponents.md) 67 | 68 | ## Join the FBPCP community 69 | * Website: https://github.com/facebookresearch/fbpcp 70 | * See the [CONTRIBUTING](https://github.com/facebookresearch/fbpcp/blob/main/CONTRIBUTING.md) file for how to help out. 71 | 72 | ## License 73 | FBPCP is [MIT](https://github.com/facebookresearch/fbpcp/blob/main/LICENSE) licensed, as found in the LICENSE file. 74 | --------------------------------------------------------------------------------