├── .bazelignore ├── .bazelrc ├── .bazelversion ├── .dockerignore ├── .gitignore ├── .gitmodules ├── BUILD ├── BUILDING.md ├── CHANGELOG.md ├── CONTRIBUTING ├── Dockerfile ├── LICENSE ├── README.md ├── WORKSPACE ├── cloudbuild.yaml ├── docs └── high-level-overview.png ├── external └── coordinator.BUILD ├── java └── src │ ├── it │ └── java │ │ └── com │ │ └── google │ │ └── ondevicepersonalization │ │ └── federatedcompute │ │ └── endtoendtests │ │ ├── BUILD │ │ ├── EndToEndArgs.java │ │ ├── EndToEndTest.java │ │ ├── README.md │ │ ├── clients │ │ ├── BUILD │ │ ├── Device.java │ │ ├── HttpClientUtils.java │ │ └── Partner.java │ │ └── resources │ │ ├── BUILD │ │ ├── checkpoint_v1 │ │ ├── checkpoint_v2 │ │ ├── client_only_plan_v1 │ │ ├── client_only_plan_v2 │ │ ├── gradient_v1 │ │ ├── gradient_v2 │ │ ├── server_plan_v1 │ │ └── server_plan_v2 │ ├── main │ ├── java │ │ └── com │ │ │ └── google │ │ │ └── ondevicepersonalization │ │ │ └── federatedcompute │ │ │ └── shuffler │ │ │ ├── aggregator │ │ │ ├── AggregatorApplication.java │ │ │ ├── BUILD │ │ │ ├── controllers │ │ │ │ ├── AggregatorController.java │ │ │ │ └── BUILD │ │ │ ├── core │ │ │ │ ├── AggregatorCore.java │ │ │ │ ├── AggregatorCoreImpl.java │ │ │ │ ├── BUILD │ │ │ │ ├── gcp │ │ │ │ │ ├── AggregatorConfig.java │ │ │ │ │ ├── BUILD │ │ │ │ │ └── PubSubMessageReceiver.java │ │ │ │ └── message │ │ │ │ │ ├── AggregatorMessage.java │ │ │ │ │ ├── AggregatorNotification.java │ │ │ │ │ └── BUILD │ │ │ └── scheduler │ │ │ │ ├── BUILD │ │ │ │ └── ScheduledTask.java │ │ │ ├── collector │ │ │ ├── BUILD │ │ │ ├── CollectorApplication.java │ │ │ ├── controllers │ │ │ │ ├── BUILD │ │ │ │ └── CollectorController.java │ │ │ ├── core │ │ │ │ ├── BUILD │ │ │ │ ├── CollectorCore.java │ │ │ │ ├── CollectorCoreImpl.java │ │ │ │ ├── CollectorCoreImplHelper.java │ │ │ │ └── gcp │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── CollectorConfig.java │ │ │ │ │ └── PubSubMessageReceiver.java │ │ │ └── scheduler │ │ │ │ ├── BUILD │ │ │ │ └── ScheduledTask.java │ │ │ ├── common │ │ │ ├── BUILD │ │ │ ├── CompressionUtils.java │ │ │ ├── Constants.java │ │ │ ├── Exceptions.java │ │ │ ├── GuidUniqueIdGenerator.java │ │ │ ├── JavaUtilRandomGenerator.java │ │ │ ├── NonRetryableException.java │ │ │ ├── ProtoParser.java │ │ │ ├── RandomGenerator.java │ │ │ ├── UniqueIdGenerator.java │ │ │ ├── config │ │ │ │ └── gcp │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── Configurations.java │ │ │ │ │ ├── EncryptionArgs.java │ │ │ │ │ ├── GcpParameterClient.java │ │ │ │ │ ├── GcpParameterConfig.java │ │ │ │ │ ├── GcpVMMetadataConfig.java │ │ │ │ │ ├── GcpVMMetadataServiceClient.java │ │ │ │ │ ├── GoogleCloudArgs.java │ │ │ │ │ └── SecurityArgs.java │ │ │ ├── converters │ │ │ │ ├── BUILD │ │ │ │ ├── TaskEntityConverter.java │ │ │ │ ├── TaskEntityStatusConverter.java │ │ │ │ └── TimestampInstantConverter.java │ │ │ ├── crypto │ │ │ │ ├── BUILD │ │ │ │ ├── Payload.java │ │ │ │ ├── PublicKey.java │ │ │ │ ├── PublicKeyEncryptionService.java │ │ │ │ ├── PublicKeyFetchingService.java │ │ │ │ ├── PublicKeys.java │ │ │ │ └── gcp │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── MultiPartyDecryptionConfig.java │ │ │ │ │ └── PublicKeyEncryptionConfig.java │ │ │ ├── dao │ │ │ │ ├── AggregationBatchDao.java │ │ │ │ ├── AggregationBatchEntity.java │ │ │ │ ├── AggregationBatchId.java │ │ │ │ ├── AssignmentDao.java │ │ │ │ ├── AssignmentEntity.java │ │ │ │ ├── AssignmentId.java │ │ │ │ ├── AuthorizationTokenDao.java │ │ │ │ ├── BUILD │ │ │ │ ├── BlobDao.java │ │ │ │ ├── BlobDescription.java │ │ │ │ ├── BlobManager.java │ │ │ │ ├── CheckInResult.java │ │ │ │ ├── IterationEntity.java │ │ │ │ ├── IterationId.java │ │ │ │ ├── ModelMetricsDao.java │ │ │ │ ├── ModelMetricsEntity.java │ │ │ │ ├── Partitioner.java │ │ │ │ ├── SinglePartitionPartitioner.java │ │ │ │ ├── TaskDao.java │ │ │ │ ├── TaskEntities.java │ │ │ │ ├── TaskEntity.java │ │ │ │ ├── TaskId.java │ │ │ │ └── gcp │ │ │ │ │ ├── AggregationBatchSpannerDao.java │ │ │ │ │ ├── AssignmentSpannerDao.java │ │ │ │ │ ├── AuthorizationTokenSpannerDao.java │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DaoConfigurations.java │ │ │ │ │ ├── GCSBlobDao.java │ │ │ │ │ ├── GCSBlobManager.java │ │ │ │ │ ├── GCSConfig.java │ │ │ │ │ ├── ModelMetricsSpannerDao.java │ │ │ │ │ ├── TaskSpannerDao.java │ │ │ │ │ └── TimestampInstantConverter.java │ │ │ ├── lock │ │ │ │ └── jdbc │ │ │ │ │ └── gcp │ │ │ │ │ ├── BUILD │ │ │ │ │ └── SpannerLockConfiguration.java │ │ │ ├── logging │ │ │ │ ├── BUILD │ │ │ │ ├── LoggingFilter.java │ │ │ │ ├── LoggingInterceptor.java │ │ │ │ ├── LoggingInterceptorConfig.java │ │ │ │ ├── LoggingInterceptors.java │ │ │ │ └── ResponseProto.java │ │ │ ├── messaging │ │ │ │ ├── BUILD │ │ │ │ ├── HttpMessageSender.java │ │ │ │ ├── MessageSender.java │ │ │ │ └── gcp │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── GcpHttpMessageSender.java │ │ │ │ │ ├── GcpHttpMessageSenderConfig.java │ │ │ │ │ ├── PubSubMessageSender.java │ │ │ │ │ ├── PubSubPublisherConfig.java │ │ │ │ │ └── PubSubSubscriberConfig.java │ │ │ ├── security │ │ │ │ ├── AndroidKeyAttestationConfig.java │ │ │ │ ├── AuthenticationFilter.java │ │ │ │ ├── AuthenticationFilterRegistrationBean.java │ │ │ │ ├── BUILD │ │ │ │ └── KeyAttestationManager.java │ │ │ └── tensorflow │ │ │ │ ├── BUILD │ │ │ │ └── TensorflowPlanSessionFactory.java │ │ │ ├── modelupdater │ │ │ ├── BUILD │ │ │ ├── ModelUpdaterApplication.java │ │ │ ├── controllers │ │ │ │ ├── BUILD │ │ │ │ └── ModelUpdaterController.java │ │ │ ├── core │ │ │ │ ├── BUILD │ │ │ │ ├── ModelUpdaterCore.java │ │ │ │ ├── ModelUpdaterCoreImpl.java │ │ │ │ ├── gcp │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── ModelUpdaterConfig.java │ │ │ │ │ └── PubSubMessageReceiver.java │ │ │ │ └── message │ │ │ │ │ ├── BUILD │ │ │ │ │ └── ModelUpdaterMessage.java │ │ │ └── scheduler │ │ │ │ ├── BUILD │ │ │ │ └── ScheduledTask.java │ │ │ ├── taskassignment │ │ │ ├── BUILD │ │ │ ├── TaskAssignmentApplication.java │ │ │ ├── controllers │ │ │ │ ├── BUILD │ │ │ │ └── TaskAssignmentController.java │ │ │ └── core │ │ │ │ ├── BUILD │ │ │ │ ├── TaskAssignmentCore.java │ │ │ │ ├── TaskAssignmentCoreHelper.java │ │ │ │ └── TaskAssignmentCoreImpl.java │ │ │ ├── taskmanagement │ │ │ ├── BUILD │ │ │ ├── TaskManagementApplication.java │ │ │ ├── controllers │ │ │ │ ├── BUILD │ │ │ │ └── TaskManagementController.java │ │ │ └── core │ │ │ │ ├── BUILD │ │ │ │ ├── TaskManagementCore.java │ │ │ │ └── TaskManagementCoreImpl.java │ │ │ └── taskscheduler │ │ │ ├── BUILD │ │ │ ├── TaskSchedulerApplication.java │ │ │ ├── controllers │ │ │ ├── BUILD │ │ │ └── TaskSchedulerController.java │ │ │ ├── core │ │ │ ├── BUILD │ │ │ ├── TaskSchedulerCore.java │ │ │ ├── TaskSchedulerCoreHelper.java │ │ │ └── TaskSchedulerCoreImpl.java │ │ │ └── scheduler │ │ │ ├── BUILD │ │ │ └── ScheduledTask.java │ └── resources │ │ ├── BUILD │ │ ├── aggregator.properties │ │ ├── collector.properties │ │ ├── logback-spring.xml │ │ ├── modelupdater.properties │ │ ├── taskassignment.properties │ │ ├── taskmanagement.properties │ │ └── taskscheduler.properties │ └── test │ └── java │ ├── com │ └── google │ │ └── ondevicepersonalization │ │ └── federatedcompute │ │ └── shuffler │ │ ├── aggregator │ │ └── core │ │ │ ├── AggregatorCoreImplTest.java │ │ │ ├── BUILD │ │ │ └── gcp │ │ │ ├── BUILD │ │ │ └── PubSubMessageReceiverTest.java │ │ ├── collector │ │ └── core │ │ │ ├── BUILD │ │ │ ├── CollectorCoreImplHelperTest.java │ │ │ ├── CollectorCoreImplTest.java │ │ │ └── gcp │ │ │ ├── BUILD │ │ │ └── PubSubMessageReceiverTest.java │ │ ├── common │ │ ├── BUILD │ │ ├── CompressionUtilsTest.java │ │ ├── ExceptionsTest.java │ │ ├── GuidUniqueIdGeneratorTest.java │ │ ├── JavaUtilRandomGeneratorTest.java │ │ ├── ProtoParserTest.java │ │ ├── config │ │ │ └── gcp │ │ │ │ ├── BUILD │ │ │ │ ├── GcpParameterClientTest.java │ │ │ │ └── GcpVMMetadataServiceClientTest.java │ │ ├── converters │ │ │ ├── BUILD │ │ │ ├── TaskEntityConverterTest.java │ │ │ ├── TaskEntityStatusConverterTest.java │ │ │ └── TimestampInstantConverterTest.java │ │ ├── crypto │ │ │ ├── BUILD │ │ │ ├── PublicKeyEncryptionServiceTest.java │ │ │ └── PublicKeyFetchingServiceTest.java │ │ ├── dao │ │ │ ├── AggregationBatchEntityTest.java │ │ │ ├── AggregationBatchIdTest.java │ │ │ ├── AssignmentEntityTest.java │ │ │ ├── AssignmentIdTest.java │ │ │ ├── BUILD │ │ │ ├── CheckInResultTest.java │ │ │ ├── IterationEntityTest.java │ │ │ ├── IterationIdTest.java │ │ │ ├── ModelMetricsEntityTest.java │ │ │ ├── SinglePartitionPartitionerTest.java │ │ │ ├── TaskEntitiesTest.java │ │ │ ├── TaskEntityStatusTest.java │ │ │ ├── TaskEntityTest.java │ │ │ ├── TaskIdTest.java │ │ │ └── gcp │ │ │ │ ├── AggregationBatchSpannerDaoTest.java │ │ │ │ ├── AssignmentSpannerDaoTest.java │ │ │ │ ├── AuthorizationTokenSpannerDaoTest.java │ │ │ │ ├── BUILD │ │ │ │ ├── GCSBlobDaoTest.java │ │ │ │ ├── GCSBlobManagerTest.java │ │ │ │ ├── GCSConfigTest.java │ │ │ │ ├── ModelMetricsSpannerDaoTest.java │ │ │ │ ├── SpannerTestHarness.java │ │ │ │ ├── TaskSpannerDaoTest.java │ │ │ │ └── TimestampInstantConverterTest.java │ │ ├── logging │ │ │ ├── BUILD │ │ │ ├── LoggingFilterTest.java │ │ │ ├── LoggingInterceptorTest.java │ │ │ ├── LoggingInterceptorsTest.java │ │ │ └── test.proto │ │ ├── messaging │ │ │ └── gcp │ │ │ │ ├── BUILD │ │ │ │ └── PubSubMessageSenderTest.java │ │ ├── security │ │ │ ├── AuthenticationFilterTest.java │ │ │ ├── BUILD │ │ │ └── KeyAttestationManagerTest.java │ │ └── tensorflow │ │ │ ├── BUILD │ │ │ └── TensorflowPlanSessionFactoryTest.java │ │ ├── modelupdater │ │ └── core │ │ │ ├── BUILD │ │ │ ├── ModelUpdaterCoreImplTest.java │ │ │ └── gcp │ │ │ ├── BUILD │ │ │ └── PubSubMessageReceiverTest.java │ │ ├── taskassignment │ │ ├── controllers │ │ │ ├── BUILD │ │ │ └── TaskAssignmentControllerTest.java │ │ └── core │ │ │ ├── BUILD │ │ │ ├── TaskAssignmentCoreHelperTest.java │ │ │ └── TaskAssignmentCoreImplTest.java │ │ ├── taskmanagement │ │ ├── controllers │ │ │ ├── BUILD │ │ │ └── TaskManagementControllerTest.java │ │ └── core │ │ │ ├── BUILD │ │ │ └── TaskManagementCoreImplTest.java │ │ └── taskscheduler │ │ └── core │ │ ├── BUILD │ │ ├── TaskSchedulerCoreHelperTest.java │ │ └── TaskSchedulerCoreImplTest.java │ └── resources │ ├── BUILD │ ├── KeyAttestationFailVerification1.json │ ├── KeyAttestationFailVerification2.json │ ├── KeyAttestationFailVerification3.json │ ├── KeyAttestationSuccessVerification.json │ ├── plan_v1 │ └── plan_v2 ├── patches ├── BUILD ├── rules_spring.patch └── rules_spring_manifest.patch ├── python └── taskbuilder │ ├── BUILD │ ├── README.md │ ├── artifact_utils.py │ ├── artifact_utils_test.py │ ├── common.py │ ├── config_validator.py │ ├── config_validator_test.py │ ├── dataset_utils.py │ ├── dataset_utils_test.py │ ├── dp_utils.py │ ├── dp_utils_test.py │ ├── example_model.py │ ├── graph_compactor.py │ ├── http_utils.py │ ├── http_utils_test.py │ ├── io_utils.py │ ├── io_utils_test.py │ ├── learning_process_utils.py │ ├── learning_process_utils_test.py │ ├── metrics_utils.py │ ├── sample │ ├── keras │ │ ├── input │ │ │ ├── keras_model │ │ │ │ ├── fingerprint.pb │ │ │ │ ├── saved_model.pb │ │ │ │ └── variables │ │ │ │ │ ├── variables.data-00000-of-00001 │ │ │ │ │ └── variables.index │ │ │ ├── keras_task_config.pbtxt │ │ │ └── keras_task_config_build_artifact_only.pbtxt │ │ └── output │ │ │ ├── client_only_plan.pb │ │ │ ├── eval_client_only_plan.pb │ │ │ ├── eval_initial_state.server.ckp │ │ │ ├── eval_plan.pb │ │ │ ├── initial_state.server.ckp │ │ │ └── plan.pb │ └── mnist │ │ ├── input │ │ ├── mnist_cnn_task_config.pbtxt │ │ ├── mnist_cnn_task_config_build_artifact_only.pbtxt │ │ └── mnist_model │ │ │ ├── fingerprint.pb │ │ │ ├── saved_model.pb │ │ │ └── variables │ │ │ ├── variables.data-00000-of-00001 │ │ │ └── variables.index │ │ └── output │ │ ├── client_only_plan.pb │ │ ├── eval_client_only_plan.pb │ │ ├── eval_initial_state.server.ckp │ │ ├── eval_plan.pb │ │ ├── initial_state.server.ckp │ │ └── plan.pb │ ├── support_ops.csv │ ├── support_ops_utils.py │ ├── task_builder_client.py │ ├── task_builder_core.py │ ├── task_builder_core_test.py │ ├── task_builder_wsgi.py │ ├── task_utils.py │ ├── task_utils_test.py │ ├── tensorflow_checkpoints.py │ ├── tensorflow_checkpoints_test.py │ ├── test_utils.py │ └── tflite_flex_ops.cc ├── requirements_lock_3_10.txt ├── scripts ├── build_images.sh ├── docker │ ├── README.md │ ├── docker-compose.yml │ ├── docker_run.sh │ ├── docker_sh.sh │ ├── install_google_cloud_sdk.bash │ ├── load_images.sh │ ├── nginx.conf │ └── run_all_services_docker.sh ├── format.sh ├── generate_coverage.sh ├── init_test_db.sh ├── run_all_tests.sh └── upload_images.sh ├── shuffler ├── dashboard │ ├── looker │ │ ├── README.md │ │ ├── dashboards │ │ │ ├── .gitkeep │ │ │ ├── model_metrics.dashboard.lookml │ │ │ └── task_status.dashboard.lookml │ │ ├── models │ │ │ ├── .gitkeep │ │ │ ├── metric.model.lkml │ │ │ └── task.model.lkml │ │ └── views │ │ │ ├── .gitkeep │ │ │ ├── metric_db_views │ │ │ ├── .gitkeep │ │ │ └── metric.view.lkml │ │ │ └── task_db_views │ │ │ ├── .gitkeep │ │ │ ├── apm_assigned.view.lkml │ │ │ ├── apm_assigned_to_upload_completed.view.lkml │ │ │ ├── apm_base.view.lkml │ │ │ ├── apm_canceled.view.lkml │ │ │ ├── apm_local_completed.view.lkml │ │ │ ├── apm_local_failed.view.lkml │ │ │ ├── apm_local_failed_example_generation.view.lkml │ │ │ ├── apm_local_failed_model_computation.view.lkml │ │ │ ├── apm_local_failed_ops_error.view.lkml │ │ │ ├── apm_local_not_eligible.view.lkml │ │ │ ├── apm_local_timeout.view.lkml │ │ │ ├── apm_remote_failed.view.lkml │ │ │ ├── apm_upload_completed.view.lkml │ │ │ ├── apm_upload_timeout.view.lkml │ │ │ ├── iteration.view.lkml │ │ │ ├── iteration_aggregating_to_complete.view.lkml │ │ │ ├── iteration_completion.view.lkml │ │ │ ├── iteration_open.view.lkml │ │ │ ├── iteration_status_base.view.lkml │ │ │ └── task.view.lkml │ └── lookerstudio │ │ ├── README.md │ │ ├── docs │ │ ├── device_training_duration_example.png │ │ ├── device_upload_duration_example.png │ │ ├── iteration_processed_time_example.png │ │ ├── local_completed_assignment_per_minute_example.png │ │ └── model_metrics_example.png │ │ └── sql │ │ ├── model │ │ └── model_metrics.sql │ │ └── ops │ │ ├── assignment_per_minute.sql │ │ ├── device_training_duration.sql │ │ ├── device_upload_duration.sql │ │ ├── iteration.sql │ │ └── iteration_processed_time.sql ├── proto │ ├── BUILD │ ├── common.proto │ ├── task.proto │ ├── task_assignments.proto │ └── task_builder.proto ├── services │ ├── aggregator │ │ └── BUILD │ ├── collector │ │ └── BUILD │ ├── modelupdater │ │ └── BUILD │ ├── taskassignment │ │ └── BUILD │ ├── taskbuilder │ │ └── BUILD │ ├── taskmanagement │ │ └── BUILD │ └── taskscheduler │ │ └── BUILD ├── spanner │ ├── BUILD │ ├── metrics_database.sdl │ └── task_database.sdl └── terraform │ └── gcp │ ├── README.md │ ├── applications │ ├── cluster │ │ ├── cluster.tf │ │ ├── outputs.tf │ │ └── variables.tf │ ├── resources │ │ ├── resources.tf │ │ ├── resources_outputs.tf │ │ └── resources_variables.tf │ └── shuffler │ │ ├── compute.tf │ │ ├── outputs.tf │ │ ├── resources.tf │ │ ├── resources_outputs.tf │ │ ├── resources_variables.tf │ │ └── variables.tf │ ├── environments │ ├── dev │ │ ├── cluster │ │ │ ├── cluster.tf │ │ │ ├── cluster_outputs.tf │ │ │ ├── cluster_variables.tf │ │ │ ├── dev.auto.tfvars │ │ │ └── main.tf │ │ ├── dev.auto.tfvars │ │ ├── main.tf │ │ ├── shuffler.tf │ │ ├── shuffler_outputs.tf │ │ └── shuffler_variables.tf │ ├── resources-only │ │ ├── shuffler.tf │ │ ├── shuffler_outputs.tf │ │ └── shuffler_variables.tf │ └── shared │ │ ├── cluster │ │ ├── cluster.tf │ │ ├── cluster_outputs.tf │ │ └── cluster_variables.tf │ │ ├── shuffler.tf │ │ ├── shuffler_outputs.tf │ │ └── shuffler_variables.tf │ └── modules │ ├── cdn │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── cluster │ ├── actuator_prometheus_monitoring.tf │ ├── collector.tf │ ├── ingress.tf │ ├── metrics_adapter.tf │ ├── monitoring_dashboard.tf │ ├── outputs.tf │ ├── taskassignment.tf │ ├── taskscheduler.tf │ ├── variables.tf │ └── workload_identity.tf │ ├── confidentialspace │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── gke │ ├── gke.tf │ ├── outputs.tf │ └── variables.tf │ ├── monitoring │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── network │ ├── network.tf │ ├── outputs.tf │ ├── routing.tf │ └── variables.tf │ ├── parameters │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── pubsub │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── serviceaccount │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── storage │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ ├── taskbuilder │ ├── main.tf │ ├── outputs.tf │ └── variables.tf │ └── taskmanagement │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── toolchain ├── BUILD ├── bookworm.lock.json ├── bookworm.yaml └── requirements_updater │ ├── BUILD │ ├── WORKSPACE │ ├── requirements.in │ └── updater.sh └── tools ├── BUILD └── toolchain_provided_tf.bzl /.bazelignore: -------------------------------------------------------------------------------- 1 | # Ignore gitsubmodules 2 | federatedcompute/ 3 | 4 | # Ignore requirements_updater. It has its own WORKSPACE file 5 | toolchain/requirements_updater/ -------------------------------------------------------------------------------- /.bazelversion: -------------------------------------------------------------------------------- 1 | 7.4.0 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # We don't need anything from docker context when building the image. 2 | /* 3 | 4 | # Allow files 5 | !/scripts/docker/install_google_cloud_sdk.bash -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore all bazel-* symlinks. There is no full list since this can change 2 | # based on the name of the directory bazel is cloned into. 3 | /bazel-* 4 | /out 5 | 6 | # Compiled libraries. 7 | **/*.a 8 | **/*.so 9 | 10 | # Bazel cache. 11 | /.cache/ 12 | 13 | # Generated vscode files. 14 | **/.vscode/ 15 | 16 | # terraform 17 | .terraform 18 | # NOTE: .terraform.lock.hcl should be checked into git 19 | # https://www.terraform.io/docs/language/dependency-lock.html#lock-file-location 20 | 21 | # emacs stuff 22 | *~ 23 | 24 | # intellij 25 | .clwb/ 26 | .ijwb/ 27 | .idea/ 28 | **/*.iml 29 | 30 | # python 31 | __pycache__/ 32 | venv/ 33 | 34 | # code coverage output 35 | genhtml/ 36 | 37 | # python cache 38 | **/__pycache__/** 39 | 40 | # mac files 41 | **/*.DS_Store 42 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "federatedcompute"] 2 | path = federatedcompute 3 | url = https://github.com/privacysandbox/federatedcompute.git 4 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /CONTRIBUTING: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Presently this project is not accepting contributions. 4 | -------------------------------------------------------------------------------- /cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Create a private pool. Default gcloud build instances are too small to build.artifacts: 16 | # gcloud builds worker-pools create odp-federatedcompute-privatepool --region us-central1 --worker-machine-type=e2-standard-32 17 | 18 | # Run build. Update ProjectId in the command below: 19 | # gcloud builds submit --substitutions=_PROJECT_ID="ProjectId" --region us-central1 20 | 21 | steps: 22 | - name: 'gcr.io/cloud-builders/docker' 23 | args: [ 'build', '--platform=linux/amd64', '-t', 24 | 'odp-federatedcompute-build:v1', 25 | '.' ] 26 | - name: 'odp-federatedcompute-build:v1' 27 | args: ['./scripts/build_images.sh'] 28 | # Uncomment to push images to specified registry 29 | # gcloud builds submit --substitutions=_PROJECT_ID="ProjectId",_REGISTRY="Registry" --region us-central1 30 | # - name: 'odp-federatedcompute-build:v1' 31 | # args: ['./scripts/upload_images.sh', '$_REGISTRY'] 32 | options: 33 | pool: 34 | name: 'projects/$_PROJECT_ID/locations/us-central1/workerPools/odp-federatedcompute-privatepool' -------------------------------------------------------------------------------- /docs/high-level-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/docs/high-level-overview.png -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | package( 15 | default_visibility = ["//visibility:public"], 16 | licenses = ["notice"], # Apache 2.0 17 | ) 18 | 19 | filegroup( 20 | name = "resources", 21 | srcs = [ 22 | "checkpoint_v1", 23 | "client_only_plan_v1", 24 | "gradient_v1", 25 | "server_plan_v1", 26 | "checkpoint_v2", 27 | "client_only_plan_v2", 28 | "gradient_v2", 29 | "server_plan_v2", 30 | ], 31 | ) 32 | -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/checkpoint_v1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/checkpoint_v1 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/checkpoint_v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/checkpoint_v2 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/client_only_plan_v1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/client_only_plan_v1 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/client_only_plan_v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/client_only_plan_v2 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/gradient_v1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/gradient_v1 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/gradient_v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/gradient_v2 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/server_plan_v1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/server_plan_v1 -------------------------------------------------------------------------------- /java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/server_plan_v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/it/java/com/google/ondevicepersonalization/federatedcompute/endtoendtests/resources/server_plan_v2 -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/controllers/AggregatorController.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.aggregator.controllers; 18 | 19 | import org.springframework.web.bind.annotation.GetMapping; 20 | import org.springframework.web.bind.annotation.RestController; 21 | 22 | /** Aggregator controller. */ 23 | @RestController 24 | public final class AggregatorController { 25 | @GetMapping("/ready") 26 | public String ready() { 27 | // TODO(291594777): Implement readiness check. 28 | return "Greetings from Aggregator Spring Boot! Ready check. \n"; 29 | } 30 | 31 | @GetMapping("/healthz") 32 | public String healthz() { 33 | // TODO(291594777): Implement health check. 34 | return "Greetings from Aggregator Spring Boot! Health check. \n"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "aggregator_controller", 19 | srcs = ["AggregatorController.java"], 20 | deps = [ 21 | "@maven//:org_springframework_boot_spring_boot", 22 | "@maven//:org_springframework_boot_spring_boot_autoconfigure", 23 | "@maven//:org_springframework_boot_spring_boot_starter_web", 24 | "@maven//:org_springframework_spring_context", 25 | "@maven//:org_springframework_spring_web", 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core/AggregatorCore.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.aggregator.core; 18 | 19 | import com.google.ondevicepersonalization.federatedcompute.shuffler.aggregator.core.message.AggregatorMessage; 20 | 21 | /** The Aggregator core. */ 22 | public interface AggregatorCore { 23 | public void process(AggregatorMessage message); 24 | } 25 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core/message/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "aggregator_message", 19 | srcs = [ 20 | "AggregatorMessage.java", 21 | ], 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common:lombok_java", 24 | ], 25 | ) 26 | 27 | java_library( 28 | name = "aggregator_notification", 29 | srcs = [ 30 | "AggregatorNotification.java", 31 | ], 32 | deps = [ 33 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common:lombok_java", 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/scheduler/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_spring//springboot:springboot.bzl", "springboot") 16 | 17 | package(default_visibility = ["//visibility:public"]) 18 | 19 | java_library( 20 | name = "scheduled_task", 21 | srcs = [ 22 | "ScheduledTask.java", 23 | ], 24 | deps = [ 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/crypto:public_key_encryption_service", 26 | "@maven//:org_slf4j_slf4j_api", 27 | "@maven//:org_springframework_spring_context", 28 | ], 29 | ) 30 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "collector_controller", 19 | srcs = ["CollectorController.java"], 20 | deps = [ 21 | "@maven//:org_springframework_boot_spring_boot", 22 | "@maven//:org_springframework_boot_spring_boot_autoconfigure", 23 | "@maven//:org_springframework_boot_spring_boot_starter_web", 24 | "@maven//:org_springframework_spring_context", 25 | "@maven//:org_springframework_spring_web", 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/controllers/CollectorController.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.collector.controllers; 18 | 19 | import org.springframework.web.bind.annotation.GetMapping; 20 | import org.springframework.web.bind.annotation.RestController; 21 | 22 | /** Collector controller. */ 23 | @RestController 24 | public final class CollectorController { 25 | @GetMapping("/ready") 26 | public String ready() { 27 | // TODO(291594777): Implement readiness check. 28 | return "Greetings from Collector Spring Boot! Ready check. \n"; 29 | } 30 | 31 | @GetMapping("/healthz") 32 | public String healthz() { 33 | // TODO(291594777): Implement health check. 34 | return "Greetings from Collector Spring Boot! Health check. \n"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/core/CollectorCore.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.collector.core; 18 | 19 | import com.google.ondevicepersonalization.federatedcompute.shuffler.aggregator.core.message.AggregatorNotification; 20 | 21 | /** Collector core interface. */ 22 | public interface CollectorCore { 23 | 24 | /** Process iterations in status COLLECTING. */ 25 | public void processCollecting(); 26 | 27 | /** Process iterations in status AGGREGATING. */ 28 | public void processAggregating(); 29 | 30 | /** Process timeouts for iteration in status COLLECTING. */ 31 | public void processTimeouts(); 32 | 33 | /** Process notifications from the Aggregator. */ 34 | public void processAggregatorNotifications(AggregatorNotification.Attributes notification); 35 | } 36 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/core/gcp/CollectorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.collector.core.gcp; 18 | 19 | import org.springframework.beans.factory.annotation.Qualifier; 20 | import org.springframework.context.annotation.Bean; 21 | import org.springframework.context.annotation.Configuration; 22 | 23 | /** Configuration for collector. */ 24 | @Configuration 25 | public class CollectorConfig { 26 | @Bean 27 | @Qualifier("pubSubSubscription") 28 | public String pubSubSubscription(String aggregatorNotificationPubsubSubscription) { 29 | // Set the pubSubSubscription bean to point to aggregatorNotificationPubsubSubscription 30 | return aggregatorNotificationPubsubSubscription; 31 | } 32 | 33 | @Bean 34 | @Qualifier("aggregatorNotificationEndpoint") 35 | public String aggregatorNotificationEndpoint( 36 | String projectId, String aggregatorNotificationPubsubTopic) { 37 | return String.format( 38 | "https://pubsub.googleapis.com/v1/projects/%s/topics/%s:publish", 39 | projectId, aggregatorNotificationPubsubTopic); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/scheduler/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_spring//springboot:springboot.bzl", "springboot") 16 | 17 | package(default_visibility = ["//visibility:public"]) 18 | 19 | java_library( 20 | name = "scheduled_task", 21 | srcs = [ 22 | "ScheduledTask.java", 23 | ], 24 | deps = [ 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/core:collector_core", 26 | "@maven//:org_springframework_spring_context", 27 | ], 28 | ) 29 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/scheduler/ScheduledTask.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.collector.scheduler; 18 | 19 | import com.google.ondevicepersonalization.federatedcompute.shuffler.collector.core.CollectorCore; 20 | import org.springframework.scheduling.annotation.Scheduled; 21 | import org.springframework.stereotype.Component; 22 | 23 | /** The scheduled task. */ 24 | @Component 25 | public class ScheduledTask { 26 | 27 | CollectorCore collector; 28 | 29 | public ScheduledTask(CollectorCore collector) { 30 | this.collector = collector; 31 | } 32 | 33 | @Scheduled(fixedDelay = 200) 34 | public void run() throws Exception { 35 | collector.processCollecting(); 36 | collector.processAggregating(); 37 | } 38 | 39 | @Scheduled(fixedDelay = 60000) 40 | public void runTimeouts() throws Exception { 41 | collector.processTimeouts(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/GuidUniqueIdGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | import java.util.UUID; 20 | import org.springframework.stereotype.Component; 21 | 22 | @Component 23 | public class GuidUniqueIdGenerator implements UniqueIdGenerator { 24 | public String generate() { 25 | return UUID.randomUUID().toString(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/JavaUtilRandomGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | import java.util.Random; 20 | import org.springframework.stereotype.Component; 21 | 22 | @Component 23 | public class JavaUtilRandomGenerator implements RandomGenerator { 24 | 25 | private final Random randGen = new Random(); 26 | 27 | @Override 28 | public long nextLong(long bound) { 29 | return randGen.nextLong(bound); 30 | } 31 | 32 | @Override 33 | public int nextInt(int bound) { 34 | return randGen.nextInt(bound); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/NonRetryableException.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | public final class NonRetryableException extends RuntimeException { 20 | public NonRetryableException(String s) { 21 | super(s); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/RandomGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | public interface RandomGenerator { 20 | 21 | /** 22 | * Returns a pseudorandomly chosen long value between zero (inclusive) and the specified bound 23 | * (exclusive). 24 | * 25 | * @param bound the upper bound (exclusive) for the returned value. Must be positive. 26 | * @return a pseudorandomly chosen long value between zero (inclusive) and the bound (exclusive) 27 | * @throws IllegalArgumentException – if bound is not positive 28 | */ 29 | public long nextLong(long bound); 30 | 31 | /** 32 | * Returns a pseudorandomly chosen long value between zero (inclusive) and the specified bound 33 | * (exclusive). 34 | * 35 | * @param bound the upper bound (exclusive) for the returned value. Must be positive. 36 | * @return a pseudorandomly chosen long value between zero (inclusive) and the bound (exclusive) 37 | * @throws IllegalArgumentException – if bound is not positive 38 | */ 39 | public int nextInt(int bound); 40 | } 41 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/UniqueIdGenerator.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | public interface UniqueIdGenerator { 20 | public String generate(); 21 | } 22 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/config/gcp/EncryptionArgs.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.config.gcp; 17 | 18 | import com.beust.jcommander.Parameter; 19 | import lombok.Getter; 20 | 21 | /** Encryption args for encrypting payloads passed from command line */ 22 | @Getter 23 | public class EncryptionArgs { 24 | @Parameter( 25 | names = "--public_key_service_base_url", 26 | description = "The base url of the public key service.") 27 | private String publicKeyServiceBaseUrl; 28 | 29 | @Parameter( 30 | names = "--should_encrypt_aggregator_output", 31 | description = "Whether the output of the aggregator should be encrypted.", 32 | arity = 1) 33 | private boolean shouldEncryptAggregatorOutput = true; 34 | } 35 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/crypto/PublicKey.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.crypto; 18 | 19 | import lombok.Builder; 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | 23 | @Getter 24 | @EqualsAndHashCode 25 | @Builder(toBuilder = true) 26 | /** POJO representing a downloaded public key. */ 27 | public class PublicKey { 28 | // ID of the key 29 | private String id; 30 | 31 | // The base64 encoded public key material 32 | private String key; 33 | } 34 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/crypto/PublicKeys.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.crypto; 18 | 19 | import java.util.List; 20 | import lombok.Builder; 21 | import lombok.EqualsAndHashCode; 22 | import lombok.Getter; 23 | 24 | @Getter 25 | @EqualsAndHashCode 26 | @Builder(toBuilder = true) 27 | /** POJO representing the list of public keys downloaded. */ 28 | public class PublicKeys { 29 | // List of public keys 30 | private List keys; 31 | } 32 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/AggregationBatchId.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import lombok.Builder; 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | 23 | /** Aggregation batch identifier. */ 24 | @Getter 25 | @EqualsAndHashCode 26 | @Builder(toBuilder = true) 27 | public class AggregationBatchId { 28 | private String populationName; 29 | private long taskId; 30 | private long iterationId; 31 | private long attemptId; 32 | private String batchId; 33 | 34 | @Override 35 | public String toString() { 36 | return populationName + "/" + taskId + "/" + iterationId + "/" + attemptId + "/" + batchId; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/AssignmentId.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import lombok.Builder; 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | 23 | /** Assignment identifier. */ 24 | @Getter 25 | @EqualsAndHashCode 26 | @Builder(toBuilder = true) 27 | public class AssignmentId { 28 | private String populationName; 29 | private long taskId; 30 | private long iterationId; 31 | private long attemptId; 32 | private String assignmentId; 33 | 34 | @Override 35 | public String toString() { 36 | return populationName + "/" + taskId + "/" + iterationId + "/" + attemptId + "/" + assignmentId; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/AuthorizationTokenDao.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | public interface AuthorizationTokenDao { 20 | 21 | TokenStatus isTokenAuthorized(String token); 22 | 23 | TokenStatus insert(String token); 24 | 25 | enum TokenStatus { 26 | /** 27 | * There are internal errors so that the token status is unknown. The server would let the 28 | * device retry later. 29 | */ 30 | UNKNOWN, 31 | /** The token is unauthorized. */ 32 | UNAUTHORIZED, 33 | /** The token is authorized. */ 34 | AUTHORIZED; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/BlobDescription.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import java.util.Map; 20 | import lombok.Builder; 21 | import lombok.EqualsAndHashCode; 22 | import lombok.Getter; 23 | 24 | /** The Blob description. */ 25 | @Getter 26 | @EqualsAndHashCode 27 | @Builder(toBuilder = true) 28 | public class BlobDescription { 29 | // It should be fixed with better approach. 30 | /** The combined URL of the resource */ 31 | String url; 32 | 33 | /** The host of the resource */ 34 | String host; 35 | 36 | /** The object path */ 37 | String resourceObject; 38 | 39 | /** The header required to upload or download. */ 40 | Map headers; 41 | } 42 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/ModelMetricsDao.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import java.util.Collection; 20 | 21 | /** Model metrics DAO. */ 22 | public interface ModelMetricsDao { 23 | 24 | /** 25 | * Insert or update model metrics entities. 26 | * 27 | * @param modelMetrics The model metrics entities to be inserted or updated. 28 | * @return true if the updated succeeded, false otherwise. 29 | */ 30 | boolean upsertModelMetrics(Collection modelMetrics); 31 | } 32 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/ModelMetricsEntity.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import lombok.Builder; 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | 23 | @Getter 24 | @EqualsAndHashCode 25 | @Builder(toBuilder = true) 26 | public class ModelMetricsEntity { 27 | 28 | private String populationName; 29 | private long taskId; 30 | private long iterationId; 31 | private String metricName; 32 | private double metricValue; 33 | } 34 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/Partitioner.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | /** A storage partioner that generates partition id. */ 20 | public interface Partitioner { 21 | 22 | int getDeviceGradientPartition(String sessionId); 23 | 24 | int getDeviceGradientPartitionCount(); 25 | 26 | int getAggregatedResultPartition(String populationName, long taskId, long iterationId); 27 | 28 | int getAggregatedResultPartitionCount(); 29 | 30 | int getPlanStoagePartition(String populationName, long taskId, long iterationId); 31 | 32 | int getPlanStoragePartitionCount(); 33 | 34 | int getCheckpointStoagePartition(String populationName, long taskId, long iterationId); 35 | 36 | int getCheckpointStoragePartitionCount(); 37 | } 38 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/TaskId.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import lombok.Builder; 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | 23 | /** Task Id. */ 24 | @Getter 25 | @EqualsAndHashCode 26 | @Builder(toBuilder = true) 27 | public class TaskId { 28 | private String populationName; 29 | private long taskId; 30 | 31 | @Override 32 | public String toString() { 33 | return populationName + "/" + taskId; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp/GCSConfig.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao.gcp; 18 | 19 | import lombok.Builder; 20 | import lombok.EqualsAndHashCode; 21 | import lombok.Getter; 22 | 23 | /** GCS Config */ 24 | @Getter 25 | @EqualsAndHashCode 26 | @Builder(toBuilder = true) 27 | public class GCSConfig { 28 | private String gradientBucketTemplate; 29 | private String aggregatedGradientBucketTemplate; 30 | private String modelBucketTemplate; 31 | private long downloadPlanTokenDurationInSecond; 32 | private long downloadCheckpointTokenDurationInSecond; 33 | private long uploadGradientTokenDurationInSecond; 34 | } 35 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp/TimestampInstantConverter.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao.gcp; 18 | 19 | import com.google.cloud.Timestamp; 20 | import com.google.common.base.Converter; 21 | import java.time.Instant; 22 | 23 | /** Converter between {@link Timestamp} and {@link Instant}. */ 24 | public final class TimestampInstantConverter extends Converter { 25 | 26 | private TimestampInstantConverter() {} 27 | 28 | public static final Converter TO_INSTANT = new TimestampInstantConverter(); 29 | public static final Converter TO_TIMESTAMP = TO_INSTANT.reverse(); 30 | 31 | @Override 32 | protected Instant doForward(Timestamp timestamp) { 33 | return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()); 34 | } 35 | 36 | @Override 37 | protected Timestamp doBackward(Instant instant) { 38 | return Timestamp.ofTimeSecondsAndNanos(instant.getEpochSecond(), instant.getNano()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/lock/jdbc/gcp/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "spanner_lock_configuration", 19 | srcs = ["SpannerLockConfiguration.java"], 20 | deps = [ 21 | "@maven//:com_google_cloud_google_cloud_spanner_pgadapter", 22 | "@maven//:org_postgresql_postgresql", 23 | "@maven//:org_springframework_boot_spring_boot_starter_jdbc", 24 | "@maven//:org_springframework_integration_spring_integration_jdbc", 25 | "@maven//:org_springframework_spring_beans", 26 | "@maven//:org_springframework_spring_context", 27 | ], 28 | ) 29 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/logging/LoggingInterceptorConfig.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.logging; 18 | 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.context.annotation.Configuration; 21 | import org.springframework.web.servlet.config.annotation.InterceptorRegistry; 22 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 23 | 24 | /** A WebMvcController to register the Logging Interceptor. */ 25 | @Configuration 26 | public class LoggingInterceptorConfig implements WebMvcConfigurer { 27 | @Autowired LoggingInterceptor loggingInterceptor; 28 | 29 | @Override 30 | public void addInterceptors(InterceptorRegistry registry) { 31 | registry.addInterceptor(loggingInterceptor); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/logging/ResponseProto.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.logging; 18 | 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Retention; 21 | import java.lang.annotation.RetentionPolicy; 22 | import java.lang.annotation.Target; 23 | 24 | /** An annotation for logging response proto. */ 25 | @Target(ElementType.METHOD) 26 | @Retention(RetentionPolicy.RUNTIME) 27 | public @interface ResponseProto { 28 | Class value(); 29 | } 30 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/messaging/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "message_sender", 19 | srcs = [ 20 | "MessageSender.java", 21 | ], 22 | ) 23 | 24 | java_library( 25 | name = "http_message_sender", 26 | srcs = [ 27 | "HttpMessageSender.java", 28 | ], 29 | ) 30 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/messaging/HttpMessageSender.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.messaging; 18 | 19 | /** Http message sender interface. */ 20 | public interface HttpMessageSender { 21 | 22 | /** Sends message to provided endpoint. */ 23 | void sendMessage(T message, String endpoint); 24 | } 25 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/messaging/MessageSender.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.messaging; 18 | 19 | /** Message sender interface for output channels. */ 20 | public interface MessageSender { 21 | 22 | /** Sends message to output channel. */ 23 | void sendMessage(T message, String topicName); 24 | } 25 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/tensorflow/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "tensorflow_plan_session_factory", 19 | srcs = [ 20 | "TensorflowPlanSessionFactory.java", 21 | ], 22 | deps = [ 23 | "@maven//:com_google_protobuf_protobuf_java", 24 | "@maven//:org_springframework_spring_context", 25 | # Import fcp last since it tries to pull in an older version of protobuf. 26 | "@federatedcompute//fcp/protos:plan_java_proto", 27 | "@federatedcompute//fcp/java_src/main/java/com/google/fcp/aggregation", 28 | "@federatedcompute//fcp/java_src/main/java/com/google/fcp/plan:phase_v2", 29 | "@federatedcompute//fcp/java_src/main/java/com/google/fcp/plan", 30 | ], 31 | ) 32 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/tensorflow/TensorflowPlanSessionFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.tensorflow; 18 | 19 | import com.google.fcp.aggregation.AggregationSession; 20 | import com.google.fcp.plan.PhaseSessionV2; 21 | import com.google.fcp.plan.PlanSession; 22 | import com.google.fcp.plan.TensorflowPhaseSessionV2; 23 | import com.google.fcp.plan.TensorflowPlanSession; 24 | import com.google.protobuf.ByteString; 25 | import org.springframework.stereotype.Component; 26 | 27 | @Component 28 | public class TensorflowPlanSessionFactory { 29 | public PlanSession createPlanSession(ByteString plan) { 30 | return new TensorflowPlanSession(plan); 31 | } 32 | 33 | public PhaseSessionV2 createPhaseSessionV2(ByteString plan) { 34 | return new TensorflowPhaseSessionV2(plan); 35 | } 36 | 37 | public AggregationSession createAggregationSession(byte[] plan) { 38 | return AggregationSession.createFromByteArray(plan); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "model_updater_controller", 19 | srcs = ["ModelUpdaterController.java"], 20 | deps = [ 21 | "@maven//:org_springframework_boot_spring_boot", 22 | "@maven//:org_springframework_boot_spring_boot_autoconfigure", 23 | "@maven//:org_springframework_boot_spring_boot_starter_web", 24 | "@maven//:org_springframework_spring_context", 25 | "@maven//:org_springframework_spring_web", 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/controllers/ModelUpdaterController.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.modelupdater.controllers; 18 | 19 | import org.springframework.web.bind.annotation.GetMapping; 20 | import org.springframework.web.bind.annotation.RestController; 21 | 22 | /** Model Updater controller. */ 23 | @RestController 24 | public final class ModelUpdaterController { 25 | @GetMapping("/ready") 26 | public String ready() { 27 | // TODO(291594777): Implement readiness check. 28 | return "Greetings from Model Updater Spring Boot! Ready check. \n"; 29 | } 30 | 31 | @GetMapping("/healthz") 32 | public String healthz() { 33 | // TODO(291594777): Implement health check. 34 | return "Greetings from Model Updater Spring Boot! Health check. \n"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core/ModelUpdaterCore.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.modelupdater.core; 18 | 19 | import com.google.ondevicepersonalization.federatedcompute.shuffler.modelupdater.core.message.ModelUpdaterMessage; 20 | 21 | /** Model updater core interface. */ 22 | public interface ModelUpdaterCore { 23 | public void process(ModelUpdaterMessage message); 24 | } 25 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core/message/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "model_updater_message", 19 | srcs = [ 20 | "ModelUpdaterMessage.java", 21 | ], 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common:lombok_java", 24 | ], 25 | ) 26 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/scheduler/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_spring//springboot:springboot.bzl", "springboot") 16 | 17 | package(default_visibility = ["//visibility:public"]) 18 | 19 | java_library( 20 | name = "scheduled_task", 21 | srcs = [ 22 | "ScheduledTask.java", 23 | ], 24 | deps = [ 25 | "@maven//:org_springframework_spring_context", 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/scheduler/ScheduledTask.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.modelupdater.scheduler; 18 | 19 | import java.util.concurrent.TimeUnit; 20 | import org.springframework.scheduling.annotation.Scheduled; 21 | import org.springframework.stereotype.Component; 22 | 23 | /** 24 | * The scheduled task. 25 | * 26 | *

This is used to keep application alive 27 | * https://github.com/GoogleCloudPlatform/spring-cloud-gcp/blob/main/docs/src/main/asciidoc/pubsub.adoc 28 | * The other main entry point of this application is via events from the inbound channel adapter. 29 | */ 30 | @Component 31 | public class ScheduledTask { 32 | @Scheduled(fixedDelay = 480, timeUnit = TimeUnit.MINUTES) 33 | public void run() throws Exception {} 34 | } 35 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskassignment/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "task_assignment_controller", 19 | srcs = ["TaskAssignmentController.java"], 20 | deps = [ 21 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common:compression_utils", 22 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao:assignment_entity", 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/logging:response_proto", 24 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskassignment/core:task_assignment_core", 25 | "//shuffler/proto:task_assignments_java_proto", 26 | "@com_google_protobuf//java/core", 27 | "@federatedcompute//fcp/protos/federatedcompute:federated_compute_java_proto", 28 | "@maven//:io_micrometer_micrometer_core", 29 | "@maven//:org_slf4j_slf4j_api", 30 | "@maven//:org_springframework_spring_context", 31 | "@maven//:org_springframework_spring_web", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskmanagement/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "task_management_controller", 19 | srcs = ["TaskManagementController.java"], 20 | deps = [ 21 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskmanagement/core:task_management_core", 22 | "//shuffler/proto:task_java_proto", 23 | "@maven//:org_springframework_boot_spring_boot", 24 | "@maven//:org_springframework_boot_spring_boot_autoconfigure", 25 | "@maven//:org_springframework_boot_spring_boot_starter_web", 26 | "@maven//:org_springframework_spring_context", 27 | "@maven//:org_springframework_spring_web", 28 | ], 29 | ) 30 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskmanagement/core/TaskManagementCore.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.taskmanagement.core; 18 | 19 | import com.google.ondevicepersonalization.federatedcompute.proto.Task; 20 | import java.util.Optional; 21 | 22 | /** Task core service interface. */ 23 | public interface TaskManagementCore { 24 | 25 | /** Get task by id. */ 26 | public Optional getTaskById(String populationName, long taskId); 27 | 28 | /** Create new task. */ 29 | public Task createTask(Task task); 30 | } 31 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskscheduler/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | java_library( 18 | name = "task_scheduler_controller", 19 | srcs = ["TaskSchedulerController.java"], 20 | deps = [ 21 | "@maven//:org_springframework_boot_spring_boot", 22 | "@maven//:org_springframework_boot_spring_boot_autoconfigure", 23 | "@maven//:org_springframework_boot_spring_boot_starter_web", 24 | "@maven//:org_springframework_spring_context", 25 | "@maven//:org_springframework_spring_web", 26 | ], 27 | ) 28 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskscheduler/controllers/TaskSchedulerController.java: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.taskscheduler.controllers; 18 | 19 | import org.springframework.web.bind.annotation.GetMapping; 20 | import org.springframework.web.bind.annotation.RestController; 21 | 22 | /** Task Scheduler controller. */ 23 | @RestController 24 | public final class TaskSchedulerController { 25 | @GetMapping("/ready") 26 | public String ready() { 27 | // TODO(291594777): Implement readiness check. 28 | return "Greetings from Task Scheduler Spring Boot! Ready check. \n"; 29 | } 30 | 31 | @GetMapping("/healthz") 32 | public String healthz() { 33 | // TODO(291594777): Implement health check. 34 | return "Greetings from Task Scheduler Spring Boot! Health check. \n"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskscheduler/core/TaskSchedulerCore.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.taskscheduler.core; 18 | 19 | /** TaskScheduler core interface. */ 20 | public interface TaskSchedulerCore { 21 | public void processActiveTasks(); 22 | 23 | public void processCreatedTasks(); 24 | 25 | public void processCompletedIterations(); 26 | } 27 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskscheduler/scheduler/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_spring//springboot:springboot.bzl", "springboot") 16 | 17 | package(default_visibility = ["//visibility:public"]) 18 | 19 | java_library( 20 | name = "scheduled_task", 21 | srcs = [ 22 | "ScheduledTask.java", 23 | ], 24 | deps = [ 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskscheduler/core:task_scheduler_core", 26 | "@maven//:org_springframework_spring_context", 27 | ], 28 | ) 29 | -------------------------------------------------------------------------------- /java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskscheduler/scheduler/ScheduledTask.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.taskscheduler.scheduler; 18 | 19 | import com.google.ondevicepersonalization.federatedcompute.shuffler.taskscheduler.core.TaskSchedulerCore; 20 | import org.springframework.scheduling.annotation.Scheduled; 21 | import org.springframework.stereotype.Component; 22 | 23 | /** The scheduled task. */ 24 | @Component 25 | public class ScheduledTask { 26 | 27 | TaskSchedulerCore taskSchedulerCore; 28 | 29 | public ScheduledTask(TaskSchedulerCore taskSchedulerCore) { 30 | this.taskSchedulerCore = taskSchedulerCore; 31 | } 32 | 33 | // TODO(b/295018999): Determine a good rate for the task. 34 | @Scheduled(fixedDelay = 200) 35 | public void run() throws Exception { 36 | taskSchedulerCore.processCreatedTasks(); 37 | taskSchedulerCore.processActiveTasks(); 38 | } 39 | 40 | @Scheduled(fixedDelay = 60000) 41 | public void processCompletedIterations() throws Exception { 42 | taskSchedulerCore.processCompletedIterations(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /java/src/main/resources/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | filegroup( 18 | name = "taskassignment_properties", 19 | srcs = ["taskassignment.properties"], 20 | ) 21 | 22 | filegroup( 23 | name = "taskmanagement_properties", 24 | srcs = ["taskmanagement.properties"], 25 | ) 26 | 27 | filegroup( 28 | name = "modelupdater_properties", 29 | srcs = ["modelupdater.properties"], 30 | ) 31 | 32 | filegroup( 33 | name = "aggregator_properties", 34 | srcs = ["aggregator.properties"], 35 | ) 36 | 37 | filegroup( 38 | name = "collector_properties", 39 | srcs = ["collector.properties"], 40 | ) 41 | 42 | filegroup( 43 | name = "taskscheduler_properties", 44 | srcs = ["taskscheduler.properties"], 45 | ) 46 | 47 | filegroup( 48 | name = "logback_spring", 49 | srcs = ["logback-spring.xml"], 50 | ) 51 | 52 | -------------------------------------------------------------------------------- /java/src/main/resources/aggregator.properties: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%-4relative [%thread] %-5level %logger{35} - activity:%X{activity.id} - requestId:%X{request.id} - %msg %n 2 | server.port=8082 3 | management.endpoint.metrics.enabled=true 4 | management.endpoints.web.exposure.include=prometheus 5 | management.metrics.export.prometheus.enabled=true 6 | spring.threads.virtual.enabled=true 7 | 8 | ########### 9 | # GCP Settings 10 | ########### 11 | 12 | # Pub/Sub settings 13 | # This property specifies the minimum amount of time that must pass before the redelivery of a message occurs. 14 | spring.cloud.gcp.pubsub.subscriber.min-duration-per-ack-extension=5 15 | # This property defines the amount of time Pub/Sub takes to redeliver a message. 16 | spring.cloud.gcp.pubsub.subscriber.max-duration-per-ack-extension=10 17 | # This property determines how long you want the subscriber clients to process messages. 10 minutes. 18 | spring.cloud.gcp.pubsub.subscriber.max-ack-extension-period=600 -------------------------------------------------------------------------------- /java/src/main/resources/collector.properties: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%-4relative [%thread] %-5level %logger{35} - activity:%X{activity.id} - iteration:%X{iteration.id} - status:%X{status.id} - %msg %n 2 | server.port=8082 3 | management.endpoint.metrics.enabled=true 4 | management.endpoints.web.exposure.include=prometheus 5 | management.metrics.export.prometheus.enabled=true 6 | spring.datasource.url=jdbc:postgresql://localhost:5432/ 7 | spring.task.scheduling.pool.size=2 8 | spring.threads.virtual.enabled=true 9 | -------------------------------------------------------------------------------- /java/src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | WARN 21 | true 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /java/src/main/resources/modelupdater.properties: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%-4relative [%thread] %-5level %logger{35} - activity:%X{activity.id} - requestId:%X{request.id} - %msg %ex %n 2 | server.port=8082 3 | management.endpoint.metrics.enabled=true 4 | management.endpoints.web.exposure.include=prometheus 5 | management.metrics.export.prometheus.enabled=true 6 | spring.threads.virtual.enabled=true 7 | 8 | ########### 9 | # GCP Settings 10 | ########### 11 | 12 | # Pub/Sub settings 13 | # This property specifies the minimum amount of time that must pass before the redelivery of a message occurs. 14 | spring.cloud.gcp.pubsub.subscriber.min-duration-per-ack-extension=5 15 | # This property defines the amount of time Pub/Sub takes to redeliver a message. 16 | spring.cloud.gcp.pubsub.subscriber.max-duration-per-ack-extension=10 17 | # This property determines how long you want the subscriber clients to process messages. 10 minutes. 18 | spring.cloud.gcp.pubsub.subscriber.max-ack-extension-period=600 19 | -------------------------------------------------------------------------------- /java/src/main/resources/taskassignment.properties: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%-4relative [%thread] %-5level %logger{35} - %X{correlation.id} - %X{activity.id} - %msg %n 2 | server.port=8083 3 | management.endpoint.metrics.enabled=true 4 | management.endpoints.web.exposure.include=prometheus 5 | management.metrics.export.prometheus.enabled=true 6 | management.metrics.distribution.percentiles-histogram.http.server.requests=true 7 | management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999 8 | spring.threads.virtual.enabled=true 9 | -------------------------------------------------------------------------------- /java/src/main/resources/taskmanagement.properties: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%-4relative [%thread] %-5level %logger{35} - %X{correlation.id} - %X{activity.id} - %msg %n 2 | server.port=8082 3 | management.endpoint.metrics.enabled=true 4 | management.endpoints.web.exposure.include=prometheus 5 | management.metrics.export.prometheus.enabled=true 6 | management.metrics.distribution.percentiles-histogram.http.server.requests=true 7 | management.metrics.distribution.percentiles.http.server.requests=0.5, 0.9, 0.95, 0.99, 0.999 8 | spring.threads.virtual.enabled=true 9 | -------------------------------------------------------------------------------- /java/src/main/resources/taskscheduler.properties: -------------------------------------------------------------------------------- 1 | logging.pattern.console=%-4relative [%thread] %-5level %logger{35} - %X{activity.id} - %msg %n 2 | server.port=8082 3 | spring.datasource.url=jdbc:postgresql://localhost:5432/ 4 | spring.task.scheduling.pool.size=2 5 | spring.threads.virtual.enabled=true 6 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core/gcp/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_java//java:defs.bzl", "java_test") 16 | 17 | java_test( 18 | name = "pub_sub_message_receiver_test", 19 | size = "small", 20 | srcs = ["PubSubMessageReceiverTest.java"], 21 | test_class = "com.google.ondevicepersonalization.federatedcompute.shuffler.aggregator.core.gcp.PubSubMessageReceiverTest", 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core:aggregator_core", 24 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core/gcp:pub_sub_message_receiver", 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core/message:aggregator_message", 26 | "@maven//:com_google_cloud_spring_cloud_gcp_pubsub", 27 | "@maven//:com_google_guava_guava", 28 | "@maven//:com_google_truth_truth", 29 | "@maven//:junit_junit", 30 | "@maven//:org_mockito_mockito_core", 31 | ], 32 | ) 33 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/core/gcp/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_java//java:defs.bzl", "java_test") 16 | 17 | java_test( 18 | name = "pub_sub_message_receiver_test", 19 | size = "small", 20 | srcs = ["PubSubMessageReceiverTest.java"], 21 | test_class = "com.google.ondevicepersonalization.federatedcompute.shuffler.collector.core.gcp.PubSubMessageReceiverTest", 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/core/gcp:pub_sub_message_receiver", 24 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/aggregator/core/message:aggregator_notification", 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector/core:collector_core", 26 | "@maven//:com_google_api_grpc_proto_google_cloud_pubsub_v1", 27 | "@maven//:com_google_cloud_spring_cloud_gcp_pubsub", 28 | "@maven//:com_google_guava_guava", 29 | "@maven//:com_google_truth_truth", 30 | "@maven//:junit_junit", 31 | "@maven//:org_mockito_mockito_core", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/CompressionUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | import static org.junit.Assert.assertArrayEquals; 20 | import static org.junit.Assert.assertThrows; 21 | import static org.junit.Assert.assertTrue; 22 | 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.JUnit4; 26 | 27 | @RunWith(JUnit4.class) 28 | public final class CompressionUtilsTest { 29 | @Test 30 | public void compressAndDecompress_success() { 31 | byte[] inputData = "HelloWorld11111111111111111111111111111111100000011".getBytes(); 32 | 33 | byte[] compressed = CompressionUtils.compressWithGzip(inputData); 34 | assertTrue(inputData.length > compressed.length); 35 | 36 | byte[] uncompressed = CompressionUtils.uncompressWithGzip(compressed); 37 | assertArrayEquals(inputData, uncompressed); 38 | } 39 | 40 | @Test 41 | public void decompress_fail() { 42 | byte[] inputData = "HelloWorld11111111".getBytes(); 43 | assertThrows( 44 | IllegalArgumentException.class, () -> CompressionUtils.uncompressWithGzip(inputData)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/GuidUniqueIdGeneratorTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | import org.junit.runners.JUnit4; 24 | 25 | @RunWith(JUnit4.class) 26 | public final class GuidUniqueIdGeneratorTest { 27 | 28 | private GuidUniqueIdGenerator generator = new GuidUniqueIdGenerator(); 29 | 30 | @Test 31 | public void testSessionIdLengthInLimit() { 32 | assertThat(generator.generate().length() <= Constants.MAX_ASSIGNMENT_ID_LENGTH).isTrue(); 33 | } 34 | 35 | @Test 36 | public void testSessionIdStringNotEqual() { 37 | String id1 = generator.generate(); 38 | String id2 = generator.generate(); 39 | assertThat(id1.equalsIgnoreCase(id2)).isFalse(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/CheckInResultTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.dao; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | import static org.junit.Assert.assertThrows; 21 | 22 | import org.junit.Test; 23 | 24 | public class CheckInResultTest { 25 | 26 | @Test 27 | public void testAndEquals() { 28 | assertThat(CheckInResult.fromCode(1)).isEqualTo(CheckInResult.SUCCESS); 29 | } 30 | 31 | @Test 32 | public void testInvalidStatusCode() { 33 | 34 | // act 35 | IllegalArgumentException expected = 36 | assertThrows(IllegalArgumentException.class, () -> CheckInResult.fromCode(9999)); 37 | 38 | // assert 39 | assertThat(expected).hasMessageThat().contains("9999"); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/logging/test.proto: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | *

Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain a 6 | * copy of the License at 7 | * 8 | *

http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | *

Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | syntax = "proto3"; 17 | 18 | package google.ondevicepersonalization.federatedcompute.test.proto; 19 | 20 | option java_package = "com.google.ondevicepersonalization.federatedcompute.test.proto"; 21 | option java_multiple_files = true; 22 | 23 | message TestRequest { 24 | TestRequestPayload payload = 1; 25 | } 26 | 27 | message TestResponse { 28 | TestResponsePayload payload = 1; 29 | } 30 | 31 | message TestRequestPayload { 32 | string name = 1; 33 | } 34 | 35 | message TestResponsePayload { 36 | string id = 1; 37 | string group_id = 2; 38 | } 39 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/messaging/gcp/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_java//java:defs.bzl", "java_test") 16 | 17 | java_test( 18 | name = "pub_sub_message_sender_test", 19 | size = "small", 20 | srcs = ["PubSubMessageSenderTest.java"], 21 | test_class = "com.google.ondevicepersonalization.federatedcompute.shuffler.common.messaging.gcp.PubSubMessageSenderTest", 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/messaging/gcp:pub_sub_message_sender", 24 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/messaging/gcp:pub_sub_publisher_config", 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core/message:model_updater_message", 26 | "@maven//:com_google_cloud_spring_cloud_gcp_pubsub", 27 | "@maven//:com_google_code_gson_gson", 28 | "@maven//:com_google_guava_guava", 29 | "@maven//:com_google_truth_truth", 30 | "@maven//:junit_junit", 31 | "@maven//:org_mockito_mockito_core", 32 | "@maven//:org_springframework_spring_messaging", 33 | ], 34 | ) 35 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/tensorflow/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_java//java:defs.bzl", "java_test") 16 | 17 | java_test( 18 | name = "tensorflow_plan_session_factory_test", 19 | size = "small", 20 | srcs = ["TensorflowPlanSessionFactoryTest.java"], 21 | test_class = "com.google.ondevicepersonalization.federatedcompute.shuffler.common.tensorflow.TensorflowPlanSessionFactoryTest", 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/tensorflow:tensorflow_plan_session_factory", 24 | "@maven//:com_google_protobuf_protobuf_java", 25 | "@maven//:com_google_truth_truth", 26 | "@maven//:junit_junit", 27 | # Import fcp last since it tries to pull in an older version of protobuf. 28 | "@federatedcompute//fcp/java_src/main/java/com/google/fcp/plan", 29 | ], 30 | ) 31 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/tensorflow/TensorflowPlanSessionFactoryTest.java: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | //////////////////////////////////////////////////////////////////////////////// 16 | 17 | package com.google.ondevicepersonalization.federatedcompute.shuffler.common.tensorflow; 18 | 19 | import static org.junit.Assert.assertThrows; 20 | 21 | import com.google.protobuf.ByteString; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.junit.runners.JUnit4; 25 | 26 | @RunWith(JUnit4.class) 27 | public final class TensorflowPlanSessionFactoryTest { 28 | 29 | private TensorflowPlanSessionFactory factory = new TensorflowPlanSessionFactory(); 30 | 31 | @Test 32 | public void createPlanSession_invalidPlan_throw() { 33 | // build an invalid plan by using an arbitrary array 34 | ByteString planByteString = ByteString.copyFrom(new byte[] {0, 1, 2}); 35 | 36 | // act and assert 37 | assertThrows(Exception.class, () -> factory.createPlanSession(planByteString)); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core/gcp/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_java//java:defs.bzl", "java_test") 16 | 17 | java_test( 18 | name = "pub_sub_message_receiver_test", 19 | size = "small", 20 | srcs = ["PubSubMessageReceiverTest.java"], 21 | test_class = "com.google.ondevicepersonalization.federatedcompute.shuffler.modelupdater.core.gcp.PubSubMessageReceiverTest", 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core:model_updater_core", 24 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core/gcp:pub_sub_message_receiver", 25 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/modelupdater/core/message:model_updater_message", 26 | "@maven//:com_google_cloud_spring_cloud_gcp_pubsub", 27 | "@maven//:com_google_guava_guava", 28 | "@maven//:com_google_truth_truth", 29 | "@maven//:junit_junit", 30 | "@maven//:org_mockito_mockito_core", 31 | ], 32 | ) 33 | -------------------------------------------------------------------------------- /java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskmanagement/controllers/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_java//java:defs.bzl", "java_test") 16 | 17 | java_test( 18 | name = "task_management_controller_test", 19 | size = "small", 20 | srcs = ["TaskManagementControllerTest.java"], 21 | test_class = "com.google.ondevicepersonalization.federatedcompute.shuffler.taskmanagement.controllers.TaskManagementControllerTest", 22 | deps = [ 23 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskmanagement/controllers:task_management_controller", 24 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/taskmanagement/core:task_management_core", 25 | "//shuffler/proto:task_java_proto", 26 | "@com_google_protobuf//java/core", 27 | "@maven//:com_google_protobuf_protobuf_java_util", 28 | "@maven//:com_google_truth_truth", 29 | "@maven//:junit_junit", 30 | "@maven//:org_mockito_mockito_core", 31 | "@maven//:org_springframework_spring_web", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /java/src/test/java/resources/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | filegroup( 18 | name = "key_attestation_verification_file_group", 19 | srcs = [ 20 | "KeyAttestationFailVerification1.json", 21 | "KeyAttestationFailVerification2.json", 22 | "KeyAttestationFailVerification3.json", 23 | "KeyAttestationSuccessVerification.json", 24 | ], 25 | ) 26 | 27 | filegroup( 28 | name = "plans", 29 | srcs = [ 30 | "plan_v1", 31 | "plan_v2", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /java/src/test/java/resources/plan_v1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/test/java/resources/plan_v1 -------------------------------------------------------------------------------- /java/src/test/java/resources/plan_v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/java/src/test/java/resources/plan_v2 -------------------------------------------------------------------------------- /patches/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /patches/rules_spring.patch: -------------------------------------------------------------------------------- 1 | diff --git springboot/springboot_pkg.sh springboot/springboot_pkg.sh 2 | index c76fcf9..2291f0f 100755 3 | --- springboot/springboot_pkg.sh 4 | +++ springboot/springboot_pkg.sh 5 | @@ -311,7 +311,7 @@ cd $working_dir 6 | # note that it does not use the manifest from the jar file, which is a bummer 7 | # so we have to respecify the manifest data 8 | # TODO we should rewrite write_manfiest.sh to produce inputs compatible for singlejar (Issue #27) 9 | -singlejar_options="--normalize --dont_change_compression" # add in --verbose for more details from command 10 | +singlejar_options="--normalize --dont_change_compression --build_target $outputjar" # add in --verbose for more details from command 11 | singlejar_mainclass="--main_class $spring_boot_launcher_class" 12 | $singlejar_cmd $singlejar_options $singlejar_mainclass \ 13 | --deploy_manifest_lines "Start-Class: $mainclass" \ 14 | -------------------------------------------------------------------------------- /patches/rules_spring_manifest.patch: -------------------------------------------------------------------------------- 1 | diff --git springboot/write_manifest.sh springboot/write_manifest.sh 2 | index 8ce58b5..2679a05 100755 3 | --- springboot/write_manifest.sh 4 | +++ springboot/write_manifest.sh 5 | @@ -52,5 +52,4 @@ echo "Main-Class: org.springframework.boot.loader.JarLauncher" >> $manifestfile 6 | echo "Spring-Boot-Classes: BOOT-INF/classes/" >> $manifestfile 7 | echo "Spring-Boot-Lib: BOOT-INF/lib/" >> $manifestfile 8 | echo "Spring-Boot-Version: $spring_version" >> $manifestfile 9 | -echo "Build-Jdk: $java_version" >> $manifestfile 10 | echo "Start-Class: $mainclass" >> $manifestfile 11 | -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/input/keras_model/fingerprint.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/input/keras_model/fingerprint.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/input/keras_model/saved_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/input/keras_model/saved_model.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/input/keras_model/variables/variables.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/input/keras_model/variables/variables.data-00000-of-00001 -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/input/keras_model/variables/variables.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/input/keras_model/variables/variables.index -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/input/keras_task_config.pbtxt: -------------------------------------------------------------------------------- 1 | # Task Configuration in pbtxt format 2 | mode: TRAINING_AND_EVAL # Task execution mode 3 | population_name: "keras_example_int64_task_builder" # Dataset name 4 | use_daf: true # Use Distributed aggregated form 5 | 6 | # Policies 7 | policies { 8 | min_separation_policy { 9 | minimum_separation: 0 # Minimum distance between examples 10 | } 11 | data_availability_policy { 12 | min_example_count: 1 # Minimum examples required 13 | } 14 | model_release_policy { 15 | num_max_training_rounds: 1000 16 | } 17 | } 18 | 19 | # Federated Learning Configuration 20 | federated_learning { 21 | learning_process { 22 | client_learning_rate: 0.1 23 | server_learning_rate: 1.0 24 | runtime_config { report_goal: 2 } # Reporting frequency 25 | } 26 | evaluation { 27 | source_training_task_id: 1 28 | checkpoint_selector: "every_1_round" 29 | evaluation_traffic: 0.2 30 | report_goal: 2 31 | } 32 | } 33 | 34 | # Differential Privacy Configuration 35 | differential_privacy { 36 | clip_norm: 0.1 37 | } 38 | -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/input/keras_task_config_build_artifact_only.pbtxt: -------------------------------------------------------------------------------- 1 | # Task Configuration in pbtxt format 2 | mode: TRAINING_AND_EVAL # Task execution mode 3 | population_name: "keras_example_int64_task_builder" # Dataset name 4 | 5 | # Policies 6 | policies { 7 | min_separation_policy { 8 | minimum_separation: 0 # Minimum distance between examples 9 | } 10 | data_availability_policy { 11 | min_example_count: 1 # Minimum examples required 12 | } 13 | model_release_policy { 14 | num_max_training_rounds: 1000 15 | } 16 | } 17 | 18 | # Federated Learning Configuration 19 | federated_learning { 20 | learning_process { 21 | client_learning_rate: 0.1 22 | server_learning_rate: 1.0 23 | runtime_config { report_goal: 2 } # Reporting frequency 24 | artifact_building { 25 | plan_url: "{storage_path}/plan.pb" 26 | client_plan_url: "{storage_path}/client_only_plan.pb" 27 | checkpoint_url: "{storage_path}/initial_state.server.ckp" 28 | } 29 | } 30 | evaluation { 31 | source_training_task_id: 1 32 | checkpoint_selector: "every_1_round" 33 | evaluation_traffic: 0.2 34 | report_goal: 2 35 | artifact_building { 36 | plan_url: "{storage_path}/eval_plan.pb" 37 | client_plan_url: "{storage_path}/eval_client_only_plan.pb" 38 | checkpoint_url: "{storage_path}/eval_initial_state.server.ckp" 39 | } 40 | } 41 | } 42 | 43 | # Differential Privacy Configuration 44 | differential_privacy { 45 | clip_norm: 0.1 46 | } 47 | -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/output/client_only_plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/output/client_only_plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/output/eval_client_only_plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/output/eval_client_only_plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/output/eval_initial_state.server.ckp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/output/eval_initial_state.server.ckp -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/output/eval_plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/output/eval_plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/output/initial_state.server.ckp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/output/initial_state.server.ckp -------------------------------------------------------------------------------- /python/taskbuilder/sample/keras/output/plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/keras/output/plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/input/mnist_cnn_task_config.pbtxt: -------------------------------------------------------------------------------- 1 | # Task Configuration in pbtxt format 2 | mode: TRAINING_AND_EVAL # Task execution mode 3 | population_name: "mnist_cnn_task" 4 | # Policies 5 | policies { 6 | min_separation_policy { 7 | minimum_separation: 0 # Minimum distance between examples 8 | } 9 | data_availability_policy { 10 | min_example_count: 1 # Minimum examples required 11 | } 12 | model_release_policy { 13 | num_max_training_rounds: 512 14 | } 15 | } 16 | 17 | # Federated Learning Configuration 18 | federated_learning { 19 | learning_process { 20 | client_optimizer: SGD 21 | client_learning_rate: 0.02 22 | server_optimizer: SGD 23 | server_learning_rate: 1.0 24 | runtime_config { 25 | report_goal: 2 26 | } 27 | metrics { 28 | name: "sparse_categorical_accuracy" 29 | } 30 | } 31 | evaluation { 32 | source_training_task_id: 1 33 | checkpoint_selector: "every_1_round" 34 | evaluation_traffic: 0.2 35 | report_goal: 2 36 | } 37 | } 38 | 39 | # Differential Privacy Configuration 40 | differential_privacy { 41 | type: FIXED_GAUSSIAN 42 | clip_norm: 0.1 43 | noise_multiplier: 0.1 44 | } 45 | -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/input/mnist_cnn_task_config_build_artifact_only.pbtxt: -------------------------------------------------------------------------------- 1 | # Task Configuration in pbtxt format 2 | mode: TRAINING_AND_EVAL # Task execution mode 3 | population_name: "mnist_cnn_task" 4 | # Policies 5 | policies { 6 | min_separation_policy { 7 | minimum_separation: 0 # Minimum distance between examples 8 | } 9 | data_availability_policy { 10 | min_example_count: 1 # Minimum examples required 11 | } 12 | model_release_policy { 13 | num_max_training_rounds: 512 14 | } 15 | } 16 | 17 | # Federated Learning Configuration 18 | federated_learning { 19 | learning_process { 20 | client_optimizer: SGD 21 | client_learning_rate: 0.02 22 | server_optimizer: SGD 23 | server_learning_rate: 1.0 24 | runtime_config { 25 | report_goal: 2 26 | } 27 | metrics { 28 | name: "sparse_categorical_accuracy" 29 | } 30 | artifact_building { 31 | plan_url: "{storage_path}/plan.pb" 32 | client_plan_url: "{storage_path}/client_only_plan.pb" 33 | checkpoint_url: "{storage_path}/initial_state.server.ckp" 34 | } 35 | } 36 | evaluation { 37 | source_training_task_id: 1 38 | checkpoint_selector: "every_1_round" 39 | evaluation_traffic: 0.2 40 | report_goal: 2 41 | artifact_building { 42 | plan_url: "{storage_path}/eval_plan.pb" 43 | client_plan_url: "{storage_path}/eval_client_only_plan.pb" 44 | checkpoint_url: "{storage_path}/eval_initial_state.server.ckp" 45 | } 46 | } 47 | } 48 | 49 | # Differential Privacy Configuration 50 | differential_privacy { 51 | type: FIXED_GAUSSIAN 52 | clip_norm: 0.1 53 | noise_multiplier: 0.1 54 | } 55 | -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/input/mnist_model/fingerprint.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/input/mnist_model/fingerprint.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/input/mnist_model/saved_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/input/mnist_model/saved_model.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/input/mnist_model/variables/variables.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/input/mnist_model/variables/variables.data-00000-of-00001 -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/input/mnist_model/variables/variables.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/input/mnist_model/variables/variables.index -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/output/client_only_plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/output/client_only_plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/output/eval_client_only_plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/output/eval_client_only_plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/output/eval_initial_state.server.ckp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/output/eval_initial_state.server.ckp -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/output/eval_plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/output/eval_plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/output/initial_state.server.ckp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/output/initial_state.server.ckp -------------------------------------------------------------------------------- /python/taskbuilder/sample/mnist/output/plan.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/python/taskbuilder/sample/mnist/output/plan.pb -------------------------------------------------------------------------------- /python/taskbuilder/support_ops.csv: -------------------------------------------------------------------------------- 1 | 341812000,AsString 2 | 341812000,AnonymousIteratorV3 3 | 341812000,BatchDatasetV2 4 | 341812000,BroadcastGradientArgs 5 | 341812000,ConcatOffset 6 | 341812000,ExternalDataset 7 | 341812000,IteratorGetNextAsOptional 8 | 341812000,MakeIterator 9 | 341812000,MapDataset 10 | 341812000,OptionalGetValue 11 | 341812000,OptionalHasValue 12 | 341812000,ParseExampleV2 13 | 341812000,RestoreV2 14 | 341812000,ReduceDataset 15 | 341812000,ReluGrad 16 | 341812000,Relu6 17 | 341812000,SaveSlices 18 | 341812000,ShapeN 19 | 341812000,Split 20 | 341812000,StridedSliceGrad 21 | 341812000,StringToHashBucketFast 22 | 341812000,TakeDataset 23 | 341812000,_TensorToHashBucketFast 24 | 341912000,L2Loss -------------------------------------------------------------------------------- /python/taskbuilder/tflite_flex_ops.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include // For converting C++ containers 17 | 18 | #include "tensorflow/lite/model.h" 19 | #include "tensorflow/lite/tools/list_flex_ops.h" 20 | 21 | namespace py = pybind11; 22 | 23 | PYBIND11_MODULE(tflite_flex_ops, m) { 24 | m.def( 25 | "AddFlexOpsFromModel", 26 | [](const std::string& model_path) { 27 | // Load the model 28 | std::unique_ptr model = 29 | tflite::FlatBufferModel::BuildFromFile(model_path.c_str()); 30 | if (!model) { 31 | throw std::runtime_error("Failed to load model"); 32 | } 33 | 34 | // Create OpKernelSet and call the function 35 | tflite::flex::OpKernelSet flex_ops; 36 | tflite::flex::AddFlexOpsFromModel(model->GetModel(), &flex_ops); 37 | std::map ops_kernel_map; 38 | for (const tflite::flex::OpKernel& op : flex_ops) { 39 | ops_kernel_map[op.op_name] = op.kernel_name; 40 | } 41 | return ops_kernel_map; 42 | }, 43 | py::arg("model_path")); 44 | } -------------------------------------------------------------------------------- /scripts/build_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Exit on failure 18 | set -e 19 | # Print commands 20 | set -x 21 | 22 | # Build all java images and store in local repository with bazel run 23 | # This should be run within the dockerbuild file to ensure deterministic builds 24 | bazel build //shuffler/services/taskscheduler:task_scheduler_image "$@" 25 | bazel build //shuffler/services/taskmanagement:task_management_image "$@" 26 | bazel build //shuffler/services/taskassignment:task_assignment_image "$@" 27 | bazel build //shuffler/services/modelupdater:model_updater_image "$@" 28 | bazel build //shuffler/services/collector:collector_image "$@" 29 | bazel build //shuffler/services/aggregator:aggregator_image "$@" 30 | bazel build //shuffler/services/taskbuilder:task_builder_image "$@" 31 | 32 | # Print container digests. 33 | set +x 34 | for file in bazel-bin/shuffler/services/*/*/index.json; do 35 | grep -H digest "$file" 36 | done 37 | 38 | -------------------------------------------------------------------------------- /scripts/docker/docker_sh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Exit on failure 18 | set -e 19 | 20 | # Print commands 21 | set -x 22 | 23 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 24 | "$SCRIPT_DIR"/docker_run.sh --tty "${@}" "bash" -------------------------------------------------------------------------------- /scripts/docker/load_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Exit on failure 18 | set -e 19 | # Print commands 20 | set -x 21 | 22 | # Build all java images and store in local repository with bazel run 23 | bazel run //shuffler/services/taskscheduler:tarball 24 | bazel run //shuffler/services/taskmanagement:tarball 25 | bazel run //shuffler/services/taskassignment:tarball 26 | bazel run //shuffler/services/modelupdater:tarball 27 | bazel run //shuffler/services/collector:tarball 28 | bazel run //shuffler/services/aggregator:tarball 29 | bazel run //shuffler/services/taskbuilder:tarball 30 | 31 | -------------------------------------------------------------------------------- /scripts/docker/nginx.conf: -------------------------------------------------------------------------------- 1 | events {} 2 | http { 3 | server { 4 | listen 8080; 5 | location /taskmanagement/ { 6 | proxy_pass http://127.0.0.1:8082; 7 | } 8 | location /taskassignment/ { 9 | proxy_pass http://127.0.0.1:8083; 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /scripts/docker/run_all_services_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Exit on failure 18 | set -e 19 | 20 | # Print commands 21 | set -x 22 | 23 | # Ensure GOOGLE_APPLICATION_CREDENTIALS is set 24 | if [[ -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then 25 | echo "GOOGLE_APPLICATION_CREDENTIALS is not set" 26 | exit 1 27 | fi 28 | 29 | # Ensure GOOGLE_CLOUD_PROJECT is set 30 | if [[ -z "${GOOGLE_CLOUD_PROJECT}" ]]; then 31 | echo "GOOGLE_CLOUD_PROJECT is not set" 32 | exit 1 33 | fi 34 | 35 | ./scripts/docker/docker_run.sh "./scripts/docker/load_images.sh" 36 | 37 | export COMPOSE_FILE="scripts/docker/docker-compose.yml" 38 | 39 | # Run all containers 40 | docker compose up ${@} 41 | -------------------------------------------------------------------------------- /scripts/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Exit on failure 18 | set -e 19 | # Print commands 20 | set -x 21 | 22 | # Java 23 | find -name '*.java' -not -path "./federatedcompute/*" | xargs google-java-format -i 24 | 25 | # C++ & Proto 26 | find -name '*.cc' -name '*.cpp' -o -name '*.proto' -not -path "./federatedcompute/*" | xargs clang-format -i 27 | 28 | # Python 29 | find -name '*.py' -not -path "./federatedcompute/*" | xargs pyformat --in_place 30 | 31 | # Bazel 32 | find -name 'BUILD' -name '*.BUILD' -name 'WORKSPACE' -name '*.bzl' -not -path "./federatedcompute/*" | xargs buildifier 33 | 34 | # Terraform 35 | terraform fmt -recursive -------------------------------------------------------------------------------- /scripts/generate_coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Fail on any error. 18 | set -e 19 | set -x 20 | 21 | # Use local toolchain for coverage to reduce shared library size 22 | bazel coverage --combined_report=lcov --incompatible_enable_cc_toolchain_resolution=false --test_output=errors -- //java/... //python/... //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp:task_spanner_dao_test //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp:assignment_spanner_dao_test //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp:aggregation_batch_spanner_dao_test 23 | 24 | genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" -------------------------------------------------------------------------------- /scripts/init_test_db.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | set -e 18 | set -x 19 | 20 | # create instance 21 | gcloud spanner instances create fcp-task-unittest \ 22 | --config=emulator-config --description="fcp task emulator" --nodes=1 23 | # create table in emulator 24 | gcloud spanner databases create test-task-dao --ddl-file=shuffler/spanner/task_database.sdl --instance=fcp-task-unittest 25 | gcloud spanner databases create test-assignment-dao --ddl-file=shuffler/spanner/task_database.sdl --instance=fcp-task-unittest 26 | gcloud spanner databases create authorization-token-dao --ddl-file=shuffler/spanner/task_database.sdl --instance=fcp-task-unittest 27 | -------------------------------------------------------------------------------- /scripts/run_all_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Fail on any error. 18 | set -e 19 | set -x 20 | 21 | if [ -z "$1" ] || [ "$1" == "java" ]; then 22 | # Run all java tests 23 | bazel test //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/... --test_output=errors 24 | 25 | # Run all java manual tests 26 | bazel test //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp:task_spanner_dao_test --test_output=errors 27 | bazel test //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp:assignment_spanner_dao_test --test_output=errors 28 | bazel test //java/src/test/java/com/google/ondevicepersonalization/federatedcompute/shuffler/common/dao/gcp:aggregation_batch_spanner_dao_test --test_output=errors 29 | fi 30 | 31 | if [ -z "$1" ] || [ "$1" == "python" ]; then 32 | # Run all python tests 33 | bazel test //python/... --test_output=errors 34 | fi -------------------------------------------------------------------------------- /scripts/upload_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Fail on any error. 18 | set -e 19 | # Print commands 20 | set -x 21 | 22 | if [[ -n $1 ]]; then 23 | echo "Updating image push repository locations to ${1}." 24 | for file in shuffler/services/*/BUILD; do 25 | sed -i "s~/~${1}~g" "$file" 26 | if [[ -n $2 ]]; then 27 | sed -i "s~\[\"latest\"\]~\[\"latest\", \"${2}\"\]~g" "$file" 28 | fi 29 | done 30 | fi 31 | 32 | 33 | # Upload all new images. 34 | # This should be run within the dockerbuild file to ensure deterministic builds 35 | bazel run //shuffler/services/taskscheduler:task_scheduler_image_publish 36 | bazel run //shuffler/services/taskmanagement:task_management_image_publish 37 | bazel run //shuffler/services/taskassignment:task_assignment_image_publish 38 | bazel run //shuffler/services/modelupdater:model_updater_image_publish 39 | bazel run //shuffler/services/collector:collector_image_publish 40 | bazel run //shuffler/services/aggregator:aggregator_image_publish 41 | bazel run //shuffler/services/taskbuilder:task_builder_image_publish 42 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/README.md: -------------------------------------------------------------------------------- 1 | # Looker (Google Cloud core) Dashboard 2 | 3 | This directory contains the [Looker](https://cloud.google.com/looker/docs) dashboards for odp-federatedcompute. 4 | 5 | ## Instructions 6 | 7 | 1. [Create a Looker instance](https://cloud.google.com/looker/docs/looker-core-create-oauth) 8 | 2. [Setup database connections](https://cloud.google.com/looker/docs/looker-core-dialects#set_up_a_database_connection) for the fcp-task-db-\ and fcp-metrics-db-\ Spanner databases 9 | 3. Create a new GitHub repository and copy the [current folder](.) into the new repository. 10 | 4. Update the [task_db](models/task.model.lkml#L1) and [model_db](models/metric.model.lkml#L1) database connection names to the created database connections. 11 | 5. [Connect to the GitHub repository to the Looker instance](https://cloud.google.com/looker/docs/setting-up-git-connection#integrating-with-git). 12 | 6. [Import the Looker project from the GitHub repository](https://cloud.google.com/looker/docs/setting-up-git-connection#connecting_a_new_lookml_project_to_a_non-empty_git_repository) 13 | 7. Two new Looker dashboards "Model Metrics" and "Task Status" will be created. -------------------------------------------------------------------------------- /shuffler/dashboard/looker/dashboards/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/looker/dashboards/.gitkeep -------------------------------------------------------------------------------- /shuffler/dashboard/looker/models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/looker/models/.gitkeep -------------------------------------------------------------------------------- /shuffler/dashboard/looker/models/metric.model.lkml: -------------------------------------------------------------------------------- 1 | connection: "metric_db" 2 | 3 | include: "/views/metric_db_views/*.view.lkml" 4 | 5 | 6 | datagroup: fcp_default_datagroup { 7 | # sql_trigger: SELECT MAX(id) FROM etl_log;; 8 | max_cache_age: "1 hour" 9 | } 10 | 11 | persist_with: fcp_default_datagroup 12 | 13 | explore: metric { 14 | } 15 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/looker/views/.gitkeep -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/metric_db_views/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/looker/views/metric_db_views/.gitkeep -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/metric_db_views/metric.view.lkml: -------------------------------------------------------------------------------- 1 | view: metric { 2 | sql_table_name: ModelMetrics ;; 3 | 4 | dimension: population_name { 5 | type: string 6 | sql: ${TABLE}.PopulationName ;; 7 | } 8 | 9 | dimension: task_id { 10 | type: number 11 | sql: ${TABLE}.TaskId ;; 12 | } 13 | 14 | dimension: iteration_id { 15 | type: number 16 | sql: ${TABLE}.IterationId ;; 17 | } 18 | 19 | 20 | dimension: metric_name { 21 | type: string 22 | sql: ${TABLE}.MetricName ;; 23 | } 24 | 25 | dimension: population_task_metric { 26 | type: string 27 | sql: CONCAT(${TABLE}.PopulationName, '-', CONCAT(CAST(${TABLE}.TaskId AS STRING), '-', ${TABLE}.MetricName)) ;; 28 | } 29 | 30 | measure: metric_value { 31 | type: min 32 | sql: ${TABLE}.MetricValue ;; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/looker/views/task_db_views/.gitkeep -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_assigned.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_assigned { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 0 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "0" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_assigned_to_upload_completed.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_assigned_to_upload_completed { 3 | extends: [apm_base] 4 | 5 | dimension: created_time_epoch { 6 | type: number 7 | sql: UNIX_SECONDS(${TABLE}.CreatedTime) ;; 8 | } 9 | 10 | measure: duration_in_seconds { 11 | type: number 12 | sql: (MAX(${created_time_epoch}) - MIN(${created_time_epoch})) ;; 13 | } 14 | 15 | measure: duration_in_minutes { 16 | type: number 17 | sql: ${duration_in_seconds} / 60 ;; 18 | } 19 | 20 | measure: average_duration { 21 | type: number 22 | sql: ${duration_in_minutes}/${unique_sessions} ;; 23 | } 24 | 25 | measure: unique_sessions { 26 | type: count_distinct 27 | sql: ${TABLE}.SessionId ;; 28 | } 29 | } 30 | 31 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_base.view.lkml: -------------------------------------------------------------------------------- 1 | view: apm_base { 2 | sql_table_name: AssignmentStatusHistory ;; 3 | 4 | dimension: population_name { 5 | type: string 6 | sql: ${TABLE}.PopulationName ;; 7 | } 8 | 9 | dimension: task_id { 10 | type: number 11 | sql: ${TABLE}.TaskId ;; 12 | } 13 | 14 | dimension: status_number { 15 | type: number 16 | sql: ${TABLE}.Status ;; 17 | } 18 | 19 | dimension: status_id_number { 20 | type: number 21 | sql: ${TABLE}.StatusId ;; 22 | } 23 | 24 | dimension: iteration_id { 25 | type: number 26 | sql: ${TABLE}.IterationId ;; 27 | } 28 | 29 | dimension: session_id { 30 | type: string 31 | sql: ${TABLE}.SessionId ;; 32 | } 33 | 34 | dimension: status { 35 | type: string 36 | sql: CASE 37 | WHEN ${TABLE}.Status = 0 THEN 'ASSIGNED' 38 | WHEN ${TABLE}.Status = 1 THEN 'LOCAL_COMPLETED' 39 | WHEN ${TABLE}.Status = 2 THEN 'UPLOAD_COMPLETED' 40 | WHEN ${TABLE}.Status = 101 THEN 'CANCELED' 41 | WHEN ${TABLE}.Status = 102 THEN 'LOCAL_FAILED' 42 | WHEN ${TABLE}.Status = 103 THEN 'LOCAL_NOT_ELIGIBLE' 43 | WHEN ${TABLE}.Status = 104 THEN 'REMOTE_FAILED' 44 | WHEN ${TABLE}.Status = 105 THEN 'LOCAL_FAILED_EXAMPLE_GENERATION' 45 | WHEN ${TABLE}.Status = 106 THEN 'LOCAL_FAILED_MODEL_COMPUTATION' 46 | WHEN ${TABLE}.Status = 107 THEN 'LOCAL_FAILED_OPS_ERROR' 47 | WHEN ${TABLE}.Status = 151 THEN 'LOCAL_TIMEOUT' 48 | WHEN ${TABLE}.Status = 152 THEN 'UPLOAD_TIMEOUT' 49 | ELSE 'Unknown' 50 | END ;; 51 | } 52 | 53 | dimension_group: created_time { 54 | type: time 55 | timeframes: [raw, minute, minute5] 56 | sql: TIMESTAMP_TRUNC(${TABLE}.CreatedTime, MINUTE) ;; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_canceled.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_canceled { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 101 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "101" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_completed.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_completed { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 1 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "1" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_failed.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_failed { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 102 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "102" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_failed_example_generation.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_failed_example_generation { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 105 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "105" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_failed_model_computation.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_failed_model_computation { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 106 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "106" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_failed_ops_error.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_failed_ops_error { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 107 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "107" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_not_eligible.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_not_eligible { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 103 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "103" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_local_timeout.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_local_timeout { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 151 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "151" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_remote_failed.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_remote_failed { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 104 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "104" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_upload_completed.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_upload_completed { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 2;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: [ 14 | status_number: "2", 15 | status_id_number: "3" 16 | ] 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/apm_upload_timeout.view.lkml: -------------------------------------------------------------------------------- 1 | include: "apm_base.view" 2 | view: apm_upload_timeout { 3 | extends: [apm_base] 4 | 5 | filter: assigned { 6 | type: yesno 7 | sql: ${TABLE}.Status = 152 ;; 8 | } 9 | 10 | measure: count_per_minute { 11 | type: count_distinct 12 | sql: ${TABLE}.SessionId ;; 13 | filters: { 14 | field: status_number 15 | value: "152" 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/iteration.view.lkml: -------------------------------------------------------------------------------- 1 | view: iteration { 2 | sql_table_name: Iteration ;; 3 | 4 | dimension: population_name { 5 | type: string 6 | sql: ${TABLE}.PopulationName ;; 7 | } 8 | 9 | dimension: task_id { 10 | type: number 11 | sql: ${TABLE}.TaskId ;; 12 | } 13 | 14 | dimension: iteration_id { 15 | type: number 16 | sql: ${TABLE}.IterationId ;; 17 | } 18 | 19 | dimension: attempt_id { 20 | type: number 21 | sql: ${TABLE}.AttemptId ;; 22 | } 23 | 24 | dimension: report_goal { 25 | type: number 26 | sql: ${TABLE}.ReportGoal ;; 27 | } 28 | 29 | dimension: max_aggregation_size { 30 | type: number 31 | sql: ${TABLE}.MaxAggregationSize ;; 32 | } 33 | 34 | dimension: status_num { 35 | type: number 36 | sql: ${TABLE}.Status ;; 37 | } 38 | 39 | dimension: status { 40 | type: string 41 | sql: CASE 42 | WHEN ${TABLE}.Status = 0 THEN 'COLLECTING' 43 | WHEN ${TABLE}.Status = 1 THEN 'AGGREGATING' 44 | WHEN ${TABLE}.Status = 2 OR ${TABLE}.Status = 50 THEN 'COMPLETED' 45 | WHEN ${TABLE}.Status = 4 THEN 'APPLYING' 46 | WHEN ${TABLE}.Status = 5 OR ${TABLE}.Status = 51 THEN 'POST_PROCESSED' 47 | WHEN ${TABLE}.Status = 101 THEN 'CANCELED' 48 | WHEN ${TABLE}.Status = 102 THEN 'AGGREGATING_FAILED' 49 | WHEN ${TABLE}.Status = 103 THEN 'APPLYING_FAILED' 50 | ELSE 'Unknown' 51 | END ;; 52 | } 53 | 54 | dimension: min_client_version { 55 | type: string 56 | sql: ${TABLE}.MinClientVersion ;; 57 | } 58 | 59 | dimension: max_client_version { 60 | type: string 61 | sql: ${TABLE}.MinClientVersion ;; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/iteration_aggregating_to_complete.view.lkml: -------------------------------------------------------------------------------- 1 | include: "iteration_status_base.view" 2 | view: iteration_aggregating_to_complete { 3 | extends: [iteration_status_base] 4 | } 5 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/iteration_completion.view.lkml: -------------------------------------------------------------------------------- 1 | include: "iteration_status_base.view" 2 | view: iteration_completion { 3 | extends: [iteration_status_base] 4 | } 5 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/iteration_open.view.lkml: -------------------------------------------------------------------------------- 1 | view: iteration_open { 2 | sql_table_name: IterationStatusHistory ;; 3 | 4 | dimension: population_name { 5 | type: string 6 | sql: ${TABLE}.PopulationName ;; 7 | } 8 | 9 | dimension: task_id { 10 | type: number 11 | sql: ${TABLE}.TaskId ;; 12 | } 13 | 14 | dimension: iteration_id { 15 | type: number 16 | sql: ${TABLE}.IterationId ;; 17 | } 18 | 19 | dimension: status { 20 | type: number 21 | sql: ${TABLE}.Status ;; 22 | } 23 | 24 | dimension_group: created_time { 25 | type: time 26 | timeframes: [time, date, week, month, minute, raw] 27 | sql: ${TABLE}.CreatedTime ;; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/iteration_status_base.view.lkml: -------------------------------------------------------------------------------- 1 | view: iteration_status_base { 2 | sql_table_name: IterationStatusHistory ;; 3 | 4 | dimension: population_name { 5 | type: string 6 | sql: ${TABLE}.PopulationName ;; 7 | } 8 | 9 | dimension: task_id { 10 | type: number 11 | sql: ${TABLE}.TaskId ;; 12 | } 13 | 14 | dimension: iteration_id { 15 | type: number 16 | sql: ${TABLE}.IterationId ;; 17 | } 18 | 19 | dimension: created_time_epoch { 20 | type: number 21 | sql: UNIX_SECONDS(${TABLE}.CreatedTime) ;; 22 | } 23 | 24 | dimension_group: created_time { 25 | type: time 26 | timeframes: [time, date, week, month, minute, second, raw] 27 | sql: ${TABLE}.CreatedTime ;; 28 | } 29 | 30 | measure: duration_in_seconds { 31 | type: number 32 | sql: (MAX(${created_time_epoch}) - MIN(${created_time_epoch})) ;; 33 | } 34 | 35 | measure: duration_in_minutes { 36 | type: number 37 | sql: ${duration_in_seconds} / 60 ;; 38 | } 39 | 40 | measure: min_created_time { 41 | type: min 42 | sql: ${created_time_epoch} ;; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /shuffler/dashboard/looker/views/task_db_views/task.view.lkml: -------------------------------------------------------------------------------- 1 | view: task { 2 | sql_table_name: Task ;; 3 | 4 | dimension: population_name { 5 | type: string 6 | sql: ${TABLE}.PopulationName ;; 7 | } 8 | 9 | dimension: task_id { 10 | type: number 11 | sql: ${TABLE}.TaskId ;; 12 | } 13 | 14 | dimension: total_iteration { 15 | type: number 16 | sql: ${TABLE}.TotalIteration ;; 17 | } 18 | 19 | dimension: min_aggregation_size { 20 | type: number 21 | sql: ${TABLE}.MinAggregationSize ;; 22 | } 23 | 24 | dimension: max_aggregation_size { 25 | type: number 26 | sql: ${TABLE}.MaxAggregationSize ;; 27 | } 28 | 29 | dimension: status_num { 30 | type: number 31 | sql: ${TABLE}.Status ;; 32 | } 33 | 34 | dimension: status { 35 | type: string 36 | sql: CASE 37 | WHEN ${TABLE}.Status = 0 THEN 'OPEN' 38 | WHEN ${TABLE}.Status = 1 THEN 'COMPLETED' 39 | WHEN ${TABLE}.Status = 2 THEN 'CREATED' 40 | WHEN ${TABLE}.Status = 101 THEN 'CANCELED' 41 | WHEN ${TABLE}.Status = 102 THEN 'FAILED' 42 | ELSE 'Unknown' 43 | END ;; 44 | } 45 | 46 | dimension_group: created_time { 47 | type: time 48 | timeframes: [time, date, week, month, raw] 49 | sql: ${TABLE}.CreatedTime ;; 50 | } 51 | 52 | 53 | dimension: min_client_version { 54 | type: string 55 | sql: ${TABLE}.MinClientVersion ;; 56 | } 57 | 58 | dimension: max_client_version { 59 | type: string 60 | sql: ${TABLE}.MinClientVersion ;; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/docs/device_training_duration_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/lookerstudio/docs/device_training_duration_example.png -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/docs/device_upload_duration_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/lookerstudio/docs/device_upload_duration_example.png -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/docs/iteration_processed_time_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/lookerstudio/docs/iteration_processed_time_example.png -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/docs/local_completed_assignment_per_minute_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/lookerstudio/docs/local_completed_assignment_per_minute_example.png -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/docs/model_metrics_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/privacysandbox/odp-federatedcompute/adb9000b8737389cc7db58966744a4a0ebc53685/shuffler/dashboard/lookerstudio/docs/model_metrics_example.png -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/sql/model/model_metrics.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | # Used by "Model Metrics" chart 16 | 17 | -- Looker cloud spanner connector has response size limit of 10 MB. See: 18 | -- https://support.google.com/looker-studio/answer/9008245?hl=en#zippy=%2Cin-this-article. 19 | -- The query selects metrics of active populations in the last 30 days. 20 | -- It is an example to avoid result exceeding the limit. 21 | -- Please update the query condition in your use case. 22 | 23 | SELECT * 24 | FROM ModelMetrics m1 25 | WHERE EXISTS ( 26 | SELECT 1 27 | FROM ModelMetrics m2 28 | WHERE m1.PopulationName = m2.PopulationName 29 | AND m2.CreatedTime >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY) 30 | ) order by CreatedTime DESC, PopulationName; -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/sql/ops/device_training_duration.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | # Used by "Device training duration" chart 16 | 17 | SELECT 18 | PopulationName, 19 | TaskId, 20 | IterationId, 21 | SessionId, 22 | MIN(UNIX_SECONDS(CreatedTime)) as CreatedTime, 23 | (MAX(UNIX_SECONDS(CreatedTime)) - MIN(UNIX_SECONDS(CreatedTime))) AS DurationInSecond, 24 | FROM AssignmentStatusHistory 25 | Where CreatedTime >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY) 26 | And (Status = 0 OR Status = 1) -- 0 is ASSIGNED and 1 is LOCAL_COMPLETED 27 | GROUP BY PopulationName, TaskId, IterationId, SessionId 28 | Order by CreatedTime DESC; -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/sql/ops/device_upload_duration.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | # Used by "Device upload duration" chart 16 | 17 | SELECT 18 | PopulationName, 19 | TaskId, 20 | IterationId, 21 | SessionId, 22 | LocalTime AS CreatedTime, 23 | COALESCE(UploadTime, 0) - COALESCE(LocalTime, 0) AS DurationInSecond 24 | FROM ( 25 | SELECT 26 | PopulationName, 27 | TaskId, 28 | IterationId, 29 | SessionId, 30 | MIN(CASE WHEN Status = 1 THEN UNIX_SECONDS(CreatedTime) END) AS LocalTime, 31 | MIN(CASE WHEN Status = 2 THEN UNIX_SECONDS(CreatedTime) END) AS UploadTime 32 | FROM AssignmentStatusHistory 33 | WHERE (Status = 1 OR Status = 2) -- 1 is LOCAL_COMPLETED and 2 is UPLOAD_COMPLETED 34 | GROUP BY PopulationName, TaskId, IterationId, SessionId 35 | HAVING COUNT(DISTINCT Status) = 2 36 | ) AS ranked_events 37 | ORDER BY LocalTime DESC; -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/sql/ops/iteration.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | SELECT 16 | I.PopulationName AS PopulationName, 17 | I.TaskId AS TaskId, 18 | I.IterationId AS IterationId, 19 | I.AttemptId AS AttemptId, 20 | CASE 21 | WHEN I.Status = 0 THEN 'COLLECTING' 22 | WHEN I.Status = 1 THEN 'AGGREGATING' 23 | WHEN I.Status = 2 OR I.Status = 50 THEN 'COMPLETED' 24 | WHEN I.Status = 4 THEN 'APPLYING' 25 | WHEN I.Status = 5 OR I.Status = 51 THEN 'POST_PROCESSED' 26 | WHEN I.Status = 101 THEN 'CANCELED' 27 | WHEN I.Status = 102 THEN 'AGGREGATING_FAILED' 28 | WHEN I.Status = 103 THEN 'APPLYING_FAILED' 29 | ELSE 'Unknown' 30 | END AS Status, 31 | I.ReportGoal AS ReportGoal, 32 | ISH.CreatedTime AS LastUpdatedTime 33 | FROM Iteration AS I 34 | FULL JOIN IterationStatusHistory AS ISH 35 | ON 36 | I.PopulationName = ISH.PopulationName 37 | AND I.TaskId = ISH.TaskId 38 | AND I.IterationId = ISH.IterationId 39 | AND I.AttemptId = ISH.AttemptId 40 | AND I.Status = ISH.Status 41 | Where I.PopulationName is not NULL 42 | And ISH.CreatedTime >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY) 43 | ORDER BY CreatedTime DESC; -------------------------------------------------------------------------------- /shuffler/dashboard/lookerstudio/sql/ops/iteration_processed_time.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | # Used by "Iteration processed time" chart 16 | 17 | SELECT 18 | PopulationName, 19 | TaskId, 20 | IterationId, 21 | MIN(UNIX_SECONDS(CreatedTime)) as CreatedTime, 22 | (MAX(UNIX_SECONDS(CreatedTime)) - MIN(UNIX_SECONDS(CreatedTime))) / 60 AS DurationInMinute, 23 | FROM IterationStatusHistory 24 | Where Status = 0 OR Status = 2 -- 0 is collecting and 2 is completed 25 | GROUP BY PopulationName, TaskId, IterationId; -------------------------------------------------------------------------------- /shuffler/services/collector/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@rules_oci//oci:defs.bzl", "oci_image", "oci_push", "oci_tarball") 16 | load("@rules_pkg//:pkg.bzl", "pkg_tar") 17 | 18 | pkg_tar( 19 | name = "binary_tar", 20 | # Bring the java_binary 21 | srcs = [ 22 | "//java/src/main/java/com/google/ondevicepersonalization/federatedcompute/shuffler/collector:collector_application", 23 | ], 24 | ) 25 | 26 | oci_image( 27 | name = "collector_image", 28 | base = "@java_base", 29 | entrypoint = [ 30 | "java", 31 | "-Dspring.profiles.active=gcp", 32 | "-XX:InitialRAMPercentage=70.0", 33 | "-XX:MaxRAMPercentage=70.0", 34 | "-jar", 35 | "/collector_application.jar", 36 | ], 37 | tars = [ 38 | ":binary_tar", 39 | ], 40 | ) 41 | 42 | oci_push( 43 | name = "collector_image_publish", 44 | image = ":collector_image", 45 | remote_tags = ["latest"], 46 | repository = "//collector_image", 47 | ) 48 | 49 | # Run a local container with: 50 | # $ bazel run :tarball 51 | # $ docker run collector_image:latest 52 | oci_tarball( 53 | name = "tarball", 54 | image = ":collector_image", 55 | repo_tags = ["collector_image:latest"], 56 | ) 57 | -------------------------------------------------------------------------------- /shuffler/spanner/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | package(default_visibility = ["//visibility:public"]) 16 | 17 | filegroup( 18 | name = "task_database_sdl", 19 | srcs = ["task_database.sdl"], 20 | ) 21 | 22 | filegroup( 23 | name = "metrics_datbase_sdl", 24 | srcs = ["metrics_database.sdl"], 25 | ) 26 | -------------------------------------------------------------------------------- /shuffler/spanner/metrics_database.sdl: -------------------------------------------------------------------------------- 1 | -- Copyright 2024 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | CREATE TABLE ModelMetrics ( 16 | PopulationName STRING(64) NOT NULL, 17 | TaskId INT64 NOT NULL, 18 | IterationId INT64 NOT NULL, 19 | MetricName STRING(64) NOT NULL, 20 | MetricValue FLOAT64 NOT NULL, 21 | CreatedTime TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true), 22 | ) PRIMARY KEY(PopulationName, TaskId, IterationId, MetricName); 23 | 24 | CREATE INDEX ModelMetricsCreatedTimeIndex ON ModelMetrics(CreatedTime); -------------------------------------------------------------------------------- /shuffler/terraform/gcp/applications/cluster/cluster.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | module "cluster" { 19 | source = "../../modules/cluster" 20 | environment = var.environment 21 | project_id = var.project_id 22 | region = var.region 23 | task_assignment_image = var.task_assignment_image 24 | task_assignment_port = var.task_assignment_port 25 | task_scheduler_image = var.task_scheduler_image 26 | collector_image = var.collector_image 27 | parent_domain_name = var.parent_domain_name 28 | static_ip_name = var.static_ip_name 29 | task_assignment_max_replicas = var.task_assignment_max_replicas 30 | task_assignment_min_replicas = var.task_assignment_min_replicas 31 | task_assignment_cpu = var.task_assignment_cpu 32 | collector_min_replicas = var.collector_min_replicas 33 | collector_max_replicas = var.collector_max_replicas 34 | collector_cpu = var.collector_cpu 35 | task_scheduler_min_replicas = var.task_scheduler_min_replicas 36 | task_scheduler_max_replicas = var.task_scheduler_max_replicas 37 | task_scheduler_cpu = var.task_scheduler_cpu 38 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/applications/cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "taskscheduler_name" { 18 | description = "Name of taskscheduler deployment" 19 | value = module.cluster.taskscheduler_name 20 | } 21 | 22 | output "taskassignment_name" { 23 | description = "Name of taskassignment deployment" 24 | value = module.cluster.taskassignment_name 25 | } 26 | 27 | output "collector_name" { 28 | description = "Name of collector deployment" 29 | value = module.cluster.collector_name 30 | } 31 | 32 | output "ingress_name" { 33 | description = "Name of cluster ingress" 34 | value = module.cluster.ingress_name 35 | } 36 | 37 | output "gcp_service_account_email" { 38 | description = "The email address of GCP service account for workloads" 39 | value = module.cluster.gcp_service_account_email 40 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/applications/shuffler/resources.tf: -------------------------------------------------------------------------------- 1 | ../resources/resources.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/applications/shuffler/resources_outputs.tf: -------------------------------------------------------------------------------- 1 | ../resources/resources_outputs.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/applications/shuffler/resources_variables.tf: -------------------------------------------------------------------------------- 1 | ../resources/resources_variables.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/cluster/cluster.tf: -------------------------------------------------------------------------------- 1 | ../../shared/cluster/cluster.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/cluster/cluster_outputs.tf: -------------------------------------------------------------------------------- 1 | ../../shared/cluster/cluster_outputs.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/cluster/cluster_variables.tf: -------------------------------------------------------------------------------- 1 | ../../shared/cluster/cluster_variables.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/cluster/dev.auto.tfvars: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # Example values required by cluster.tf 18 | # 19 | # These values should be modified for each of your environments. 20 | 21 | environment = "demo-env" 22 | project_id = "demo-project" 23 | region = "us-central1" 24 | 25 | # Workload Images 26 | task_assignment_image = "///:" 27 | task_scheduler_image = "///:" 28 | collector_image = "///:" 29 | 30 | # Output from shuffler 31 | parent_domain_name = "demo-domain-xyz.com" 32 | static_ip_name = "" 33 | gke_cluster_ca_certificate = "" 34 | gke_host = "" -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/cluster/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # Demo main.tf 18 | # 19 | # This file is meant to show an example of what necessary environment-specific 20 | # configuration is necessary in each environment. Terraform backend 21 | # configuration cannot reference Terraform variables so this file must be 22 | # customized for each environment. 23 | 24 | terraform { 25 | # Note: the following lines should be uncommented in order to store Terraform 26 | # state in a remote backend. 27 | 28 | # backend "gcs" { 29 | # bucket = "[bucket name goes here]" 30 | # prefix = "[key name goes here].tfstate" 31 | # } 32 | 33 | required_providers { 34 | google = { 35 | source = "hashicorp/google" 36 | version = ">= 4.0" 37 | } 38 | kubernetes = { 39 | source = "hashicorp/kubernetes" 40 | version = ">= 2.13" 41 | } 42 | } 43 | required_version = ">= 0.13" 44 | } 45 | 46 | data "google_client_config" "default" {} 47 | 48 | provider "kubernetes" { 49 | host = "https://${var.gke_host}" 50 | token = data.google_client_config.default.access_token 51 | cluster_ca_certificate = base64decode(var.gke_cluster_ca_certificate) 52 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # Demo main.tf 18 | # 19 | # This file is meant to show an example of what necessary environment-specific 20 | # configuration is necessary in each environment. Terraform backend 21 | # configuration cannot reference Terraform variables so this file must be 22 | # customized for each environment. 23 | 24 | terraform { 25 | # Note: the following lines should be uncommented in order to store Terraform 26 | # state in a remote backend. 27 | 28 | # backend "gcs" { 29 | # bucket = "[bucket name goes here]" 30 | # prefix = "[key name goes here].tfstate" 31 | # } 32 | 33 | required_providers { 34 | google = { 35 | source = "hashicorp/google" 36 | version = ">= 5.36" 37 | } 38 | kubernetes = { 39 | source = "hashicorp/kubernetes" 40 | version = ">= 2.13" 41 | } 42 | } 43 | required_version = ">= 0.13" 44 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/shuffler.tf: -------------------------------------------------------------------------------- 1 | ../shared/shuffler.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/shuffler_outputs.tf: -------------------------------------------------------------------------------- 1 | ../shared/shuffler_outputs.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/dev/shuffler_variables.tf: -------------------------------------------------------------------------------- 1 | ../shared/shuffler_variables.tf -------------------------------------------------------------------------------- /shuffler/terraform/gcp/environments/shared/cluster/cluster_outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "taskscheduler_name" { 18 | description = "Name of taskscheduler deployment" 19 | value = module.cluster.taskscheduler_name 20 | } 21 | 22 | output "taskassignment_name" { 23 | description = "Name of taskassignment deployment" 24 | value = module.cluster.taskassignment_name 25 | } 26 | 27 | output "collector_name" { 28 | description = "Name of collector deployment" 29 | value = module.cluster.collector_name 30 | } 31 | 32 | output "ingress_name" { 33 | description = "Name of cluster ingress" 34 | value = module.cluster.ingress_name 35 | } 36 | 37 | output "gcp_service_account_email" { 38 | description = "The email address of GCP service account for workloads" 39 | value = module.cluster.gcp_service_account_email 40 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/cdn/outputs.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | output "cdn_domain" { 16 | value = local.domain 17 | } 18 | 19 | output "key_value_a" { 20 | value = google_compute_backend_bucket_signed_url_key.backend_key_a.key_value 21 | } 22 | 23 | output "key_value_b" { 24 | value = google_compute_backend_bucket_signed_url_key.backend_key_b.key_value 25 | } 26 | 27 | output "key_name" { 28 | value = toggles_leapfrog.toggle.alpha ? google_compute_backend_bucket_signed_url_key.backend_key_a.name : google_compute_backend_bucket_signed_url_key.backend_key_b.name 29 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/cdn/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | ################################################################################ 18 | # Global Variables. 19 | ################################################################################ 20 | 21 | variable "project_id" { 22 | description = "GCP Project ID in which this module will be created." 23 | type = string 24 | } 25 | 26 | variable "environment" { 27 | description = "Description for the environment, e.g. dev, staging, production" 28 | type = string 29 | } 30 | 31 | variable "backend_bucket_name" { 32 | description = "Name of bucket to use as cdn backend" 33 | type = string 34 | } 35 | 36 | variable "parent_domain_name" { 37 | description = "Parent domain name to use for cdn" 38 | type = string 39 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/cluster/ingress.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "kubernetes_ingress_v1" "ingress" { 18 | metadata { 19 | name = "${var.environment}-ingress" 20 | annotations = { 21 | "kubernetes.io/ingress.allow-http" = "false" 22 | "kubernetes.io/ingress.global-static-ip-name" = var.static_ip_name 23 | "ingress.gcp.kubernetes.io/pre-shared-cert" = google_compute_managed_ssl_certificate.default.name 24 | } 25 | labels = { 26 | maintained_by = "terraform" 27 | } 28 | } 29 | 30 | spec { 31 | rule { 32 | http { 33 | path { 34 | backend { 35 | service { 36 | name = kubernetes_service_v1.taskassignment.metadata[0].name 37 | port { 38 | number = var.task_assignment_port 39 | } 40 | } 41 | } 42 | 43 | path = "/taskassignment/*" 44 | } 45 | } 46 | } 47 | } 48 | } 49 | 50 | resource "google_compute_managed_ssl_certificate" "default" { 51 | name = "${var.environment}-cert" 52 | provider = google 53 | managed { 54 | domains = [var.parent_domain_name] 55 | } 56 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "taskscheduler_name" { 18 | description = "Name of taskscheduler deployment" 19 | value = kubernetes_deployment_v1.taskscheduler.metadata[0].name 20 | } 21 | 22 | output "taskassignment_name" { 23 | description = "Name of taskassignment deployment" 24 | value = kubernetes_deployment_v1.taskassignment.metadata[0].name 25 | } 26 | 27 | output "collector_name" { 28 | description = "Name of collector deployment" 29 | value = kubernetes_deployment_v1.collector.metadata[0].name 30 | } 31 | 32 | output "ingress_name" { 33 | description = "Name of cluster ingress" 34 | value = kubernetes_ingress_v1.ingress.metadata[0].name 35 | } 36 | 37 | output "gcp_service_account_email" { 38 | description = "The email address of GCP service account for workloads" 39 | value = module.gke-workload-identity.gcp_service_account_email 40 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/cluster/workload_identity.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "gke-workload-identity" { 18 | source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" 19 | name = "${var.environment}-gke-wi" 20 | namespace = "default" 21 | project_id = var.project_id 22 | roles = ["roles/spanner.databaseUser", "roles/logging.logWriter", "roles/iam.serviceAccountTokenCreator", "roles/storage.objectUser", "roles/pubsub.publisher", "roles/pubsub.subscriber", "roles/gkehub.serviceAgent", "roles/secretmanager.secretAccessor"] 23 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/confidentialspace/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/gke/gke.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "gke" { 18 | source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-public-cluster" 19 | project_id = var.project_id 20 | name = "${var.environment}-cluster" 21 | regional = true 22 | region = var.region 23 | network = var.network_name 24 | subnetwork = var.subnets_names[index(var.subnets_names, var.subnet_name)] 25 | ip_range_pods = var.pods_range_name 26 | ip_range_services = var.svc_range_name 27 | release_channel = "REGULAR" 28 | horizontal_pod_autoscaling = true 29 | http_load_balancing = true 30 | network_tags = [var.environment] 31 | service_account = var.cluster_service_account 32 | cluster_resource_labels = { 33 | environment = var.environment 34 | } 35 | deletion_protection = var.cluster_deletion_protection 36 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/monitoring/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/network/network.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "gcp-network" { 18 | source = "terraform-google-modules/network/google" 19 | version = ">= 4.0.1" 20 | 21 | project_id = var.project_id 22 | network_name = var.network_name 23 | 24 | subnets = [ 25 | { 26 | subnet_name = var.subnet_name 27 | subnet_ip = "10.0.0.0/17" 28 | subnet_region = var.region 29 | }, 30 | { 31 | subnet_name = var.master_auth_subnetwork 32 | subnet_ip = "10.60.0.0/17" 33 | subnet_region = var.region 34 | }, 35 | ] 36 | 37 | secondary_ranges = { 38 | (var.subnet_name) = [ 39 | { 40 | range_name = var.pods_range_name 41 | ip_cidr_range = "192.168.0.0/18" 42 | }, 43 | { 44 | range_name = var.svc_range_name 45 | ip_cidr_range = "192.168.64.0/18" 46 | }, 47 | ] 48 | } 49 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/network/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "network_name" { 18 | description = "The name of the VPC being created" 19 | value = module.gcp-network.network_name 20 | } 21 | 22 | output "subnets_names" { 23 | description = "The names of the subnet being created" 24 | value = module.gcp-network.subnets_names 25 | } 26 | 27 | output "subnets_self_links" { 28 | description = "The self-links of subnets being created" 29 | value = module.gcp-network.subnets_self_links 30 | } 31 | 32 | output "static_ip" { 33 | description = "The allocated static ip address" 34 | value = google_compute_global_address.default.address 35 | } 36 | 37 | output "static_ip_name" { 38 | description = "The name of the allocated static ip address" 39 | value = google_compute_global_address.default.name 40 | } 41 | 42 | 43 | -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/network/routing.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # reserved IP address 18 | resource "google_compute_global_address" "default" { 19 | name = "${var.environment}-ip" 20 | ip_version = "IPV4" 21 | provider = google 22 | } 23 | 24 | # to register web-server's ip address in DNS 25 | resource "google_dns_record_set" "a" { 26 | name = "${var.parent_domain_name}." 27 | managed_zone = replace(var.parent_domain_name, ".", "-") 28 | type = "A" 29 | ttl = 300 30 | rrdatas = [ 31 | google_compute_global_address.default.address 32 | ] 33 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/parameters/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_secret_manager_secret" "worker_parameter" { 18 | secret_id = format("fc-%s-%s", var.environment, var.parameter_name) 19 | replication { 20 | auto {} 21 | } 22 | } 23 | 24 | resource "google_secret_manager_secret_version" "worker_parameter_value" { 25 | secret = google_secret_manager_secret.worker_parameter.id 26 | secret_data = var.parameter_value 27 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/parameters/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "secret_name" { 18 | description = "Name of created secret" 19 | value = google_secret_manager_secret.worker_parameter.id 20 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/parameters/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "environment" { 18 | type = string 19 | description = "Description for the environment, e.g. dev, staging, production" 20 | } 21 | 22 | variable "parameter_name" { 23 | type = string 24 | description = "Name of the parameter." 25 | } 26 | 27 | variable "parameter_value" { 28 | type = string 29 | description = "Value of the parameter." 30 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/pubsub/outputs.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | output "aggregator_topic_name" { 16 | value = google_pubsub_topic.aggregator-topic.name 17 | } 18 | 19 | output "aggregator_subscription_name" { 20 | value = google_pubsub_subscription.aggregator-subscription.name 21 | } 22 | 23 | output "aggregator_notifications_topic_name" { 24 | value = google_pubsub_topic.aggregator-notifications-topic.name 25 | } 26 | 27 | output "aggregator_notifications_subscription_name" { 28 | value = google_pubsub_subscription.aggregator-notifications-subscription.name 29 | } 30 | 31 | output "model_updater_topic_name" { 32 | value = google_pubsub_topic.model-updater-topic.name 33 | } 34 | 35 | output "model_updater_subscription_name" { 36 | value = google_pubsub_subscription.model-updater-subscription.name 37 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/pubsub/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | ################################################################################ 18 | # Global Variables. 19 | ################################################################################ 20 | 21 | variable "project_id" { 22 | description = "GCP Project ID in which this module will be created." 23 | type = string 24 | } 25 | 26 | variable "environment" { 27 | description = "Description for the environment, e.g. dev, staging, production" 28 | type = string 29 | } 30 | 31 | variable "region" { 32 | description = "Region where all services will be created." 33 | type = string 34 | } 35 | 36 | variable "enable_exactly_once_delivery" { 37 | description = "Enable exactly once delivery on pubsub subscriptions. Consider disabling for improved performance at the cost of potentially redelivered messages. https://cloud.google.com/pubsub/docs/exactly-once-delivery" 38 | type = bool 39 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/serviceaccount/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | service_account_email = var.service_account_email == "" ? google_service_account.service_account[0].email : var.service_account_email 19 | } 20 | 21 | resource "google_service_account" "service_account" { 22 | count = var.service_account_email == "" ? 1 : 0 23 | # Service account id has a 30 character limit 24 | account_id = "${var.environment}-${var.service_account_name}" 25 | } 26 | 27 | resource "google_project_iam_member" "worker_logging_iam" { 28 | for_each = toset(var.roles) 29 | role = each.value 30 | member = "serviceAccount:${local.service_account_email}" 31 | project = var.project_id 32 | } 33 | 34 | 35 | -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/serviceaccount/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "service_account_email" { 18 | description = "Email of the service account" 19 | value = local.service_account_email 20 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/serviceaccount/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "GCP Project ID in which this module will be created." 19 | type = string 20 | } 21 | 22 | variable "environment" { 23 | description = "Description for the environment, e.g. dev, staging, production" 24 | type = string 25 | } 26 | 27 | variable "service_account_email" { 28 | description = "Service account email to use. If blank one will be created" 29 | type = string 30 | } 31 | 32 | variable "service_account_name" { 33 | description = "Service account name to use for creation" 34 | type = string 35 | } 36 | 37 | variable "roles" { 38 | description = "List of roles to grant the service account." 39 | type = list(any) 40 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/taskbuilder/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "task_builder_name" { 18 | description = "Name of task builder cloud run deployment" 19 | value = google_cloud_run_v2_service.taskbuilder.name 20 | } 21 | 22 | output "task_builder_url" { 23 | description = "URL of the task builder service. The terraform may need to be re-run if blank to populate." 24 | value = "https://${google_api_gateway_gateway.api_gw.default_hostname}" 25 | } 26 | 27 | output "task_builder_managed_service_name" { 28 | description = "Task builder API managed service name" 29 | value = google_api_gateway_api.api_gw.managed_service 30 | } -------------------------------------------------------------------------------- /shuffler/terraform/gcp/modules/taskmanagement/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "task_management_name" { 18 | description = "Name of task management cloud run deployment" 19 | value = google_cloud_run_v2_service.taskmanagement.name 20 | } 21 | 22 | output "task_management_url" { 23 | description = "URL of the task management service. The terraform may need to be re-run if blank to populate." 24 | value = google_cloud_run_v2_service.taskmanagement.uri 25 | } -------------------------------------------------------------------------------- /toolchain/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /toolchain/bookworm.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | version: 1 16 | 17 | sources: 18 | - channel: bookworm main contrib 19 | url: https://snapshot.debian.org/archive/debian/20240210T223313Z 20 | - channel: bookworm-security main 21 | url: https://snapshot.debian.org/archive/debian-security/20240210T223313Z 22 | - channel: bookworm-updates main 23 | url: https://snapshot.debian.org/archive/debian/20240210T223313Z 24 | 25 | archs: 26 | - "amd64" 27 | 28 | packages: 29 | - "libc++1-16" -------------------------------------------------------------------------------- /toolchain/requirements_updater/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | load("@python//:defs.bzl", "compile_pip_requirements") 16 | 17 | compile_pip_requirements( 18 | name = "requirements_3_10", 19 | extra_args = ["--allow-unsafe"], 20 | requirements_in = "requirements.in", 21 | requirements_txt = "requirements_lock_3_10.txt", 22 | ) 23 | -------------------------------------------------------------------------------- /toolchain/requirements_updater/WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "requirements_updater") 2 | 3 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 4 | 5 | http_archive( 6 | name = "bazel_skylib", 7 | sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", 8 | urls = [ 9 | "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", 10 | "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", 11 | ], 12 | ) 13 | 14 | http_archive( 15 | name = "rules_python", 16 | sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", 17 | strip_prefix = "rules_python-0.26.0", 18 | url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", 19 | ) 20 | 21 | load("@rules_python//python:repositories.bzl", "py_repositories") 22 | 23 | py_repositories() 24 | 25 | load("@rules_python//python:repositories.bzl", "python_register_toolchains") 26 | load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") 27 | 28 | default_python_version = "3.10" 29 | 30 | python_register_toolchains( 31 | name = "python", 32 | ignore_root_user_error = True, 33 | python_version = default_python_version, 34 | register_coverage_tool = True, 35 | ) 36 | 37 | pip_install_dependencies() 38 | -------------------------------------------------------------------------------- /toolchain/requirements_updater/requirements.in: -------------------------------------------------------------------------------- 1 | # Requirements from federated-compute 2 | attrs~=23.1 3 | dm-tree~=0.1.8 4 | absl-py~=1.4 5 | protobuf>=4.23 6 | 7 | # Server requirements 8 | responses>=0.24.0 9 | google-cloud-storage>=2.13.0 10 | google-cloud-secret-manager>=2.20.0 11 | google-api-core>=2.19.0 12 | google-auth>=2.29.0 13 | google-crc32c>=1.5.0 14 | flask==3.0.2 15 | 16 | # Google DP 17 | mpmath~=1.2 18 | numpy~=1.21 19 | scipy~=1.7 20 | 21 | # The TensorFlow version should match what's specified in the WORKSPACE file for 22 | # C++ targets. 23 | tensorflow==2.14.0 24 | tensorflow-federated==0.68.0 25 | -------------------------------------------------------------------------------- /toolchain/requirements_updater/updater.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2024 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # Exit on failure 18 | set -e 19 | # Print commands 20 | set -x 21 | # All commands run relative to this directory 22 | cd "$(dirname "${BASH_SOURCE[0]}")" 23 | 24 | VERSION="3_10" 25 | 26 | touch "requirements_lock_$VERSION.txt" 27 | bazel run --experimental_convenience_symlinks=ignore //:requirements_"$VERSION".update 28 | sed -i '/^#/d' requirements_lock_"$VERSION".txt 29 | mv requirements_lock_"$VERSION".txt ../../requirements_lock_"$VERSION".txt 30 | -------------------------------------------------------------------------------- /tools/BUILD: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --------------------------------------------------------------------------------