├── .dockerignore ├── .github └── workflows │ ├── checks.yml │ └── upgrade_automation.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── auth ├── auth_context.go ├── authzserver │ ├── authorize.go │ ├── authorize_test.go │ ├── claims_verifier.go │ ├── claims_verifier_test.go │ ├── doc.go │ ├── encryptor.go │ ├── initialize.go │ ├── initialize_test.go │ ├── metadata.go │ ├── metadata_provider.go │ ├── metadata_provider_test.go │ ├── metadata_test.go │ ├── provider.go │ ├── provider_test.go │ ├── resource_server.go │ ├── resource_server_test.go │ ├── stateless_token_store.go │ ├── token.go │ ├── token_test.go │ └── utils.go ├── config │ ├── authorizationservertype_enumer.go │ ├── config.go │ ├── config_flags.go │ ├── config_flags_test.go │ ├── config_test.go │ ├── samesite_enumer.go │ ├── testdata │ │ ├── config.yaml │ │ └── secret.yaml │ └── third_party_config.go ├── constants.go ├── cookie.go ├── cookie_manager.go ├── cookie_manager_test.go ├── cookie_test.go ├── create_secrets.go ├── handler_utils.go ├── handler_utils_test.go ├── handlers.go ├── handlers_test.go ├── identity_context.go ├── identity_context_test.go ├── init_secrets.go ├── interceptor.go ├── interceptor_test.go ├── interfaces │ ├── context.go │ ├── cookie.go │ └── mocks │ │ ├── authentication_context.go │ │ ├── cookie_handler.go │ │ ├── handler_registerer.go │ │ ├── identity_context.go │ │ ├── o_auth2_provider.go │ │ └── o_auth2_resource_server.go ├── token.go ├── token_test.go └── user_info_provider.go ├── boilerplate ├── flyte │ ├── code_of_conduct │ │ ├── CODE_OF_CONDUCT.md │ │ ├── README.rst │ │ └── update.sh │ ├── docker_build │ │ ├── Makefile │ │ ├── Readme.rst │ │ └── docker_build.sh │ ├── end2end │ │ ├── Makefile │ │ ├── end2end.sh │ │ ├── functional-test-config.yaml │ │ └── run-tests.py │ ├── flyte_golang_compile │ │ ├── Readme.rst │ │ ├── flyte_golang_compile.Template │ │ └── update.sh │ ├── github_workflows │ │ ├── Readme.rst │ │ ├── master.yml │ │ ├── pull_request.yml │ │ └── update.sh │ ├── golang_dockerfile │ │ ├── Dockerfile.GoTemplate │ │ ├── Readme.rst │ │ └── update.sh │ ├── golang_support_tools │ │ ├── go.mod │ │ ├── go.sum │ │ └── tools.go │ ├── golang_test_targets │ │ ├── Makefile │ │ ├── Readme.rst │ │ ├── download_tooling.sh │ │ ├── go-gen.sh │ │ └── goimports │ ├── golangci_file │ │ ├── .golangci.yml │ │ ├── Readme.rst │ │ └── update.sh │ └── pull_request_template │ │ ├── Readme.rst │ │ ├── pull_request_template.md │ │ └── update.sh ├── update.cfg └── update.sh ├── cmd ├── entrypoints │ ├── clusterresource.go │ ├── k8s_secret.go │ ├── migrate.go │ ├── root.go │ ├── serve.go │ └── serve_test.go ├── main.go └── scheduler │ ├── entrypoints │ ├── precheck.go │ ├── root.go │ └── scheduler.go │ └── main.go ├── codecov.yml ├── dataproxy ├── service.go └── service_test.go ├── flyteadmin_config.yaml ├── go.mod ├── go.sum ├── pkg ├── async │ ├── cloudevent │ │ ├── factory.go │ │ ├── factory_test.go │ │ ├── implementations │ │ │ ├── cloudevent_publisher.go │ │ │ ├── cloudevent_publisher_test.go │ │ │ ├── sender.go │ │ │ └── sender_test.go │ │ ├── interfaces │ │ │ ├── publisher.go │ │ │ └── sender.go │ │ └── mocks │ │ │ ├── publisher.go │ │ │ └── sender.go │ ├── events │ │ ├── implementations │ │ │ ├── node_execution_event_writer.go │ │ │ ├── node_execution_event_writer_test.go │ │ │ ├── workflow_execution_event_writer.go │ │ │ └── workflow_execution_event_writer_test.go │ │ ├── interfaces │ │ │ ├── node_execution.go │ │ │ └── workflow_execution.go │ │ └── mocks │ │ │ ├── node_execution_event_writer.go │ │ │ └── workflow_execution_event_writer.go │ ├── notifications │ │ ├── email.go │ │ ├── email_test.go │ │ ├── factory.go │ │ ├── factory_test.go │ │ ├── implementations │ │ │ ├── aws_emailer.go │ │ │ ├── aws_emailer_test.go │ │ │ ├── aws_processor.go │ │ │ ├── aws_processor_test.go │ │ │ ├── email_metrics.go │ │ │ ├── emailers.go │ │ │ ├── event_publisher.go │ │ │ ├── event_publisher_test.go │ │ │ ├── gcp_processor.go │ │ │ ├── gcp_processor_test.go │ │ │ ├── noop_notifications.go │ │ │ ├── processor_metrics.go │ │ │ ├── publisher.go │ │ │ ├── publisher_test.go │ │ │ ├── sandbox_processor.go │ │ │ ├── sandbox_processor_test.go │ │ │ ├── sandbox_publisher.go │ │ │ ├── sandbox_publisher_test.go │ │ │ ├── sendgrid_emailer.go │ │ │ └── sendgrid_emailer_test.go │ │ ├── interfaces │ │ │ ├── emailer.go │ │ │ ├── processor.go │ │ │ └── publisher.go │ │ └── mocks │ │ │ ├── emailer.go │ │ │ ├── processor.go │ │ │ └── publisher.go │ ├── schedule │ │ ├── aws │ │ │ ├── cloud_watch_scheduler.go │ │ │ ├── cloud_watch_scheduler_test.go │ │ │ ├── interfaces │ │ │ │ └── cloud_watch_event_client.go │ │ │ ├── mocks │ │ │ │ └── mock_cloud_watch_event_client.go │ │ │ ├── serialization.go │ │ │ ├── serialization_test.go │ │ │ ├── shared.go │ │ │ ├── shared_test.go │ │ │ ├── workflow_executor.go │ │ │ └── workflow_executor_test.go │ │ ├── factory.go │ │ ├── interfaces │ │ │ ├── event_scheduler.go │ │ │ └── workflow_executor.go │ │ ├── mocks │ │ │ ├── mock_event_scheduler.go │ │ │ └── mock_workflow_executor.go │ │ └── noop │ │ │ ├── event_scheduler.go │ │ │ └── workflow_executor.go │ ├── shared.go │ └── shared_test.go ├── clusterresource │ ├── controller.go │ ├── controller_test.go │ ├── impl │ │ ├── admin_service_data_provider.go │ │ ├── admin_service_data_provider_test.go │ │ ├── db_admin_data_provider.go │ │ ├── db_admin_data_provider_test.go │ │ └── shared.go │ ├── interfaces │ │ └── admin.go │ ├── mocks │ │ └── flyte_admin_data_provider.go │ ├── sync_stats.go │ ├── testdata │ │ ├── docker.yaml │ │ ├── gsa.yaml │ │ ├── imagepullsecrets.yaml │ │ └── namespace.yaml │ └── utils.go ├── common │ ├── artifacttype_enumer.go │ ├── cloud.go │ ├── constants.go │ ├── data_store.go │ ├── data_store_test.go │ ├── entity.go │ ├── executions.go │ ├── executions_test.go │ ├── filters.go │ ├── filters_test.go │ ├── flyte_url.go │ ├── flyte_url_test.go │ ├── mocks │ │ └── storage.go │ ├── namespace.go │ ├── namespace_test.go │ ├── sorting.go │ ├── sorting_test.go │ └── testutils │ │ └── common.go ├── config │ ├── config.go │ ├── serverconfig_flags.go │ └── serverconfig_flags_test.go ├── data │ ├── factory.go │ ├── implementations │ │ ├── aws_remote_url.go │ │ ├── aws_remote_url_test.go │ │ ├── gcp_remote_url.go │ │ ├── gcp_remote_url_test.go │ │ ├── noop_remote_url.go │ │ └── noop_remote_url_test.go │ ├── interfaces │ │ └── remote.go │ └── mocks │ │ └── remote.go ├── errors │ ├── errors.go │ └── errors_test.go ├── executioncluster │ ├── execution_target.go │ ├── impl │ │ ├── cluster_execution_target_provider.go │ │ ├── factory.go │ │ ├── in_cluster.go │ │ ├── in_cluster_test.go │ │ ├── list_targets.go │ │ ├── list_targets_test.go │ │ ├── random_cluster_selector.go │ │ ├── random_cluster_selector_test.go │ │ └── testdata │ │ │ ├── clusters_config.yaml │ │ │ ├── clusters_config2.yaml │ │ │ └── clusters_config2_default_label.yaml │ ├── interfaces │ │ ├── cluster.go │ │ └── execution_target_provider.go │ └── mocks │ │ ├── cluster_interface.go │ │ ├── execution_target_provider.go │ │ ├── get_target_interface.go │ │ ├── list_targets_interface.go │ │ └── mock_cluster.go ├── flytek8s │ └── client.go ├── manager │ ├── impl │ │ ├── description_entity_manager.go │ │ ├── description_entity_manager_test.go │ │ ├── execution_manager.go │ │ ├── execution_manager_test.go │ │ ├── executions │ │ │ ├── quality_of_service.go │ │ │ ├── quality_of_service_test.go │ │ │ ├── queues.go │ │ │ └── queues_test.go │ │ ├── launch_plan_manager.go │ │ ├── launch_plan_manager_test.go │ │ ├── metrics_manager.go │ │ ├── metrics_manager_test.go │ │ ├── named_entity_manager.go │ │ ├── named_entity_manager_test.go │ │ ├── node_execution_manager.go │ │ ├── node_execution_manager_test.go │ │ ├── project_manager.go │ │ ├── project_manager_test.go │ │ ├── resources │ │ │ ├── resource_manager.go │ │ │ └── resource_manager_test.go │ │ ├── shared │ │ │ ├── constants.go │ │ │ ├── errors.go │ │ │ └── iface.go │ │ ├── signal_manager.go │ │ ├── signal_manager_test.go │ │ ├── task_execution_manager.go │ │ ├── task_execution_manager_test.go │ │ ├── task_manager.go │ │ ├── task_manager_test.go │ │ ├── test_constants.go │ │ ├── testutils │ │ │ ├── attributes.go │ │ │ ├── config.go │ │ │ ├── constants.go │ │ │ ├── mock_closures.go │ │ │ ├── mock_requests.go │ │ │ └── repository.go │ │ ├── util │ │ │ ├── data.go │ │ │ ├── data_test.go │ │ │ ├── digests.go │ │ │ ├── digests_test.go │ │ │ ├── filters.go │ │ │ ├── filters_test.go │ │ │ ├── resources.go │ │ │ ├── resources_test.go │ │ │ ├── shared.go │ │ │ ├── shared_test.go │ │ │ ├── single_task_execution.go │ │ │ ├── single_task_execution_test.go │ │ │ └── testdata │ │ │ │ └── workflow.json │ │ ├── validation │ │ │ ├── attributes_validator.go │ │ │ ├── attributes_validator_test.go │ │ │ ├── execution_validator.go │ │ │ ├── execution_validator_test.go │ │ │ ├── launch_plan_validator.go │ │ │ ├── launch_plan_validator_test.go │ │ │ ├── named_entity_validator.go │ │ │ ├── named_entity_validator_test.go │ │ │ ├── node_execution_validator.go │ │ │ ├── node_execution_validator_test.go │ │ │ ├── notifications_validator.go │ │ │ ├── notifications_validator_test.go │ │ │ ├── project_validator.go │ │ │ ├── project_validator_test.go │ │ │ ├── shared_execution.go │ │ │ ├── shared_execution_test.go │ │ │ ├── signal_validator.go │ │ │ ├── signal_validator_test.go │ │ │ ├── task_execution_validator.go │ │ │ ├── task_execution_validator_test.go │ │ │ ├── task_validator.go │ │ │ ├── task_validator_test.go │ │ │ ├── validation.go │ │ │ ├── validation_test.go │ │ │ ├── workflow_validator.go │ │ │ └── workflow_validator_test.go │ │ ├── version_manager.go │ │ ├── version_manager_test.go │ │ ├── workflow_manager.go │ │ └── workflow_manager_test.go │ ├── interfaces │ │ ├── description_entity.go │ │ ├── execution.go │ │ ├── launch_plan.go │ │ ├── metrics.go │ │ ├── named_entity.go │ │ ├── node_execution.go │ │ ├── project.go │ │ ├── resource.go │ │ ├── signal.go │ │ ├── task.go │ │ ├── task_execution.go │ │ ├── version.go │ │ └── workflow.go │ └── mocks │ │ ├── execution.go │ │ ├── launch_plan.go │ │ ├── metrics_interface.go │ │ ├── named_entity.go │ │ ├── node_execution.go │ │ ├── project.go │ │ ├── resource.go │ │ ├── signal_interface.go │ │ ├── task.go │ │ ├── task_execution.go │ │ ├── version_interface.go │ │ └── workflow.go ├── repositories │ ├── config │ │ ├── migration_models.go │ │ ├── migrations.go │ │ ├── migrations_test.go │ │ └── seed_data.go │ ├── database.go │ ├── database_integration_test.go │ ├── database_test.go │ ├── errors │ │ ├── error_transformer.go │ │ ├── errors.go │ │ ├── postgres.go │ │ ├── postgres_test.go │ │ └── test_error_transformer.go │ ├── gen │ │ └── models │ │ │ └── node_execution.pb.go │ ├── gorm_repo.go │ ├── gormimpl │ │ ├── common.go │ │ ├── description_entity_repo.go │ │ ├── description_entity_repo_test.go │ │ ├── execution_event_repo.go │ │ ├── execution_event_repo_test.go │ │ ├── execution_repo.go │ │ ├── execution_repo_test.go │ │ ├── launch_plan_repo.go │ │ ├── launch_plan_repo_test.go │ │ ├── metrics.go │ │ ├── named_entity_repo.go │ │ ├── named_entity_repo_test.go │ │ ├── node_execution_event_repo.go │ │ ├── node_execution_event_repo_test.go │ │ ├── node_execution_repo.go │ │ ├── node_execution_repo_test.go │ │ ├── project_repo.go │ │ ├── project_repo_test.go │ │ ├── resource_repo.go │ │ ├── resource_repo_test.go │ │ ├── signal_repo.go │ │ ├── signal_repo_test.go │ │ ├── task_execution_repo.go │ │ ├── task_execution_repo_test.go │ │ ├── task_repo.go │ │ ├── task_repo_test.go │ │ ├── utils_for_test.go │ │ ├── workflow_repo.go │ │ └── workflow_repo_test.go │ ├── interfaces │ │ ├── common.go │ │ ├── description_entity.go │ │ ├── execution_event_repo.go │ │ ├── execution_repo.go │ │ ├── launch_plan_repo.go │ │ ├── named_entity_repo.go │ │ ├── node_execution_event_repo.go │ │ ├── node_execution_repo.go │ │ ├── project_repo.go │ │ ├── repository.go │ │ ├── resource_repo.go │ │ ├── signal_repo.go │ │ ├── task_execution_repo.go │ │ ├── task_repo.go │ │ └── workflow_repo.go │ ├── mocks │ │ ├── description_entity_repo.go │ │ ├── execution_event_repo_interface.go │ │ ├── execution_repo.go │ │ ├── launch_plan_repo.go │ │ ├── named_entity_metadata_repo.go │ │ ├── node_execution_event_repo_interface.go │ │ ├── node_execution_repo.go │ │ ├── project_repo.go │ │ ├── repository.go │ │ ├── resource.go │ │ ├── signal_repo_interface.go │ │ ├── task_execution_repo.go │ │ ├── task_repo.go │ │ └── workflow_repo.go │ ├── models │ │ ├── base_model.go │ │ ├── description_entity.go │ │ ├── execution.go │ │ ├── execution_event.go │ │ ├── launch_plan.go │ │ ├── named_entity.go │ │ ├── node_execution.go │ │ ├── node_execution_event.go │ │ ├── project.go │ │ ├── protos │ │ │ └── node_execution.proto │ │ ├── resource.go │ │ ├── signal.go │ │ ├── task.go │ │ ├── task_execution.go │ │ └── workflow.go │ └── transformers │ │ ├── constants.go │ │ ├── description_entity.go │ │ ├── description_entity_test.go │ │ ├── execution.go │ │ ├── execution_event.go │ │ ├── execution_event_test.go │ │ ├── execution_test.go │ │ ├── init_test.go │ │ ├── launch_plan.go │ │ ├── launch_plan_test.go │ │ ├── named_entity.go │ │ ├── named_entity_test.go │ │ ├── node_execution.go │ │ ├── node_execution_event.go │ │ ├── node_execution_event_test.go │ │ ├── node_execution_test.go │ │ ├── project.go │ │ ├── project_test.go │ │ ├── resource.go │ │ ├── resource_test.go │ │ ├── signal.go │ │ ├── signal_test.go │ │ ├── task.go │ │ ├── task_execution.go │ │ ├── task_execution_test.go │ │ ├── task_test.go │ │ ├── workflow.go │ │ └── workflow_test.go ├── rpc │ ├── adminservice │ │ ├── attributes.go │ │ ├── base.go │ │ ├── base_test.go │ │ ├── description_entity.go │ │ ├── execution.go │ │ ├── launch_plan.go │ │ ├── metrics.go │ │ ├── named_entity.go │ │ ├── node_execution.go │ │ ├── project.go │ │ ├── task.go │ │ ├── task_execution.go │ │ ├── tests │ │ │ ├── execution_test.go │ │ │ ├── launch_plan_test.go │ │ │ ├── node_execution_test.go │ │ │ ├── project_domain_test.go │ │ │ ├── project_test.go │ │ │ ├── task_execution_test.go │ │ │ ├── task_test.go │ │ │ ├── util.go │ │ │ └── workflow_test.go │ │ ├── util │ │ │ ├── metrics.go │ │ │ ├── transformers.go │ │ │ └── transformers_test.go │ │ ├── version.go │ │ └── workflow.go │ ├── signal_service.go │ └── signal_service_test.go ├── runtime │ ├── application_config_provider.go │ ├── cluster_config_provider.go │ ├── cluster_pool_assignment_provider.go │ ├── cluster_resource_provider.go │ ├── cluster_resource_provider_test.go │ ├── config_provider_test.go │ ├── configuration_provider.go │ ├── execution_queue_provider.go │ ├── interfaces │ │ ├── application_configuration.go │ │ ├── cluster_configuration.go │ │ ├── cluster_pools.go │ │ ├── cluster_resource_configuration.go │ │ ├── configuration.go │ │ ├── inlineeventdatapolicy_enumer.go │ │ ├── mocks │ │ │ ├── cluster_pool_assignment_configuration.go │ │ │ └── quality_of_service_configuration.go │ │ ├── namespace_configuration.go │ │ ├── quality_of_service_configuration.go │ │ ├── queue_configuration.go │ │ ├── registration_validation_provider.go │ │ ├── task_resource_configuration.go │ │ └── whitelist.go │ ├── mocks │ │ ├── cluster_configuration.go │ │ ├── mock_application_provider.go │ │ ├── mock_cluster_resource_provider.go │ │ ├── mock_configuration_provider.go │ │ ├── mock_execution_queue_provider.go │ │ ├── mock_registration_validation_provider.go │ │ ├── mock_task_resource_provider.go │ │ ├── mock_whitelist_provider.go │ │ └── namespace_mapping_configuration.go │ ├── namespace_config_provider.go │ ├── quality_of_service_provider.go │ ├── registration_validation_provider.go │ ├── task_resource_provider.go │ ├── testdata │ │ ├── cluster_resource_config.yaml │ │ ├── clusters_config.yaml │ │ ├── config.yaml │ │ ├── event.yaml │ │ ├── postgres_config.yaml │ │ └── sqlite_config.yaml │ └── whitelist_provider.go ├── server │ ├── cert_utils.go │ ├── initialize.go │ └── service.go └── workflowengine │ ├── impl │ ├── compiler.go │ ├── interface_provider.go │ ├── interface_provider_test.go │ ├── k8s_executor.go │ ├── k8s_executor_test.go │ ├── prepare_execution.go │ ├── prepare_execution_test.go │ └── workflow_builder.go │ ├── interfaces │ ├── builder.go │ ├── compiler.go │ └── executor.go │ └── mocks │ ├── flyte_workflow_builder.go │ ├── mock_compiler.go │ └── workflow_executor.go ├── plugins ├── registry.go └── registry_test.go ├── pull_request_template.md ├── sampleresourcetemplates ├── docker.yaml ├── gsa.yaml ├── imagepullsecrets.yaml └── namespace.yaml ├── scheduler.Dockerfile ├── scheduler ├── core │ ├── doc.go │ ├── gocron_job.go │ ├── gocron_scheduler.go │ ├── gocron_scheduler_test.go │ ├── scheduler.go │ ├── snapshot_runner.go │ └── updater.go ├── dbapi │ ├── doc.go │ ├── event_scheduler_impl.go │ └── event_scheduler_impl_test.go ├── doc.go ├── executor │ ├── doc.go │ ├── executor.go │ ├── executor_impl.go │ ├── executor_impl_test.go │ └── mocks │ │ └── executor.go ├── identifier │ ├── doc.go │ └── identifier.go ├── repositories │ ├── doc.go │ ├── gormimpl │ │ ├── metrics.go │ │ ├── schedulable_entity_repo.go │ │ └── schedule_entities_snapshot_repo.go │ ├── interfaces │ │ ├── schedulable_entity_repo.go │ │ ├── schedule_entities_snapshot_repo.go │ │ └── scheduler.go │ ├── mocks │ │ ├── schedulable_entity_repo_interface.go │ │ └── schedule_entities_snap_shot_repo_interface.go │ └── models │ │ ├── schedulable_entity.go │ │ └── schedule_entities_snapshot.go ├── schedule_executor.go ├── schedule_executor_test.go ├── snapshoter │ ├── doc.go │ ├── persistence.go │ ├── reader.go │ ├── snapshot.go │ ├── snapshot_v1.go │ ├── snapshoter.go │ ├── snapshoter_test.go │ ├── versioned_snapshot.go │ ├── versioned_snapshot_test.go │ └── writer.go └── start.go ├── script ├── gen_proto.sh └── integration │ ├── execute.sh │ ├── k8s │ ├── integration.yaml │ └── main.sh │ └── launch.sh └── tests ├── attributes_test.go ├── bootstrap.go ├── execution_test.go ├── helpers.go ├── launch_plan_test.go ├── named_entity_test.go ├── node_execution_test.go ├── project_test.go ├── scheduler_test.go ├── shared.go ├── task_execution_test.go ├── task_test.go └── workflow_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | vendor/* 2 | -------------------------------------------------------------------------------- /.github/workflows/upgrade_automation.yml: -------------------------------------------------------------------------------- 1 | name: Upgrade Automation 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | component: 6 | description: "Name of Flyte component" 7 | required: true 8 | default: "boilerplate" 9 | type: choice 10 | options: 11 | - boilerplate 12 | - flyteidl 13 | jobs: 14 | trigger-upgrade: 15 | name: ${{ github.event.inputs.component }} Upgrade 16 | uses: flyteorg/flytetools/.github/workflows/flyte_automation.yml@master 17 | with: 18 | component: ${{ github.event.inputs.component }} 19 | secrets: 20 | FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }} 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .idea/ 3 | .DS_Store 4 | .vscode/ 5 | .vendor-new/ 6 | .swp 7 | 8 | vendor/ 9 | node_modules/ 10 | 11 | .virtualgo 12 | boilerplate/lyft/end2end/tmp 13 | dist 14 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | skip-files: 3 | # because we're skipping TLS verification - for now 4 | - cmd/entrypoints/serve.go 5 | - pkg/async/messages/sqs.go 6 | 7 | linters: 8 | disable-all: true 9 | enable: 10 | - deadcode 11 | - errcheck 12 | - gas 13 | - goconst 14 | - goimports 15 | - golint 16 | - gosimple 17 | - govet 18 | - ineffassign 19 | - misspell 20 | - nakedret 21 | - staticcheck 22 | - structcheck 23 | - typecheck 24 | - unconvert 25 | - unparam 26 | - unused 27 | - varcheck 28 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: flyteadmin 2 | before: 3 | hooks: 4 | - go mod download 5 | builds: 6 | - id: flyteadmin 7 | env: 8 | - CGO_ENABLED=0 9 | main: ./cmd/main.go 10 | ldflags: 11 | - -s -w -X github.com/flyteorg/flytestdlib/version.Version={{.Version}} -X github.com/flyteorg/flytestdlib/version.Build={{.ShortCommit}} -X github.com/flyteorg/flytestdlib/version.BuildTime={{.Date}} 12 | 13 | binary: flyteadmin 14 | goos: 15 | - linux 16 | - windows 17 | - darwin 18 | - id: flytescheduler 19 | env: 20 | - CGO_ENABLED=0 21 | main: ./cmd/scheduler/main.go 22 | ldflags: 23 | - -s -w -X github.com/flyteorg/flytestdlib/version.Version={{.Version}} -X github.com/flyteorg/flytestdlib/version.Build={{.ShortCommit}} -X github.com/flyteorg/flytestdlib/version.BuildTime={{.Date}} 24 | 25 | binary: flytescheduler 26 | goos: 27 | - linux 28 | - windows 29 | - darwin 30 | archives: 31 | - id: flyteadmin-archive 32 | name_template: |- 33 | flyteadmin_{{ .Tag }}_{{ .Os }}_ 34 | {{- if eq .Arch "amd64" }}x86_64 35 | {{- else if eq .Arch "386" }}i386 36 | {{- else }}{{ .Arch }}{{ end }} 37 | builds: 38 | - flyteadmin 39 | - flytescheduler 40 | format_overrides: 41 | - goos: windows 42 | format: zip 43 | 44 | checksum: 45 | name_template: 'checksums.txt' 46 | changelog: 47 | sort: asc 48 | filters: 49 | exclude: 50 | - '^docs:' 51 | - '^test:' 52 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in 2 | # the repo. Unless a later match takes precedence. 3 | * @katrogan @wild-endeavor @EngHabu @kumare3 @pmahindrakar-oss @hamersaw @eapolinario 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/). 2 | All contributors and participants agree to abide by its terms. 3 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | flyteadmin 2 | Copyright 2019 Lyft Inc. 3 | 4 | This product includes software developed at Lyft Inc. 5 | -------------------------------------------------------------------------------- /auth/authzserver/doc.go: -------------------------------------------------------------------------------- 1 | // OAuthServer implementation that serve oauth2 authorize and client_credentials flows. 2 | package authzserver 3 | -------------------------------------------------------------------------------- /auth/authzserver/encryptor.go: -------------------------------------------------------------------------------- 1 | package authzserver 2 | 3 | type Encryptor interface { 4 | Encrypt(raw string) (cypher string, err error) 5 | Decrypt(cypher string) (raw string, err error) 6 | } 7 | -------------------------------------------------------------------------------- /auth/authzserver/initialize_test.go: -------------------------------------------------------------------------------- 1 | package authzserver 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ory/fosite/storage" 7 | 8 | "github.com/ory/fosite/compose" 9 | 10 | "github.com/stretchr/testify/mock" 11 | 12 | "github.com/flyteorg/flyteadmin/auth" 13 | "github.com/flyteorg/flyteadmin/auth/interfaces/mocks" 14 | ) 15 | 16 | func TestRegisterHandlers(t *testing.T) { 17 | t.Run("No OAuth2 Provider, no registration required", func(t *testing.T) { 18 | registerer := &mocks.HandlerRegisterer{} 19 | RegisterHandlers(registerer, auth.Context{}) 20 | }) 21 | 22 | t.Run("Register 4 endpoints", func(t *testing.T) { 23 | registerer := &mocks.HandlerRegisterer{} 24 | registerer.On("HandleFunc", "/oauth2/authorize", mock.Anything) 25 | registerer.On("HandleFunc", "/oauth2/authorize_callback", mock.Anything) 26 | registerer.On("HandleFunc", "/oauth2/jwks", mock.Anything) 27 | registerer.On("HandleFunc", "/oauth2/token", mock.Anything) 28 | authCtx := &mocks.AuthenticationContext{} 29 | oauth2Provider := &mocks.OAuth2Provider{} 30 | authCtx.OnOAuth2Provider().Return(oauth2Provider) 31 | RegisterHandlers(registerer, authCtx) 32 | }) 33 | } 34 | 35 | func Test_composeOAuth2Provider(t *testing.T) { 36 | composeOAuth2Provider(nil, &compose.Config{}, &storage.MemoryStore{}, nil) 37 | } 38 | -------------------------------------------------------------------------------- /auth/config/testdata/config.yaml: -------------------------------------------------------------------------------- 1 | appAuth: 2 | selfAuthServer: 3 | accessTokenLifespan: 5h 4 | refreshTokenLifespan: 6h 5 | staticClients: 6 | - my-client: 7 | id: my-client 8 | client_secret: JDJhJDA2JHB4czFBa0c4MUt2cmhwbWwxUWlMU09RYVRrOWVlUHJVLzdZYWI5eTA3aDN4MFRnbGJhb1Q2 9 | -------------------------------------------------------------------------------- /auth/config/testdata/secret.yaml: -------------------------------------------------------------------------------- 1 | myval: "JDJhJDA2JHB4czFBa0c4MUt2cmhwbWwxUWlMU09RYVRrOWVlUHJVLzdZYWI5eTA3aDN4MFRnbGJhb1Q2" 2 | -------------------------------------------------------------------------------- /auth/config/third_party_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // This struct encapsulates config options for bootstrapping various Flyte applications with config values 4 | // For example, FlyteClientConfig contains application-specific values to initialize the config required by flyte client 5 | type ThirdPartyConfigOptions struct { 6 | FlyteClientConfig FlyteClientConfig `json:"flyteClient"` 7 | } 8 | 9 | type FlyteClientConfig struct { 10 | ClientID string `json:"clientId" pflag:",public identifier for the app which handles authorization for a Flyte deployment"` 11 | RedirectURI string `json:"redirectUri" pflag:",This is the callback uri registered with the app which handles authorization for a Flyte deployment"` 12 | Scopes []string `json:"scopes" pflag:",Recommended scopes for the client to request."` 13 | Audience string `json:"audience" pflag:",Audience to use when initiating OAuth2 authorization requests."` 14 | } 15 | 16 | func (o ThirdPartyConfigOptions) IsEmpty() bool { 17 | return len(o.FlyteClientConfig.ClientID) == 0 && len(o.FlyteClientConfig.RedirectURI) == 0 && len(o.FlyteClientConfig.Scopes) == 0 18 | } 19 | -------------------------------------------------------------------------------- /auth/constants.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import "github.com/flyteorg/flytestdlib/contextutils" 4 | 5 | const ( 6 | // OAuth2 Parameters 7 | CsrfFormKey = "state" 8 | AuthorizationResponseCodeType = "code" 9 | DefaultAuthorizationHeader = "authorization" 10 | BearerScheme = "Bearer" 11 | IDTokenScheme = "IDToken" 12 | // Add the -bin suffix so that the header value is automatically base64 encoded 13 | UserInfoMDKey = "UserInfo-bin" 14 | 15 | // https://tools.ietf.org/html/rfc8414 16 | // This should be defined without a leading slash. If there is one, the url library's ResolveReference will make it a root path 17 | OAuth2MetadataEndpoint = ".well-known/oauth-authorization-server" 18 | 19 | // https://openid.net/specs/openid-connect-discovery-1_0.html 20 | // This should be defined without a leading slash. If there is one, the url library's ResolveReference will make it a root path 21 | OIdCMetadataEndpoint = ".well-known/openid-configuration" 22 | 23 | ContextKeyIdentityContext = contextutils.Key("identity_context") 24 | ScopeAll = "all" 25 | ) 26 | -------------------------------------------------------------------------------- /auth/identity_context_test.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | ) 10 | 11 | func TestGetClaims(t *testing.T) { 12 | noClaims := map[string]interface{}(nil) 13 | noClaimsCtx, err := NewIdentityContext("", "", "", time.Now(), nil, nil, nil) 14 | assert.NoError(t, err) 15 | assert.EqualValues(t, noClaims, noClaimsCtx.Claims()) 16 | 17 | claims := map[string]interface{}{ 18 | "groups": []string{"g1", "g2"}, 19 | "something": "else", 20 | } 21 | withClaimsCtx, err := NewIdentityContext("", "", "", time.Now(), nil, nil, claims) 22 | assert.NoError(t, err) 23 | assert.EqualValues(t, claims, withClaimsCtx.Claims()) 24 | 25 | assert.NotEmpty(t, withClaimsCtx.UserInfo().AdditionalClaims) 26 | } 27 | 28 | func TestWithExecutionUserIdentifier(t *testing.T) { 29 | idctx, err := NewIdentityContext("", "", "", time.Now(), sets.String{}, nil, nil) 30 | assert.NoError(t, err) 31 | newIDCtx := idctx.WithExecutionUserIdentifier("byhsu") 32 | // make sure the original one is intact 33 | assert.Equal(t, "", idctx.ExecutionIdentity()) 34 | assert.Equal(t, "byhsu", newIDCtx.ExecutionIdentity()) 35 | } 36 | -------------------------------------------------------------------------------- /auth/interceptor.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flytestdlib/logger" 7 | "google.golang.org/grpc" 8 | "google.golang.org/grpc/codes" 9 | "google.golang.org/grpc/status" 10 | ) 11 | 12 | func BlanketAuthorization(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) ( 13 | resp interface{}, err error) { 14 | 15 | identityContext := IdentityContextFromContext(ctx) 16 | if identityContext.IsEmpty() { 17 | return handler(ctx, req) 18 | } 19 | 20 | if !identityContext.Scopes().Has(ScopeAll) { 21 | logger.Debugf(ctx, "authenticated user doesn't have required scope") 22 | return nil, status.Errorf(codes.Unauthenticated, "authenticated user doesn't have required scope") 23 | } 24 | 25 | return handler(ctx, req) 26 | } 27 | 28 | // ExecutionUserIdentifierInterceptor injects identityContext.UserID() to identityContext.executionIdentity 29 | func ExecutionUserIdentifierInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) ( 30 | resp interface{}, err error) { 31 | identityContext := IdentityContextFromContext(ctx) 32 | identityContext = identityContext.WithExecutionUserIdentifier(identityContext.UserID()) 33 | ctx = identityContext.WithContext(ctx) 34 | return handler(ctx, req) 35 | } 36 | -------------------------------------------------------------------------------- /auth/interfaces/cookie.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" 8 | 9 | "golang.org/x/oauth2" 10 | ) 11 | 12 | //go:generate mockery -name=CookieHandler -output=mocks/ -case=underscore 13 | 14 | type CookieHandler interface { 15 | SetTokenCookies(ctx context.Context, writer http.ResponseWriter, token *oauth2.Token) error 16 | RetrieveTokenValues(ctx context.Context, request *http.Request) (idToken, accessToken, refreshToken string, err error) 17 | 18 | SetUserInfoCookie(ctx context.Context, writer http.ResponseWriter, userInfo *service.UserInfoResponse) error 19 | RetrieveUserInfo(ctx context.Context, request *http.Request) (*service.UserInfoResponse, error) 20 | 21 | // SetAuthCodeCookie stores, in a cookie, the /authorize request url initiated by an app before executing OIdC protocol. 22 | // This enables the service to recover it after the user completes the login process in an external OIdC provider. 23 | SetAuthCodeCookie(ctx context.Context, writer http.ResponseWriter, authRequestURL string) error 24 | 25 | // RetrieveAuthCodeRequest retrieves the /authorize request url from stored cookie to complete the OAuth2 app auth 26 | // flow. 27 | RetrieveAuthCodeRequest(ctx context.Context, request *http.Request) (authRequestURL string, err error) 28 | DeleteCookies(ctx context.Context, writer http.ResponseWriter) 29 | } 30 | -------------------------------------------------------------------------------- /auth/interfaces/mocks/handler_registerer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | http "net/http" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | ) 10 | 11 | // HandlerRegisterer is an autogenerated mock type for the HandlerRegisterer type 12 | type HandlerRegisterer struct { 13 | mock.Mock 14 | } 15 | 16 | // HandleFunc provides a mock function with given fields: pattern, handler 17 | func (_m *HandlerRegisterer) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) { 18 | _m.Called(pattern, handler) 19 | } 20 | -------------------------------------------------------------------------------- /auth/user_info_provider.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" 7 | ) 8 | 9 | type UserInfoProvider struct { 10 | } 11 | 12 | // UserInfo returns user_info claims about the currently logged in user. 13 | // See the OpenID Connect spec at https://openid.net/specs/openid-connect-core-1_0.html#UserInfoResponse for more information. 14 | func (s UserInfoProvider) UserInfo(ctx context.Context, _ *service.UserInfoRequest) (*service.UserInfoResponse, error) { 15 | identityContext := IdentityContextFromContext(ctx) 16 | return identityContext.UserInfo(), nil 17 | } 18 | 19 | func NewUserInfoProvider() UserInfoProvider { 20 | return UserInfoProvider{} 21 | } 22 | -------------------------------------------------------------------------------- /boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/). 2 | All contributors and participants agree to abide by its terms. 3 | -------------------------------------------------------------------------------- /boilerplate/flyte/code_of_conduct/README.rst: -------------------------------------------------------------------------------- 1 | CODE OF CONDUCT 2 | ~~~~~~~~~~~~~~~ 3 | -------------------------------------------------------------------------------- /boilerplate/flyte/code_of_conduct/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | cp ${DIR}/CODE_OF_CONDUCT.md ${DIR}/../../../CODE_OF_CONDUCT.md 13 | -------------------------------------------------------------------------------- /boilerplate/flyte/docker_build/Makefile: -------------------------------------------------------------------------------- 1 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 2 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 3 | # 4 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 5 | 6 | .PHONY: docker_build 7 | docker_build: 8 | IMAGE_NAME=$$REPOSITORY ./boilerplate/flyte/docker_build/docker_build.sh 9 | 10 | .PHONY: dockerhub_push 11 | dockerhub_push: 12 | IMAGE_NAME=flyteorg/$$REPOSITORY REGISTRY=docker.io ./boilerplate/flyte/docker_build/docker_build.sh 13 | -------------------------------------------------------------------------------- /boilerplate/flyte/docker_build/Readme.rst: -------------------------------------------------------------------------------- 1 | Docker Build and Push 2 | ~~~~~~~~~~~~~~~~~~~~~ 3 | 4 | Provides a ``make docker_build`` target that builds your image locally. 5 | 6 | Provides a ``make dockerhub_push`` target that pushes your final image to Dockerhub. 7 | 8 | The Dockerhub image will tagged ``:`` 9 | 10 | If git head has a git tag, the Dockerhub image will also be tagged ``:``. 11 | 12 | **To Enable:** 13 | 14 | Add ``flyteorg/docker_build`` to your ``boilerplate/update.cfg`` file. 15 | 16 | Add ``include boilerplate/flyte/docker_build/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable 17 | 18 | :: 19 | 20 | REPOSITORY= 21 | include boilerplate/flyte/docker_build/Makefile 22 | 23 | (this ensures the extra Make targets get included in your main Makefile) 24 | -------------------------------------------------------------------------------- /boilerplate/flyte/end2end/Makefile: -------------------------------------------------------------------------------- 1 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 2 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 3 | # 4 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 5 | 6 | .PHONY: end2end_execute 7 | end2end_execute: export FLYTESNACKS_PRIORITIES ?= P0 8 | end2end_execute: export FLYTESNACKS_VERSION ?= $(shell curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name) 9 | end2end_execute: 10 | ./boilerplate/flyte/end2end/end2end.sh ./boilerplate/flyte/end2end/functional-test-config.yaml --return_non_zero_on_failure 11 | 12 | .PHONY: k8s_integration_execute 13 | k8s_integration_execute: 14 | echo "pass" 15 | -------------------------------------------------------------------------------- /boilerplate/flyte/end2end/end2end.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | set -eu 8 | 9 | CONFIG_FILE=$1; shift 10 | EXTRA_FLAGS=( "$@" ) 11 | 12 | python ./boilerplate/flyte/end2end/run-tests.py $FLYTESNACKS_VERSION $FLYTESNACKS_PRIORITIES $CONFIG_FILE ${EXTRA_FLAGS[@]} 13 | -------------------------------------------------------------------------------- /boilerplate/flyte/end2end/functional-test-config.yaml: -------------------------------------------------------------------------------- 1 | admin: 2 | # For GRPC endpoints you might want to use dns:///flyte.myexample.com 3 | endpoint: dns:///localhost:30080 4 | authType: Pkce 5 | insecure: true 6 | -------------------------------------------------------------------------------- /boilerplate/flyte/flyte_golang_compile/Readme.rst: -------------------------------------------------------------------------------- 1 | Flyte Golang Compile 2 | ~~~~~~~~~~~~~~~~~~~~ 3 | 4 | Common compile script for Flyte golang services. 5 | 6 | **To Enable:** 7 | 8 | Add ``flyteorg/flyte_golang_compile`` to your ``boilerplate/update.cfg`` file. 9 | 10 | Add the following to your Makefile 11 | 12 | :: 13 | 14 | .PHONY: compile_linux 15 | compile_linux: 16 | PACKAGES={{ *your packages }} OUTPUT={{ /path/to/output }} ./boilerplate/flyte/flyte_golang_compile.sh 17 | -------------------------------------------------------------------------------- /boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template: -------------------------------------------------------------------------------- 1 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 2 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 3 | # 4 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 5 | 6 | if [ -z "$PACKAGES" ]; then 7 | echo "PACKAGES environment VAR not set" 8 | exit 1 9 | fi 10 | 11 | if [ -z "$OUTPUT" ]; then 12 | echo "OUTPUT environment VAR not set" 13 | exit 1 14 | fi 15 | 16 | # get the GIT_SHA and RELEASE_SEMVER 17 | 18 | GIT_SHA=$(git rev-parse HEAD) 19 | RELEASE_SEMVER=$(git describe --tags --exact-match $GIT_SHA 2>/dev/null) 20 | 21 | CURRENT_PKG=github.com/flyteorg/{{ REPOSITORY }} 22 | VERSION_PKG="${CURRENT_PKG}/vendor/github.com/flyteorg/flytestdlib" 23 | 24 | LDFLAGS="-X ${VERSION_PKG}/version.Build=${GIT_SHA} -X ${VERSION_PKG}/version.Version=${RELEASE_SEMVER}" 25 | 26 | GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "$LDFLAGS" -o "$OUTPUT" "$PACKAGES" 27 | -------------------------------------------------------------------------------- /boilerplate/flyte/flyte_golang_compile/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | echo " - generating ${DIR}/flyte_golang_compile.sh" 13 | sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/flyte_golang_compile.Template > ${DIR}/flyte_golang_compile.sh 14 | -------------------------------------------------------------------------------- /boilerplate/flyte/github_workflows/Readme.rst: -------------------------------------------------------------------------------- 1 | Golang Github Actions 2 | ~~~~~~~~~~~~~~~~~ 3 | 4 | Provides a two github actions workflows. 5 | 6 | **To Enable:** 7 | 8 | Add ``flyteorg/github_workflows`` to your ``boilerplate/update.cfg`` file. 9 | 10 | Add a github secret ``package_name`` with the name to use for publishing (e.g. ``flytepropeller``). Typicaly, this will be the same name as the repository. 11 | 12 | *Note*: If you are working on a fork, include that prefix in your package name (``myfork/flytepropeller``). 13 | 14 | The actions will push to 2 repos: 15 | 16 | 1. ``docker.pkg.github.com/flyteorg//`` 17 | 2. ``docker.pkg.github.com/flyteorg//-stages`` : this repo is used to cache build stages to speed up iterative builds after. 18 | 19 | There are two workflows that get deployed: 20 | 21 | 1. A workflow that runs on Pull Requests to build and push images to github registy tagged with the commit sha. 22 | 2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest" 23 | -------------------------------------------------------------------------------- /boilerplate/flyte/github_workflows/master.yml: -------------------------------------------------------------------------------- 1 | name: Master 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@master 13 | with: 14 | fetch-depth: '0' 15 | - name: Bump version and push tag 16 | id: bump-version 17 | uses: anothrNick/github-tag-action@1.36.0 18 | env: 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | WITH_V: true 21 | DEFAULT_BUMP: patch 22 | - name: Push Docker Image to Github Registry 23 | uses: whoan/docker-build-with-cache-action@v5 24 | with: 25 | username: "${{ github.actor }}" 26 | password: "${{ secrets.GITHUB_TOKEN }}" 27 | image_name: ${{ secrets.package_name }} 28 | image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }} 29 | push_git_tag: true 30 | registry: docker.pkg.github.com 31 | build_extra_args: "--compress=true" 32 | -------------------------------------------------------------------------------- /boilerplate/flyte/github_workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | name: Pull Request 2 | 3 | on: 4 | pull_request 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v1 11 | - name: Push Docker Image to Github Registry 12 | uses: whoan/docker-build-with-cache-action@v5 13 | with: 14 | username: "${{ github.actor }}" 15 | password: "${{ secrets.GITHUB_TOKEN }}" 16 | image_name: ${{ secrets.package_name }} 17 | image_tag: ${{ github.sha }} 18 | push_git_tag: true 19 | registry: docker.pkg.github.com 20 | -------------------------------------------------------------------------------- /boilerplate/flyte/github_workflows/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | mkdir -p ${DIR}/../../../.github/workflows 13 | 14 | echo " - generating github action workflows in root directory." 15 | sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/master.yml > ${DIR}/../../../.github/workflows/master.yml 16 | sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/pull_request.yml > ${DIR}/../../../.github/workflows/pull_request.yml 17 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_dockerfile/Dockerfile.GoTemplate: -------------------------------------------------------------------------------- 1 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 2 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 3 | # 4 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 5 | 6 | FROM golang:1.13.3-alpine3.10 as builder 7 | RUN apk add git openssh-client make curl 8 | 9 | # COPY only the go mod files for efficient caching 10 | COPY go.mod go.sum /go/src/github.com/flyteorg/{{REPOSITORY}}/ 11 | WORKDIR /go/src/github.com/flyteorg/{{REPOSITORY}} 12 | 13 | # Pull dependencies 14 | RUN go mod download 15 | 16 | # COPY the rest of the source code 17 | COPY . /go/src/github.com/flyteorg/{{REPOSITORY}}/ 18 | 19 | # This 'linux_compile' target should compile binaries to the /artifacts directory 20 | # The main entrypoint should be compiled to /artifacts/{{REPOSITORY}} 21 | RUN make linux_compile 22 | 23 | # update the PATH to include the /artifacts directory 24 | ENV PATH="/artifacts:${PATH}" 25 | 26 | # This will eventually move to centurylink/ca-certs:latest for minimum possible image size 27 | FROM alpine:3.13.7 28 | COPY --from=builder /artifacts /bin 29 | 30 | RUN apk --update add ca-certificates 31 | 32 | CMD ["{{REPOSITORY}}"] 33 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_dockerfile/Readme.rst: -------------------------------------------------------------------------------- 1 | Golang Dockerfile 2 | ~~~~~~~~~~~~~~~~~ 3 | 4 | Provides a Dockerfile that produces a small image. 5 | 6 | **To Enable:** 7 | 8 | Add ``flyteorg/golang_dockerfile`` to your ``boilerplate/update.cfg`` file. 9 | 10 | Create and configure a ``make linux_compile`` target that compiles your go binaries to the ``/artifacts`` directory :: 11 | 12 | .PHONY: linux_compile 13 | linux_compile: 14 | RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts {{ packages }} 15 | 16 | All binaries compiled to ``/artifacts`` will be available at ``/bin`` in your final image. 17 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_dockerfile/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | echo " - generating Dockerfile in root directory." 13 | sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/Dockerfile.GoTemplate > ${DIR}/../../../Dockerfile 14 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_support_tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | package tools 5 | 6 | import ( 7 | _ "github.com/EngHabu/mockery/cmd/mockery" 8 | _ "github.com/alvaroloes/enumer" 9 | _ "github.com/flyteorg/flytestdlib/cli/pflags" 10 | _ "github.com/golangci/golangci-lint/cmd/golangci-lint" 11 | _ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" 12 | ) 13 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_test_targets/Readme.rst: -------------------------------------------------------------------------------- 1 | Golang Test Targets 2 | ~~~~~~~~~~~~~~~~~~~ 3 | 4 | Provides an ``install`` make target that uses ``go mod`` to install golang dependencies. 5 | 6 | Provides a ``lint`` make target that uses golangci to lint your code. 7 | 8 | Provides a ``test_unit`` target for unit tests. 9 | 10 | Provides a ``test_unit_cover`` target for analysing coverage of unit tests, which will output the coverage of each function and total statement coverage. 11 | 12 | Provides a ``test_unit_visual`` target for visualizing coverage of unit tests through an interactive html code heat map. 13 | 14 | Provides a ``test_benchmark`` target for benchmark tests. 15 | 16 | **To Enable:** 17 | 18 | Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file. 19 | 20 | Make sure you're using ``go mod`` for dependency management. 21 | 22 | Provide a ``.golangci`` configuration (the lint target requires it). 23 | 24 | Add ``include boilerplate/flyte/golang_test_targets/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable 25 | 26 | :: 27 | 28 | REPOSITORY= 29 | include boilerplate/flyte/golang_test_targets/Makefile 30 | 31 | (this ensures the extra make targets get included in your main Makefile) 32 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_test_targets/go-gen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | echo "Running go generate" 6 | go generate ./... 7 | 8 | # This section is used by GitHub workflow to ensure that the generation step was run 9 | if [ -n "$DELTA_CHECK" ]; then 10 | DIRTY=$(git status --porcelain) 11 | if [ -n "$DIRTY" ]; then 12 | echo "FAILED: Go code updated without commiting generated code." 13 | echo "Ensure make generate has run and all changes are committed." 14 | DIFF=$(git diff) 15 | echo "diff detected: $DIFF" 16 | DIFF=$(git diff --name-only) 17 | echo "files different: $DIFF" 18 | exit 1 19 | else 20 | echo "SUCCESS: Generated code is up to date." 21 | fi 22 | fi 23 | -------------------------------------------------------------------------------- /boilerplate/flyte/golang_test_targets/goimports: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | goimports -w $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*" -not -path "./boilerplate/*") 9 | -------------------------------------------------------------------------------- /boilerplate/flyte/golangci_file/.golangci.yml: -------------------------------------------------------------------------------- 1 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 2 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 3 | # 4 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 5 | 6 | run: 7 | skip-dirs: 8 | - pkg/client 9 | 10 | linters: 11 | disable-all: true 12 | enable: 13 | - deadcode 14 | - errcheck 15 | - gas 16 | - goconst 17 | - goimports 18 | - golint 19 | - gosimple 20 | - govet 21 | - ineffassign 22 | - misspell 23 | - nakedret 24 | - staticcheck 25 | - structcheck 26 | - typecheck 27 | - unconvert 28 | - unparam 29 | - unused 30 | - varcheck 31 | -------------------------------------------------------------------------------- /boilerplate/flyte/golangci_file/Readme.rst: -------------------------------------------------------------------------------- 1 | GolangCI File 2 | ~~~~~~~~~~~~~ 3 | 4 | Provides a ``.golangci`` file with the linters we've agreed upon. 5 | 6 | **To Enable:** 7 | 8 | Add ``flyteorg/golangci_file`` to your ``boilerplate/update.cfg`` file. 9 | -------------------------------------------------------------------------------- /boilerplate/flyte/golangci_file/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | # Clone the .golangci file 13 | echo " - copying ${DIR}/.golangci to the root directory." 14 | cp ${DIR}/.golangci.yml ${DIR}/../../../.golangci.yml 15 | -------------------------------------------------------------------------------- /boilerplate/flyte/pull_request_template/Readme.rst: -------------------------------------------------------------------------------- 1 | Pull Request Template 2 | ~~~~~~~~~~~~~~~~~~~~~ 3 | 4 | Provides a Pull Request template. 5 | 6 | **To Enable:** 7 | 8 | Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file. 9 | -------------------------------------------------------------------------------- /boilerplate/flyte/pull_request_template/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## _Read then delete this section_ 2 | 3 | _- Make sure to use a concise title for the pull-request._ 4 | 5 | _- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version 6 | will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump)_ 7 | 8 | # TL;DR 9 | _Please replace this text with a description of what this PR accomplishes._ 10 | 11 | ## Type 12 | - [ ] Bug Fix 13 | - [ ] Feature 14 | - [ ] Plugin 15 | 16 | ## Are all requirements met? 17 | 18 | - [ ] Code completed 19 | - [ ] Smoke tested 20 | - [ ] Unit tests added 21 | - [ ] Code documentation added 22 | - [ ] Any pending items have an associated Issue 23 | 24 | ## Complete description 25 | _How did you fix the bug, make the feature etc. Link to any design docs etc_ 26 | 27 | ## Tracking Issue 28 | _Remove the '*fixes*' keyword if there will be multiple PRs to fix the linked issue_ 29 | 30 | fixes https://github.com/flyteorg/flyte/issues/ 31 | 32 | ## Follow-up issue 33 | _NA_ 34 | OR 35 | _https://github.com/flyteorg/flyte/issues/_ 36 | -------------------------------------------------------------------------------- /boilerplate/flyte/pull_request_template/update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | cp ${DIR}/pull_request_template.md ${DIR}/../../../pull_request_template.md 13 | -------------------------------------------------------------------------------- /boilerplate/update.cfg: -------------------------------------------------------------------------------- 1 | flyte/docker_build 2 | flyte/golang_test_targets 3 | flyte/golang_support_tools 4 | flyte/pull_request_template 5 | flyte/end2end 6 | flyte/code_of_conduct 7 | -------------------------------------------------------------------------------- /cmd/entrypoints/k8s_secret.go: -------------------------------------------------------------------------------- 1 | package entrypoints 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/auth" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | var secretsCmd = &cobra.Command{ 9 | Use: "secret", 10 | Aliases: []string{"secrets"}, 11 | } 12 | 13 | func init() { 14 | secretsCmd.AddCommand(auth.GetCreateSecretsCommand()) 15 | secretsCmd.AddCommand(auth.GetInitSecretsCommand()) 16 | RootCmd.AddCommand(secretsCmd) 17 | } 18 | -------------------------------------------------------------------------------- /cmd/entrypoints/migrate.go: -------------------------------------------------------------------------------- 1 | package entrypoints 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/server" 7 | 8 | "github.com/spf13/cobra" 9 | _ "gorm.io/driver/postgres" // Required to import database driver. 10 | ) 11 | 12 | var parentMigrateCmd = &cobra.Command{ 13 | Use: "migrate", 14 | Short: "This command controls migration behavior for the Flyte admin database. Please choose a subcommand.", 15 | } 16 | 17 | // This runs all the migrations 18 | var migrateCmd = &cobra.Command{ 19 | Use: "run", 20 | Short: "This command will run all the migrations for the database", 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | ctx := context.Background() 23 | return server.Migrate(ctx) 24 | }, 25 | } 26 | 27 | // Rollback the latest migration 28 | var rollbackCmd = &cobra.Command{ 29 | Use: "rollback", 30 | Short: "This command will rollback one migration", 31 | RunE: func(cmd *cobra.Command, args []string) error { 32 | ctx := context.Background() 33 | return server.Rollback(ctx) 34 | }, 35 | } 36 | 37 | // This seeds the database with project values 38 | var seedProjectsCmd = &cobra.Command{ 39 | Use: "seed-projects", 40 | Short: "Seed projects in the database.", 41 | RunE: func(cmd *cobra.Command, args []string) error { 42 | ctx := context.Background() 43 | return server.SeedProjects(ctx, args) 44 | }, 45 | } 46 | 47 | func init() { 48 | RootCmd.AddCommand(parentMigrateCmd) 49 | parentMigrateCmd.AddCommand(migrateCmd) 50 | parentMigrateCmd.AddCommand(rollbackCmd) 51 | parentMigrateCmd.AddCommand(seedProjectsCmd) 52 | } 53 | -------------------------------------------------------------------------------- /cmd/entrypoints/serve.go: -------------------------------------------------------------------------------- 1 | package entrypoints 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/plugins" 7 | 8 | "github.com/flyteorg/flytestdlib/profutils" 9 | 10 | _ "net/http/pprof" // Required to serve application. 11 | 12 | "github.com/flyteorg/flyteadmin/pkg/server" 13 | 14 | "github.com/flyteorg/flytestdlib/logger" 15 | "github.com/spf13/cobra" 16 | 17 | runtimeConfig "github.com/flyteorg/flyteadmin/pkg/runtime" 18 | ) 19 | 20 | var pluginRegistryStore = plugins.NewAtomicRegistry(plugins.NewRegistry()) 21 | 22 | // serveCmd represents the serve command 23 | var serveCmd = &cobra.Command{ 24 | Use: "serve", 25 | Short: "Launches the Flyte admin server", 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | ctx := context.Background() 28 | // Serve profiling endpoints. 29 | cfg := runtimeConfig.NewConfigurationProvider() 30 | go func() { 31 | err := profutils.StartProfilingServerWithDefaultHandlers( 32 | ctx, cfg.ApplicationConfiguration().GetTopLevelConfig().GetProfilerPort(), nil) 33 | if err != nil { 34 | logger.Panicf(ctx, "Failed to Start profiling and Metrics server. Error, %v", err) 35 | } 36 | }() 37 | server.SetMetricKeys(cfg.ApplicationConfiguration().GetTopLevelConfig()) 38 | 39 | return server.Serve(ctx, pluginRegistryStore.Load(), nil) 40 | }, 41 | } 42 | 43 | func init() { 44 | // Command information 45 | RootCmd.AddCommand(serveCmd) 46 | RootCmd.AddCommand(secretsCmd) 47 | } 48 | -------------------------------------------------------------------------------- /cmd/entrypoints/serve_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package entrypoints 5 | 6 | // This is an integration test because the token will show up as expired, you will need a live token 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "testing" 12 | 13 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 14 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" 15 | "github.com/stretchr/testify/assert" 16 | "golang.org/x/oauth2" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/credentials" 19 | "google.golang.org/grpc/credentials/oauth" 20 | ) 21 | 22 | func TestClient(t *testing.T) { 23 | ctx := context.Background() 24 | endpoint := "localhost:8088" 25 | 26 | var opts []grpc.DialOption 27 | 28 | creds, err := credentials.NewClientTLSFromFile("/path/to/server.pem", ":8088") 29 | assert.NoError(t, err) 30 | opts = append(opts, grpc.WithTransportCredentials(creds)) 31 | 32 | token := oauth2.Token{ 33 | AccessToken: "j.w.t", 34 | } 35 | tokenRpcCredentials := oauth.NewOauthAccess(&token) 36 | tokenDialOption := grpc.WithPerRPCCredentials(tokenRpcCredentials) 37 | opts = append(opts, tokenDialOption) 38 | 39 | conn, err := grpc.Dial(endpoint, opts...) 40 | if err != nil { 41 | fmt.Printf("Dial error %v\n", err) 42 | } 43 | assert.NoError(t, err) 44 | client := service.NewAdminServiceClient(conn) 45 | resp, err := client.ListProjects(ctx, &admin.ProjectListRequest{}) 46 | if err != nil { 47 | fmt.Printf("Error %v\n", err) 48 | } 49 | assert.NoError(t, err) 50 | fmt.Printf("Response: %v\n", resp) 51 | } 52 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/cmd/entrypoints" 5 | "github.com/flyteorg/flyteadmin/plugins" 6 | "github.com/golang/glog" 7 | ) 8 | 9 | func main() { 10 | glog.V(2).Info("Beginning Flyte Controller") 11 | err := entrypoints.Execute(plugins.NewRegistry()) 12 | if err != nil { 13 | panic(err) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /cmd/scheduler/entrypoints/precheck.go: -------------------------------------------------------------------------------- 1 | package entrypoints 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/flyteorg/flyteidl/clients/go/admin" 8 | "github.com/flyteorg/flytestdlib/logger" 9 | 10 | "github.com/spf13/cobra" 11 | "google.golang.org/grpc/health/grpc_health_v1" 12 | ) 13 | 14 | const ( 15 | healthCheckSuccess = "Health check passed, Flyteadmin is up and running" 16 | healthCheckError = "health check failed with status %v" 17 | ) 18 | 19 | var preCheckRunCmd = &cobra.Command{ 20 | Use: "precheck", 21 | Short: "This command will check pre requirement for scheduler", 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | ctx := context.Background() 24 | 25 | clientSet, err := admin.ClientSetBuilder().WithConfig(admin.GetConfig(ctx)).Build(ctx) 26 | 27 | if err != nil { 28 | logger.Errorf(ctx, "Flyte native scheduler precheck failed due to %v\n", err) 29 | return err 30 | } 31 | 32 | healthCheckResponse, err := clientSet.HealthServiceClient().Check(ctx, 33 | &grpc_health_v1.HealthCheckRequest{Service: "flyteadmin"}) 34 | if err != nil { 35 | return err 36 | } 37 | if healthCheckResponse.GetStatus() != grpc_health_v1.HealthCheckResponse_SERVING { 38 | logger.Errorf(ctx, healthCheckError, healthCheckResponse.GetStatus()) 39 | return fmt.Errorf(healthCheckError, healthCheckResponse.GetStatus()) 40 | } 41 | logger.Infof(ctx, "Health check response is %v", healthCheckResponse) 42 | logger.Infof(ctx, healthCheckSuccess) 43 | return nil 44 | }, 45 | } 46 | 47 | func init() { 48 | RootCmd.AddCommand(preCheckRunCmd) 49 | } 50 | -------------------------------------------------------------------------------- /cmd/scheduler/entrypoints/scheduler.go: -------------------------------------------------------------------------------- 1 | package entrypoints 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/server" 7 | 8 | "github.com/flyteorg/flyteadmin/pkg/runtime" 9 | "github.com/flyteorg/flyteadmin/scheduler" 10 | "github.com/flyteorg/flytestdlib/logger" 11 | "github.com/flyteorg/flytestdlib/profutils" 12 | "github.com/spf13/cobra" 13 | _ "gorm.io/driver/postgres" // Required to import database driver. 14 | ) 15 | 16 | var schedulerRunCmd = &cobra.Command{ 17 | Use: "run", 18 | Short: "This command will start the flyte native scheduler and periodically get new schedules from the db for scheduling", 19 | RunE: func(cmd *cobra.Command, args []string) error { 20 | ctx := context.Background() 21 | schedulerConfiguration := runtime.NewConfigurationProvider().ApplicationConfiguration().GetSchedulerConfig() 22 | // Serve profiling endpoints. 23 | go func() { 24 | err := profutils.StartProfilingServerWithDefaultHandlers( 25 | ctx, schedulerConfiguration.ProfilerPort.Port, nil) 26 | if err != nil { 27 | logger.Panicf(ctx, "Failed to Start profiling and Metrics server. Error, %v", err) 28 | } 29 | }() 30 | 31 | configuration := runtime.NewConfigurationProvider() 32 | applicationConfiguration := configuration.ApplicationConfiguration().GetTopLevelConfig() 33 | server.SetMetricKeys(applicationConfiguration) 34 | 35 | return scheduler.StartScheduler(ctx) 36 | }, 37 | } 38 | 39 | func init() { 40 | RootCmd.AddCommand(schedulerRunCmd) 41 | } 42 | -------------------------------------------------------------------------------- /cmd/scheduler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/cmd/scheduler/entrypoints" 5 | "github.com/golang/glog" 6 | ) 7 | 8 | func main() { 9 | glog.V(2).Info("Beginning Flyte Scheduler") 10 | err := entrypoints.Execute() 11 | if err != nil { 12 | panic(err) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "pkg/executioncluster" 3 | - "pkg/clusterresource" 4 | -------------------------------------------------------------------------------- /pkg/async/cloudevent/implementations/sender_test.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/NYTimes/gizmo/pubsub/pubsubtest" 8 | 9 | cloudevents "github.com/cloudevents/sdk-go/v2" 10 | "github.com/cloudevents/sdk-go/v2/event" 11 | "github.com/cloudevents/sdk-go/v2/protocol" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | type mockCloudEventClient struct{} 16 | 17 | func (s mockCloudEventClient) Request(ctx context.Context, event event.Event) (*event.Event, protocol.Result) { 18 | return nil, nil 19 | } 20 | 21 | func (s mockCloudEventClient) StartReceiver(ctx context.Context, fn interface{}) error { 22 | return nil 23 | } 24 | 25 | func (s mockCloudEventClient) Send(ctx context.Context, event event.Event) protocol.Result { 26 | return nil 27 | } 28 | 29 | func TestPubSubSender(t *testing.T) { 30 | pubSubSender := PubSubSender{&pubsubtest.TestPublisher{}} 31 | cloudEvent := cloudevents.NewEvent() 32 | err := pubSubSender.Send(context.Background(), "test", cloudEvent) 33 | assert.Nil(t, err) 34 | } 35 | 36 | func TestKafkaSender(t *testing.T) { 37 | kafkaSender := KafkaSender{mockCloudEventClient{}} 38 | cloudEvent := cloudevents.NewEvent() 39 | err := kafkaSender.Send(context.Background(), "test", cloudEvent) 40 | assert.Nil(t, err) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/async/cloudevent/interfaces/publisher.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golang/protobuf/proto" 7 | ) 8 | 9 | //go:generate mockery -name=Publisher -output=../mocks -case=underscore 10 | 11 | // Publisher Defines the interface for Publishing execution event to other services (AWS pub/sub, Kafka). 12 | type Publisher interface { 13 | // Publish The notificationType is inferred from the Notification object in the Execution Spec. 14 | Publish(ctx context.Context, notificationType string, msg proto.Message) error 15 | } 16 | -------------------------------------------------------------------------------- /pkg/async/cloudevent/interfaces/sender.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | cloudevents "github.com/cloudevents/sdk-go/v2" 7 | ) 8 | 9 | //go:generate mockery -name=Sender -output=../mocks -case=underscore 10 | 11 | // Sender Defines the interface for sending cloudevents. 12 | type Sender interface { 13 | // Send a cloud event to other services (AWS pub/sub, Kafka). 14 | Send(ctx context.Context, notificationType string, event cloudevents.Event) error 15 | } 16 | -------------------------------------------------------------------------------- /pkg/async/cloudevent/mocks/publisher.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | 10 | protoiface "google.golang.org/protobuf/runtime/protoiface" 11 | ) 12 | 13 | // Publisher is an autogenerated mock type for the Publisher type 14 | type Publisher struct { 15 | mock.Mock 16 | } 17 | 18 | type Publisher_Publish struct { 19 | *mock.Call 20 | } 21 | 22 | func (_m Publisher_Publish) Return(_a0 error) *Publisher_Publish { 23 | return &Publisher_Publish{Call: _m.Call.Return(_a0)} 24 | } 25 | 26 | func (_m *Publisher) OnPublish(ctx context.Context, notificationType string, msg protoiface.MessageV1) *Publisher_Publish { 27 | c_call := _m.On("Publish", ctx, notificationType, msg) 28 | return &Publisher_Publish{Call: c_call} 29 | } 30 | 31 | func (_m *Publisher) OnPublishMatch(matchers ...interface{}) *Publisher_Publish { 32 | c_call := _m.On("Publish", matchers...) 33 | return &Publisher_Publish{Call: c_call} 34 | } 35 | 36 | // Publish provides a mock function with given fields: ctx, notificationType, msg 37 | func (_m *Publisher) Publish(ctx context.Context, notificationType string, msg protoiface.MessageV1) error { 38 | ret := _m.Called(ctx, notificationType, msg) 39 | 40 | var r0 error 41 | if rf, ok := ret.Get(0).(func(context.Context, string, protoiface.MessageV1) error); ok { 42 | r0 = rf(ctx, notificationType, msg) 43 | } else { 44 | r0 = ret.Error(0) 45 | } 46 | 47 | return r0 48 | } 49 | -------------------------------------------------------------------------------- /pkg/async/cloudevent/mocks/sender.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | event "github.com/cloudevents/sdk-go/v2/event" 9 | 10 | mock "github.com/stretchr/testify/mock" 11 | ) 12 | 13 | // Sender is an autogenerated mock type for the Sender type 14 | type Sender struct { 15 | mock.Mock 16 | } 17 | 18 | type Sender_Send struct { 19 | *mock.Call 20 | } 21 | 22 | func (_m Sender_Send) Return(_a0 error) *Sender_Send { 23 | return &Sender_Send{Call: _m.Call.Return(_a0)} 24 | } 25 | 26 | func (_m *Sender) OnSend(ctx context.Context, notificationType string, _a2 event.Event) *Sender_Send { 27 | c_call := _m.On("Send", ctx, notificationType, _a2) 28 | return &Sender_Send{Call: c_call} 29 | } 30 | 31 | func (_m *Sender) OnSendMatch(matchers ...interface{}) *Sender_Send { 32 | c_call := _m.On("Send", matchers...) 33 | return &Sender_Send{Call: c_call} 34 | } 35 | 36 | // Send provides a mock function with given fields: ctx, notificationType, _a2 37 | func (_m *Sender) Send(ctx context.Context, notificationType string, _a2 event.Event) error { 38 | ret := _m.Called(ctx, notificationType, _a2) 39 | 40 | var r0 error 41 | if rf, ok := ret.Get(0).(func(context.Context, string, event.Event) error); ok { 42 | r0 = rf(ctx, notificationType, _a2) 43 | } else { 44 | r0 = ret.Error(0) 45 | } 46 | 47 | return r0 48 | } 49 | -------------------------------------------------------------------------------- /pkg/async/events/implementations/node_execution_event_writer_test.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/mocks" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 9 | event2 "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event" 10 | ) 11 | 12 | func TestNodeExecutionEventWriter(t *testing.T) { 13 | db := mocks.NewMockRepository() 14 | 15 | event := admin.NodeExecutionEventRequest{ 16 | RequestId: "request_id", 17 | Event: &event2.NodeExecutionEvent{ 18 | Id: &core.NodeExecutionIdentifier{ 19 | NodeId: "node_id", 20 | ExecutionId: &core.WorkflowExecutionIdentifier{ 21 | Project: "project", 22 | Domain: "domain", 23 | Name: "exec_name", 24 | }, 25 | }, 26 | }, 27 | } 28 | 29 | nodeExecEventRepo := mocks.NodeExecutionEventRepoInterface{} 30 | nodeExecEventRepo.On("Create", event).Return(nil) 31 | db.(*mocks.MockRepository).NodeExecutionEventRepoIface = &nodeExecEventRepo 32 | writer := NewNodeExecutionEventWriter(db, 100) 33 | // Assert we can write an event using the buffered channel without holding up this process. 34 | writer.Write(event) 35 | go func() { writer.Run() }() 36 | close(writer.(*nodeExecutionEventWriter).events) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/async/events/implementations/workflow_execution_event_writer_test.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/mocks" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 9 | event2 "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event" 10 | ) 11 | 12 | func TestWorkflowExecutionEventWriter(t *testing.T) { 13 | db := mocks.NewMockRepository() 14 | 15 | event := admin.WorkflowExecutionEventRequest{ 16 | RequestId: "request_id", 17 | Event: &event2.WorkflowExecutionEvent{ 18 | ExecutionId: &core.WorkflowExecutionIdentifier{ 19 | Project: "project", 20 | Domain: "domain", 21 | Name: "exec_name", 22 | }, 23 | }, 24 | } 25 | 26 | workflowExecEventRepo := mocks.ExecutionEventRepoInterface{} 27 | workflowExecEventRepo.On("Create", event).Return(nil) 28 | db.(*mocks.MockRepository).ExecutionEventRepoIface = &workflowExecEventRepo 29 | writer := NewWorkflowExecutionEventWriter(db, 100) 30 | // Assert we can write an event using the buffered channel without holding up this process. 31 | writer.Write(event) 32 | go func() { writer.Run() }() 33 | close(writer.(*workflowExecutionEventWriter).events) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/async/events/interfaces/node_execution.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 5 | ) 6 | 7 | //go:generate mockery -name=NodeExecutionEventWriter -output=../mocks -case=underscore 8 | 9 | type NodeExecutionEventWriter interface { 10 | Run() 11 | Write(nodeExecutionEvent admin.NodeExecutionEventRequest) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/async/events/interfaces/workflow_execution.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 5 | ) 6 | 7 | //go:generate mockery -name=WorkflowExecutionEventWriter -output=../mocks -case=underscore 8 | 9 | type WorkflowExecutionEventWriter interface { 10 | Run() 11 | Write(workflowExecutionEvent admin.WorkflowExecutionEventRequest) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/async/events/mocks/node_execution_event_writer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | admin "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | ) 10 | 11 | // NodeExecutionEventWriter is an autogenerated mock type for the NodeExecutionEventWriter type 12 | type NodeExecutionEventWriter struct { 13 | mock.Mock 14 | } 15 | 16 | // Run provides a mock function with given fields: 17 | func (_m *NodeExecutionEventWriter) Run() { 18 | _m.Called() 19 | } 20 | 21 | // Write provides a mock function with given fields: nodeExecutionEvent 22 | func (_m *NodeExecutionEventWriter) Write(nodeExecutionEvent admin.NodeExecutionEventRequest) { 23 | _m.Called(nodeExecutionEvent) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/async/events/mocks/workflow_execution_event_writer.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | admin "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | ) 10 | 11 | // WorkflowExecutionEventWriter is an autogenerated mock type for the WorkflowExecutionEventWriter type 12 | type WorkflowExecutionEventWriter struct { 13 | mock.Mock 14 | } 15 | 16 | // Run provides a mock function with given fields: 17 | func (_m *WorkflowExecutionEventWriter) Run() { 18 | _m.Called() 19 | } 20 | 21 | // Write provides a mock function with given fields: workflowExecutionEvent 22 | func (_m *WorkflowExecutionEventWriter) Write(workflowExecutionEvent admin.WorkflowExecutionEventRequest) { 23 | _m.Called(workflowExecutionEvent) 24 | } 25 | -------------------------------------------------------------------------------- /pkg/async/notifications/implementations/email_metrics.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "github.com/flyteorg/flytestdlib/promutils" 5 | "github.com/prometheus/client_golang/prometheus" 6 | ) 7 | 8 | type emailMetrics struct { 9 | Scope promutils.Scope 10 | SendSuccess prometheus.Counter 11 | SendError prometheus.Counter 12 | SendTotal prometheus.Counter 13 | } 14 | 15 | func newEmailMetrics(scope promutils.Scope) emailMetrics { 16 | return emailMetrics{ 17 | Scope: scope, 18 | SendSuccess: scope.MustNewCounter("send_success", "Number of successful emails sent via Emailer."), 19 | SendError: scope.MustNewCounter("send_error", "Number of errors when sending email via Emailer"), 20 | SendTotal: scope.MustNewCounter("send_total", "Total number of emails attempted to be sent"), 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /pkg/async/notifications/implementations/emailers.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | type ExternalEmailer = string 4 | 5 | const ( 6 | Sendgrid ExternalEmailer = "sendgrid" 7 | ) 8 | -------------------------------------------------------------------------------- /pkg/async/notifications/implementations/sandbox_publisher.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flytestdlib/logger" 7 | "github.com/golang/protobuf/proto" 8 | ) 9 | 10 | type SandboxPublisher struct { 11 | pubChan chan<- []byte 12 | } 13 | 14 | func (p *SandboxPublisher) Publish(ctx context.Context, notificationType string, msg proto.Message) error { 15 | logger.Debugf(ctx, "Publishing the following message [%s]", msg.String()) 16 | 17 | data, err := proto.Marshal(msg) 18 | 19 | if err != nil { 20 | logger.Errorf(ctx, "Failed to publish a message with key [%s] and message [%s] and error: %v", notificationType, msg.String(), err) 21 | return err 22 | } 23 | 24 | p.pubChan <- data 25 | 26 | return nil 27 | } 28 | 29 | func NewSandboxPublisher(pubChan chan<- []byte) *SandboxPublisher { 30 | return &SandboxPublisher{ 31 | pubChan: pubChan, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/async/notifications/implementations/sandbox_publisher_test.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | // mockMessage is a dummy proto message that will always fail to marshal 12 | type mockMessage struct{} 13 | 14 | func (m *mockMessage) Reset() {} 15 | func (m *mockMessage) String() string { return "mockMessage" } 16 | func (m *mockMessage) ProtoMessage() {} 17 | func (m *mockMessage) Marshal() ([]byte, error) { return nil, errors.New("forced marshal error") } 18 | 19 | func TestSandboxPublisher_Publish(t *testing.T) { 20 | msgChan := make(chan []byte, 1) 21 | publisher := NewSandboxPublisher(msgChan) 22 | 23 | err := publisher.Publish(context.Background(), "NOTIFICATION_TYPE", &testEmail) 24 | 25 | assert.NotZero(t, len(msgChan)) 26 | assert.Nil(t, err) 27 | } 28 | 29 | func TestSandboxPublisher_PublishMarshalError(t *testing.T) { 30 | msgChan := make(chan []byte, 1) 31 | publisher := NewSandboxPublisher(msgChan) 32 | 33 | err := publisher.Publish(context.Background(), "testMarshallError", &mockMessage{}) 34 | assert.Error(t, err) 35 | assert.Equal(t, "forced marshal error", err.Error()) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/async/notifications/interfaces/emailer.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // The implementation of Emailer needs to be passed to the implementation of Processor 10 | // in order for emails to be sent. 11 | type Emailer interface { 12 | SendEmail(ctx context.Context, email admin.EmailMessage) error 13 | } 14 | -------------------------------------------------------------------------------- /pkg/async/notifications/interfaces/processor.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | // Exposes the common methods required for a subscriber. 4 | // There is one ProcessNotification per type. 5 | type Processor interface { 6 | 7 | // Starts processing messages from the underlying subscriber. 8 | // If the channel closes gracefully, no error will be returned. 9 | // If the underlying channel experiences errors, 10 | // an error is returned and the channel is closed. 11 | StartProcessing() 12 | 13 | // This should be invoked when the application is shutting down. 14 | // If StartProcessing() returned an error, StopProcessing() will return an error because 15 | // the channel was already closed. 16 | StopProcessing() error 17 | } 18 | -------------------------------------------------------------------------------- /pkg/async/notifications/interfaces/publisher.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golang/protobuf/proto" 7 | ) 8 | 9 | // Note on Notifications 10 | 11 | // Notifications are handled in two steps. 12 | // 1. Publishing a notification 13 | // 2. Processing a notification 14 | 15 | // Publishing a notification enqueues a notification message to be processed. Currently there is only 16 | // one publisher for all notification types with the type differing based on the key. 17 | // The notification hasn't been delivered at this stage. 18 | // Processing a notification takes a notification message from the publisher and will pass 19 | // the notification using the desired delivery method (ex: email). There is one processor per 20 | // notification type. 21 | 22 | // Publish a notification will differ between different types of notifications using the key 23 | // The contract requires one subscription per type i.e. one for email one for slack, etc... 24 | type Publisher interface { 25 | // The notification type is inferred from the Notification object in the Execution Spec. 26 | Publish(ctx context.Context, notificationType string, msg proto.Message) error 27 | } 28 | -------------------------------------------------------------------------------- /pkg/async/notifications/mocks/emailer.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/service/ses" 5 | "github.com/aws/aws-sdk-go/service/ses/sesiface" 6 | ) 7 | 8 | type AwsSendEmailFunc func(input *ses.SendEmailInput) (*ses.SendEmailOutput, error) 9 | 10 | type SESClient struct { 11 | sesiface.SESAPI 12 | sendEmail AwsSendEmailFunc 13 | } 14 | 15 | func (m *SESClient) SetSendEmailFunc(emailFunc AwsSendEmailFunc) { 16 | m.sendEmail = emailFunc 17 | } 18 | 19 | func (m *SESClient) SendEmail(input *ses.SendEmailInput) (*ses.SendEmailOutput, error) { 20 | if m.sendEmail != nil { 21 | return m.sendEmail(input) 22 | } 23 | return &ses.SendEmailOutput{}, nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/async/notifications/mocks/processor.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | type RunFunc func() error 10 | 11 | type StopFunc func() error 12 | 13 | type MockSubscriber struct { 14 | runFunc RunFunc 15 | stopFunc StopFunc 16 | } 17 | 18 | func (m *MockSubscriber) Run() error { 19 | if m.runFunc != nil { 20 | return m.runFunc() 21 | } 22 | return nil 23 | } 24 | 25 | func (m *MockSubscriber) Stop() error { 26 | if m.stopFunc != nil { 27 | return m.stopFunc() 28 | } 29 | return nil 30 | } 31 | 32 | type SendEmailFunc func(ctx context.Context, email admin.EmailMessage) error 33 | 34 | type MockEmailer struct { 35 | sendEmailFunc SendEmailFunc 36 | } 37 | 38 | func (m *MockEmailer) SetSendEmailFunc(sendEmail SendEmailFunc) { 39 | m.sendEmailFunc = sendEmail 40 | } 41 | 42 | func (m *MockEmailer) SendEmail(ctx context.Context, email admin.EmailMessage) error { 43 | if m.sendEmailFunc != nil { 44 | return m.sendEmailFunc(ctx, email) 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/async/notifications/mocks/publisher.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golang/protobuf/proto" 7 | ) 8 | 9 | type PublishFunc func(ctx context.Context, key string, msg proto.Message) error 10 | 11 | type MockPublisher struct { 12 | publishFunc PublishFunc 13 | } 14 | 15 | func (m *MockPublisher) SetPublishCallback(publishFunction PublishFunc) { 16 | m.publishFunc = publishFunction 17 | } 18 | 19 | func (m *MockPublisher) Publish(ctx context.Context, notificationType string, msg proto.Message) error { 20 | if m.publishFunc != nil { 21 | return m.publishFunc(ctx, notificationType, msg) 22 | } 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/async/schedule/aws/interfaces/cloud_watch_event_client.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import "github.com/aws/aws-sdk-go/service/cloudwatchevents" 4 | 5 | // A subset of the AWS CloudWatchEvents service client. 6 | type CloudWatchEventClient interface { 7 | PutRule(input *cloudwatchevents.PutRuleInput) (*cloudwatchevents.PutRuleOutput, error) 8 | PutTargets(input *cloudwatchevents.PutTargetsInput) (*cloudwatchevents.PutTargetsOutput, error) 9 | DeleteRule(input *cloudwatchevents.DeleteRuleInput) (*cloudwatchevents.DeleteRuleOutput, error) 10 | RemoveTargets(input *cloudwatchevents.RemoveTargetsInput) (*cloudwatchevents.RemoveTargetsOutput, error) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/async/schedule/aws/shared.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "hash/fnv" 7 | 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 9 | "github.com/flyteorg/flytestdlib/logger" 10 | ) 11 | 12 | func hashIdentifier(identifier core.Identifier) uint64 { 13 | h := fnv.New64() 14 | _, err := h.Write([]byte(fmt.Sprintf(scheduleNameInputsFormat, 15 | identifier.Project, identifier.Domain, identifier.Name))) 16 | if err != nil { 17 | // This shouldn't occur. 18 | logger.Errorf(context.Background(), 19 | "failed to hash launch plan identifier: %+v to get schedule name with err: %v", identifier, err) 20 | return 0 21 | } 22 | logger.Debugf(context.Background(), "Returning hash for [%+v]: %d", identifier, h.Sum64()) 23 | return h.Sum64() 24 | } 25 | -------------------------------------------------------------------------------- /pkg/async/schedule/aws/shared_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 7 | 8 | "github.com/magiconair/properties/assert" 9 | ) 10 | 11 | func TestHashIdentifier(t *testing.T) { 12 | identifier := core.Identifier{ 13 | Project: "project", 14 | Domain: "domain", 15 | Name: "name", 16 | Version: "ignored", 17 | } 18 | hashedValue := hashIdentifier(identifier) 19 | assert.Equal(t, uint64(16301494360130577061), hashedValue) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/async/schedule/interfaces/event_scheduler.go: -------------------------------------------------------------------------------- 1 | // Defines an event scheduler interface 2 | package interfaces 3 | 4 | import ( 5 | "context" 6 | 7 | appInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 9 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 10 | ) 11 | 12 | type AddScheduleInput struct { 13 | // Defines the unique identifier associated with the schedule 14 | Identifier core.Identifier 15 | // Defines the schedule expression. 16 | ScheduleExpression admin.Schedule 17 | // Message payload encoded as an CloudWatch event rule InputTemplate. 18 | Payload *string 19 | // Optional: The application-wide prefix to be applied for schedule names. 20 | ScheduleNamePrefix string 21 | } 22 | 23 | type RemoveScheduleInput struct { 24 | // Defines the unique identifier associated with the schedule 25 | Identifier core.Identifier 26 | // Optional: The application-wide prefix to be applied for schedule names. 27 | ScheduleNamePrefix string 28 | } 29 | 30 | type EventScheduler interface { 31 | // Schedules an event. 32 | AddSchedule(ctx context.Context, input AddScheduleInput) error 33 | 34 | // CreateScheduleInput using the scheduler config and launch plan identifier and schedule 35 | CreateScheduleInput(ctx context.Context, appConfig *appInterfaces.SchedulerConfig, identifier core.Identifier, 36 | schedule *admin.Schedule) (AddScheduleInput, error) 37 | 38 | // Removes an existing schedule. 39 | RemoveSchedule(ctx context.Context, input RemoveScheduleInput) error 40 | } 41 | -------------------------------------------------------------------------------- /pkg/async/schedule/interfaces/workflow_executor.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | // Handles responding to scheduled workflow execution events and creating executions. 4 | type WorkflowExecutor interface { 5 | Run() 6 | Stop() error 7 | } 8 | -------------------------------------------------------------------------------- /pkg/async/schedule/mocks/mock_workflow_executor.go: -------------------------------------------------------------------------------- 1 | // Mock implementation of a WorkflowExecutor for use in tests. 2 | package mocks 3 | 4 | type MockWorkflowExecutor struct { 5 | runFunc func() 6 | stopFunc func() error 7 | } 8 | 9 | func (e *MockWorkflowExecutor) SetRunFunc(runFunc func()) { 10 | e.runFunc = runFunc 11 | } 12 | 13 | func (e *MockWorkflowExecutor) Run() { 14 | if e.runFunc != nil { 15 | e.runFunc() 16 | } 17 | } 18 | 19 | func (e *MockWorkflowExecutor) SetStopFunc(stopFunc func() error) { 20 | e.stopFunc = stopFunc 21 | } 22 | 23 | func (e *MockWorkflowExecutor) Stop() error { 24 | if e.stopFunc != nil { 25 | return e.stopFunc() 26 | } 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /pkg/async/schedule/noop/event_scheduler.go: -------------------------------------------------------------------------------- 1 | // No-op event event scheduler for use in development. 2 | package noop 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/async/schedule/interfaces" 8 | runtimeInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 9 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 10 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 11 | "github.com/flyteorg/flytestdlib/logger" 12 | ) 13 | 14 | type EventScheduler struct{} 15 | 16 | func (s *EventScheduler) CreateScheduleInput(ctx context.Context, appConfig *runtimeInterfaces.SchedulerConfig, identifier core.Identifier, schedule *admin.Schedule) (interfaces.AddScheduleInput, error) { 17 | panic("implement me") 18 | } 19 | 20 | func (s *EventScheduler) AddSchedule(ctx context.Context, input interfaces.AddScheduleInput) error { 21 | logger.Debugf(ctx, "Received call to add schedule [%+v]", input) 22 | logger.Debug(ctx, "Not scheduling anything") 23 | return nil 24 | } 25 | 26 | func (s *EventScheduler) RemoveSchedule(ctx context.Context, input interfaces.RemoveScheduleInput) error { 27 | logger.Debugf(ctx, "Received call to remove schedule [%+v]", input.Identifier) 28 | logger.Debug(ctx, "Not scheduling anything") 29 | return nil 30 | } 31 | 32 | func NewNoopEventScheduler() interfaces.EventScheduler { 33 | return &EventScheduler{} 34 | } 35 | -------------------------------------------------------------------------------- /pkg/async/schedule/noop/workflow_executor.go: -------------------------------------------------------------------------------- 1 | package noop 2 | 3 | import "github.com/flyteorg/flyteadmin/pkg/async/schedule/interfaces" 4 | 5 | type workflowExecutor struct{} 6 | 7 | func (w *workflowExecutor) Run() {} 8 | 9 | func (w *workflowExecutor) Stop() error { 10 | return nil 11 | } 12 | 13 | func NewWorkflowExecutor() interfaces.WorkflowExecutor { 14 | return &workflowExecutor{} 15 | } 16 | -------------------------------------------------------------------------------- /pkg/async/shared.go: -------------------------------------------------------------------------------- 1 | package async 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/flyteorg/flytestdlib/logger" 8 | ) 9 | 10 | // RetryDelay indicates how long to wait between restarting a subscriber connection in the case of network failures. 11 | var RetryDelay = 30 * time.Second 12 | 13 | func RetryOnSpecificErrors(attempts int, delay time.Duration, f func() error, IsErrorRetryable func(error) bool) error { 14 | var err error 15 | for attempt := 0; attempt <= attempts; attempt++ { 16 | err = f() 17 | if err == nil { 18 | return nil 19 | } 20 | if !IsErrorRetryable(err) { 21 | return err 22 | } 23 | logger.Warningf(context.Background(), 24 | "Failed [%v] on attempt %d of %d", err, attempt, attempts) 25 | time.Sleep(delay) 26 | } 27 | return err 28 | } 29 | 30 | func retryOnAllErrors(err error) bool { 31 | return true 32 | } 33 | 34 | func Retry(attempts int, delay time.Duration, f func() error) error { 35 | return RetryOnSpecificErrors(attempts, delay, f, retryOnAllErrors) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/async/shared_test.go: -------------------------------------------------------------------------------- 1 | package async 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestRetry(t *testing.T) { 12 | attemptsRecorded := 0 13 | err := Retry(3, time.Millisecond, func() error { 14 | if attemptsRecorded == 3 { 15 | return nil 16 | } 17 | attemptsRecorded++ 18 | return errors.New("foo") 19 | }) 20 | assert.Nil(t, err) 21 | } 22 | 23 | func TestRetry_RetriesExhausted(t *testing.T) { 24 | attemptsRecorded := 0 25 | err := Retry(2, time.Millisecond, func() error { 26 | if attemptsRecorded == 3 { 27 | return nil 28 | } 29 | attemptsRecorded++ 30 | return errors.New("foo") 31 | }) 32 | assert.EqualError(t, err, "foo") 33 | } 34 | 35 | func TestRetryOnlyOnRetryableExceptions(t *testing.T) { 36 | attemptsRecorded := 0 37 | err := RetryOnSpecificErrors(3, time.Millisecond, func() error { 38 | attemptsRecorded++ 39 | if attemptsRecorded <= 1 { 40 | return errors.New("foo") 41 | } 42 | return errors.New("not-foo") 43 | }, func(err error) bool { 44 | return err.Error() == "foo" 45 | }) 46 | assert.EqualValues(t, attemptsRecorded, 2) 47 | assert.EqualError(t, err, "not-foo") 48 | } 49 | -------------------------------------------------------------------------------- /pkg/clusterresource/impl/shared.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/errors" 5 | "google.golang.org/grpc/codes" 6 | ) 7 | 8 | func NewMissingEntityError(entity string) error { 9 | return errors.NewFlyteAdminErrorf(codes.NotFound, "Failed to find [%s]", entity) 10 | } 11 | -------------------------------------------------------------------------------- /pkg/clusterresource/interfaces/admin.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | //go:generate mockery -name FlyteAdminDataProvider -output=../mocks -case=underscore 10 | 11 | type FlyteAdminDataProvider interface { 12 | GetClusterResourceAttributes(ctx context.Context, project, domain string) (*admin.ClusterResourceAttributes, error) 13 | GetProjects(ctx context.Context) (*admin.Projects, error) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/clusterresource/sync_stats.go: -------------------------------------------------------------------------------- 1 | package clusterresource 2 | 3 | // ResourceSyncStats is a simple struct to track the number of resources created, updated, already there, and errored 4 | type ResourceSyncStats struct { 5 | Created int 6 | Updated int 7 | AlreadyThere int 8 | Errored int 9 | } 10 | 11 | // Add adds the values of the other ResourceSyncStats to this one 12 | func (m *ResourceSyncStats) Add(other ResourceSyncStats) { 13 | m.Created += other.Created 14 | m.Updated += other.Updated 15 | m.AlreadyThere += other.AlreadyThere 16 | m.Errored += other.Errored 17 | } 18 | -------------------------------------------------------------------------------- /pkg/clusterresource/testdata/docker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: dockerhub 5 | namespace: {{ namespace }} 6 | stringData: 7 | .dockerconfigjson: '{"auths":{"docker.io":{"username":"mydockerusername","password":"{{ dockerSecret }}","email":"none","auth":" {{ dockerAuth }}"}}}' 8 | type: kubernetes.io/dockerconfigjson 9 | -------------------------------------------------------------------------------- /pkg/clusterresource/testdata/gsa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: {{ namespace }}-gsa 5 | namespace: {{ namespace }} 6 | -------------------------------------------------------------------------------- /pkg/clusterresource/testdata/imagepullsecrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: default 5 | namespace: {{ namespace }} 6 | imagePullSecrets: 7 | - name: dockerhub 8 | -------------------------------------------------------------------------------- /pkg/clusterresource/testdata/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ namespace }} 5 | spec: 6 | finalizers: 7 | - kubernetes 8 | -------------------------------------------------------------------------------- /pkg/clusterresource/utils.go: -------------------------------------------------------------------------------- 1 | package clusterresource 2 | 3 | import ( 4 | jsonpatch "github.com/evanphx/json-patch" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/apimachinery/pkg/types" 7 | "k8s.io/apimachinery/pkg/util/json" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | ) 10 | 11 | type strategicMergeFromPatch struct { 12 | from runtime.Object 13 | } 14 | 15 | // Type implements patch. 16 | func (s *strategicMergeFromPatch) Type() types.PatchType { 17 | return types.StrategicMergePatchType 18 | } 19 | 20 | // Data implements Patch. 21 | func (s *strategicMergeFromPatch) Data(obj client.Object) ([]byte, error) { 22 | originalJSON, err := json.Marshal(s.from) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | modifiedJSON, err := json.Marshal(obj) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return jsonpatch.CreateMergePatch(originalJSON, modifiedJSON) 33 | } 34 | 35 | // StrategicMergeFrom creates a Patch using the strategic-merge-patch strategy with the given object as base. 36 | func StrategicMergeFrom(obj runtime.Object) client.Patch { 37 | return &strategicMergeFromPatch{obj} 38 | } 39 | -------------------------------------------------------------------------------- /pkg/common/cloud.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Common configuration parameters for initializing back-end cloud clients. 4 | 5 | type CloudProvider = string 6 | 7 | const ( 8 | AWS CloudProvider = "aws" 9 | GCP CloudProvider = "gcp" 10 | Sandbox CloudProvider = "sandbox" 11 | Local CloudProvider = "local" 12 | None CloudProvider = "none" 13 | ) 14 | -------------------------------------------------------------------------------- /pkg/common/constants.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/flyteorg/flytestdlib/contextutils" 4 | 5 | var RuntimeTypeKey = contextutils.Key("runtime_type") 6 | var RuntimeVersionKey = contextutils.Key("runtime_version") 7 | 8 | const ( 9 | AuditFieldsContextKey contextutils.Key = "audit_fields" 10 | PrincipalContextKey contextutils.Key = "principal" 11 | ErrorKindKey contextutils.Key = "error_kind" 12 | ) 13 | 14 | const MaxResponseStatusBytes = 32000 15 | 16 | // DefaultProducerID is used in older versions of propeller which hard code this producer id. 17 | // See https://github.com/flyteorg/flytepropeller/blob/eaf084934de5d630cd4c11aae15ecae780cc787e/pkg/controller/nodes/task/transformer.go#L114 18 | const DefaultProducerID = "propeller" 19 | -------------------------------------------------------------------------------- /pkg/common/entity.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 5 | ) 6 | 7 | type Entity = string 8 | 9 | const ( 10 | Execution = "e" 11 | LaunchPlan = "l" 12 | NodeExecution = "ne" 13 | NodeExecutionEvent = "nee" 14 | Task = "t" 15 | TaskExecution = "te" 16 | Workflow = "w" 17 | NamedEntity = "nen" 18 | NamedEntityMetadata = "nem" 19 | Project = "p" 20 | Signal = "s" 21 | AdminTag = "at" 22 | ExecutionAdminTag = "eat" 23 | ) 24 | 25 | // ResourceTypeToEntity maps a resource type to an entity suitable for use with Database filters 26 | var ResourceTypeToEntity = map[core.ResourceType]Entity{ 27 | core.ResourceType_LAUNCH_PLAN: LaunchPlan, 28 | core.ResourceType_TASK: Task, 29 | core.ResourceType_WORKFLOW: Workflow, 30 | } 31 | -------------------------------------------------------------------------------- /pkg/common/executions_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | const AllowedExecutionIDStartCharStr = "abcdefghijklmnopqrstuvwxyz" 11 | const AllowedExecutionIDStr = "abcdefghijklmnopqrstuvwxyz1234567890" 12 | 13 | var AllowedExecutionIDStartChars = []rune(AllowedExecutionIDStartCharStr) 14 | var AllowedExecutionIDChars = []rune(AllowedExecutionIDStr) 15 | 16 | func TestGetExecutionName(t *testing.T) { 17 | randString := GetExecutionName(time.Now().UnixNano()) 18 | assert.Len(t, randString, ExecutionIDLength) 19 | assert.Contains(t, AllowedExecutionIDStartChars, rune(randString[0])) 20 | for i := 1; i < len(randString); i++ { 21 | assert.Contains(t, AllowedExecutionIDChars, rune(randString[i])) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pkg/common/namespace.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | const projectTemplate = "{{ project }}" 8 | const domainTemplate = "{{ domain }}" 9 | 10 | const replaceAllInstancesOfString = -1 11 | 12 | // GetNamespaceName returns kubernetes namespace name according to user defined template from config 13 | func GetNamespaceName(template string, project, domain string) string { 14 | var namespace = template 15 | namespace = strings.Replace(namespace, projectTemplate, project, replaceAllInstancesOfString) 16 | namespace = strings.Replace(namespace, domainTemplate, domain, replaceAllInstancesOfString) 17 | 18 | return namespace 19 | } 20 | -------------------------------------------------------------------------------- /pkg/common/namespace_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetNamespaceName(t *testing.T) { 10 | testCases := []struct { 11 | template string 12 | project string 13 | domain string 14 | want string 15 | }{ 16 | {"prefix-{{ project }}-{{ domain }}", "flytesnacks", "production", "prefix-flytesnacks-production"}, 17 | {"{{ domain }}", "flytesnacks", "production", "production"}, 18 | {"{{ project }}", "flytesnacks", "production", "flytesnacks"}, 19 | } 20 | 21 | for _, tc := range testCases { 22 | got := GetNamespaceName(tc.template, tc.project, tc.domain) 23 | assert.Equal(t, got, tc.want) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pkg/common/sorting.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | "google.golang.org/grpc/codes" 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | 10 | "github.com/flyteorg/flyteadmin/pkg/errors" 11 | ) 12 | 13 | const gormDescending = "%s desc" 14 | const gormAscending = "%s asc" 15 | 16 | type SortParameter interface { 17 | GetGormOrderExpr() string 18 | } 19 | 20 | type sortParamImpl struct { 21 | gormOrderExpression string 22 | } 23 | 24 | func (s *sortParamImpl) GetGormOrderExpr() string { 25 | return s.gormOrderExpression 26 | } 27 | 28 | func NewSortParameter(sort *admin.Sort, allowed sets.String) (SortParameter, error) { 29 | if sort == nil { 30 | return nil, nil 31 | } 32 | 33 | key := sort.Key 34 | if !allowed.Has(key) { 35 | return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid sort key '%s'", key) 36 | } 37 | 38 | var gormOrderExpression string 39 | switch sort.Direction { 40 | case admin.Sort_DESCENDING: 41 | gormOrderExpression = fmt.Sprintf(gormDescending, key) 42 | case admin.Sort_ASCENDING: 43 | gormOrderExpression = fmt.Sprintf(gormAscending, key) 44 | default: 45 | return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "invalid sort order specified: %v", sort) 46 | } 47 | return &sortParamImpl{ 48 | gormOrderExpression: gormOrderExpression, 49 | }, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/common/testutils/common.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 4 | 5 | // Convenience method to wrap verbose boilerplate for initializing a PluginOverrides MatchingAttributes. 6 | func GetPluginOverridesAttributes(vals map[string][]string) *admin.MatchingAttributes { 7 | overrides := make([]*admin.PluginOverride, 0, len(vals)) 8 | for taskType, pluginIDs := range vals { 9 | overrides = append(overrides, &admin.PluginOverride{ 10 | TaskType: taskType, 11 | PluginId: pluginIDs, 12 | }) 13 | } 14 | return &admin.MatchingAttributes{ 15 | Target: &admin.MatchingAttributes_PluginOverrides{ 16 | PluginOverrides: &admin.PluginOverrides{ 17 | Overrides: overrides, 18 | }, 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pkg/data/implementations/noop_remote_url.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/data/interfaces" 7 | "github.com/flyteorg/flyteadmin/pkg/errors" 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 9 | "github.com/flyteorg/flytestdlib/storage" 10 | "google.golang.org/grpc/codes" 11 | ) 12 | 13 | // No-op implementation of a RemoteURLInterface 14 | type NoopRemoteURL struct { 15 | remoteDataStoreClient storage.DataStore 16 | } 17 | 18 | func (n *NoopRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { 19 | metadata, err := n.remoteDataStoreClient.Head(ctx, storage.DataReference(uri)) 20 | if err != nil { 21 | return admin.UrlBlob{}, errors.NewFlyteAdminErrorf(codes.Internal, 22 | "failed to get metadata for uri: %s with err: %v", uri, err) 23 | } 24 | return admin.UrlBlob{ 25 | Url: uri, 26 | Bytes: metadata.Size(), 27 | }, nil 28 | } 29 | 30 | func NewNoopRemoteURL(remoteDataStoreClient storage.DataStore) interfaces.RemoteURLInterface { 31 | return &NoopRemoteURL{ 32 | remoteDataStoreClient: remoteDataStoreClient, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /pkg/data/implementations/noop_remote_url_test.go: -------------------------------------------------------------------------------- 1 | package implementations 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/flyteorg/flytestdlib/storage" 8 | 9 | "github.com/stretchr/testify/assert" 10 | 11 | commonMocks "github.com/flyteorg/flyteadmin/pkg/common/mocks" 12 | ) 13 | 14 | const noopFileSize = int64(1256) 15 | 16 | type MockMetadata struct{} 17 | 18 | func (m MockMetadata) Exists() bool { 19 | return true 20 | } 21 | 22 | func (m MockMetadata) Size() int64 { 23 | return noopFileSize 24 | } 25 | 26 | func (m MockMetadata) Etag() string { 27 | return "etag" 28 | } 29 | 30 | func getMockStorage() storage.DataStore { 31 | mockStorage := commonMocks.GetMockStorageClient() 32 | mockStorage.ComposedProtobufStore.(*commonMocks.TestDataStore).HeadCb = 33 | func(ctx context.Context, reference storage.DataReference) (storage.Metadata, error) { 34 | return MockMetadata{}, nil 35 | } 36 | return *mockStorage 37 | } 38 | 39 | func TestNoopRemoteURLGet(t *testing.T) { 40 | noopRemoteURL := NewNoopRemoteURL(getMockStorage()) 41 | urlBlob, err := noopRemoteURL.Get(context.Background(), "uri") 42 | assert.Nil(t, err) 43 | assert.NotEmpty(t, urlBlob) 44 | assert.Equal(t, "uri", urlBlob.Url) 45 | assert.Equal(t, noopFileSize, urlBlob.Bytes) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/data/interfaces/remote.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Defines an interface for fetching pre-signed URLs. 10 | type RemoteURLInterface interface { 11 | // TODO: Refactor for URI to be of type DataReference. We should package a FromString-like function in flytestdlib 12 | Get(ctx context.Context, uri string) (admin.UrlBlob, error) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/data/mocks/remote.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/data/interfaces" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 8 | ) 9 | 10 | // Mock implementation of a RemoteURLInterface 11 | type MockRemoteURL struct { 12 | GetCallback func(ctx context.Context, uri string) (admin.UrlBlob, error) 13 | } 14 | 15 | func (m *MockRemoteURL) Get(ctx context.Context, uri string) (admin.UrlBlob, error) { 16 | if m.GetCallback != nil { 17 | return m.GetCallback(ctx, uri) 18 | } 19 | return admin.UrlBlob{}, nil 20 | } 21 | 22 | func NewMockRemoteURL() interfaces.RemoteURLInterface { 23 | return &MockRemoteURL{} 24 | } 25 | -------------------------------------------------------------------------------- /pkg/executioncluster/execution_target.go: -------------------------------------------------------------------------------- 1 | package executioncluster 2 | 3 | import ( 4 | flyteclient "github.com/flyteorg/flytepropeller/pkg/client/clientset/versioned" 5 | "github.com/flyteorg/flytestdlib/random" 6 | "k8s.io/client-go/dynamic" 7 | restclient "k8s.io/client-go/rest" 8 | 9 | "sigs.k8s.io/controller-runtime/pkg/client" 10 | ) 11 | 12 | // Spec to determine the execution target 13 | type ExecutionTargetSpec struct { 14 | TargetID string 15 | ExecutionID string 16 | Project string 17 | Domain string 18 | Workflow string 19 | LaunchPlan string 20 | } 21 | 22 | // Client object of the target execution cluster 23 | type ExecutionTarget struct { 24 | ID string 25 | FlyteClient flyteclient.Interface 26 | Client client.Client 27 | DynamicClient dynamic.Interface 28 | Enabled bool 29 | Config restclient.Config 30 | } 31 | 32 | func (e ExecutionTarget) Compare(to random.Comparable) bool { 33 | return e.ID < to.(ExecutionTarget).ID 34 | } 35 | -------------------------------------------------------------------------------- /pkg/executioncluster/impl/factory.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | executioncluster_interface "github.com/flyteorg/flyteadmin/pkg/executioncluster/interfaces" 5 | repositoryInterfaces "github.com/flyteorg/flyteadmin/pkg/repositories/interfaces" 6 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 7 | "github.com/flyteorg/flytestdlib/promutils" 8 | ) 9 | 10 | func GetExecutionCluster(scope promutils.Scope, kubeConfig, master string, config interfaces.Configuration, db repositoryInterfaces.Repository) executioncluster_interface.ClusterInterface { 11 | initializationErrorCounter := scope.MustNewCounter( 12 | "flyteclient_initialization_error", 13 | "count of errors encountered initializing a flyte client from kube config") 14 | switch len(config.ClusterConfiguration().GetClusterConfigs()) { 15 | case 0: 16 | cluster, err := NewInCluster(initializationErrorCounter, kubeConfig, master) 17 | if err != nil { 18 | panic(err) 19 | } 20 | return cluster 21 | default: 22 | listTargetsProvider, err := NewListTargets(initializationErrorCounter, NewExecutionTargetProvider(), config.ClusterConfiguration()) 23 | if err != nil { 24 | panic(err) 25 | } 26 | cluster, err := NewRandomClusterSelector(listTargetsProvider, config, db) 27 | if err != nil { 28 | panic(err) 29 | } 30 | return cluster 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /pkg/executioncluster/impl/testdata/clusters_config.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | labelClusterMap: 3 | test: 4 | - id: testcluster 5 | weight: 1 6 | all: 7 | - id: testcluster2 8 | weight: 0.5 9 | - id: testcluster3 10 | weight: 0.5 11 | clusterConfigs: 12 | - name: "testcluster" 13 | endpoint: "testcluster_endpoint" 14 | auth: 15 | type: "file_path" 16 | tokenPath: "/path/to/testcluster/token" 17 | certPath: "/path/to/testcluster/cert" 18 | - name: "testcluster2" 19 | endpoint: "testcluster2_endpoint" 20 | enabled: true 21 | auth: 22 | type: "file_path" 23 | tokenPath: "/path/to/testcluster2/token" 24 | certPath: "/path/to/testcluster2/cert" 25 | - name: "testcluster3" 26 | endpoint: "testcluster3_endpoint" 27 | enabled: true 28 | auth: 29 | type: "file_path" 30 | tokenPath: "/path/to/testcluster3/token" 31 | certPath: "/path/to/testcluster3/cert" 32 | -------------------------------------------------------------------------------- /pkg/executioncluster/impl/testdata/clusters_config2.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | labelClusterMap: 3 | one: 4 | - id: testcluster1 5 | weight: 1 6 | two: 7 | - id: testcluster2 8 | weight: 1 9 | three: 10 | - id: testcluster3 11 | weight: 1 12 | clusterConfigs: 13 | - name: "testcluster1" 14 | endpoint: "testcluster1_endpoint" 15 | enabled: true 16 | auth: 17 | type: "file_path" 18 | tokenPath: "/path/to/testcluster1/token" 19 | certPath: "/path/to/testcluster1/cert" 20 | - name: "testcluster2" 21 | endpoint: "testcluster2_endpoint" 22 | enabled: true 23 | auth: 24 | type: "file_path" 25 | tokenPath: "/path/to/testcluster2/token" 26 | certPath: "/path/to/testcluster2/cert" 27 | - name: "testcluster3" 28 | endpoint: "testcluster2_endpoint" 29 | enabled: true 30 | auth: 31 | type: "file_path" 32 | tokenPath: "/path/to/testcluster2/token" 33 | certPath: "/path/to/testcluster2/cert" 34 | 35 | -------------------------------------------------------------------------------- /pkg/executioncluster/impl/testdata/clusters_config2_default_label.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | defaultExecutionLabel: one 3 | labelClusterMap: 4 | one: 5 | - id: testcluster1 6 | weight: 1 7 | two: 8 | - id: testcluster2 9 | weight: 1 10 | three: 11 | - id: testcluster3 12 | weight: 1 13 | clusterConfigs: 14 | - name: "testcluster1" 15 | endpoint: "testcluster1_endpoint" 16 | enabled: true 17 | auth: 18 | type: "file_path" 19 | tokenPath: "/path/to/testcluster1/token" 20 | certPath: "/path/to/testcluster1/cert" 21 | - name: "testcluster2" 22 | endpoint: "testcluster2_endpoint" 23 | enabled: true 24 | auth: 25 | type: "file_path" 26 | tokenPath: "/path/to/testcluster2/token" 27 | certPath: "/path/to/testcluster2/cert" 28 | - name: "testcluster3" 29 | endpoint: "testcluster2_endpoint" 30 | enabled: true 31 | auth: 32 | type: "file_path" 33 | tokenPath: "/path/to/testcluster2/token" 34 | certPath: "/path/to/testcluster2/cert" 35 | 36 | -------------------------------------------------------------------------------- /pkg/executioncluster/interfaces/cluster.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/executioncluster" 7 | ) 8 | 9 | type GetTargetInterface interface { 10 | GetTarget(context.Context, *executioncluster.ExecutionTargetSpec) (*executioncluster.ExecutionTarget, error) 11 | } 12 | 13 | type ListTargetsInterface interface { 14 | GetAllTargets() map[string]*executioncluster.ExecutionTarget 15 | // Returns all enabled targets. 16 | GetValidTargets() map[string]*executioncluster.ExecutionTarget 17 | } 18 | 19 | // Interface for the Execution Cluster 20 | type ClusterInterface interface { 21 | GetTargetInterface 22 | ListTargetsInterface 23 | } 24 | -------------------------------------------------------------------------------- /pkg/executioncluster/interfaces/execution_target_provider.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/executioncluster" 5 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | //go:generate mockery -all -case=underscore -output=../mocks -case=underscore 10 | 11 | type ExecutionTargetProvider interface { 12 | GetExecutionTarget(initializationErrorCounter prometheus.Counter, k8sCluster interfaces.ClusterConfig) (*executioncluster.ExecutionTarget, error) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/manager/impl/shared/constants.go: -------------------------------------------------------------------------------- 1 | // Shared constants for the manager implementation. 2 | package shared 3 | 4 | // Field names for reference 5 | const ( 6 | Project = "project" 7 | Domain = "domain" 8 | Name = "name" 9 | ID = "id" 10 | Version = "version" 11 | ResourceType = "resource_type" 12 | Spec = "spec" 13 | Type = "type" 14 | RuntimeVersion = "runtime version" 15 | Metadata = "metadata" 16 | TypedInterface = "typed interface" 17 | Image = "image" 18 | Limit = "limit" 19 | Filters = "filters" 20 | ExpectedInputs = "expected_inputs" 21 | FixedInputs = "fixed_inputs" 22 | DefaultInputs = "default_inputs" 23 | Inputs = "inputs" 24 | State = "state" 25 | ExecutionID = "execution_id" 26 | NodeID = "node_id" 27 | NodeExecutionID = "node_execution_id" 28 | TaskID = "task_id" 29 | OccurredAt = "occurred_at" 30 | Event = "event" 31 | ParentTaskExecutionID = "parent_task_execution_id" 32 | UserInputs = "user_inputs" 33 | Attributes = "attributes" 34 | MatchingAttributes = "matching_attributes" 35 | // Parent of a node execution in the node executions table 36 | ParentID = "parent_id" 37 | WorkflowClosure = "workflow_closure" 38 | ) 39 | -------------------------------------------------------------------------------- /pkg/manager/impl/shared/errors.go: -------------------------------------------------------------------------------- 1 | // Convenience methods for shared errors. 2 | package shared 3 | 4 | import ( 5 | "fmt" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/errors" 8 | 9 | "google.golang.org/grpc/codes" 10 | ) 11 | 12 | const missingFieldFormat = "missing %s" 13 | const invalidArgFormat = "invalid value for %s" 14 | 15 | func GetMissingArgumentError(field string) error { 16 | return errors.NewFlyteAdminError(codes.InvalidArgument, fmt.Sprintf(missingFieldFormat, field)) 17 | } 18 | 19 | func GetInvalidArgumentError(field string) error { 20 | return errors.NewFlyteAdminError(codes.InvalidArgument, fmt.Sprintf(invalidArgFormat, field)) 21 | } 22 | -------------------------------------------------------------------------------- /pkg/manager/impl/test_constants.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 5 | ) 6 | 7 | const project = "project" 8 | const domain = "domain" 9 | const name = "name" 10 | const version = "version" 11 | const description = "description" 12 | const resourceType = core.ResourceType_WORKFLOW 13 | -------------------------------------------------------------------------------- /pkg/manager/impl/testutils/attributes.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 4 | 5 | var ExecutionQueueAttributes = &admin.MatchingAttributes{ 6 | Target: &admin.MatchingAttributes_ExecutionQueueAttributes{ 7 | ExecutionQueueAttributes: &admin.ExecutionQueueAttributes{ 8 | Tags: []string{ 9 | "foo", "bar", "baz", 10 | }, 11 | }, 12 | }, 13 | } 14 | 15 | var WorkflowExecutionConfigSample = &admin.MatchingAttributes{ 16 | Target: &admin.MatchingAttributes_WorkflowExecutionConfig{ 17 | WorkflowExecutionConfig: &admin.WorkflowExecutionConfig{ 18 | MaxParallelism: 5, 19 | RawOutputDataConfig: &admin.RawOutputDataConfig{ 20 | OutputLocationPrefix: "s3://test-bucket", 21 | }, 22 | Labels: &admin.Labels{ 23 | Values: map[string]string{"lab1": "val1"}, 24 | }, 25 | Annotations: nil, 26 | }, 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /pkg/manager/impl/testutils/config.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/common" 5 | runtimeInterfaces "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 6 | runtimeMocks "github.com/flyteorg/flyteadmin/pkg/runtime/mocks" 7 | ) 8 | 9 | func GetApplicationConfigWithDefaultDomains() runtimeInterfaces.ApplicationConfiguration { 10 | config := runtimeMocks.MockApplicationProvider{} 11 | config.SetDomainsConfig(runtimeInterfaces.DomainsConfig{ 12 | { 13 | ID: "development", 14 | Name: "development", 15 | }, 16 | { 17 | ID: "staging", 18 | Name: "staging", 19 | }, 20 | { 21 | ID: "production", 22 | Name: "production", 23 | }, 24 | { 25 | ID: "domain", 26 | Name: "domain", 27 | }, 28 | }) 29 | config.SetRemoteDataConfig(runtimeInterfaces.RemoteDataConfig{ 30 | Scheme: common.Local, SignedURL: runtimeInterfaces.SignedURL{ 31 | Enabled: true, 32 | }}) 33 | return &config 34 | } 35 | -------------------------------------------------------------------------------- /pkg/manager/impl/testutils/constants.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | const ( 4 | ProjectQueryPattern = "project = ?" 5 | DomainQueryPattern = "domain = ?" 6 | NameQueryPattern = "name = ?" 7 | StateQueryPattern = "state = ?" 8 | ) 9 | -------------------------------------------------------------------------------- /pkg/manager/impl/testutils/repository.go: -------------------------------------------------------------------------------- 1 | package testutils 2 | 3 | import ( 4 | "context" 5 | 6 | repositoryInterfaces "github.com/flyteorg/flyteadmin/pkg/repositories/interfaces" 7 | 8 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 9 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 10 | 11 | repositoryMocks "github.com/flyteorg/flyteadmin/pkg/repositories/mocks" 12 | ) 13 | 14 | func GetRepoWithDefaultProjectAndErr(err error) repositoryInterfaces.Repository { 15 | repo := repositoryMocks.NewMockRepository() 16 | repo.ProjectRepo().(*repositoryMocks.MockProjectRepo).GetFunction = func( 17 | ctx context.Context, projectID string) (models.Project, error) { 18 | activeState := int32(admin.Project_ACTIVE) 19 | return models.Project{State: &activeState}, err 20 | } 21 | return repo 22 | } 23 | 24 | func GetRepoWithDefaultProject() repositoryInterfaces.Repository { 25 | repo := repositoryMocks.NewMockRepository() 26 | repo.ProjectRepo().(*repositoryMocks.MockProjectRepo).GetFunction = func( 27 | ctx context.Context, projectID string) (models.Project, error) { 28 | activeState := int32(admin.Project_ACTIVE) 29 | return models.Project{State: &activeState}, nil 30 | } 31 | return repo 32 | } 33 | -------------------------------------------------------------------------------- /pkg/manager/impl/validation/notifications_validator.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/common" 5 | "github.com/flyteorg/flyteadmin/pkg/manager/impl/shared" 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // TODO: maybe add more sophisticated email validation. 10 | func validateRecipientsEmail(recipients []string) error { 11 | if len(recipients) == 0 { 12 | return shared.GetMissingArgumentError("recipients") 13 | } 14 | for _, recipient := range recipients { 15 | if len(recipient) == 0 { 16 | return shared.GetMissingArgumentError("recipient") 17 | } 18 | } 19 | return nil 20 | } 21 | 22 | func validateNotifications(notifications []*admin.Notification) error { 23 | for _, notif := range notifications { 24 | switch { 25 | case notif.GetEmail() != nil: 26 | if err := validateRecipientsEmail(notif.GetEmail().RecipientsEmail); err != nil { 27 | return err 28 | } 29 | case notif.GetSlack() != nil: 30 | if err := validateRecipientsEmail(notif.GetSlack().RecipientsEmail); err != nil { 31 | return err 32 | } 33 | case notif.GetPagerDuty() != nil: 34 | if err := validateRecipientsEmail(notif.GetPagerDuty().RecipientsEmail); err != nil { 35 | return err 36 | } 37 | default: 38 | return shared.GetInvalidArgumentError("notification type") 39 | } 40 | 41 | for _, phase := range notif.Phases { 42 | if !common.IsExecutionTerminal(phase) { 43 | return shared.GetInvalidArgumentError("phase") 44 | } 45 | } 46 | } 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /pkg/manager/impl/version_manager.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/manager/interfaces" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 8 | adminversion "github.com/flyteorg/flytestdlib/version" 9 | ) 10 | 11 | type VersionManager struct { 12 | Version string 13 | Build string 14 | BuildTime string 15 | } 16 | 17 | func (v *VersionManager) GetVersion(ctx context.Context, r *admin.GetVersionRequest) (*admin.GetVersionResponse, error) { 18 | return &admin.GetVersionResponse{ 19 | ControlPlaneVersion: &admin.Version{ 20 | Version: v.Version, 21 | Build: v.Build, 22 | BuildTime: v.BuildTime, 23 | }, 24 | }, nil 25 | } 26 | 27 | func NewVersionManager() interfaces.VersionInterface { 28 | return &VersionManager{ 29 | Build: adminversion.Build, 30 | Version: adminversion.Version, 31 | BuildTime: adminversion.BuildTime, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/manager/impl/version_manager_test.go: -------------------------------------------------------------------------------- 1 | package impl 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 8 | adminversion "github.com/flyteorg/flytestdlib/version" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | var ( 13 | build = "fa70a14" 14 | buildTime = "2021-03-25" 15 | appversion = "v0.3.7-47-gfa70a14" 16 | ) 17 | 18 | func TestVersionManager_GetVersion(t *testing.T) { 19 | adminversion.Build = build 20 | adminversion.BuildTime = buildTime 21 | adminversion.Version = appversion 22 | vmanager := NewVersionManager() 23 | 24 | v, err := vmanager.GetVersion(context.Background(), &admin.GetVersionRequest{}) 25 | assert.Nil(t, err) 26 | assert.Equal(t, v.ControlPlaneVersion.BuildTime, buildTime) 27 | assert.Equal(t, v.ControlPlaneVersion.Build, build) 28 | assert.Equal(t, v.ControlPlaneVersion.Version, appversion) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/description_entity.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // DescriptionEntityInterface for managing DescriptionEntity 10 | type DescriptionEntityInterface interface { 11 | GetDescriptionEntity(ctx context.Context, request admin.ObjectGetRequest) (*admin.DescriptionEntity, error) 12 | ListDescriptionEntity(ctx context.Context, request admin.DescriptionEntityListRequest) (*admin.DescriptionEntityList, error) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/launch_plan.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing Flyte Launch Plans 10 | type LaunchPlanInterface interface { 11 | // Interface to create Launch Plans based on the request. 12 | CreateLaunchPlan(ctx context.Context, request admin.LaunchPlanCreateRequest) ( 13 | *admin.LaunchPlanCreateResponse, error) 14 | UpdateLaunchPlan(ctx context.Context, request admin.LaunchPlanUpdateRequest) ( 15 | *admin.LaunchPlanUpdateResponse, error) 16 | GetLaunchPlan(ctx context.Context, request admin.ObjectGetRequest) ( 17 | *admin.LaunchPlan, error) 18 | GetActiveLaunchPlan(ctx context.Context, request admin.ActiveLaunchPlanRequest) ( 19 | *admin.LaunchPlan, error) 20 | ListLaunchPlans(ctx context.Context, request admin.ResourceListRequest) ( 21 | *admin.LaunchPlanList, error) 22 | ListActiveLaunchPlans(ctx context.Context, request admin.ActiveLaunchPlanListRequest) ( 23 | *admin.LaunchPlanList, error) 24 | ListLaunchPlanIds(ctx context.Context, request admin.NamedEntityIdentifierListRequest) ( 25 | *admin.NamedEntityIdentifierList, error) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/metrics.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | //go:generate mockery -name=MetricsInterface -output=../mocks -case=underscore 10 | 11 | // Interface for managing Flyte execution metrics 12 | type MetricsInterface interface { 13 | GetExecutionMetrics(ctx context.Context, request admin.WorkflowExecutionGetMetricsRequest) ( 14 | *admin.WorkflowExecutionGetMetricsResponse, error) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/named_entity.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing metadata associated with NamedEntityIdentifiers 10 | type NamedEntityInterface interface { 11 | GetNamedEntity(ctx context.Context, request admin.NamedEntityGetRequest) (*admin.NamedEntity, error) 12 | UpdateNamedEntity(ctx context.Context, request admin.NamedEntityUpdateRequest) (*admin.NamedEntityUpdateResponse, error) 13 | ListNamedEntities(ctx context.Context, request admin.NamedEntityListRequest) (*admin.NamedEntityList, error) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/node_execution.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing Flyte Workflow NodeExecutions 10 | type NodeExecutionInterface interface { 11 | CreateNodeEvent(ctx context.Context, request admin.NodeExecutionEventRequest) ( 12 | *admin.NodeExecutionEventResponse, error) 13 | GetNodeExecution(ctx context.Context, request admin.NodeExecutionGetRequest) (*admin.NodeExecution, error) 14 | ListNodeExecutions(ctx context.Context, request admin.NodeExecutionListRequest) (*admin.NodeExecutionList, error) 15 | ListNodeExecutionsForTask(ctx context.Context, request admin.NodeExecutionForTaskListRequest) (*admin.NodeExecutionList, error) 16 | GetNodeExecutionData( 17 | ctx context.Context, request admin.NodeExecutionGetDataRequest) (*admin.NodeExecutionGetDataResponse, error) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/project.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing projects (and domains). 10 | type ProjectInterface interface { 11 | CreateProject(ctx context.Context, request admin.ProjectRegisterRequest) (*admin.ProjectRegisterResponse, error) 12 | ListProjects(ctx context.Context, request admin.ProjectListRequest) (*admin.Projects, error) 13 | UpdateProject(ctx context.Context, request admin.Project) (*admin.ProjectUpdateResponse, error) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/signal.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | //go:generate mockery -name=SignalInterface -output=../mocks -case=underscore 10 | 11 | // Interface for managing Flyte Signals 12 | type SignalInterface interface { 13 | GetOrCreateSignal(ctx context.Context, request admin.SignalGetOrCreateRequest) (*admin.Signal, error) 14 | ListSignals(ctx context.Context, request admin.SignalListRequest) (*admin.SignalList, error) 15 | SetSignal(ctx context.Context, request admin.SignalSetRequest) (*admin.SignalSetResponse, error) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/task.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing Flyte Tasks 10 | type TaskInterface interface { 11 | CreateTask(ctx context.Context, request admin.TaskCreateRequest) (*admin.TaskCreateResponse, error) 12 | GetTask(ctx context.Context, request admin.ObjectGetRequest) (*admin.Task, error) 13 | ListTasks(ctx context.Context, request admin.ResourceListRequest) (*admin.TaskList, error) 14 | ListUniqueTaskIdentifiers(ctx context.Context, request admin.NamedEntityIdentifierListRequest) ( 15 | *admin.NamedEntityIdentifierList, error) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/task_execution.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing Flyte Workflow TaskExecutions 10 | type TaskExecutionInterface interface { 11 | CreateTaskExecutionEvent(ctx context.Context, request admin.TaskExecutionEventRequest) ( 12 | *admin.TaskExecutionEventResponse, error) 13 | GetTaskExecution(ctx context.Context, request admin.TaskExecutionGetRequest) (*admin.TaskExecution, error) 14 | ListTaskExecutions(ctx context.Context, request admin.TaskExecutionListRequest) (*admin.TaskExecutionList, error) 15 | GetTaskExecutionData( 16 | ctx context.Context, request admin.TaskExecutionGetDataRequest) (*admin.TaskExecutionGetDataResponse, error) 17 | } 18 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/version.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | //go:generate mockery -name VersionInterface -output=../mocks -case=underscore 10 | 11 | // Interface for managing Flyte admin version 12 | type VersionInterface interface { 13 | GetVersion(ctx context.Context, r *admin.GetVersionRequest) (*admin.GetVersionResponse, error) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/manager/interfaces/workflow.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | // Interface for managing Flyte Workflows 10 | type WorkflowInterface interface { 11 | CreateWorkflow(ctx context.Context, request admin.WorkflowCreateRequest) (*admin.WorkflowCreateResponse, error) 12 | GetWorkflow(ctx context.Context, request admin.ObjectGetRequest) (*admin.Workflow, error) 13 | ListWorkflows(ctx context.Context, request admin.ResourceListRequest) (*admin.WorkflowList, error) 14 | ListWorkflowIdentifiers(ctx context.Context, request admin.NamedEntityIdentifierListRequest) ( 15 | *admin.NamedEntityIdentifierList, error) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/manager/mocks/named_entity.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | type GetNamedEntityFunc func(ctx context.Context, request admin.NamedEntityGetRequest) (*admin.NamedEntity, error) 10 | type UpdateNamedEntityFunc func(ctx context.Context, request admin.NamedEntityUpdateRequest) (*admin.NamedEntityUpdateResponse, error) 11 | type ListNamedEntitiesFunc func(ctx context.Context, request admin.NamedEntityListRequest) (*admin.NamedEntityList, error) 12 | 13 | type NamedEntityManager struct { 14 | GetNamedEntityFunc GetNamedEntityFunc 15 | UpdateNamedEntityFunc UpdateNamedEntityFunc 16 | ListNamedEntitiesFunc ListNamedEntitiesFunc 17 | } 18 | 19 | func (m *NamedEntityManager) GetNamedEntity(ctx context.Context, request admin.NamedEntityGetRequest) (*admin.NamedEntity, error) { 20 | if m.GetNamedEntityFunc != nil { 21 | return m.GetNamedEntityFunc(ctx, request) 22 | } 23 | return nil, nil 24 | } 25 | 26 | func (m *NamedEntityManager) UpdateNamedEntity(ctx context.Context, request admin.NamedEntityUpdateRequest) (*admin.NamedEntityUpdateResponse, error) { 27 | if m.UpdateNamedEntityFunc != nil { 28 | return m.UpdateNamedEntityFunc(ctx, request) 29 | } 30 | return nil, nil 31 | } 32 | 33 | func (m *NamedEntityManager) ListNamedEntities(ctx context.Context, request admin.NamedEntityListRequest) (*admin.NamedEntityList, error) { 34 | if m.ListNamedEntitiesFunc != nil { 35 | return m.ListNamedEntitiesFunc(ctx, request) 36 | } 37 | return nil, nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/repositories/config/seed_data.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 8 | "github.com/flyteorg/flytestdlib/logger" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | // Returns a function to seed the database with default values. 13 | func SeedProjects(db *gorm.DB, projects []string) error { 14 | tx := db.Begin() 15 | for _, project := range projects { 16 | projectModel := models.Project{ 17 | Identifier: project, 18 | Name: project, 19 | Description: fmt.Sprintf("%s description", project), 20 | } 21 | if err := tx.Where(models.Project{Identifier: project}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil { 22 | logger.Warningf(context.Background(), "failed to save project [%s]", project) 23 | tx.Rollback() 24 | return err 25 | } 26 | } 27 | return tx.Commit().Error 28 | } 29 | -------------------------------------------------------------------------------- /pkg/repositories/database_integration_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package repositories 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/flyteorg/flyteadmin/pkg/config" 10 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestInsertionOfTask(t *testing.T) { 15 | // Because this is an integration test, we'll need to load the version of configuration that's actually being run 16 | // instead of the sample one in this repo 17 | config.Init("/etc/flyte/config/flyteadmin_config.yaml") 18 | 19 | repository := GetRepository(POSTGRES) 20 | taskRepository := repository.TaskRepo() 21 | 22 | err := taskRepository.Create(models.Task{ 23 | Name: "sometesttask", 24 | Closure: []byte("in bytes"), 25 | Domain: "testdev", 26 | Project: "flyte", 27 | Version: "0.0.0", 28 | }) 29 | 30 | assert.NoError(t, err) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/repositories/errors/error_transformer.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | admin_errors "github.com/flyteorg/flyteadmin/pkg/errors" 5 | ) 6 | 7 | // Defines the basic error transformer interface that all database types must implement. 8 | type ErrorTransformer interface { 9 | ToFlyteAdminError(err error) admin_errors.FlyteAdminError 10 | } 11 | -------------------------------------------------------------------------------- /pkg/repositories/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Generic errors used across files in repositories/ 2 | package errors 3 | 4 | import ( 5 | "github.com/flyteorg/flyteadmin/pkg/errors" 6 | "github.com/golang/protobuf/proto" 7 | "google.golang.org/grpc/codes" 8 | ) 9 | 10 | const ( 11 | singletonNotFound = "missing singleton entity of type %s" 12 | notFound = "missing entity of type %s with identifier %v" 13 | idNotFound = "missing entity of type %s" 14 | invalidInput = "missing and/or invalid parameters: %s" 15 | ) 16 | 17 | func GetMissingEntityError(entityType string, identifier proto.Message) errors.FlyteAdminError { 18 | return errors.NewFlyteAdminErrorf(codes.NotFound, notFound, entityType, identifier) 19 | } 20 | 21 | func GetSingletonMissingEntityError(entityType string) errors.FlyteAdminError { 22 | return errors.NewFlyteAdminErrorf(codes.NotFound, singletonNotFound, entityType) 23 | } 24 | 25 | func GetMissingEntityByIDError(entityType string) errors.FlyteAdminError { 26 | return errors.NewFlyteAdminErrorf(codes.NotFound, idNotFound, entityType) 27 | } 28 | 29 | func GetInvalidInputError(input string) errors.FlyteAdminError { 30 | return errors.NewFlyteAdminErrorf(codes.InvalidArgument, invalidInput, input) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/repositories/errors/postgres_test.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | mockScope "github.com/flyteorg/flytestdlib/promutils" 8 | 9 | "github.com/jackc/pgconn" 10 | "github.com/magiconair/properties/assert" 11 | "google.golang.org/grpc/codes" 12 | ) 13 | 14 | func TestToFlyteAdminError_InvalidPqError(t *testing.T) { 15 | err := errors.New("foo") 16 | transformedErr := NewPostgresErrorTransformer(mockScope.NewTestScope()).ToFlyteAdminError(err) 17 | assert.Equal(t, codes.Internal, transformedErr.Code()) 18 | assert.Equal(t, "unexpected error type for: foo", transformedErr.Error()) 19 | } 20 | 21 | func TestToFlyteAdminError_UniqueConstraintViolation(t *testing.T) { 22 | err := &pgconn.PgError{ 23 | Code: "23505", 24 | Message: "message", 25 | } 26 | transformedErr := NewPostgresErrorTransformer(mockScope.NewTestScope()).ToFlyteAdminError(err) 27 | assert.Equal(t, codes.AlreadyExists, transformedErr.Code()) 28 | assert.Equal(t, "value with matching already exists (message)", 29 | transformedErr.Error()) 30 | } 31 | 32 | func TestToFlyteAdminError_UnrecognizedPostgresError(t *testing.T) { 33 | err := &pgconn.PgError{ 34 | Code: "foo", 35 | Message: "message", 36 | } 37 | transformedErr := NewPostgresErrorTransformer(mockScope.NewTestScope()).ToFlyteAdminError(err) 38 | assert.Equal(t, codes.Unknown, transformedErr.Code()) 39 | assert.Equal(t, "failed database operation with message", 40 | transformedErr.Error()) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/repositories/errors/test_error_transformer.go: -------------------------------------------------------------------------------- 1 | // Implementation of an error transformer for test. 2 | package errors 3 | 4 | import ( 5 | "github.com/flyteorg/flyteadmin/pkg/errors" 6 | "google.golang.org/grpc/codes" 7 | ) 8 | 9 | type transformFunc func(err error) errors.FlyteAdminError 10 | 11 | type testErrorTransformer struct { 12 | transformers []transformFunc 13 | } 14 | 15 | // Special method for test. 16 | func (t *testErrorTransformer) AddTransform(transformer transformFunc) { 17 | t.transformers = append(t.transformers, transformer) 18 | } 19 | 20 | func (t *testErrorTransformer) ToFlyteAdminError(err error) errors.FlyteAdminError { 21 | for _, transformer := range t.transformers { 22 | if adminErr := transformer(err); adminErr != nil { 23 | return adminErr 24 | } 25 | } 26 | return errors.NewFlyteAdminError(codes.Unknown, "Test transformer failed to find transformation to apply") 27 | } 28 | 29 | func NewTestErrorTransformer() ErrorTransformer { 30 | return &testErrorTransformer{} 31 | } 32 | -------------------------------------------------------------------------------- /pkg/repositories/gormimpl/execution_event_repo.go: -------------------------------------------------------------------------------- 1 | package gormimpl 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/errors" 7 | "github.com/flyteorg/flyteadmin/pkg/repositories/interfaces" 8 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 9 | "github.com/flyteorg/flytestdlib/promutils" 10 | 11 | "gorm.io/gorm" 12 | ) 13 | 14 | type ExecutionEventRepo struct { 15 | db *gorm.DB 16 | errorTransformer errors.ErrorTransformer 17 | metrics gormMetrics 18 | } 19 | 20 | func (r *ExecutionEventRepo) Create(ctx context.Context, input models.ExecutionEvent) error { 21 | timer := r.metrics.CreateDuration.Start() 22 | tx := r.db.Omit("id").Create(&input) 23 | timer.Stop() 24 | if tx.Error != nil { 25 | return r.errorTransformer.ToFlyteAdminError(tx.Error) 26 | } 27 | return nil 28 | } 29 | 30 | // Returns an instance of ExecutionRepoInterface 31 | func NewExecutionEventRepo( 32 | db *gorm.DB, errorTransformer errors.ErrorTransformer, scope promutils.Scope) interfaces.ExecutionEventRepoInterface { 33 | metrics := newMetrics(scope) 34 | return &ExecutionEventRepo{ 35 | db: db, 36 | errorTransformer: errorTransformer, 37 | metrics: metrics, 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/repositories/gormimpl/execution_event_repo_test.go: -------------------------------------------------------------------------------- 1 | package gormimpl 2 | 3 | import ( 4 | "context" 5 | "database/sql/driver" 6 | "testing" 7 | "time" 8 | 9 | mocket "github.com/Selvatico/go-mocket" 10 | "github.com/flyteorg/flyteadmin/pkg/repositories/errors" 11 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 12 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 13 | mockScope "github.com/flyteorg/flytestdlib/promutils" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestCreateExecutionEvent(t *testing.T) { 18 | GlobalMock := mocket.Catcher.Reset() 19 | GlobalMock.Logging = true 20 | created := false 21 | 22 | // Only match on queries that append expected filters 23 | GlobalMock.NewMock().WithQuery(`INSERT INTO "execution_events" ("created_at","updated_at","deleted_at","execution_project","execution_domain","execution_name","request_id","occurred_at","phase") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`).WithCallback( 24 | func(s string, values []driver.NamedValue) { 25 | created = true 26 | }, 27 | ) 28 | execEventRepo := NewExecutionEventRepo(GetDbForTest(t), errors.NewTestErrorTransformer(), mockScope.NewTestScope()) 29 | err := execEventRepo.Create(context.Background(), models.ExecutionEvent{ 30 | RequestID: "request id 1", 31 | ExecutionKey: models.ExecutionKey{ 32 | Project: "project", 33 | Domain: "domain", 34 | Name: "1", 35 | }, 36 | OccurredAt: time.Now(), 37 | Phase: core.WorkflowExecution_SUCCEEDED.String(), 38 | }) 39 | assert.NoError(t, err) 40 | assert.True(t, created) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/repositories/gormimpl/node_execution_event_repo.go: -------------------------------------------------------------------------------- 1 | package gormimpl 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flytestdlib/promutils" 7 | "gorm.io/gorm" 8 | 9 | "github.com/flyteorg/flyteadmin/pkg/repositories/errors" 10 | "github.com/flyteorg/flyteadmin/pkg/repositories/interfaces" 11 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 12 | ) 13 | 14 | type NodeExecutionEventRepo struct { 15 | db *gorm.DB 16 | errorTransformer errors.ErrorTransformer 17 | metrics gormMetrics 18 | } 19 | 20 | func (r *NodeExecutionEventRepo) Create(ctx context.Context, input models.NodeExecutionEvent) error { 21 | timer := r.metrics.CreateDuration.Start() 22 | tx := r.db.Omit("id").Create(&input) 23 | timer.Stop() 24 | if tx.Error != nil { 25 | return r.errorTransformer.ToFlyteAdminError(tx.Error) 26 | } 27 | return nil 28 | } 29 | 30 | // Returns an instance of NodeExecutionRepoInterface 31 | func NewNodeExecutionEventRepo( 32 | db *gorm.DB, errorTransformer errors.ErrorTransformer, scope promutils.Scope) interfaces.NodeExecutionEventRepoInterface { 33 | metrics := newMetrics(scope) 34 | return &NodeExecutionEventRepo{ 35 | db: db, 36 | errorTransformer: errorTransformer, 37 | metrics: metrics, 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/repositories/gormimpl/node_execution_event_repo_test.go: -------------------------------------------------------------------------------- 1 | package gormimpl 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | mocket "github.com/Selvatico/go-mocket" 8 | "github.com/flyteorg/flyteadmin/pkg/repositories/errors" 9 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 10 | mockScope "github.com/flyteorg/flytestdlib/promutils" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestCreateNodeExecutionEvent(t *testing.T) { 15 | GlobalMock := mocket.Catcher.Reset() 16 | nodeExecutionEventQuery := GlobalMock.NewMock() 17 | nodeExecutionEventQuery.WithQuery(`INSERT INTO "node_execution_events" ("created_at","updated_at","deleted_at","execution_project","execution_domain","execution_name","node_id","request_id","occurred_at","phase") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`) 18 | nodeExecEventRepo := NewNodeExecutionEventRepo(GetDbForTest(t), errors.NewTestErrorTransformer(), mockScope.NewTestScope()) 19 | err := nodeExecEventRepo.Create(context.Background(), models.NodeExecutionEvent{ 20 | NodeExecutionKey: models.NodeExecutionKey{ 21 | NodeID: "1", 22 | ExecutionKey: models.ExecutionKey{ 23 | Project: "project", 24 | Domain: "domain", 25 | Name: "1", 26 | }, 27 | }, 28 | RequestID: "xxyzz", 29 | Phase: nodePhase, 30 | OccurredAt: nodeStartedAt, 31 | }) 32 | assert.NoError(t, err) 33 | assert.True(t, nodeExecutionEventQuery.Triggered) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/repositories/gormimpl/utils_for_test.go: -------------------------------------------------------------------------------- 1 | // Shared utils for postgresql tests. 2 | package gormimpl 3 | 4 | import ( 5 | "testing" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/common" 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 9 | 10 | mocket "github.com/Selvatico/go-mocket" 11 | "gorm.io/driver/postgres" 12 | "gorm.io/gorm" 13 | ) 14 | 15 | const project = "project" 16 | const domain = "domain" 17 | const name = "name" 18 | const description = "description" 19 | const resourceType = core.ResourceType_WORKFLOW 20 | const version = "XYZ" 21 | 22 | func GetDbForTest(t *testing.T) *gorm.DB { 23 | mocket.Catcher.Register() 24 | db, err := gorm.Open(postgres.New(postgres.Config{DriverName: mocket.DriverName})) 25 | if err != nil { 26 | t.Fatalf("Failed to open mock db with err %v", err) 27 | } 28 | return db 29 | } 30 | 31 | func getEqualityFilter(entity common.Entity, field string, value interface{}) common.InlineFilter { 32 | filter, _ := common.NewSingleValueFilter(entity, common.Equal, field, value) 33 | return filter 34 | } 35 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/description_entity.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 8 | ) 9 | 10 | type GetDescriptionEntityInput struct { 11 | ResourceType core.ResourceType 12 | Project string 13 | Domain string 14 | Name string 15 | Version string 16 | } 17 | 18 | type DescriptionEntityCollectionOutput struct { 19 | Entities []models.DescriptionEntity 20 | } 21 | 22 | // DescriptionEntityRepoInterface Defines the interface for interacting with Description models. 23 | type DescriptionEntityRepoInterface interface { 24 | // Get Returns a matching DescriptionEntity if it exists. 25 | Get(ctx context.Context, input GetDescriptionEntityInput) (models.DescriptionEntity, error) 26 | // List Returns DescriptionEntity matching query parameters. A limit must be provided for the results page size 27 | List(ctx context.Context, input ListResourceInput) (DescriptionEntityCollectionOutput, error) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/execution_event_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | //go:generate mockery -name=ExecutionEventRepoInterface -output=../mocks -case=underscore 10 | 11 | type ExecutionEventRepoInterface interface { 12 | // Inserts a workflow execution event into the database store. 13 | Create(ctx context.Context, input models.ExecutionEvent) error 14 | } 15 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/execution_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | // Defines the interface for interacting with workflow execution models. 10 | type ExecutionRepoInterface interface { 11 | // Inserts a workflow execution model into the database store. 12 | Create(ctx context.Context, input models.Execution) error 13 | // This updates only an existing execution model with all non-empty fields in the input. 14 | Update(ctx context.Context, execution models.Execution) error 15 | // Returns a matching execution if it exists. 16 | Get(ctx context.Context, input Identifier) (models.Execution, error) 17 | // Returns executions matching query parameters. A limit must be provided for the results page size. 18 | List(ctx context.Context, input ListResourceInput) (ExecutionCollectionOutput, error) 19 | // Returns count of executions matching query parameters. 20 | Count(ctx context.Context, input CountResourceInput) (int64, error) 21 | } 22 | 23 | // Response format for a query on workflows. 24 | type ExecutionCollectionOutput struct { 25 | Executions []models.Execution 26 | } 27 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/named_entity_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 7 | 8 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 9 | ) 10 | 11 | type GetNamedEntityInput struct { 12 | ResourceType core.ResourceType 13 | Project string 14 | Domain string 15 | Name string 16 | } 17 | 18 | // Parameters for querying multiple resources. 19 | type ListNamedEntityInput struct { 20 | ListResourceInput 21 | Project string 22 | Domain string 23 | ResourceType core.ResourceType 24 | } 25 | 26 | type NamedEntityCollectionOutput struct { 27 | Entities []models.NamedEntity 28 | } 29 | 30 | // Defines the interface for interacting with NamedEntity models 31 | type NamedEntityRepoInterface interface { 32 | // Returns NamedEntity objects matching the provided query. A limit is 33 | // required 34 | List(ctx context.Context, input ListNamedEntityInput) (NamedEntityCollectionOutput, error) 35 | // Updates NamedEntity record, will create metadata if it does not exist 36 | Update(ctx context.Context, input models.NamedEntity) error 37 | // Gets metadata (if available) associated with a NamedEntity 38 | Get(ctx context.Context, input GetNamedEntityInput) (models.NamedEntity, error) 39 | } 40 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/node_execution_event_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | //go:generate mockery -name=NodeExecutionEventRepoInterface -output=../mocks -case=underscore 10 | 11 | type NodeExecutionEventRepoInterface interface { 12 | // Inserts a node execution event into the database store. 13 | Create(ctx context.Context, input models.NodeExecutionEvent) error 14 | } 15 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/project_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | type ProjectRepoInterface interface { 10 | // Inserts a namespace model into the database store. 11 | Create(ctx context.Context, project models.Project) error 12 | // Returns a matching project when it exists. 13 | Get(ctx context.Context, projectID string) (models.Project, error) 14 | // Returns projects matching query parameters. 15 | List(ctx context.Context, input ListResourceInput) ([]models.Project, error) 16 | // Given a project that exists in the DB and a partial set of fields to update 17 | // as a second project (projectUpdate), updates the original project which already 18 | // exists in the DB. 19 | UpdateProject(ctx context.Context, projectUpdate models.Project) error 20 | } 21 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/repository.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | schedulerInterfaces "github.com/flyteorg/flyteadmin/scheduler/repositories/interfaces" 5 | "gorm.io/gorm" 6 | ) 7 | 8 | // The Repository indicates the methods that each Repository must support. 9 | // A Repository indicates a Database which is collection of Tables/models. 10 | // The goal is allow databases to be Plugged in easily. 11 | type Repository interface { 12 | TaskRepo() TaskRepoInterface 13 | WorkflowRepo() WorkflowRepoInterface 14 | LaunchPlanRepo() LaunchPlanRepoInterface 15 | ExecutionRepo() ExecutionRepoInterface 16 | ExecutionEventRepo() ExecutionEventRepoInterface 17 | ProjectRepo() ProjectRepoInterface 18 | ResourceRepo() ResourceRepoInterface 19 | NodeExecutionRepo() NodeExecutionRepoInterface 20 | NodeExecutionEventRepo() NodeExecutionEventRepoInterface 21 | TaskExecutionRepo() TaskExecutionRepoInterface 22 | NamedEntityRepo() NamedEntityRepoInterface 23 | DescriptionEntityRepo() DescriptionEntityRepoInterface 24 | SchedulableEntityRepo() schedulerInterfaces.SchedulableEntityRepoInterface 25 | ScheduleEntitiesSnapshotRepo() schedulerInterfaces.ScheduleEntitiesSnapShotRepoInterface 26 | SignalRepo() SignalRepoInterface 27 | 28 | GetGormDB() *gorm.DB 29 | } 30 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/resource_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | type ResourceRepoInterface interface { 10 | // Inserts or updates an existing Type model into the database store. 11 | CreateOrUpdate(ctx context.Context, input models.Resource) error 12 | // Returns a matching Type model based on hierarchical resolution. 13 | Get(ctx context.Context, ID ResourceID) (models.Resource, error) 14 | // Returns a matching Type model. 15 | GetRaw(ctx context.Context, ID ResourceID) (models.Resource, error) 16 | // GetProjectLevel returns the Project level resource entry, if any, even if there is a higher 17 | // specificity resource. 18 | GetProjectLevel(ctx context.Context, ID ResourceID) (models.Resource, error) 19 | // Lists all resources 20 | ListAll(ctx context.Context, resourceType string) ([]models.Resource, error) 21 | // Deletes a matching Type model when it exists. 22 | Delete(ctx context.Context, ID ResourceID) error 23 | } 24 | 25 | type ResourceID struct { 26 | Project string 27 | Domain string 28 | Workflow string 29 | LaunchPlan string 30 | ResourceType string 31 | } 32 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/signal_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 8 | ) 9 | 10 | //go:generate mockery -name=SignalRepoInterface -output=../mocks -case=underscore 11 | 12 | // Defines the interface for interacting with signal models. 13 | type SignalRepoInterface interface { 14 | // Get retrieves a signal model from the database store. 15 | Get(ctx context.Context, input models.SignalKey) (models.Signal, error) 16 | // GetOrCreate inserts a signal model into the database store or returns one if it already exists. 17 | GetOrCreate(ctx context.Context, input *models.Signal) error 18 | // List all signals that match the input values. 19 | List(ctx context.Context, input ListResourceInput) ([]models.Signal, error) 20 | // Update sets the value on a signal in the database store. 21 | Update(ctx context.Context, input models.SignalKey, value []byte) error 22 | } 23 | 24 | type GetSignalInput struct { 25 | SignalID core.SignalIdentifier 26 | } 27 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/task_execution_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 8 | ) 9 | 10 | // Defines the interface for interacting with task execution models. 11 | type TaskExecutionRepoInterface interface { 12 | // Inserts a task execution model into the database store. 13 | Create(ctx context.Context, input models.TaskExecution) error 14 | // Updates an existing task execution in the database store with all non-empty fields in the input. 15 | Update(ctx context.Context, execution models.TaskExecution) error 16 | // Returns a matching execution if it exists. 17 | Get(ctx context.Context, input GetTaskExecutionInput) (models.TaskExecution, error) 18 | // Returns task executions matching query parameters. A limit must be provided for the results page size. 19 | List(ctx context.Context, input ListResourceInput) (TaskExecutionCollectionOutput, error) 20 | // Returns count of task executions matching query parameters. 21 | Count(ctx context.Context, input CountResourceInput) (int64, error) 22 | } 23 | 24 | type GetTaskExecutionInput struct { 25 | TaskExecutionID core.TaskExecutionIdentifier 26 | } 27 | 28 | // Response format for a query on task executions. 29 | type TaskExecutionCollectionOutput struct { 30 | TaskExecutions []models.TaskExecution 31 | } 32 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/task_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | // Defines the interface for interacting with Task models. 10 | type TaskRepoInterface interface { 11 | // Inserts a task model into the database store. 12 | Create(ctx context.Context, input models.Task, descriptionEntity *models.DescriptionEntity) error 13 | // Returns a matching task if it exists. 14 | Get(ctx context.Context, input Identifier) (models.Task, error) 15 | // Returns task revisions matching query parameters. A limit must be provided for the results page size. 16 | List(ctx context.Context, input ListResourceInput) (TaskCollectionOutput, error) 17 | // Returns tasks with only the project, name, and domain filled in. 18 | // A limit must be provided. 19 | ListTaskIdentifiers(ctx context.Context, input ListResourceInput) (TaskCollectionOutput, error) 20 | } 21 | 22 | // Response format for a query on tasks. 23 | type TaskCollectionOutput struct { 24 | Tasks []models.Task 25 | } 26 | -------------------------------------------------------------------------------- /pkg/repositories/interfaces/workflow_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 7 | ) 8 | 9 | // Defines the interface for interacting with Workflow models. 10 | type WorkflowRepoInterface interface { 11 | // Inserts a workflow model into the database store. 12 | Create(ctx context.Context, input models.Workflow, descriptionEntity *models.DescriptionEntity) error 13 | // Returns a matching workflow if it exists. 14 | Get(ctx context.Context, input Identifier) (models.Workflow, error) 15 | // Returns workflow revisions matching query parameters. A limit must be provided for the results page size. 16 | List(ctx context.Context, input ListResourceInput) (WorkflowCollectionOutput, error) 17 | ListIdentifiers(ctx context.Context, input ListResourceInput) (WorkflowCollectionOutput, error) 18 | } 19 | 20 | // Response format for a query on workflows. 21 | type WorkflowCollectionOutput struct { 22 | Workflows []models.Workflow 23 | } 24 | -------------------------------------------------------------------------------- /pkg/repositories/models/base_model.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "gorm.io/gorm/schema" 8 | "k8s.io/apimachinery/pkg/util/sets" 9 | ) 10 | 11 | // This is the base model definition every flyteadmin model embeds. 12 | // This is nearly identical to http://doc.gorm.io/models.html#conventions except that flyteadmin models define their 13 | // own primary keys rather than use the ID as the primary key 14 | type BaseModel struct { 15 | ID uint `gorm:"index;autoIncrement"` 16 | CreatedAt time.Time 17 | UpdatedAt time.Time 18 | DeletedAt *time.Time `gorm:"index"` 19 | } 20 | 21 | func modelColumns(v any) sets.String { 22 | s, err := schema.Parse(v, &sync.Map{}, schema.NamingStrategy{}) 23 | if err != nil { 24 | panic(err) 25 | } 26 | return sets.NewString(s.DBNames...) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/repositories/models/description_entity.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 4 | 5 | // DescriptionEntityKey DescriptionEntity primary key 6 | type DescriptionEntityKey struct { 7 | ResourceType core.ResourceType `gorm:"primary_key;index:description_entity_project_domain_name_version_idx" valid:"length(0|255)"` 8 | Project string `gorm:"primary_key;index:description_entity_project_domain_name_version_idx" valid:"length(0|255)"` 9 | Domain string `gorm:"primary_key;index:description_entity_project_domain_name_version_idx" valid:"length(0|255)"` 10 | Name string `gorm:"primary_key;index:description_entity_project_domain_name_version_idx" valid:"length(0|255)"` 11 | Version string `gorm:"primary_key;index:description_entity_project_domain_name_version_idx" valid:"length(0|255)"` 12 | } 13 | 14 | // SourceCode Database model to encapsulate a SourceCode. 15 | type SourceCode struct { 16 | Link string `valid:"length(0|255)"` 17 | } 18 | 19 | // DescriptionEntity Database model to encapsulate a DescriptionEntity. 20 | type DescriptionEntity struct { 21 | DescriptionEntityKey 22 | 23 | BaseModel 24 | 25 | ShortDescription string 26 | 27 | LongDescription []byte 28 | 29 | SourceCode 30 | } 31 | 32 | var DescriptionEntityColumns = modelColumns(DescriptionEntity{}) 33 | -------------------------------------------------------------------------------- /pkg/repositories/models/execution_event.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type ExecutionEvent struct { 8 | BaseModel 9 | ExecutionKey 10 | RequestID string `valid:"length(0|255)"` 11 | OccurredAt time.Time 12 | Phase string `gorm:"primary_key"` 13 | } 14 | -------------------------------------------------------------------------------- /pkg/repositories/models/launch_plan.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // Launch plan primary key 4 | type LaunchPlanKey struct { 5 | Project string `gorm:"primary_key;index:lp_project_domain_name_idx,lp_project_domain_idx" valid:"length(0|255)"` 6 | Domain string `gorm:"primary_key;index:lp_project_domain_name_idx,lp_project_domain_idx" valid:"length(0|255)"` 7 | Name string `gorm:"primary_key;index:lp_project_domain_name_idx" valid:"length(0|255)"` 8 | Version string `gorm:"primary_key" valid:"length(0|255)"` 9 | } 10 | 11 | type LaunchPlanScheduleType string 12 | 13 | const ( 14 | // LaunchPlanScheduleTypeNONE is the const representing the launch plan does not have a schedule 15 | LaunchPlanScheduleTypeNONE LaunchPlanScheduleType = "NONE" 16 | // LaunchPlanScheduleTypeCRON is the const representing the launch plan has a CRON type of schedule 17 | LaunchPlanScheduleTypeCRON LaunchPlanScheduleType = "CRON" 18 | // LaunchPlanScheduleTypeRATE is the launch plan has a RATE type of schedule 19 | LaunchPlanScheduleTypeRATE LaunchPlanScheduleType = "RATE" 20 | ) 21 | 22 | // Database model to encapsulate a launch plan. 23 | type LaunchPlan struct { 24 | BaseModel 25 | LaunchPlanKey 26 | Spec []byte `gorm:"not null"` 27 | WorkflowID uint `gorm:"index"` 28 | Closure []byte `gorm:"not null"` 29 | // GORM doesn't save the zero value for ints, so we use a pointer for the State field 30 | State *int32 `gorm:"default:0"` 31 | // Hash of the launch plan 32 | Digest []byte 33 | ScheduleType LaunchPlanScheduleType 34 | } 35 | 36 | var LaunchPlanColumns = modelColumns(LaunchPlan{}) 37 | -------------------------------------------------------------------------------- /pkg/repositories/models/node_execution_event.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type NodeExecutionEvent struct { 8 | BaseModel 9 | NodeExecutionKey 10 | RequestID string 11 | OccurredAt time.Time 12 | Phase string `gorm:"primary_key"` 13 | } 14 | 15 | var NodeExecutionEventColumns = modelColumns(NodeExecutionEvent{}) 16 | -------------------------------------------------------------------------------- /pkg/repositories/models/project.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | type Project struct { 4 | BaseModel 5 | Identifier string `gorm:"primary_key"` 6 | Name string `valid:"length(0|255)"` // Human-readable name, not a unique identifier. 7 | Description string `gorm:"type:varchar(300)"` 8 | Labels []byte 9 | // GORM doesn't save the zero value for ints, so we use a pointer for the State field 10 | State *int32 `gorm:"default:0;index"` 11 | } 12 | 13 | var ProjectColumns = modelColumns(Project{}) 14 | -------------------------------------------------------------------------------- /pkg/repositories/models/protos/node_execution.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package flyteadmin.models; 4 | option go_package = "/models"; 5 | 6 | // Data about the node execution that is only referenced by FlyteAdmin and never by external callers. 7 | message NodeExecutionInternalData { 8 | // Event version > 0 indicates that node execution events can now include populated IsParent and IsDynamic fields. 9 | int32 event_version = 1; 10 | } 11 | -------------------------------------------------------------------------------- /pkg/repositories/models/resource.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "time" 4 | 5 | type ResourcePriority int32 6 | 7 | const ( 8 | ResourcePriorityProjectLevel ResourcePriority = 5 // use this 9 | ResourcePriorityProjectDomainLevel ResourcePriority = 10 10 | ResourcePriorityWorkflowLevel ResourcePriority = 100 11 | ResourcePriorityLaunchPlanLevel ResourcePriority = 1000 12 | ) 13 | 14 | // Represents Flyte resources repository. 15 | // In this model, the combination of (Project, Domain, Workflow, LaunchPlan, ResourceType) is unique 16 | type Resource struct { 17 | ID int64 `gorm:"AUTO_INCREMENT;column:id;primary_key"` 18 | CreatedAt time.Time 19 | UpdatedAt time.Time 20 | DeletedAt *time.Time `sql:"index"` 21 | Project string `gorm:"uniqueIndex:resource_idx" valid:"length(0|255)"` 22 | Domain string `gorm:"uniqueIndex:resource_idx" valid:"length(0|255)"` 23 | Workflow string `gorm:"uniqueIndex:resource_idx" valid:"length(0|255)"` 24 | LaunchPlan string `gorm:"uniqueIndex:resource_idx" valid:"length(0|255)"` 25 | ResourceType string `gorm:"uniqueIndex:resource_idx" valid:"length(0|255)"` 26 | Priority ResourcePriority 27 | // Serialized flyteidl.admin.MatchingAttributes. 28 | Attributes []byte 29 | } 30 | -------------------------------------------------------------------------------- /pkg/repositories/models/signal.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // Signal primary key 4 | type SignalKey struct { 5 | ExecutionKey 6 | SignalID string `gorm:"primary_key;index" valid:"length(0|255)"` 7 | } 8 | 9 | // Database model to encapsulate a signal. 10 | type Signal struct { 11 | BaseModel 12 | SignalKey 13 | Type []byte `gorm:"not null"` 14 | Value []byte 15 | } 16 | 17 | var SignalColumns = modelColumns(Signal{}) 18 | -------------------------------------------------------------------------------- /pkg/repositories/models/task.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // IMPORTANT: If you update the model below, be sure to double check model definitions in 4 | // pkg/repositories/config/migration_models.go 5 | 6 | // Task primary key 7 | type TaskKey struct { 8 | Project string `gorm:"primary_key;index:task_project_domain_name_idx;index:task_project_domain_idx" valid:"length(0|255)"` 9 | Domain string `gorm:"primary_key;index:task_project_domain_name_idx;index:task_project_domain_idx" valid:"length(0|255)"` 10 | Name string `gorm:"primary_key;index:task_project_domain_name_idx" valid:"length(0|255)"` 11 | Version string `gorm:"primary_key" valid:"length(0|255)"` 12 | } 13 | 14 | // Database model to encapsulate a task. 15 | type Task struct { 16 | BaseModel 17 | TaskKey 18 | Closure []byte `gorm:"not null"` 19 | // Hash of the compiled task closure 20 | Digest []byte 21 | // Task type (also stored in the closure put promoted as a column for filtering). 22 | Type string `valid:"length(0|255)"` 23 | // ShortDescription for the task. 24 | ShortDescription string 25 | } 26 | 27 | var TaskColumns = modelColumns(Task{}) 28 | -------------------------------------------------------------------------------- /pkg/repositories/models/workflow.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // Workflow primary key 4 | type WorkflowKey struct { 5 | Project string `gorm:"primary_key;index:workflow_project_domain_name_idx;index:workflow_project_domain_idx" valid:"length(0|255)"` 6 | Domain string `gorm:"primary_key;index:workflow_project_domain_name_idx;index:workflow_project_domain_idx" valid:"length(0|255)"` 7 | Name string `gorm:"primary_key;index:workflow_project_domain_name_idx" valid:"length(0|255)"` 8 | Version string `gorm:"primary_key"` 9 | } 10 | 11 | // Database model to encapsulate a workflow. 12 | type Workflow struct { 13 | BaseModel 14 | WorkflowKey 15 | TypedInterface []byte 16 | RemoteClosureIdentifier string `gorm:"not null" valid:"length(0|255)"` 17 | // Hash of the compiled workflow closure 18 | Digest []byte 19 | // ShortDescription for the workflow. 20 | ShortDescription string 21 | } 22 | 23 | var WorkflowColumns = modelColumns(Workflow{}) 24 | -------------------------------------------------------------------------------- /pkg/repositories/transformers/constants.go: -------------------------------------------------------------------------------- 1 | package transformers 2 | 3 | // InputsObjectSuffix is used when execution event data includes inline events and admin offloads the data. 4 | const InputsObjectSuffix = "offloaded_inputs" 5 | 6 | // OutputsObjectSuffix is used when execution event data includes inline outputs but the admin deployment is configured 7 | // to offload such data. The generated file path for the offloaded data will include the execution identifier and this suffix. 8 | const OutputsObjectSuffix = "offloaded_outputs" 9 | -------------------------------------------------------------------------------- /pkg/repositories/transformers/execution_event.go: -------------------------------------------------------------------------------- 1 | package transformers 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/errors" 5 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | "github.com/golang/protobuf/ptypes" 8 | "google.golang.org/grpc/codes" 9 | ) 10 | 11 | // Transforms a ExecutionEventCreateRequest to a ExecutionEvent model 12 | func CreateExecutionEventModel(request admin.WorkflowExecutionEventRequest) (*models.ExecutionEvent, error) { 13 | occurredAt, err := ptypes.Timestamp(request.Event.OccurredAt) 14 | if err != nil { 15 | return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to marshal occurred at timestamp") 16 | } 17 | return &models.ExecutionEvent{ 18 | ExecutionKey: models.ExecutionKey{ 19 | Project: request.Event.ExecutionId.Project, 20 | Domain: request.Event.ExecutionId.Domain, 21 | Name: request.Event.ExecutionId.Name, 22 | }, 23 | RequestID: request.RequestId, 24 | OccurredAt: occurredAt, 25 | Phase: request.Event.Phase.String(), 26 | }, nil 27 | } 28 | -------------------------------------------------------------------------------- /pkg/repositories/transformers/execution_event_test.go: -------------------------------------------------------------------------------- 1 | package transformers 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 8 | 9 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 10 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 11 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event" 12 | "github.com/golang/protobuf/ptypes" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestCreateExecutionEventModel(t *testing.T) { 17 | requestID := "foo" 18 | phase := core.WorkflowExecution_RUNNING 19 | 20 | timestamp := time.Now().UTC() 21 | occurredAt, _ := ptypes.TimestampProto(timestamp) 22 | executionEvent, err := CreateExecutionEventModel( 23 | admin.WorkflowExecutionEventRequest{ 24 | RequestId: requestID, 25 | Event: &event.WorkflowExecutionEvent{ 26 | ExecutionId: &core.WorkflowExecutionIdentifier{ 27 | Project: "project", 28 | Domain: "domain", 29 | Name: "name", 30 | }, 31 | Phase: phase, 32 | OccurredAt: occurredAt, 33 | }, 34 | }) 35 | 36 | assert.Nil(t, err) 37 | assert.Equal(t, requestID, executionEvent.RequestID) 38 | assert.Equal(t, models.ExecutionKey{ 39 | Project: "project", 40 | Domain: "domain", 41 | Name: "name", 42 | }, executionEvent.ExecutionKey) 43 | assert.Equal(t, timestamp, executionEvent.OccurredAt) 44 | assert.Equal(t, phase.String(), executionEvent.Phase) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/repositories/transformers/init_test.go: -------------------------------------------------------------------------------- 1 | package transformers 2 | 3 | import ( 4 | "github.com/flyteorg/flytestdlib/contextutils" 5 | "github.com/flyteorg/flytestdlib/promutils/labeled" 6 | ) 7 | 8 | func init() { 9 | labeled.SetMetricKeys(contextutils.ProjectKey, contextutils.DomainKey, contextutils.WorkflowIDKey, contextutils.TaskIDKey) 10 | } 11 | -------------------------------------------------------------------------------- /pkg/repositories/transformers/node_execution_event.go: -------------------------------------------------------------------------------- 1 | package transformers 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/errors" 5 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | "github.com/golang/protobuf/ptypes" 8 | "google.golang.org/grpc/codes" 9 | ) 10 | 11 | // Transforms a NodeExecutionEventRequest to a NodeExecutionEvent model 12 | func CreateNodeExecutionEventModel(request admin.NodeExecutionEventRequest) (*models.NodeExecutionEvent, error) { 13 | occurredAt, err := ptypes.Timestamp(request.Event.OccurredAt) 14 | if err != nil { 15 | return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to marshal occurred at timestamp") 16 | } 17 | return &models.NodeExecutionEvent{ 18 | NodeExecutionKey: models.NodeExecutionKey{ 19 | NodeID: request.Event.Id.NodeId, 20 | ExecutionKey: models.ExecutionKey{ 21 | Project: request.Event.Id.ExecutionId.Project, 22 | Domain: request.Event.Id.ExecutionId.Domain, 23 | Name: request.Event.Id.ExecutionId.Name, 24 | }, 25 | }, 26 | RequestID: request.RequestId, 27 | OccurredAt: occurredAt, 28 | Phase: request.Event.Phase.String(), 29 | }, nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/repositories/transformers/node_execution_event_test.go: -------------------------------------------------------------------------------- 1 | package transformers 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 8 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 9 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 10 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/event" 11 | "github.com/golang/protobuf/ptypes" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestCreateNodeExecutionEventModel(t *testing.T) { 16 | occurredAt := time.Now().UTC() 17 | occurredAtProto, _ := ptypes.TimestampProto(occurredAt) 18 | request := admin.NodeExecutionEventRequest{ 19 | RequestId: "request id", 20 | Event: &event.NodeExecutionEvent{ 21 | Id: &core.NodeExecutionIdentifier{ 22 | NodeId: "nodey", 23 | ExecutionId: &core.WorkflowExecutionIdentifier{ 24 | Project: "project", 25 | Domain: "domain", 26 | Name: "name", 27 | }, 28 | }, 29 | Phase: core.NodeExecution_ABORTED, 30 | OccurredAt: occurredAtProto, 31 | }, 32 | } 33 | 34 | nodeExecutionEventModel, err := CreateNodeExecutionEventModel(request) 35 | assert.Nil(t, err) 36 | assert.Equal(t, &models.NodeExecutionEvent{ 37 | NodeExecutionKey: models.NodeExecutionKey{ 38 | NodeID: "nodey", 39 | ExecutionKey: models.ExecutionKey{ 40 | Project: "project", 41 | Domain: "domain", 42 | Name: "name", 43 | }, 44 | }, 45 | RequestID: "request id", 46 | OccurredAt: occurredAt, 47 | Phase: "ABORTED", 48 | }, nodeExecutionEventModel) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/rpc/adminservice/base_test.go: -------------------------------------------------------------------------------- 1 | package adminservice 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/flyteorg/flytestdlib/logger" 8 | 9 | "github.com/flyteorg/flytestdlib/promutils" 10 | "github.com/golang/protobuf/proto" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func Test_interceptPanic(t *testing.T) { 15 | m := AdminService{ 16 | Metrics: InitMetrics(promutils.NewTestScope()), 17 | } 18 | 19 | ctx := context.Background() 20 | 21 | // Mute logs to avoid .Fatal() (called in interceptPanic) causing the process to close 22 | assert.NoError(t, logger.SetConfig(&logger.Config{Mute: true})) 23 | 24 | func() { 25 | defer func() { 26 | if err := recover(); err != nil { 27 | assert.Fail(t, "Unexpected error", err) 28 | } 29 | }() 30 | 31 | a := func() { 32 | defer m.interceptPanic(ctx, proto.Message(nil)) 33 | 34 | var x *int 35 | *x = 10 36 | } 37 | 38 | a() 39 | }() 40 | } 41 | -------------------------------------------------------------------------------- /pkg/rpc/adminservice/tests/util.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/manager/mocks" 5 | "github.com/flyteorg/flyteadmin/pkg/rpc/adminservice" 6 | mockScope "github.com/flyteorg/flytestdlib/promutils" 7 | ) 8 | 9 | type NewMockAdminServerInput struct { 10 | executionManager *mocks.MockExecutionManager 11 | launchPlanManager *mocks.MockLaunchPlanManager 12 | nodeExecutionManager *mocks.MockNodeExecutionManager 13 | projectManager *mocks.MockProjectManager 14 | resourceManager *mocks.MockResourceManager 15 | taskManager *mocks.MockTaskManager 16 | workflowManager *mocks.MockWorkflowManager 17 | taskExecutionManager *mocks.MockTaskExecutionManager 18 | } 19 | 20 | func NewMockAdminServer(input NewMockAdminServerInput) *adminservice.AdminService { 21 | var testScope = mockScope.NewTestScope() 22 | return &adminservice.AdminService{ 23 | ExecutionManager: input.executionManager, 24 | LaunchPlanManager: input.launchPlanManager, 25 | NodeExecutionManager: input.nodeExecutionManager, 26 | TaskManager: input.taskManager, 27 | ProjectManager: input.projectManager, 28 | ResourceManager: input.resourceManager, 29 | WorkflowManager: input.workflowManager, 30 | TaskExecutionManager: input.taskExecutionManager, 31 | Metrics: adminservice.InitMetrics(testScope), 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/rpc/adminservice/util/transformers.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "google.golang.org/grpc/codes" 5 | "google.golang.org/grpc/status" 6 | 7 | "github.com/flyteorg/flyteadmin/pkg/common" 8 | "github.com/flyteorg/flyteadmin/pkg/errors" 9 | ) 10 | 11 | // TransformAndRecordError transforms errors to grpc-compatible error types and optionally truncates it if necessary. 12 | func TransformAndRecordError(err error, metrics *RequestMetrics) error { 13 | errMsg := err.Error() 14 | shouldTruncate := false 15 | if len(errMsg) > common.MaxResponseStatusBytes { 16 | errMsg = errMsg[:common.MaxResponseStatusBytes] 17 | shouldTruncate = true 18 | } 19 | 20 | adminErr, isAdminErr := err.(errors.FlyteAdminError) 21 | grpcStatus, isStatus := status.FromError(err) 22 | switch { 23 | case isAdminErr: 24 | if shouldTruncate { 25 | adminErr = errors.NewFlyteAdminError(adminErr.Code(), errMsg) 26 | } 27 | case isStatus: 28 | adminErr = errors.NewFlyteAdminError(grpcStatus.Code(), errMsg) 29 | default: 30 | adminErr = errors.NewFlyteAdminError(codes.Internal, errMsg) 31 | } 32 | 33 | metrics.Record(adminErr.Code()) 34 | return adminErr 35 | } 36 | -------------------------------------------------------------------------------- /pkg/rpc/adminservice/version.go: -------------------------------------------------------------------------------- 1 | package adminservice 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 7 | ) 8 | 9 | func (m *AdminService) GetVersion(ctx context.Context, request *admin.GetVersionRequest) (*admin.GetVersionResponse, error) { 10 | 11 | defer m.interceptPanic(ctx, request) 12 | response, err := m.VersionManager.GetVersion(ctx, request) 13 | if err != nil { 14 | return nil, err 15 | } 16 | return response, nil 17 | } 18 | -------------------------------------------------------------------------------- /pkg/runtime/cluster_pool_assignment_provider.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 5 | 6 | "github.com/flyteorg/flytestdlib/config" 7 | ) 8 | 9 | const clusterPoolsKey = "clusterPools" 10 | 11 | var clusterPoolsConfig = config.MustRegisterSection(clusterPoolsKey, &interfaces.ClusterPoolAssignmentConfig{ 12 | ClusterPoolAssignments: make(interfaces.ClusterPoolAssignments), 13 | }) 14 | 15 | // Implementation of an interfaces.ClusterPoolAssignmentConfiguration 16 | type ClusterPoolAssignmentConfigurationProvider struct{} 17 | 18 | func (p *ClusterPoolAssignmentConfigurationProvider) GetClusterPoolAssignments() interfaces.ClusterPoolAssignments { 19 | return clusterPoolsConfig.GetConfig().(*interfaces.ClusterPoolAssignmentConfig).ClusterPoolAssignments 20 | } 21 | 22 | func NewClusterPoolAssignmentConfigurationProvider() interfaces.ClusterPoolAssignmentConfiguration { 23 | return &ClusterPoolAssignmentConfigurationProvider{} 24 | } 25 | -------------------------------------------------------------------------------- /pkg/runtime/cluster_resource_provider_test.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "path/filepath" 9 | 10 | "github.com/flyteorg/flytestdlib/config" 11 | "github.com/flyteorg/flytestdlib/config/viper" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func initTestClusterResourceConfig() error { 16 | pwd, err := os.Getwd() 17 | if err != nil { 18 | return err 19 | } 20 | 21 | configAccessor := viper.NewAccessor(config.Options{ 22 | SearchPaths: []string{filepath.Join(pwd, "testdata/cluster_resource_config.yaml")}, 23 | StrictMode: false, 24 | }) 25 | return configAccessor.UpdateConfig(context.TODO()) 26 | } 27 | 28 | func TestClusterResourceConfig(t *testing.T) { 29 | err := initTestClusterResourceConfig() 30 | assert.NoError(t, err) 31 | 32 | configProvider := NewConfigurationProvider() 33 | clusterResourceConfig := configProvider.ClusterResourceConfiguration() 34 | assert.Equal(t, "/etc/flyte/clusterresource/templates", clusterResourceConfig.GetTemplatePath()) 35 | assert.Equal(t, "flyte_user", clusterResourceConfig.GetTemplateData()["user"].Value) 36 | assert.Equal(t, "TEST_SECRET", clusterResourceConfig.GetTemplateData()["secret"].ValueFrom.EnvVar) 37 | assert.Equal(t, "/etc/flyte/misc.txt", clusterResourceConfig.GetTemplateData()["file"].ValueFrom.FilePath) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/runtime/execution_queue_provider.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 5 | 6 | "github.com/flyteorg/flytestdlib/config" 7 | ) 8 | 9 | const queuesKey = "queues" 10 | 11 | var executionQueuesConfig = config.MustRegisterSection(queuesKey, &interfaces.QueueConfig{ 12 | ExecutionQueues: make([]interfaces.ExecutionQueue, 0), 13 | WorkflowConfigs: make([]interfaces.WorkflowConfig, 0), 14 | }) 15 | 16 | // Implementation of an interfaces.QueueConfiguration 17 | type QueueConfigurationProvider struct{} 18 | 19 | func (p *QueueConfigurationProvider) GetExecutionQueues() []interfaces.ExecutionQueue { 20 | return executionQueuesConfig.GetConfig().(*interfaces.QueueConfig).ExecutionQueues 21 | } 22 | 23 | func (p *QueueConfigurationProvider) GetWorkflowConfigs() []interfaces.WorkflowConfig { 24 | return executionQueuesConfig.GetConfig().(*interfaces.QueueConfig).WorkflowConfigs 25 | } 26 | 27 | func NewQueueConfigurationProvider() interfaces.QueueConfiguration { 28 | return &QueueConfigurationProvider{} 29 | } 30 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/cluster_pools.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | //go:generate mockery -name ClusterPoolAssignmentConfiguration -output=mocks -case=underscore 4 | 5 | type ClusterPoolAssignment struct { 6 | Pool string `json:"pool"` 7 | } 8 | 9 | type ClusterPoolAssignments = map[DomainName]ClusterPoolAssignment 10 | 11 | type ClusterPoolAssignmentConfig struct { 12 | ClusterPoolAssignments ClusterPoolAssignments `json:"clusterPoolAssignments"` 13 | } 14 | 15 | type ClusterPoolAssignmentConfiguration interface { 16 | GetClusterPoolAssignments() ClusterPoolAssignments 17 | } 18 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/configuration.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | // Interface for getting parsed values from a configuration file 4 | type Configuration interface { 5 | ApplicationConfiguration() ApplicationConfiguration 6 | QueueConfiguration() QueueConfiguration 7 | ClusterConfiguration() ClusterConfiguration 8 | TaskResourceConfiguration() TaskResourceConfiguration 9 | WhitelistConfiguration() WhitelistConfiguration 10 | RegistrationValidationConfiguration() RegistrationValidationConfiguration 11 | ClusterResourceConfiguration() ClusterResourceConfiguration 12 | NamespaceMappingConfiguration() NamespaceMappingConfiguration 13 | QualityOfServiceConfiguration() QualityOfServiceConfiguration 14 | ClusterPoolAssignmentConfiguration() ClusterPoolAssignmentConfiguration 15 | } 16 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/namespace_configuration.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | type NamespaceMappingConfig struct { 4 | Mapping string `json:"mapping"` // Deprecated 5 | Template string `json:"template"` 6 | TemplateData TemplateData `json:"templateData"` 7 | } 8 | 9 | //go:generate mockery -name NamespaceMappingConfiguration -output=../mocks -case=underscore 10 | 11 | type NamespaceMappingConfiguration interface { 12 | GetNamespaceTemplate() string 13 | } 14 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/quality_of_service_configuration.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 5 | "github.com/flyteorg/flytestdlib/config" 6 | ) 7 | 8 | type TierName = string 9 | 10 | // Just incrementally start using mockery, replace with -all when working on https://github.com/flyteorg/flyte/issues/149 11 | //go:generate mockery -name QualityOfServiceConfiguration -output=mocks -case=underscore 12 | 13 | type QualityOfServiceSpec struct { 14 | QueueingBudget config.Duration `json:"queueingBudget"` 15 | } 16 | 17 | type QualityOfServiceConfig struct { 18 | TierExecutionValues map[TierName]QualityOfServiceSpec `json:"tierExecutionValues"` 19 | DefaultTiers map[DomainName]TierName `json:"defaultTiers"` 20 | } 21 | 22 | type QualityOfServiceConfiguration interface { 23 | GetTierExecutionValues() map[core.QualityOfService_Tier]core.QualityOfServiceSpec 24 | GetDefaultTiers() map[DomainName]core.QualityOfService_Tier 25 | } 26 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/queue_configuration.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | // Holds details about a queue used for task execution. 4 | // Matching attributes determine which workflows' tasks will run where. 5 | type ExecutionQueue struct { 6 | Dynamic string 7 | Attributes []string 8 | } 9 | 10 | func (q ExecutionQueue) GetAttributes() []string { 11 | return q.Attributes 12 | } 13 | 14 | type ExecutionQueues []ExecutionQueue 15 | 16 | // Defines the specific resource attributes (tags) a workflow requires to run. 17 | type WorkflowConfig struct { 18 | Domain string `json:"domain"` 19 | Tags []string `json:"tags"` 20 | } 21 | 22 | type WorkflowConfigs []WorkflowConfig 23 | 24 | type QueueConfig struct { 25 | ExecutionQueues ExecutionQueues `json:"executionQueues"` 26 | WorkflowConfigs WorkflowConfigs `json:"workflowConfigs"` 27 | } 28 | 29 | // Provides values set in runtime configuration files. 30 | // These files can be changed without requiring a full server restart. 31 | type QueueConfiguration interface { 32 | // Returns executions queues defined in runtime configuration files. 33 | GetExecutionQueues() []ExecutionQueue 34 | // Returns workflow configurations defined in runtime configuration files. 35 | GetWorkflowConfigs() []WorkflowConfig 36 | } 37 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/registration_validation_provider.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | type RegistrationValidationConfig struct { 4 | MaxWorkflowNodes int `json:"maxWorkflowNodes"` 5 | MaxLabelEntries int `json:"maxLabelEntries"` 6 | MaxAnnotationEntries int `json:"maxAnnotationEntries"` 7 | WorkflowSizeLimit string `json:"workflowSizeLimit"` 8 | } 9 | 10 | // Provides validation limits used at entity registration 11 | type RegistrationValidationConfiguration interface { 12 | GetWorkflowNodeLimit() int 13 | GetMaxLabelEntries() int 14 | GetMaxAnnotationEntries() int 15 | GetWorkflowSizeLimit() string 16 | } 17 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/task_resource_configuration.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import "k8s.io/apimachinery/pkg/api/resource" 4 | 5 | type TaskResourceSet struct { 6 | CPU resource.Quantity `json:"cpu"` 7 | GPU resource.Quantity `json:"gpu"` 8 | Memory resource.Quantity `json:"memory"` 9 | Storage resource.Quantity `json:"storage"` 10 | EphemeralStorage resource.Quantity `json:"ephemeralStorage"` 11 | } 12 | 13 | // Provides default values for task resource limits and defaults. 14 | type TaskResourceConfiguration interface { 15 | GetDefaults() TaskResourceSet 16 | GetLimits() TaskResourceSet 17 | } 18 | -------------------------------------------------------------------------------- /pkg/runtime/interfaces/whitelist.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | type WhitelistScope struct { 4 | Project string `json:"project"` 5 | Domain string `json:"domain"` 6 | } 7 | 8 | // Defines specific task types whitelisted for support. 9 | type TaskTypeWhitelist = map[string][]WhitelistScope 10 | 11 | type WhitelistConfiguration interface { 12 | // Returns whitelisted task types defined in runtime configuration files. 13 | GetTaskTypeWhitelist() TaskTypeWhitelist 14 | } 15 | -------------------------------------------------------------------------------- /pkg/runtime/mocks/mock_cluster_resource_provider.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 7 | ) 8 | 9 | type MockClusterResourceConfiguration struct { 10 | TemplatePath string 11 | TemplateData interfaces.TemplateData 12 | RefreshInterval time.Duration 13 | CustomTemplateData map[interfaces.DomainName]interfaces.TemplateData 14 | StandaloneDeployment bool 15 | } 16 | 17 | func (c MockClusterResourceConfiguration) GetTemplatePath() string { 18 | return c.TemplatePath 19 | } 20 | func (c MockClusterResourceConfiguration) GetTemplateData() interfaces.TemplateData { 21 | return c.TemplateData 22 | } 23 | 24 | func (c MockClusterResourceConfiguration) GetRefreshInterval() time.Duration { 25 | return c.RefreshInterval 26 | } 27 | 28 | func (c MockClusterResourceConfiguration) GetCustomTemplateData() map[interfaces.DomainName]interfaces.TemplateData { 29 | return c.CustomTemplateData 30 | } 31 | 32 | func (c MockClusterResourceConfiguration) IsStandaloneDeployment() bool { 33 | return c.StandaloneDeployment 34 | } 35 | 36 | func NewMockClusterResourceConfiguration() interfaces.ClusterResourceConfiguration { 37 | return &MockClusterResourceConfiguration{} 38 | } 39 | -------------------------------------------------------------------------------- /pkg/runtime/mocks/mock_execution_queue_provider.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 4 | 5 | type MockQueueConfigurationProvider struct { 6 | executionQueues []interfaces.ExecutionQueue 7 | workflowConfigs []interfaces.WorkflowConfig 8 | } 9 | 10 | func (p *MockQueueConfigurationProvider) GetExecutionQueues() []interfaces.ExecutionQueue { 11 | return p.executionQueues 12 | } 13 | 14 | func (p *MockQueueConfigurationProvider) GetWorkflowConfigs() []interfaces.WorkflowConfig { 15 | return p.workflowConfigs 16 | } 17 | 18 | func NewMockQueueConfigurationProvider( 19 | executionQueues []interfaces.ExecutionQueue, 20 | workflowConfigs []interfaces.WorkflowConfig) interfaces.QueueConfiguration { 21 | return &MockQueueConfigurationProvider{ 22 | executionQueues: executionQueues, 23 | workflowConfigs: workflowConfigs, 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pkg/runtime/mocks/mock_registration_validation_provider.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 4 | 5 | type MockRegistrationValidationProvider struct { 6 | WorkflowNodeLimit int 7 | MaxLabelEntries int 8 | MaxAnnotationEntries int 9 | WorkflowSizeLimit string 10 | } 11 | 12 | func (c *MockRegistrationValidationProvider) GetWorkflowNodeLimit() int { 13 | return c.WorkflowNodeLimit 14 | } 15 | 16 | func (c *MockRegistrationValidationProvider) GetMaxLabelEntries() int { 17 | return c.MaxLabelEntries 18 | } 19 | 20 | func (c *MockRegistrationValidationProvider) GetMaxAnnotationEntries() int { 21 | return c.MaxAnnotationEntries 22 | } 23 | 24 | func (c *MockRegistrationValidationProvider) GetWorkflowSizeLimit() string { 25 | return c.WorkflowSizeLimit 26 | } 27 | 28 | func NewMockRegistrationValidationProvider() interfaces.RegistrationValidationConfiguration { 29 | return &MockRegistrationValidationProvider{} 30 | } 31 | -------------------------------------------------------------------------------- /pkg/runtime/mocks/mock_task_resource_provider.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 4 | 5 | type MockTaskResourceConfiguration struct { 6 | Defaults interfaces.TaskResourceSet 7 | Limits interfaces.TaskResourceSet 8 | } 9 | 10 | func (c *MockTaskResourceConfiguration) GetDefaults() interfaces.TaskResourceSet { 11 | return c.Defaults 12 | } 13 | func (c *MockTaskResourceConfiguration) GetLimits() interfaces.TaskResourceSet { 14 | return c.Limits 15 | } 16 | 17 | func NewMockTaskResourceConfiguration(defaults, limits interfaces.TaskResourceSet) interfaces.TaskResourceConfiguration { 18 | return &MockTaskResourceConfiguration{ 19 | Defaults: defaults, 20 | Limits: limits, 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /pkg/runtime/mocks/mock_whitelist_provider.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 4 | 5 | type MockWhitelistConfiguration struct { 6 | TaskTypeWhitelist interfaces.TaskTypeWhitelist 7 | } 8 | 9 | func (c *MockWhitelistConfiguration) GetTaskTypeWhitelist() interfaces.TaskTypeWhitelist { 10 | return c.TaskTypeWhitelist 11 | } 12 | 13 | func NewMockWhitelistConfiguration() interfaces.WhitelistConfiguration { 14 | return &MockWhitelistConfiguration{} 15 | } 16 | -------------------------------------------------------------------------------- /pkg/runtime/namespace_config_provider.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 7 | "github.com/flyteorg/flytestdlib/config" 8 | "github.com/flyteorg/flytestdlib/logger" 9 | ) 10 | 11 | const ( 12 | namespaceMappingKey = "namespace_mapping" 13 | defaultTemplate = "{{ project }}-{{ domain }}" 14 | ) 15 | 16 | var namespaceMappingConfig = config.MustRegisterSection(namespaceMappingKey, &interfaces.NamespaceMappingConfig{ 17 | Template: defaultTemplate, 18 | }) 19 | 20 | type NamespaceMappingConfigurationProvider struct{} 21 | 22 | func (p *NamespaceMappingConfigurationProvider) GetNamespaceTemplate() string { 23 | var template string 24 | if namespaceMappingConfig != nil && namespaceMappingConfig.GetConfig() != nil { 25 | template = namespaceMappingConfig.GetConfig().(*interfaces.NamespaceMappingConfig).Template 26 | if len(namespaceMappingConfig.GetConfig().(*interfaces.NamespaceMappingConfig).Mapping) > 0 { 27 | logger.Errorf(context.TODO(), "Using `mapping` in namespace configs is deprecated. "+ 28 | "Please use a custom string template like `{{ project }}-{{ domain }}` instead") 29 | } 30 | } 31 | if len(template) == 0 { 32 | logger.Infof(context.TODO(), "No namespace template specified in config. Using [%+s] by default", defaultTemplate) 33 | template = defaultTemplate 34 | } 35 | return template 36 | } 37 | 38 | func NewNamespaceMappingConfigurationProvider() interfaces.NamespaceMappingConfiguration { 39 | return &NamespaceMappingConfigurationProvider{} 40 | } 41 | -------------------------------------------------------------------------------- /pkg/runtime/registration_validation_provider.go: -------------------------------------------------------------------------------- 1 | // Interface for configurable values used in entity registration and validation 2 | package runtime 3 | 4 | import ( 5 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 6 | "github.com/flyteorg/flytestdlib/config" 7 | ) 8 | 9 | const registration = "registration" 10 | 11 | var registrationValidationConfig = config.MustRegisterSection(registration, &interfaces.RegistrationValidationConfig{ 12 | MaxWorkflowNodes: 100, 13 | }) 14 | 15 | // Implementation of an interfaces.TaskResourceConfiguration 16 | type RegistrationValidationProvider struct{} 17 | 18 | func (p *RegistrationValidationProvider) GetWorkflowNodeLimit() int { 19 | return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxWorkflowNodes 20 | } 21 | 22 | func (p *RegistrationValidationProvider) GetMaxLabelEntries() int { 23 | return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxLabelEntries 24 | } 25 | 26 | func (p *RegistrationValidationProvider) GetMaxAnnotationEntries() int { 27 | return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxAnnotationEntries 28 | } 29 | 30 | func (p *RegistrationValidationProvider) GetWorkflowSizeLimit() string { 31 | return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).WorkflowSizeLimit 32 | } 33 | 34 | func NewRegistrationValidationProvider() interfaces.RegistrationValidationConfiguration { 35 | return &RegistrationValidationProvider{} 36 | } 37 | -------------------------------------------------------------------------------- /pkg/runtime/task_resource_provider.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 5 | "github.com/flyteorg/flytestdlib/config" 6 | "k8s.io/apimachinery/pkg/api/resource" 7 | ) 8 | 9 | const taskResourceKey = "task_resources" 10 | 11 | var taskResourceConfig = config.MustRegisterSection(taskResourceKey, &TaskResourceSpec{ 12 | Defaults: interfaces.TaskResourceSet{ 13 | CPU: resource.MustParse("2"), 14 | Memory: resource.MustParse("200Mi"), 15 | }, 16 | Limits: interfaces.TaskResourceSet{ 17 | CPU: resource.MustParse("2"), 18 | Memory: resource.MustParse("1Gi"), 19 | GPU: resource.MustParse("1"), 20 | }, 21 | }) 22 | 23 | type TaskResourceSpec struct { 24 | Defaults interfaces.TaskResourceSet `json:"defaults"` 25 | Limits interfaces.TaskResourceSet `json:"limits"` 26 | } 27 | 28 | // Implementation of an interfaces.TaskResourceConfiguration 29 | type TaskResourceProvider struct{} 30 | 31 | func (p *TaskResourceProvider) GetDefaults() interfaces.TaskResourceSet { 32 | return taskResourceConfig.GetConfig().(*TaskResourceSpec).Defaults 33 | } 34 | 35 | func (p *TaskResourceProvider) GetLimits() interfaces.TaskResourceSet { 36 | return taskResourceConfig.GetConfig().(*TaskResourceSpec).Limits 37 | } 38 | 39 | func NewTaskResourceProvider() interfaces.TaskResourceConfiguration { 40 | return &TaskResourceProvider{} 41 | } 42 | -------------------------------------------------------------------------------- /pkg/runtime/testdata/cluster_resource_config.yaml: -------------------------------------------------------------------------------- 1 | cluster_resources: 2 | templatePath: "/etc/flyte/clusterresource/templates" 3 | templateData: 4 | user: 5 | value: "flyte_user" 6 | secret: 7 | valueFrom: 8 | env: "TEST_SECRET" 9 | file: 10 | valueFrom: 11 | filePath: "/etc/flyte/misc.txt" 12 | -------------------------------------------------------------------------------- /pkg/runtime/testdata/clusters_config.yaml: -------------------------------------------------------------------------------- 1 | clusters: 2 | clusterConfigs: 3 | - name: "testcluster" 4 | endpoint: "testcluster_endpoint" 5 | auth: 6 | type: "file_path" 7 | tokenPath: "/path/to/testcluster/token" 8 | certPath: "/path/to/testcluster/cert" 9 | - name: "testcluster2" 10 | endpoint: "testcluster2_endpoint" 11 | enabled: true 12 | auth: 13 | type: "file_path" 14 | tokenPath: "/path/to/testcluster2/token" 15 | certPath: "/path/to/testcluster2/cert" 16 | -------------------------------------------------------------------------------- /pkg/runtime/testdata/config.yaml: -------------------------------------------------------------------------------- 1 | queues: 2 | - name: "critical" 3 | attributes: 4 | - critical 5 | - name: "gpu" 6 | attributes: 7 | - awesome 8 | - gpu 9 | workflow_configs: 10 | - project: myProject 11 | domain: production 12 | workflowName: "foo" 13 | tags: 14 | - critical 15 | - important 16 | - project: myProject2 17 | domain: production 18 | workflowName: "bar" 19 | tags: 20 | - gpu 21 | -------------------------------------------------------------------------------- /pkg/runtime/testdata/event.yaml: -------------------------------------------------------------------------------- 1 | cloudevents: 2 | enable: true 3 | type: aws 4 | aws: 5 | region: us-east-1 6 | eventsPublisher: 7 | topicName: topic -------------------------------------------------------------------------------- /pkg/runtime/testdata/postgres_config.yaml: -------------------------------------------------------------------------------- 1 | database: 2 | postgres: 3 | port: 5432 4 | username: postgres 5 | host: postgres 6 | dbname: postgres 7 | options: "sslmode=disable" 8 | -------------------------------------------------------------------------------- /pkg/runtime/testdata/sqlite_config.yaml: -------------------------------------------------------------------------------- 1 | database: 2 | sqlite: 3 | file: admin.db 4 | -------------------------------------------------------------------------------- /pkg/runtime/whitelist_provider.go: -------------------------------------------------------------------------------- 1 | package runtime 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" 5 | "github.com/flyteorg/flytestdlib/config" 6 | ) 7 | 8 | const whitelistKey = "task_type_whitelist" 9 | 10 | var whiteListProviderDefault = make(map[string][]interfaces.WhitelistScope) 11 | 12 | var whitelistConfig = config.MustRegisterSection(whitelistKey, &whiteListProviderDefault) 13 | 14 | // Implementation of an interfaces.QueueConfiguration 15 | type WhitelistConfigurationProvider struct{} 16 | 17 | func (p *WhitelistConfigurationProvider) GetTaskTypeWhitelist() interfaces.TaskTypeWhitelist { 18 | whitelists := whitelistConfig.GetConfig().(*interfaces.TaskTypeWhitelist) 19 | return *whitelists 20 | } 21 | 22 | func NewWhitelistConfigurationProvider() interfaces.WhitelistConfiguration { 23 | return &WhitelistConfigurationProvider{} 24 | } 25 | -------------------------------------------------------------------------------- /pkg/server/cert_utils.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | 8 | "github.com/flyteorg/flytestdlib/errors" 9 | "github.com/flyteorg/flytestdlib/logger" 10 | 11 | "io/ioutil" 12 | ) 13 | 14 | const ( 15 | ErrCertificate errors.ErrorCode = "CERTIFICATE_FAILURE" 16 | ) 17 | 18 | func GetSslCredentials(ctx context.Context, certFile, keyFile string) (*x509.CertPool, *tls.Certificate, error) { 19 | cert, err := tls.LoadX509KeyPair(certFile, keyFile) 20 | if err != nil { 21 | return nil, nil, errors.Wrapf(ErrCertificate, err, "failed to load X509 key pair: %s", certFile) 22 | } 23 | logger.Infof(ctx, "Constructing SSL credentials") 24 | 25 | certPool := x509.NewCertPool() 26 | data, err := ioutil.ReadFile(certFile) 27 | if err != nil { 28 | return nil, nil, errors.Wrapf(ErrCertificate, err, "failed to read server cert file: %s", certFile) 29 | } 30 | if ok := certPool.AppendCertsFromPEM(data); !ok { 31 | return nil, nil, errors.Errorf(ErrCertificate, "failed to load certificate into the pool") 32 | } 33 | 34 | return certPool, &cert, nil 35 | } 36 | -------------------------------------------------------------------------------- /pkg/workflowengine/interfaces/builder.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 5 | "github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1" 6 | ) 7 | 8 | //go:generate mockery -name FlyteWorkflowBuilder -output=../mocks -case=underscore 9 | 10 | // FlyteWorkflowBuilder produces a v1alpha1.FlyteWorkflow definition from a compiled workflow closure and execution inputs 11 | type FlyteWorkflowBuilder interface { 12 | Build( 13 | wfClosure *core.CompiledWorkflowClosure, inputs *core.LiteralMap, executionID *core.WorkflowExecutionIdentifier, 14 | namespace string) (*v1alpha1.FlyteWorkflow, error) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/workflowengine/interfaces/compiler.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" 5 | "github.com/flyteorg/flytepropeller/pkg/compiler" 6 | "github.com/flyteorg/flytepropeller/pkg/compiler/common" 7 | ) 8 | 9 | // Workflow compiler interface. 10 | type Compiler interface { 11 | CompileTask(task *core.TaskTemplate) (*core.CompiledTask, error) 12 | GetRequirements(fg *core.WorkflowTemplate, subWfs []*core.WorkflowTemplate) ( 13 | compiler.WorkflowExecutionRequirements, error) 14 | CompileWorkflow(primaryWf *core.WorkflowTemplate, subworkflows []*core.WorkflowTemplate, tasks []*core.CompiledTask, 15 | launchPlans []common.InterfaceProvider) (*core.CompiledWorkflowClosure, error) 16 | } 17 | -------------------------------------------------------------------------------- /pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## _Read then delete this section_ 2 | 3 | _- Make sure to use a concise title for the pull-request._ 4 | 5 | _- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version 6 | will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump)_ 7 | 8 | # TL;DR 9 | _Please replace this text with a description of what this PR accomplishes._ 10 | 11 | ## Type 12 | - [ ] Bug Fix 13 | - [ ] Feature 14 | - [ ] Plugin 15 | 16 | ## Are all requirements met? 17 | 18 | - [ ] Code completed 19 | - [ ] Smoke tested 20 | - [ ] Unit tests added 21 | - [ ] Code documentation added 22 | - [ ] Any pending items have an associated Issue 23 | 24 | ## Complete description 25 | _How did you fix the bug, make the feature etc. Link to any design docs etc_ 26 | 27 | ## Tracking Issue 28 | _Remove the '*fixes*' keyword if there will be multiple PRs to fix the linked issue_ 29 | 30 | fixes https://github.com/flyteorg/flyte/issues/ 31 | 32 | ## Follow-up issue 33 | _NA_ 34 | OR 35 | _https://github.com/flyteorg/flyte/issues/_ 36 | -------------------------------------------------------------------------------- /sampleresourcetemplates/docker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: dockerhub 5 | namespace: {{ namespace }} 6 | stringData: 7 | .dockerconfigjson: '{"auths":{"docker.io":{"username":"mydockerusername","password":"{{ dockerSecret }}","email":"none","auth":" {{ dockerAuth }}"}}}' 8 | type: kubernetes.io/dockerconfigjson 9 | -------------------------------------------------------------------------------- /sampleresourcetemplates/gsa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: {{ namespace }}-gsa 5 | namespace: {{ namespace }} 6 | spec: 7 | displayName: Service account for {{ namespace }} -------------------------------------------------------------------------------- /sampleresourcetemplates/imagepullsecrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: default 5 | namespace: {{ namespace }} 6 | imagePullSecrets: 7 | - name: dockerhub 8 | -------------------------------------------------------------------------------- /sampleresourcetemplates/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ namespace }} 5 | spec: 6 | finalizers: 7 | - kubernetes 8 | -------------------------------------------------------------------------------- /scheduler.Dockerfile: -------------------------------------------------------------------------------- 1 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 2 | # ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: 3 | # 4 | # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst 5 | 6 | FROM --platform=${BUILDPLATFORM} golang:1.18-alpine3.15 as builder 7 | 8 | ARG TARGETARCH 9 | ENV GOARCH "${TARGETARCH}" 10 | ENV GOOS linux 11 | 12 | RUN apk add git openssh-client make curl 13 | 14 | # COPY only the go mod files for efficient caching 15 | COPY go.mod go.sum /go/src/github.com/flyteorg/flyteadmin/ 16 | WORKDIR /go/src/github.com/flyteorg/flyteadmin 17 | 18 | # Pull dependencies 19 | RUN go mod download 20 | 21 | # COPY the rest of the source code 22 | COPY . /go/src/github.com/flyteorg/flyteadmin/ 23 | 24 | # This 'linux_compile_scheduler' target should compile binaries to the /artifacts directory 25 | # The main entrypoint should be compiled to /artifacts/flytescheduler 26 | RUN make linux_compile_scheduler 27 | 28 | # update the PATH to include the /artifacts directory 29 | ENV PATH="/artifacts:${PATH}" 30 | 31 | # This will eventually move to centurylink/ca-certs:latest for minimum possible image size 32 | FROM alpine:3.15 33 | LABEL org.opencontainers.image.source https://github.com/flyteorg/flyteadmin 34 | 35 | COPY --from=builder /artifacts /bin 36 | 37 | # Ensure the latest CA certs are present to authenticate SSL connections. 38 | RUN apk --update add ca-certificates 39 | 40 | RUN addgroup -S flyte && adduser -S flyte -G flyte 41 | USER flyte 42 | 43 | CMD ["flytescheduler"] 44 | 45 | -------------------------------------------------------------------------------- /scheduler/core/doc.go: -------------------------------------------------------------------------------- 1 | // Package core 2 | // This is core package for the scheduler which includes 3 | // - scheduler interface 4 | // - scheduler implementation using gocron https://github.com/robfig/cron 5 | // - updater which updates the schedules in the scheduler by reading periodically from the DB 6 | // - snapshot runner which snapshot the schedules with there last exec times so that it can be used as check point 7 | // in case of a crash. After a crash the scheduler replays the schedules from the last recorded snapshot. 8 | // It relies on the admin idempotency aspect to fail executions if the execution with a scheduled time already exists with it. 9 | package core 10 | -------------------------------------------------------------------------------- /scheduler/core/gocron_job.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "runtime/pprof" 7 | "time" 8 | 9 | "github.com/flyteorg/flyteadmin/scheduler/repositories/models" 10 | "github.com/flyteorg/flytestdlib/contextutils" 11 | "github.com/flyteorg/flytestdlib/logger" 12 | 13 | "github.com/robfig/cron/v3" 14 | ) 15 | 16 | // GoCronJob this provides a wrapper around the go cron libraries job function. 17 | type GoCronJob struct { 18 | ctx context.Context 19 | nameOfSchedule string 20 | schedule models.SchedulableEntity 21 | funcWithSchedule TimedFuncWithSchedule 22 | lastExecTime *time.Time 23 | catchupFromTime *time.Time 24 | entryID cron.EntryID 25 | } 26 | 27 | func (g *GoCronJob) Run(t time.Time) { 28 | // Create job function label to be used for creating the child context 29 | jobFuncLabel := fmt.Sprintf("jobfunc-%v", g.nameOfSchedule) 30 | jobFuncCtxWithLabel := contextutils.WithGoroutineLabel(g.ctx, jobFuncLabel) 31 | // TODO : add panic counter metric 32 | 33 | pprof.SetGoroutineLabels(jobFuncCtxWithLabel) 34 | if err := g.funcWithSchedule(jobFuncCtxWithLabel, g.schedule, t); err != nil { 35 | logger.Errorf(jobFuncCtxWithLabel, "Got error while scheduling %v", err) 36 | } 37 | // Update the lastExecTime only if new trigger time t is after lastExecTime. 38 | if g.lastExecTime == nil || g.lastExecTime.Before(t) { 39 | g.lastExecTime = &t 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /scheduler/core/snapshot_runner.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | 6 | sImpl "github.com/flyteorg/flyteadmin/scheduler/snapshoter" 7 | ) 8 | 9 | const snapShotVersion = 1 10 | 11 | // Snapshotrunner allows the ability to snapshot the scheduler state and save it to the db. 12 | // Its invoked periodically from the scheduledExecutor 13 | type Snapshotrunner struct { 14 | snapshoter sImpl.Persistence 15 | scheduler Scheduler 16 | } 17 | 18 | func (u Snapshotrunner) Run(ctx context.Context) { 19 | snapshot := u.scheduler.CalculateSnapshot(ctx) 20 | snapshotWriter := &sImpl.VersionedSnapshot{Version: snapShotVersion} 21 | u.snapshoter.Save(ctx, snapshotWriter, snapshot) 22 | } 23 | 24 | func NewSnapshotRunner(snapshoter sImpl.Persistence, scheduler Scheduler) Snapshotrunner { 25 | return Snapshotrunner{snapshoter: snapshoter, scheduler: scheduler} 26 | } 27 | -------------------------------------------------------------------------------- /scheduler/core/updater.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | 6 | repositoryInterfaces "github.com/flyteorg/flyteadmin/scheduler/repositories/interfaces" 7 | 8 | "github.com/flyteorg/flytestdlib/logger" 9 | ) 10 | 11 | // Updater this updates the scheduler with the latest state of schedules from the DB. 12 | type Updater struct { 13 | db repositoryInterfaces.SchedulerRepoInterface 14 | scheduler Scheduler 15 | } 16 | 17 | func (u Updater) UpdateGoCronSchedules(ctx context.Context) { 18 | schedules, err := u.db.SchedulableEntityRepo().GetAll(ctx) 19 | if err != nil { 20 | logger.Errorf(ctx, "Failed to fetch the schedules in this round due to %v", err) 21 | return 22 | } 23 | u.scheduler.UpdateSchedules(ctx, schedules) 24 | } 25 | 26 | func NewUpdater(db repositoryInterfaces.SchedulerRepoInterface, 27 | scheduler Scheduler) Updater { 28 | return Updater{db: db, scheduler: scheduler} 29 | } 30 | -------------------------------------------------------------------------------- /scheduler/dbapi/doc.go: -------------------------------------------------------------------------------- 1 | // Package dbapi 2 | // This package implements the event scheduler interface which is called whenever a launchplan is enabled or disabled. 3 | // Using this api the launchplan manager chooses to activate or deactivate a schedule if it has a cron or fixed rate 4 | // schedule. 5 | package dbapi 6 | -------------------------------------------------------------------------------- /scheduler/executor/doc.go: -------------------------------------------------------------------------------- 1 | // Package executor 2 | // This package provides an interface to talk to admin for scheduled executions. 3 | // The implementation constructs a request using the schedule details and the passed in schedule time to be sent to admin 4 | // for execution 5 | package executor 6 | -------------------------------------------------------------------------------- /scheduler/executor/executor.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/flyteorg/flyteadmin/scheduler/repositories/models" 8 | ) 9 | 10 | //go:generate mockery -name Executor -output=mocks -case=underscore 11 | 12 | // Executor allows the ability to create scheduled executions on admin 13 | type Executor interface { 14 | // Execute sends a scheduled execution request to admin 15 | Execute(ctx context.Context, scheduledTime time.Time, s models.SchedulableEntity) error 16 | } 17 | -------------------------------------------------------------------------------- /scheduler/executor/mocks/executor.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | context "context" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | 10 | models "github.com/flyteorg/flyteadmin/scheduler/repositories/models" 11 | 12 | time "time" 13 | ) 14 | 15 | // Executor is an autogenerated mock type for the Executor type 16 | type Executor struct { 17 | mock.Mock 18 | } 19 | 20 | type Executor_Execute struct { 21 | *mock.Call 22 | } 23 | 24 | func (_m Executor_Execute) Return(_a0 error) *Executor_Execute { 25 | return &Executor_Execute{Call: _m.Call.Return(_a0)} 26 | } 27 | 28 | func (_m *Executor) OnExecute(ctx context.Context, scheduledTime time.Time, s models.SchedulableEntity) *Executor_Execute { 29 | c_call := _m.On("Execute", ctx, scheduledTime, s) 30 | return &Executor_Execute{Call: c_call} 31 | } 32 | 33 | func (_m *Executor) OnExecuteMatch(matchers ...interface{}) *Executor_Execute { 34 | c_call := _m.On("Execute", matchers...) 35 | return &Executor_Execute{Call: c_call} 36 | } 37 | 38 | // Execute provides a mock function with given fields: ctx, scheduledTime, s 39 | func (_m *Executor) Execute(ctx context.Context, scheduledTime time.Time, s models.SchedulableEntity) error { 40 | ret := _m.Called(ctx, scheduledTime, s) 41 | 42 | var r0 error 43 | if rf, ok := ret.Get(0).(func(context.Context, time.Time, models.SchedulableEntity) error); ok { 44 | r0 = rf(ctx, scheduledTime, s) 45 | } else { 46 | r0 = ret.Error(0) 47 | } 48 | 49 | return r0 50 | } 51 | -------------------------------------------------------------------------------- /scheduler/identifier/doc.go: -------------------------------------------------------------------------------- 1 | // Package identifier 2 | // This package provides utility functions for creating a unique schedule name and execution name 3 | package identifier 4 | -------------------------------------------------------------------------------- /scheduler/repositories/doc.go: -------------------------------------------------------------------------------- 1 | // Package repositories 2 | // This package provides the interfaces and implementations to save and retrieve schedules and snapshots from the DB. 3 | // Along with activating and deactivating the schedules. 4 | package repositories 5 | -------------------------------------------------------------------------------- /scheduler/repositories/gormimpl/metrics.go: -------------------------------------------------------------------------------- 1 | package gormimpl 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/flyteorg/flytestdlib/promutils" 7 | ) 8 | 9 | // Common metrics emitted by gormimpl repos. 10 | type gormMetrics struct { 11 | Scope promutils.Scope 12 | CreateDuration promutils.StopWatch 13 | GetDuration promutils.StopWatch 14 | UpdateDuration promutils.StopWatch 15 | ListDuration promutils.StopWatch 16 | ListIdentifiersDuration promutils.StopWatch 17 | DeleteDuration promutils.StopWatch 18 | ExistsDuration promutils.StopWatch 19 | } 20 | 21 | func newMetrics(scope promutils.Scope) gormMetrics { 22 | return gormMetrics{ 23 | Scope: scope, 24 | CreateDuration: scope.MustNewStopWatch( 25 | "create", "time taken to create a new entry", time.Millisecond), 26 | GetDuration: scope.MustNewStopWatch( 27 | "get", "time taken to get an entry", time.Millisecond), 28 | UpdateDuration: scope.MustNewStopWatch( 29 | "update", "time taken to update an entry", time.Millisecond), 30 | ListDuration: scope.MustNewStopWatch( 31 | "list", "time taken to list entries", time.Millisecond), 32 | ListIdentifiersDuration: scope.MustNewStopWatch( 33 | "list_identifiers", "time taken to list identifier entries", time.Millisecond), 34 | DeleteDuration: scope.MustNewStopWatch("delete", "time taken to delete an individual entry", time.Millisecond), 35 | ExistsDuration: scope.MustNewStopWatch("exists", "time taken to determine whether an individual entry exists", time.Millisecond), 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /scheduler/repositories/interfaces/schedulable_entity_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/scheduler/repositories/models" 7 | ) 8 | 9 | //go:generate mockery -name=SchedulableEntityRepoInterface -output=../mocks -case=underscore 10 | 11 | // SchedulableEntityRepoInterface : An Interface for interacting with the schedulable entity in the database 12 | type SchedulableEntityRepoInterface interface { 13 | 14 | // Create a schedulable entity in the database store 15 | Create(ctx context.Context, input models.SchedulableEntity) error 16 | 17 | // Activate a schedulable entity in the database store. 18 | Activate(ctx context.Context, input models.SchedulableEntity) error 19 | 20 | // Deactivate a schedulable entity in the database store. 21 | Deactivate(ctx context.Context, ID models.SchedulableEntityKey) error 22 | 23 | // Get a schedulable entity from the database store using the schedulable entity id. 24 | Get(ctx context.Context, ID models.SchedulableEntityKey) (models.SchedulableEntity, error) 25 | 26 | // GetAll Gets all the active schedulable entities from the db 27 | GetAll(ctx context.Context) ([]models.SchedulableEntity, error) 28 | } 29 | -------------------------------------------------------------------------------- /scheduler/repositories/interfaces/schedule_entities_snapshot_repo.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/flyteorg/flyteadmin/scheduler/repositories/models" 7 | ) 8 | 9 | //go:generate mockery -name=ScheduleEntitiesSnapShotRepoInterface -output=../mocks -case=underscore 10 | 11 | // ScheduleEntitiesSnapShotRepoInterface : An Interface for interacting with the snapshot of schedulable entities in the database 12 | type ScheduleEntitiesSnapShotRepoInterface interface { 13 | 14 | // Create/ Update the snapshot in the database store 15 | Write(ctx context.Context, input models.ScheduleEntitiesSnapshot) error 16 | 17 | // Get the latest snapshot from the database store. 18 | Read(ctx context.Context) (models.ScheduleEntitiesSnapshot, error) 19 | } 20 | -------------------------------------------------------------------------------- /scheduler/repositories/interfaces/scheduler.go: -------------------------------------------------------------------------------- 1 | package interfaces 2 | 3 | type SchedulerRepoInterface interface { 4 | SchedulableEntityRepo() SchedulableEntityRepoInterface 5 | ScheduleEntitiesSnapshotRepo() ScheduleEntitiesSnapShotRepoInterface 6 | } 7 | -------------------------------------------------------------------------------- /scheduler/repositories/models/schedulable_entity.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/flyteorg/flyteadmin/pkg/repositories/models" 5 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin" 6 | ) 7 | 8 | // Database model to encapsulate metadata associated with a SchedulableEntity 9 | type SchedulableEntity struct { 10 | models.BaseModel 11 | SchedulableEntityKey 12 | CronExpression string 13 | FixedRateValue uint32 14 | Unit admin.FixedRateUnit 15 | KickoffTimeInputArg string 16 | Active *bool 17 | } 18 | 19 | // Schedulable entity primary key 20 | type SchedulableEntityKey struct { 21 | Project string `gorm:"primary_key"` 22 | Domain string `gorm:"primary_key"` 23 | Name string `gorm:"primary_key"` 24 | Version string `gorm:"primary_key"` 25 | } 26 | -------------------------------------------------------------------------------- /scheduler/repositories/models/schedule_entities_snapshot.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "github.com/flyteorg/flyteadmin/pkg/repositories/models" 4 | 5 | // Database model to save the snapshot for the schedulable entities in the db 6 | type ScheduleEntitiesSnapshot struct { 7 | models.BaseModel 8 | Snapshot []byte `gorm:"column:snapshot" schema:"-"` 9 | } 10 | 11 | type ScheduleEntitiesSnapshotCollectionOutput struct { 12 | Snapshots []ScheduleEntitiesSnapshot 13 | } 14 | -------------------------------------------------------------------------------- /scheduler/snapshoter/doc.go: -------------------------------------------------------------------------------- 1 | // Package snapshoter 2 | // This package provides the ability to snapshot all the schedules in the scheduler job store and persist them in the DB 3 | // in GOB binary format. Also it provides ability to bootstrap the scheduler from this snapshot so that the scheduler 4 | // can run catchup for all the schedules from the snapshoted time. 5 | package snapshoter 6 | -------------------------------------------------------------------------------- /scheduler/snapshoter/persistence.go: -------------------------------------------------------------------------------- 1 | package snapshoter 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Persistence allows to read and save the serialized form of the snapshot from a storage. 8 | // Currently we have DB implementation for it. 9 | type Persistence interface { 10 | // Save Run(ctx context.Context) 11 | // Save saves the snapshot to the storage in a serialized form. 12 | Save(ctx context.Context, writer Writer, snapshot Snapshot) 13 | // Read reads the serialized snapshot from the storage and deserializes to its in memory format. 14 | Read(ctx context.Context, reader Reader) (Snapshot, error) 15 | } 16 | -------------------------------------------------------------------------------- /scheduler/snapshoter/reader.go: -------------------------------------------------------------------------------- 1 | package snapshoter 2 | 3 | import "io" 4 | 5 | // Reader provides an interface to read the snapshot and deserialize it to its in memory format. 6 | type Reader interface { 7 | // ReadSnapshot reads the snapshot from the reader 8 | ReadSnapshot(reader io.Reader) (Snapshot, error) 9 | } 10 | -------------------------------------------------------------------------------- /scheduler/snapshoter/snapshot.go: -------------------------------------------------------------------------------- 1 | package snapshoter 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Snapshot used by the scheduler for creating, updating and reading snapshots of the schedules. 8 | type Snapshot interface { 9 | // GetLastExecutionTime of the schedule given by the key 10 | GetLastExecutionTime(key string) *time.Time 11 | // UpdateLastExecutionTime of the schedule given by key to the lastExecTime 12 | UpdateLastExecutionTime(key string, lastExecTime *time.Time) 13 | // CreateSnapshot creates the snapshot of all the schedules and there execution times. 14 | Serialize() ([]byte, error) 15 | // BootstrapFrom bootstraps the snapshot from a byte array 16 | Deserialize(snapshot []byte) error 17 | // GetVersion gets the version number of snapshot written 18 | GetVersion() int 19 | // IsEmpty returns true if the snapshot contains no schedules 20 | IsEmpty() bool 21 | // Create an empty snapshot 22 | Create() Snapshot 23 | } 24 | -------------------------------------------------------------------------------- /scheduler/snapshoter/versioned_snapshot.go: -------------------------------------------------------------------------------- 1 | package snapshoter 2 | 3 | import ( 4 | "encoding/gob" 5 | "fmt" 6 | "io" 7 | "time" 8 | ) 9 | 10 | // VersionedSnapshot stores the version and gob serialized form of the snapshot 11 | // Provides a read and write methods to serialize and deserialize the gob format of the snapshot. 12 | // Including a version provides compatibility check 13 | type VersionedSnapshot struct { 14 | Version int 15 | Ser []byte 16 | } 17 | 18 | func (s *VersionedSnapshot) WriteSnapshot(w io.Writer, snapshot Snapshot) error { 19 | byteContents, err := snapshot.Serialize() 20 | if err != nil { 21 | return err 22 | } 23 | s.Version = snapshot.GetVersion() 24 | s.Ser = byteContents 25 | enc := gob.NewEncoder(w) 26 | return enc.Encode(s) 27 | } 28 | 29 | func (s *VersionedSnapshot) ReadSnapshot(r io.Reader) (Snapshot, error) { 30 | err := gob.NewDecoder(r).Decode(s) 31 | if err != nil { 32 | return nil, err 33 | } 34 | if s.Version == 1 { 35 | snapShotV1 := SnapshotV1{LastTimes: map[string]*time.Time{}} 36 | err = snapShotV1.Deserialize(s.Ser) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return &snapShotV1, nil 41 | } 42 | return nil, fmt.Errorf("unsupported version %v", s.Version) 43 | } 44 | -------------------------------------------------------------------------------- /scheduler/snapshoter/versioned_snapshot_test.go: -------------------------------------------------------------------------------- 1 | package snapshoter 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestReadWriteSnapshot(t *testing.T) { 12 | t.Run("successful read write", func(t *testing.T) { 13 | var bytesArray []byte 14 | f := bytes.NewBuffer(bytesArray) 15 | writer := VersionedSnapshot{} 16 | snapshot := &SnapshotV1{ 17 | LastTimes: map[string]*time.Time{}, 18 | } 19 | currTime := time.Now() 20 | snapshot.LastTimes["schedule1"] = &currTime 21 | err := writer.WriteSnapshot(f, snapshot) 22 | assert.Nil(t, err) 23 | r := bytes.NewReader(f.Bytes()) 24 | reader := VersionedSnapshot{} 25 | s, err := reader.ReadSnapshot(r) 26 | assert.Nil(t, err) 27 | assert.Equal(t, s.IsEmpty(), false) 28 | assert.NotNil(t, s.GetLastExecutionTime("schedule1")) 29 | }) 30 | 31 | t.Run("successful write unsuccessful read", func(t *testing.T) { 32 | var bytesArray []byte 33 | f := bytes.NewBuffer(bytesArray) 34 | writer := VersionedSnapshot{} 35 | snapshot := &SnapshotV1{ 36 | LastTimes: map[string]*time.Time{}, 37 | } 38 | currTime := time.Now() 39 | snapshot.LastTimes["schedule1"] = &currTime 40 | err := writer.WriteSnapshot(f, snapshot) 41 | assert.Nil(t, err) 42 | 43 | bytesArray = f.Bytes() 44 | bytesArray[len(bytesArray)-1] = 1 45 | r := bytes.NewReader(f.Bytes()) 46 | reader := VersionedSnapshot{} 47 | _, err = reader.ReadSnapshot(r) 48 | assert.NotNil(t, err) 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /scheduler/snapshoter/writer.go: -------------------------------------------------------------------------------- 1 | package snapshoter 2 | 3 | import "io" 4 | 5 | // Writer provides an interface to write the serialized form of the snapshot to a writer 6 | type Writer interface { 7 | // WriteSnapshot writes the serialized form of the snapshot to the writer 8 | WriteSnapshot(writer io.Writer, snapshot Snapshot) error 9 | } 10 | -------------------------------------------------------------------------------- /script/gen_proto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 4 | 5 | protoc --proto_path=pkg/repositories/models/protos --go_out=pkg/repositories/gen node_execution.proto -------------------------------------------------------------------------------- /script/integration/execute.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. 4 | # ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: 5 | # 6 | # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst 7 | 8 | set -e 9 | 10 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 11 | 12 | # TODO: load all images 13 | docker tag ${IMAGE} "flyteadmin:test" 14 | kind load docker-image flyteadmin:test 15 | 16 | # start flyteadmin and dependencies 17 | kubectl apply -f "${DIR}/k8s/integration.yaml" 18 | 19 | # This is a separate function so that we can potentially reuse in the future when we have more than one test 20 | function wait_for_flyte_deploys() { 21 | SECONDS=0 22 | echo "" 23 | echo "waiting for flyte deploy to complete..." 24 | echo "" 25 | 26 | # wait for flyteadmin deployment to complete 27 | kubectl -n flyte rollout status deployment flyteadmin 28 | echo "" 29 | 30 | echo "Flyte deployed in $SECONDS seconds." 31 | } 32 | 33 | wait_for_flyte_deploys 34 | 35 | ## get the name of the flyteadmin pod 36 | POD_NAME=$(kubectl get pods -n flyte --field-selector=status.phase=Running -o go-template="{{range .items}}{{.metadata.name}}:{{end}}" | tr ":" "\n" | grep flyteadmin) 37 | 38 | echo $POD_NAME 39 | 40 | # launch the integration tests 41 | kubectl exec -it -n flyte "$POD_NAME" -- make -C /go/src/github.com/flyteorg/flyteadmin integration 42 | -------------------------------------------------------------------------------- /script/integration/k8s/main.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 5 | 6 | echo "" 7 | echo "waiting up to 5 minutes for kubernetes to start..." 8 | 9 | K8S_TIMEOUT="300" 10 | 11 | SECONDS=0 12 | while ! systemctl is-active --quiet multi-user.target; do 13 | sleep 2 14 | if [ "$SECONDS" -gt "$K8S_TIMEOUT" ]; then 15 | echo "ERROR: timed out waiting for kubernetes to start." 16 | exit 1 17 | fi 18 | done 19 | 20 | echo "kubernetes started in $SECONDS seconds." 21 | echo "" 22 | 23 | # Load the locally-built flyteadmin image 24 | docker load -i /images/flyteadmin 25 | 26 | # Start flyteadmin and dependencies 27 | kubectl create -f "${DIR}/integration.yaml" 28 | 29 | # In debug mode, run bash instead of running the tests 30 | if [ -n "$DOCKERNETES_DEBUG" ]; then 31 | bash 32 | fi 33 | 34 | # Wait for flyteadmin deployment to complete 35 | kubectl -n flyte rollout status deployment flyteadmin 36 | 37 | # Get the name of the flyteadmin pod 38 | POD_NAME=$(kubectl get pods -n flyte -o go-template="{{range .items}}{{.metadata.name}}:{{end}}" | tr ":" "\n" | grep flyteadmin) 39 | 40 | # Launch the integration tests 41 | kubectl exec -it -n flyte "$POD_NAME" -- make -C /go/src/github.com/flyteorg/flyteadmin integration 42 | -------------------------------------------------------------------------------- /script/integration/launch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 5 | 6 | GIT_SHA=$(git rev-parse HEAD) 7 | 8 | if ! docker images | grep "${GIT_SHA}-builder"; then 9 | echo "ERROR:" 10 | echo "flyteadmin:${GIT_SHA}-builder image does not exist." 11 | echo "build the image first with BUILD_PHASE=builder make docker_build" 12 | fi 13 | 14 | docker tag "flyteadmin:${GIT_SHA}-builder" "flyteadmin:test" 15 | 16 | docker save -o "/tmp/flyteadmin" "flyteadmin:test" 17 | 18 | # stop any existing test container that might be running 19 | docker kill dockernetes || true 20 | 21 | # The container must start with systemd (/sbin/init) as PID 1 22 | 23 | docker run \ 24 | --detach \ 25 | --rm \ 26 | --privileged \ 27 | --volume /var/lib/docker \ 28 | --volume /lib/modules:/lib/modules \ 29 | --volume ${DIR}/../..:/flyteadmin \ 30 | --volume /tmp/flyteadmin:/images/flyteadmin \ 31 | --name dockernetes \ 32 | --env "DOCKER_REGISTRY_USERNAME=${DOCKER_REGISTRY_USERNAME}" \ 33 | --env "DOCKER_REGISTRY_PASSWORD=${DOCKER_REGISTRY_PASSWORD}" \ 34 | --env "DOCKERNETES_DEBUG=${DOCKERNETES_DEBUG}" \ 35 | lyft/dockernetes:1.10.1-v0.1 /sbin/init 36 | 37 | # wait for the system to initalize, then run execute.sh 38 | docker exec \ 39 | -it \ 40 | dockernetes /flyteadmin/script/integration/k8s/main.sh 41 | -------------------------------------------------------------------------------- /tests/helpers.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service" 7 | "google.golang.org/grpc" 8 | "google.golang.org/grpc/grpclog" 9 | ) 10 | 11 | // This returns a gRPC client configured to hit the locally running instance of Flyte admin 12 | // This also returns a gRPC connection - be sure to defer a Close() call!!! 13 | func GetTestAdminServiceClient() (service.AdminServiceClient, *grpc.ClientConn) { 14 | // Load the running configuration in order to talk to the running flyteadmin instance 15 | 16 | var opts []grpc.DialOption 17 | opts = append(opts, grpc.WithInsecure()) 18 | conn, err := grpc.Dial(fmt.Sprintf("0.0.0.0:%d", 8089), opts...) 19 | if err != nil { 20 | grpclog.Fatalf("fail to dial: %v", err) 21 | } 22 | client := service.NewAdminServiceClient(conn) 23 | return client, conn 24 | } 25 | 26 | func GetTestHostEndpoint() string { 27 | return "http://localhost:8088" 28 | } 29 | --------------------------------------------------------------------------------