├── manifest ├── .gitignore ├── service-account.json ├── secret-onepanel-default-env-defaultnamespace.json ├── configmap-onepanel-defaultnamespace.json ├── abs │ ├── service.json │ └── deployment.json ├── gcs │ ├── service.json │ └── deployment.json ├── clusterrolebinding-models.json ├── clusterrolebinding-onepanel-namespaces-defaultnamespace.json ├── kfserving │ └── secret.json ├── secret-onepanel-defaultnamespace.json ├── rolebinding-onepanel-defaultnamespace.json ├── service-minio-onepanel.json └── networkpolicy-onepanel-defaultnamespace.json ├── .github ├── semantic.yml ├── issue_label_bot.yaml ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── push_tag.yaml │ └── push_dev_branch.yaml ├── pkg ├── util │ ├── validate │ │ ├── validate_test.go │ │ └── validate.go │ ├── number │ │ └── number.go │ ├── ptr │ │ └── ptr.go │ ├── uid │ │ └── uid.go │ ├── router │ │ ├── api.go │ │ └── router.go │ ├── s3 │ │ └── s3.go │ ├── request │ │ ├── request.go │ │ ├── sort │ │ │ └── sort.go │ │ └── pagination │ │ │ └── pagination.go │ ├── sql │ │ ├── sql_test.go │ │ └── sql.go │ ├── data │ │ └── migration.go │ ├── types │ │ └── types.go │ ├── env │ │ └── env.go │ ├── error.go │ ├── collection │ │ └── collection.go │ ├── gcs │ │ └── gcs.go │ └── label │ │ └── label.go ├── service_types.go ├── secret_test.go ├── namespace_test.go ├── filter.go ├── workspace_types_test.go ├── database.go ├── istio.go ├── types_test.go ├── label_types_test.go ├── file_types.go ├── common_types_test.go ├── workflow_template_version_types.go ├── workflow_execution_test.go ├── inference_service.go └── label_types.go ├── img ├── logo.png ├── features.png └── onepanel.gif ├── .gitignore ├── db ├── sql │ ├── 20200512182928_add_updated_at_to_workspaces.sql │ ├── 20200616194847_remove_workflow_template_versions_uid.sql │ ├── 20200204103708_add_is_archived_to_workflow_templates.sql │ ├── 20200511164307_add_path_to_workspaces.sql │ ├── 20200617122509_remove_url_from_workspaces.sql │ ├── 20210802222820_configurable_capture_workspace_node.sql │ ├── 20200420112057_cache_workflow_template_versions_count.sql │ ├── 20200522171359_alter_cron_workflows_column_sizes.sql │ ├── 20200504161141_add_parameters_to_workflow_executions.sql │ ├── 20200510202248_add_is_system_to_workflow_templates.sql │ ├── 20200521124859_add_unique_index_cron_workflows_uid_namespace.sql │ ├── 20200522164526_alter_workflow_executions_column_sizes.sql │ ├── 20200810132237_add_description_to_workspace_templates.sql │ ├── 20200727155027_add_parameters_col_to_workflow_template_version.sql │ ├── 20210326180613_add_description_to_workflow_template_versions.sql │ ├── 20201118200215_fix_default_migration.sql │ ├── 20200422160902_add_labels.sql │ ├── 20201116105825_add_metrics_to_workflows.sql │ ├── 20200123130105_add_latest_workflow_template_version.sql │ ├── 20200521145827_add_is_archived_to_cron_wfs_and_wf_execs.sql │ ├── 20191210141652_create_workflow_templates.sql │ ├── 20191210181732_create_workflow_template_versions.sql │ ├── 20200407113554_remove_workflow_template_versions_table.sql │ ├── 20200630112657_update_version_types.sql │ ├── 20200425173049_workspace_template_versions.sql │ ├── 20200524114047_update_is_archived_to_not_null.sql │ ├── 20201023121927_fix_null_labels.sql │ ├── 20200422140125_add_cron_workflows.sql │ ├── 20200424132932_add_started_and_version_to_workflow_executions.sql │ ├── 20200413141037_create_workflow_execution.sql │ ├── 20200521112757_add_namespace_cron_workflows.sql │ ├── 20200421103946_add_back_workflow_template_versions.sql │ ├── 20200518163745_add_conditional_unique_index_to_workflow_templates.sql │ ├── 20200425172611_workspace_templates.sql │ ├── 20200429141030_add_manifest_to_crons.sql │ └── 20200502184729_workspaces.sql ├── go │ ├── 20201016170415_update_cvat.go │ ├── 20211028205201_cvat_1_6.go │ ├── 20201115145814_add_tensorboard_to_maskrcnn.go │ ├── 20201115134934_add_tensorboard_to_tfod.go │ ├── 20210107094725_update_cvat_workspace.go │ ├── 20210129094725_update_cvat_workspace.go │ ├── 20201115133046_update_cvat_env_vars.go │ ├── 20201214133458_fix_jupyterlab_gpu.go │ ├── 20201211161117_fix_resources_gke.go │ ├── 20201229205644_fix_jupyter_workspace_yaml.go │ ├── 20210129142057_update_jupyter_workspace.go │ ├── 20201028145442_update_jupyter_lab_template.go │ ├── 20201113094916_update_cvat_onepanel_sdk.go │ ├── 20210414165510_add_deep_learning_desktop_workspace.go │ ├── 20201221194344_pytorch_update.go │ ├── 20201102104048_update_cvat_reduce_vols.go │ ├── 20201031165106_add_tensorboard_env_var_to_jupyterlab_template.go │ ├── 20201209124226_update_tensorflow_training.go │ ├── 20201130130433_update_tfod_path.go │ ├── 20201225172926_add_hyperparameter_tuning_example_workflow_template.go │ ├── 20201208155805_replace_tty_with_env_var_for_maskrcnn.go │ ├── 20201223062947_tf_training_update_node_pool.go │ ├── 20201028145443_update_vscode_template.go │ ├── 20210129152427_update_vscode_template.go │ ├── 20201221195937_maskrcnn_update_parameters_to_select_nodepool.go │ ├── 20201208155115_replace_tty_with_env_var_for_tfod.go │ ├── 20201223202929_tfod_update_node_pool.go │ ├── 20200728190804_update_workflow_template_labels.go │ ├── 20200814160856_add_description_to_jupyter_template.go │ ├── 20210329171739_update_workspaces.go │ ├── 20210224180017_update_workspace_templates.go │ ├── 20200922103448_add_jupyter_lab_description.go │ ├── 20210323175655_update_templates_for_pns.go │ ├── 20210719190719_update_filesyncer.go │ └── 20200727144157_add_parameters_to_workflow_template_version.go └── yaml │ ├── workspaces │ ├── vscode │ │ ├── 20200929144301.yaml │ │ ├── 20210129152427.yaml │ │ ├── 20210224180017.yaml │ │ ├── 20210719190719.yaml │ │ ├── 20210323175655.yaml │ │ └── 20201028145443.yaml │ ├── vnc │ │ ├── 20210414165510.yaml │ │ └── 20210719190719.yaml │ └── jupyterlab │ │ ├── 20200525160514.yaml │ │ ├── 20200821162630.yaml │ │ └── 20200929153931.yaml │ └── workflows │ ├── pytorch-mnist-training │ └── 20200605090509.yaml │ └── tensorflow-mnist-training │ └── 20200605090535.yaml ├── api ├── proto │ ├── metric.proto │ ├── common.proto │ ├── api.proto │ ├── namespace.proto │ ├── config.proto │ ├── files.proto │ ├── services.proto │ ├── auth.proto │ ├── label.proto │ ├── inference_service.proto │ └── cron_workflow.proto └── third_party │ ├── README.grpc-gateway │ ├── google │ └── api │ │ ├── annotations.proto │ │ └── httpbody.proto │ └── protoc-gen-openapiv2 │ └── options │ └── annotations.proto ├── helper ├── go.mod ├── tools.go ├── README.md └── Dockerfile ├── server ├── server.go ├── service_server.go ├── config_server.go └── namespace_server.go ├── cmd ├── README.md └── goose │ └── goose.go ├── Dockerfile ├── design-docs └── Persist Packages during Workspace Switch or Update.md ├── Makefile ├── go.mod └── README.md /manifest/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/semantic.yml: -------------------------------------------------------------------------------- 1 | titleOnly: true 2 | -------------------------------------------------------------------------------- /pkg/util/validate/validate_test.go: -------------------------------------------------------------------------------- 1 | package validate 2 | -------------------------------------------------------------------------------- /img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onepanelio/onepanel/HEAD/img/logo.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .DS_Store 3 | .idea 4 | pocs 5 | __debug_bin 6 | vendor 7 | venv3 -------------------------------------------------------------------------------- /img/features.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onepanelio/onepanel/HEAD/img/features.png -------------------------------------------------------------------------------- /img/onepanel.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onepanelio/onepanel/HEAD/img/onepanel.gif -------------------------------------------------------------------------------- /.github/issue_label_bot.yaml: -------------------------------------------------------------------------------- 1 | label-alias: 2 | bug: 'kind/bug' 3 | feature_request: 'kind/enhancement' 4 | question: 'kind/question' 5 | -------------------------------------------------------------------------------- /db/sql/20200512182928_add_updated_at_to_workspaces.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workspaces ADD COLUMN updated_at timestamp; 3 | 4 | -- +goose Down 5 | ALTER TABLE workspaces DROP COLUMN updated_at; -------------------------------------------------------------------------------- /db/sql/20200616194847_remove_workflow_template_versions_uid.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_template_versions DROP COLUMN uid; 3 | 4 | -- +goose Down 5 | UPDATE workflow_template_versions SET uid = version::text; -------------------------------------------------------------------------------- /manifest/service-account.json: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "ServiceAccount", 3 | "apiVersion": "v1", 4 | "metadata": { 5 | "name": "$(applicationDefaultNamespace)", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | } 8 | } -------------------------------------------------------------------------------- /api/proto/metric.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | message Metric { 7 | string name = 1; 8 | double value = 2; 9 | string format = 3; 10 | } -------------------------------------------------------------------------------- /db/sql/20200204103708_add_is_archived_to_workflow_templates.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_templates ADD COLUMN is_archived boolean DEFAULT false; 3 | 4 | -- +goose Down 5 | ALTER TABLE workflow_templates DROP COLUMN is_archived; 6 | -------------------------------------------------------------------------------- /helper/go.mod: -------------------------------------------------------------------------------- 1 | module core-helper/m/v2 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.0.1 7 | google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.1 8 | google.golang.org/protobuf v1.25.0 9 | ) 10 | -------------------------------------------------------------------------------- /manifest/secret-onepanel-default-env-defaultnamespace.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Secret", 4 | "metadata": { 5 | "name": "onepanel-default-env", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "type": "Opaque" 9 | } -------------------------------------------------------------------------------- /db/sql/20200511164307_add_path_to_workspaces.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workspaces ADD COLUMN url TEXT; 3 | UPDATE workspaces set url = ''; 4 | ALTER TABLE workspaces ALTER COLUMN url SET NOT NULL; 5 | 6 | -- +goose Down 7 | ALTER TABLE workspaces DROP COLUMN url; -------------------------------------------------------------------------------- /pkg/service_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | // Service represents an installable "service" added to the system. 4 | // This can be something like modeldb, or some other service that complements the main system. 5 | type Service struct { 6 | Name string 7 | URL string 8 | } 9 | -------------------------------------------------------------------------------- /db/sql/20200617122509_remove_url_from_workspaces.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workspaces DROP COLUMN url; 3 | 4 | -- +goose Down 5 | ALTER TABLE workspaces ADD COLUMN url TEXT; 6 | UPDATE workspaces set url = ''; 7 | ALTER TABLE workspaces ALTER COLUMN url SET NOT NULL; 8 | -------------------------------------------------------------------------------- /db/sql/20210802222820_configurable_capture_workspace_node.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE workspaces ADD COLUMN capture_node boolean; 4 | UPDATE workspaces SET capture_node = false; 5 | 6 | -- +goose Down 7 | ALTER TABLE workspaces DROP COLUMN capture_node; 8 | -------------------------------------------------------------------------------- /manifest/configmap-onepanel-defaultnamespace.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "ConfigMap", 4 | "metadata": { 5 | "name": "onepanel", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "data": { 9 | "artifactRepository": "archiveLogs: true\n$(artifactRepositoryProvider)\n" 10 | } 11 | } -------------------------------------------------------------------------------- /db/sql/20200420112057_cache_workflow_template_versions_count.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_templates ADD COLUMN versions INTEGER; 3 | UPDATE workflow_templates SET versions = 1; 4 | ALTER TABLE workflow_templates ALTER COLUMN versions SET NOT NULL; 5 | 6 | -- +goose Down 7 | ALTER TABLE workflow_templates DROP COLUMN versions; -------------------------------------------------------------------------------- /db/sql/20200522171359_alter_cron_workflows_column_sizes.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE cron_workflows ALTER COLUMN uid TYPE varchar(63); 3 | ALTER TABLE cron_workflows ALTER COLUMN name TYPE varchar(63); 4 | 5 | -- +goose Down 6 | ALTER TABLE cron_workflows ALTER COLUMN uid TYPE varchar(30); 7 | ALTER TABLE cron_workflows ALTER COLUMN name TYPE varchar(30); -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | v1 "github.com/onepanelio/core/pkg" 6 | "github.com/onepanelio/core/server/auth" 7 | ) 8 | 9 | const ( 10 | TimeLayout = "2006-01-02 15:04:05" 11 | ) 12 | 13 | func getClient(ctx context.Context) *v1.Client { 14 | return ctx.Value(auth.ContextClientKey).(*v1.Client) 15 | } 16 | -------------------------------------------------------------------------------- /db/sql/20200504161141_add_parameters_to_workflow_executions.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_executions ADD COLUMN parameters JSONB; 3 | UPDATE workflow_executions SET parameters = '[]'::JSONB; 4 | ALTER TABLE workflow_executions ALTER COLUMN parameters SET NOT NULL; 5 | 6 | -- +goose Down 7 | ALTER TABLE workflow_executions DROP COLUMN parameters; 8 | -------------------------------------------------------------------------------- /db/sql/20200510202248_add_is_system_to_workflow_templates.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_templates ADD COLUMN is_system BOOLEAN DEFAULT false; 3 | UPDATE workflow_templates SET is_system = false; 4 | ALTER TABLE workflow_templates ALTER COLUMN is_system SET NOT NULL; 5 | 6 | -- +goose Down 7 | ALTER TABLE workflow_templates DROP COLUMN is_system; 8 | -------------------------------------------------------------------------------- /pkg/util/validate/validate.go: -------------------------------------------------------------------------------- 1 | package validate 2 | 3 | import ( 4 | "github.com/asaskevich/govalidator" 5 | "strings" 6 | ) 7 | 8 | func IsDNSHost(str string) bool { 9 | if str == "" || len(strings.Replace(str, ".", "", -1)) > 63 { 10 | // constraints already violated 11 | return false 12 | } 13 | return govalidator.IsDNSName(str) 14 | } 15 | -------------------------------------------------------------------------------- /db/sql/20200521124859_add_unique_index_cron_workflows_uid_namespace.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | CREATE UNIQUE INDEX cron_workflow_namespace_uid ON cron_workflows (uid, namespace); 4 | -- +goose Down 5 | -- SQL in this section is executed when the migration is rolled back. 6 | DROP INDEX cron_workflow_namespace_uid; -------------------------------------------------------------------------------- /db/sql/20200522164526_alter_workflow_executions_column_sizes.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_executions ALTER COLUMN uid TYPE varchar(63); 3 | ALTER TABLE workflow_executions ALTER COLUMN name TYPE varchar(63); 4 | 5 | -- +goose Down 6 | ALTER TABLE workflow_executions ALTER COLUMN uid TYPE varchar(30); 7 | ALTER TABLE workflow_executions ALTER COLUMN name TYPE text; -------------------------------------------------------------------------------- /db/sql/20200810132237_add_description_to_workspace_templates.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE workspace_templates ADD COLUMN description TEXT DEFAULT ''; 4 | 5 | -- +goose Down 6 | -- SQL in this section is executed when the migration is rolled back. 7 | ALTER TABLE workspace_templates DROP COLUMN description; 8 | -------------------------------------------------------------------------------- /db/sql/20200727155027_add_parameters_col_to_workflow_template_version.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_template_versions ADD COLUMN parameters JSONB; 3 | UPDATE workflow_template_versions SET parameters = '[]'::JSONB; 4 | ALTER TABLE workflow_template_versions ALTER COLUMN parameters SET NOT NULL; 5 | 6 | -- +goose Down 7 | ALTER TABLE workflow_template_versions DROP COLUMN parameters; 8 | -------------------------------------------------------------------------------- /db/sql/20210326180613_add_description_to_workflow_template_versions.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE workflow_template_versions ADD COLUMN description TEXT DEFAULT ''; 4 | 5 | -- +goose Down 6 | -- SQL in this section is executed when the migration is rolled back. 7 | ALTER TABLE workflow_template_versions DROP COLUMN description; -------------------------------------------------------------------------------- /db/sql/20201118200215_fix_default_migration.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | UPDATE workflow_executions SET metrics = '[]'::JSONB 4 | WHERE metrics = '{}'::JSONB; 5 | 6 | -- +goose Down 7 | -- SQL in this section is executed when the migration is rolled back. 8 | UPDATE workflow_executions SET metrics = '{}'::JSONB 9 | WHERE metrics = '[]'::JSONB; -------------------------------------------------------------------------------- /manifest/abs/service.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Service", 4 | "metadata": { 5 | "name": "minio-gateway", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "spec": { 9 | "selector": { 10 | "app": "minio-gateway" 11 | }, 12 | "ports": [ 13 | { 14 | "port": 9000, 15 | "targetPort": 9000 16 | } 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /manifest/gcs/service.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Service", 4 | "metadata": { 5 | "name": "minio-gateway", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "spec": { 9 | "selector": { 10 | "app": "minio-gateway" 11 | }, 12 | "ports": [ 13 | { 14 | "port": 9000, 15 | "targetPort": 9000 16 | } 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /db/sql/20200422160902_add_labels.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE labels 3 | ( 4 | id serial PRIMARY KEY, 5 | key varchar(255), 6 | value varchar(255), 7 | resource varchar(255), 8 | resource_id INTEGER, 9 | 10 | -- auditing info 11 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc') 12 | ); 13 | 14 | -- +goose Down 15 | DROP TABLE labels; 16 | -------------------------------------------------------------------------------- /cmd/README.md: -------------------------------------------------------------------------------- 1 | # Helper scripts 2 | 3 | ## gen-release-md.go 4 | Generates markdown for releases. 5 | 6 | Usage: 7 | ```bash 8 | go run cmd/gen-release-md/gen-release-md.go -v=0.10.0 -u=[github-username] > /tmp/release.md 9 | ``` 10 | 11 | ## goose.go 12 | Supports both Go and SQL migrations. 13 | 14 | ```bash 15 | go run cmd/goose/goose up # run up migrations 16 | go run cmd/goose/goose down # run down migrations 17 | ``` -------------------------------------------------------------------------------- /pkg/util/number/number.go: -------------------------------------------------------------------------------- 1 | package number 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | // Takes the input value, increments it, and formats it as a string. 9 | func IncrementStringInt(value string) (string, error) { 10 | numericValue, err := strconv.Atoi(value) 11 | if err != nil { 12 | return value, err 13 | } 14 | 15 | numericValue++ 16 | 17 | stringValue := fmt.Sprintf("%v", numericValue) 18 | 19 | return stringValue, nil 20 | } 21 | -------------------------------------------------------------------------------- /db/sql/20201116105825_add_metrics_to_workflows.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE workflow_executions ADD COLUMN metrics JSONB; 4 | UPDATE workflow_executions SET metrics = '{}'::JSONB; 5 | ALTER TABLE workflow_executions ALTER COLUMN metrics SET NOT NULL; 6 | 7 | -- +goose Down 8 | -- SQL in this section is executed when the migration is rolled back. 9 | ALTER TABLE workflow_executions DROP COLUMN metrics; -------------------------------------------------------------------------------- /pkg/util/ptr/ptr.go: -------------------------------------------------------------------------------- 1 | package ptr 2 | 3 | import "time" 4 | 5 | func Bool(value bool) *bool { 6 | return &value 7 | } 8 | 9 | func Int32(value int32) *int32 { 10 | return &value 11 | } 12 | 13 | func Uint64(value uint64) *uint64 { 14 | return &value 15 | } 16 | 17 | func Int64(value int64) *int64 { 18 | return &value 19 | } 20 | 21 | func String(value string) *string { 22 | return &value 23 | } 24 | 25 | func Time(value time.Time) *time.Time { 26 | return &value 27 | } 28 | -------------------------------------------------------------------------------- /pkg/util/uid/uid.go: -------------------------------------------------------------------------------- 1 | package uid 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | func GenerateUID(input string, max int) (string, error) { 11 | re, _ := regexp.Compile(`[^a-zA-Z0-9-]{1,}`) 12 | cleanUp := strings.ToLower(re.ReplaceAllString(input, `-`)) 13 | if len(cleanUp) > max { 14 | return "", errors.New(fmt.Sprintf("Length of string exceeds %d", max)) 15 | } 16 | return strings.ToLower(re.ReplaceAllString(input, `-`)), nil 17 | } 18 | -------------------------------------------------------------------------------- /db/sql/20200123130105_add_latest_workflow_template_version.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_template_versions ADD COLUMN is_latest boolean; 3 | 4 | UPDATE workflow_template_versions 5 | SET is_latest = false; 6 | 7 | UPDATE workflow_template_versions 8 | SET is_latest = true 9 | WHERE id IN ( 10 | SELECT max(id) 11 | FROM workflow_template_versions 12 | GROUP BY workflow_template_id, id 13 | ); 14 | 15 | -- +goose Down 16 | ALTER TABLE workflow_template_versions DROP COLUMN is_latest; 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.5 AS builder 2 | 3 | WORKDIR /go/src 4 | COPY . . 5 | 6 | RUN go get -d -v ./... 7 | RUN go install -v ./... 8 | RUN go get -u github.com/pressly/goose/cmd/goose 9 | RUN go build -o /go/bin/goose ./cmd/goose/goose.go 10 | 11 | FROM golang:1.15.5 12 | COPY --from=builder /go/bin/core . 13 | COPY --from=builder /go/src/db ./db 14 | COPY --from=builder /go/bin/goose . 15 | COPY --from=builder /go/src/manifest ./manifest 16 | 17 | EXPOSE 8888 18 | EXPOSE 8887 19 | 20 | CMD ["./core"] 21 | -------------------------------------------------------------------------------- /db/sql/20200521145827_add_is_archived_to_cron_wfs_and_wf_execs.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE cron_workflows ADD COLUMN is_archived BOOL DEFAULT false NOT NULL; 4 | ALTER TABLE workflow_executions ADD COLUMN is_archived BOOL DEFAULT false NOT NULL; 5 | -- +goose Down 6 | -- SQL in this section is executed when the migration is rolled back. 7 | ALTER TABLE cron_workflows DROP COLUMN is_archived; 8 | ALTER TABLE workflow_executions DROP COLUMN is_archived; -------------------------------------------------------------------------------- /helper/tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package corehelper 4 | 5 | import ( 6 | // this is for finding the package versions 7 | _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway" 8 | // this is for finding the package versions 9 | _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2" 10 | // this is for finding the package versions 11 | _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" 12 | // this is for finding the package versions 13 | _ "google.golang.org/protobuf/cmd/protoc-gen-go" 14 | ) 15 | -------------------------------------------------------------------------------- /db/sql/20191210141652_create_workflow_templates.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE workflow_templates 3 | ( 4 | id serial PRIMARY KEY, 5 | uid varchar(30) UNIQUE NOT NULL CHECK(uid <> ''), 6 | name text NOT NULL CHECK(name <> ''), 7 | namespace varchar(30) NOT NULL, 8 | 9 | -- auditing info 10 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 11 | modified_at timestamp, 12 | 13 | UNIQUE (uid, namespace) 14 | ); 15 | 16 | -- +goose Down 17 | DROP TABLE workflow_templates; -------------------------------------------------------------------------------- /api/third_party/README.grpc-gateway: -------------------------------------------------------------------------------- 1 | Google APIs 2 | ============ 3 | 4 | Project: Google APIs 5 | URL: https://github.com/google/googleapis 6 | Revision: 3544ab16c3342d790b00764251e348705991ea4b 7 | License: Apache License 2.0 8 | 9 | 10 | Imported Files 11 | --------------- 12 | 13 | - google/api/annotations.proto 14 | - google/api/http.proto 15 | - google/api/httpbody.proto 16 | 17 | 18 | Generated Files 19 | ---------------- 20 | 21 | They are generated from the .proto files by protoc-gen-go. 22 | - google/api/annotations.pb.go 23 | - google/api/http.pb.go 24 | -------------------------------------------------------------------------------- /manifest/clusterrolebinding-models.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "rbac.authorization.k8s.io/v1", 3 | "kind": "ClusterRoleBinding", 4 | "metadata": { 5 | "name": "onepanel-kfserving-$(applicationDefaultNamespace)" 6 | }, 7 | "subjects": [ 8 | { 9 | "kind": "ServiceAccount", 10 | "name": "$(applicationDefaultNamespace)", 11 | "namespace": "$(applicationDefaultNamespace)" 12 | } 13 | ], 14 | "roleRef": { 15 | "apiGroup": "rbac.authorization.k8s.io", 16 | "kind": "ClusterRole", 17 | "name": "onepanel-models" 18 | } 19 | } -------------------------------------------------------------------------------- /db/sql/20191210181732_create_workflow_template_versions.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE workflow_template_versions 3 | ( 4 | id serial PRIMARY KEY, 5 | workflow_template_id integer NOT NULL REFERENCES workflow_templates ON DELETE CASCADE, 6 | version integer NOT NULL, 7 | manifest text NOT NULL, 8 | 9 | -- auditing info 10 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 11 | modified_at timestamp 12 | ); 13 | 14 | -- +goose Down 15 | DROP TABLE workflow_template_versions; -------------------------------------------------------------------------------- /manifest/clusterrolebinding-onepanel-namespaces-defaultnamespace.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "rbac.authorization.k8s.io/v1beta1", 3 | "kind": "ClusterRoleBinding", 4 | "metadata": { 5 | "labels": { 6 | "app": "onepanel" 7 | }, 8 | "name": "onepanel-namespaces" 9 | }, 10 | "roleRef": { 11 | "apiGroup": "rbac.authorization.k8s.io", 12 | "kind": "ClusterRole", 13 | "name": "onepanel-namespaces" 14 | }, 15 | "subjects": [ 16 | { 17 | "kind": "ServiceAccount", 18 | "name": "$(applicationDefaultNamespace)", 19 | "namespace": "$(applicationDefaultNamespace)" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /db/sql/20200407113554_remove_workflow_template_versions_table.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | DROP TABLE workflow_template_versions; 3 | 4 | -- +goose Down 5 | CREATE TABLE workflow_template_versions 6 | ( 7 | id serial PRIMARY KEY, 8 | workflow_template_id integer NOT NULL REFERENCES workflow_templates ON DELETE CASCADE, 9 | version integer NOT NULL, 10 | is_latest boolean NOT NULL, 11 | manifest text NOT NULL, 12 | 13 | -- auditing info 14 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 15 | modified_at timestamp 16 | ); 17 | 18 | -------------------------------------------------------------------------------- /pkg/secret_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestClient_CreateSecret(t *testing.T) { 10 | c := DefaultTestClient() 11 | 12 | err := c.CreateSecret("namespace", &Secret{ 13 | Name: "name", 14 | }) 15 | assert.Nil(t, err) 16 | } 17 | 18 | func TestClient_GetSecret(t *testing.T) { 19 | c := DefaultTestClient() 20 | 21 | err := c.CreateSecret("namespace", &Secret{ 22 | Name: "name", 23 | }) 24 | assert.Nil(t, err) 25 | 26 | s, err := c.GetSecret("namespace", "name") 27 | assert.Nil(t, err) 28 | 29 | assert.NotNil(t, s) 30 | assert.Equal(t, s.Name, "name") 31 | } 32 | -------------------------------------------------------------------------------- /db/sql/20200630112657_update_version_types.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE workflow_template_versions ALTER COLUMN version TYPE BIGINT; 4 | ALTER TABLE workspace_template_versions ALTER COLUMN version TYPE BIGINT; 5 | ALTER TABLE workspaces ALTER COLUMN workspace_template_version TYPE BIGINT; 6 | 7 | -- +goose Down 8 | -- SQL in this section is executed when the migration is rolled back. 9 | ALTER TABLE workflow_template_versions ALTER COLUMN version TYPE INT; 10 | ALTER TABLE workspace_template_versions ALTER COLUMN version TYPE INT; 11 | ALTER TABLE workspaces ALTER COLUMN workspace_template_version TYPE INT; -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /manifest/kfserving/secret.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Secret", 4 | "metadata": { 5 | "name": "kfserving-storage", 6 | "namespace": "$(applicationDefaultNamespace)", 7 | "annotations": { 8 | "serving.kubeflow.org/s3-endpoint": "$(artifactRepositoryS3Endpoint)", 9 | "serving.kubeflow.org/s3-usehttps": "0", 10 | "serving.kubeflow.org/s3-region": "$(artifactRepositoryS3Region)", 11 | "serving.kubeflow.org/s3-useanoncredential": "false" 12 | } 13 | }, 14 | "type": "Opaque", 15 | "data": { 16 | "AWS_ACCESS_KEY_ID": "$(artifactRepositoryS3AccessKey)", 17 | "AWS_SECRET_ACCESS_KEY": "$(artifactRepositoryS3SecretKey)" 18 | } 19 | } -------------------------------------------------------------------------------- /db/sql/20200425173049_workspace_template_versions.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE workspace_template_versions 3 | ( 4 | id serial PRIMARY KEY, 5 | workspace_template_id integer NOT NULL REFERENCES workspace_templates ON DELETE CASCADE, 6 | version integer NOT NULL, 7 | manifest text NOT NULL, 8 | is_latest boolean DEFAULT false, 9 | 10 | -- auditing info 11 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 12 | modified_at timestamp 13 | ); 14 | 15 | -- +goose Down 16 | DROP TABLE workspace_template_versions; -------------------------------------------------------------------------------- /db/sql/20200524114047_update_is_archived_to_not_null.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_templates ALTER COLUMN is_archived SET DEFAULT false; 3 | ALTER TABLE workflow_templates ALTER COLUMN is_archived SET NOT NULL; 4 | 5 | ALTER TABLE workspace_templates ALTER COLUMN is_archived SET DEFAULT false; 6 | ALTER TABLE workspace_templates ALTER COLUMN is_archived SET NOT NULL; 7 | 8 | -- +goose Down 9 | ALTER TABLE workspace_templates ALTER COLUMN is_archived DROP NOT NULL; 10 | ALTER TABLE workspace_templates ALTER COLUMN is_archived DROP DEFAULT; 11 | 12 | ALTER TABLE workflow_templates ALTER COLUMN is_archived DROP NOT NULL; 13 | ALTER TABLE workflow_templates ALTER COLUMN is_archived DROP DEFAULT; 14 | -------------------------------------------------------------------------------- /db/sql/20201023121927_fix_null_labels.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | UPDATE workflow_executions 4 | SET labels = '{}'::jsonb 5 | WHERE labels = 'null'::jsonb; 6 | 7 | UPDATE workflow_templates 8 | SET labels = '{}'::jsonb 9 | WHERE labels = 'null'::jsonb; 10 | 11 | UPDATE workspace_templates 12 | SET labels = '{}'::jsonb 13 | WHERE labels = 'null'::jsonb; 14 | 15 | UPDATE workflow_template_versions 16 | SET labels = '{}'::jsonb 17 | WHERE labels = 'null'::jsonb; 18 | 19 | UPDATE workspace_template_Versions 20 | SET labels = '{}'::jsonb 21 | WHERE labels = 'null'::jsonb; 22 | 23 | -- +goose Down 24 | -- SQL in this section is executed when the migration is rolled back. 25 | -------------------------------------------------------------------------------- /manifest/secret-onepanel-defaultnamespace.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "v1", 3 | "kind": "Secret", 4 | "metadata": { 5 | "name": "onepanel", 6 | "namespace": "$(applicationDefaultNamespace)", 7 | "labels": { 8 | "app.kubernetes.io/component": "onepanel", 9 | "app.kubernetes.io/instance": "onepanel-v0.5.0", 10 | "app.kubernetes.io/managed-by": "onepanel-cli", 11 | "app.kubernetes.io/name": "onepanel", 12 | "app.kubernetes.io/part-of": "onepanel", 13 | "app.kubernetes.io/version": "v0.5.0" 14 | } 15 | }, 16 | "data": { 17 | "artifactRepositoryS3AccessKey": "$(artifactRepositoryS3AccessKey)", 18 | "artifactRepositoryS3SecretKey": "$(artifactRepositoryS3SecretKey)" 19 | }, 20 | "type": "Opaque" 21 | } -------------------------------------------------------------------------------- /api/proto/common.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | message Parameter { 7 | string name = 1; 8 | string value = 2; 9 | string type = 3; 10 | string displayName = 4; 11 | string hint = 5; 12 | bool required = 6; 13 | string visibility = 7; 14 | 15 | repeated ParameterOption options = 8; 16 | } 17 | 18 | message ParameterOption { 19 | string name = 1; 20 | string value = 2; 21 | } 22 | 23 | message LogStreamResponse { 24 | repeated LogEntry logEntries = 1; 25 | } 26 | 27 | message LogEntry { 28 | string timestamp = 1; 29 | string content = 2; 30 | } 31 | 32 | message MachineType { 33 | string name = 1; 34 | string value = 2; 35 | } -------------------------------------------------------------------------------- /manifest/rolebinding-onepanel-defaultnamespace.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "rbac.authorization.k8s.io/v1beta1", 3 | "kind": "RoleBinding", 4 | "metadata": { 5 | "labels": { 6 | "app": "onepanel" 7 | }, 8 | "name": "onepanel", 9 | "namespace": "$(applicationDefaultNamespace)" 10 | }, 11 | "roleRef": { 12 | "apiGroup": "rbac.authorization.k8s.io", 13 | "kind": "Role", 14 | "name": "onepanel" 15 | }, 16 | "subjects": [ 17 | { 18 | "kind": "ServiceAccount", 19 | "name": "default", 20 | "namespace": "$(applicationDefaultNamespace)" 21 | }, 22 | { 23 | "kind": "ServiceAccount", 24 | "name": "$(applicationDefaultNamespace)", 25 | "namespace": "$(applicationDefaultNamespace)" 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /db/sql/20200422140125_add_cron_workflows.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE cron_workflows 3 | ( 4 | id serial PRIMARY KEY, 5 | uid varchar(30) UNIQUE NOT NULL CHECK(uid <> ''), 6 | name varchar(30), 7 | workflow_template_version_id INT REFERENCES workflow_template_versions, 8 | schedule varchar(255), 9 | timezone varchar(255), 10 | suspend boolean, 11 | concurrency_policy varchar(255), 12 | starting_deadline_seconds INT, 13 | successful_jobs_history_limit INT, 14 | failed_jobs_history_limit INT, 15 | workflow_spec TEXT, 16 | 17 | -- auditing info 18 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 19 | modified_at timestamp 20 | ); 21 | 22 | -- +goose Down 23 | DROP TABLE cron_workflows; 24 | -------------------------------------------------------------------------------- /helper/README.md: -------------------------------------------------------------------------------- 1 | # Onepanel Core Helper 2 | 3 | Helper provides the files to build a Docker image to assist 4 | with code generation for the onepanel-core project. 5 | 6 | In particular, you can use the docker image to generate all of the gRPC related files, as well as create migrations. 7 | 8 | ## Build 9 | 10 | To build, it should be sufficient to run 11 | ```bash 12 | docker build -t onepanel/helper:v1.0.0 . 13 | ``` 14 | 15 | ### Updating 16 | 17 | Create a new directory somewhere outside of this project. 18 | 19 | Copy the `tools.go` file there and change into that directory in your terminal. 20 | 21 | Run 22 | ```bash 23 | go mod init onepanel-tools 24 | go mod tidy 25 | ``` 26 | 27 | Then, take the `go.mod` and `go.sum` files and copy them back here. 28 | 29 | Then run the # Build steps. -------------------------------------------------------------------------------- /pkg/namespace_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | corev1 "k8s.io/api/core/v1" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | ) 11 | 12 | func testCreateNamespace(c *Client) { 13 | for i := 0; i < 5; i++ { 14 | c.CoreV1().Namespaces().Create(&corev1.Namespace{ 15 | ObjectMeta: metav1.ObjectMeta{ 16 | Name: "namespace-" + strconv.Itoa(i), 17 | Labels: map[string]string{ 18 | "onepanel.io/enabled": "true", 19 | }, 20 | }, 21 | }) 22 | } 23 | } 24 | 25 | func TestClient_ListNamespace(t *testing.T) { 26 | c := DefaultTestClient() 27 | 28 | testCreateNamespace(c) 29 | 30 | n, err := c.ListNamespaces() 31 | assert.Nil(t, err) 32 | assert.NotEmpty(t, n) 33 | assert.Equal(t, len(n), 5) 34 | } 35 | -------------------------------------------------------------------------------- /db/sql/20200424132932_add_started_and_version_to_workflow_executions.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_executions 3 | ADD COLUMN started_at TIMESTAMP, 4 | ADD COLUMN workflow_template_version_id INT REFERENCES workflow_template_versions, 5 | ADD COLUMN phase VARCHAR(50), 6 | ADD COLUMN cron_workflow_id INT REFERENCES cron_workflows, 7 | DROP COLUMN failed_at, 8 | DROP COLUMN workflow_template_id 9 | ; 10 | 11 | UPDATE workflow_executions 12 | SET started_at = created_at, 13 | phase = 'Succeeded' 14 | ; 15 | 16 | -- +goose Down 17 | ALTER TABLE workflow_executions 18 | DROP COLUMN started_at, 19 | DROP COLUMN workflow_template_version_id, 20 | DROP COLUMN phase, 21 | DROP COLUMN cron_workflow_id, 22 | ADD COLUMN failed_at TIMESTAMP, 23 | ADD COLUMN workflow_template_id INT 24 | ; 25 | -------------------------------------------------------------------------------- /db/go/20201016170415_update_cvat.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201016170415() { 10 | if _, ok := initializedMigrations[20201016170415]; !ok { 11 | goose.AddMigration(Up20201016170415, Down20201016170415) 12 | initializedMigrations[20201016170415] = true 13 | } 14 | } 15 | 16 | // Up20201016170415 updates cvat to a new version 17 | func Up20201016170415(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20201016170415.yaml"), 21 | cvatTemplateName) 22 | } 23 | 24 | // Down20201016170415 does nothing 25 | func Down20201016170415(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /db/sql/20200413141037_create_workflow_execution.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE workflow_executions 4 | ( 5 | id serial PRIMARY KEY, 6 | uid varchar(30) UNIQUE NOT NULL CHECK(uid <> ''), 7 | workflow_template_id integer NOT NULL REFERENCES workflow_templates ON DELETE CASCADE, 8 | name text NOT NULL CHECK (name <> ''), 9 | namespace varchar(30) NOT NULL, 10 | 11 | -- auditing info 12 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 13 | finished_at timestamp DEFAULT NULL, 14 | failed_at timestamp DEFAULT NULL 15 | ); 16 | -- +goose StatementEnd 17 | 18 | -- +goose Down 19 | -- +goose StatementBegin 20 | DROP TABLE workflow_executions; 21 | -- +goose StatementEnd 22 | -------------------------------------------------------------------------------- /db/sql/20200521112757_add_namespace_cron_workflows.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- SQL in this section is executed when the migration is applied. 3 | ALTER TABLE cron_workflows ADD COLUMN namespace varchar(30); 4 | UPDATE cron_workflows cwfs 5 | SET namespace = q.namespace 6 | FROM ( 7 | SELECT wft.id, wft.namespace, wtv.id as wtv_id, wtv.workflow_template_id, cw.id 8 | FROM workflow_templates wft 9 | INNER JOIN workflow_template_versions wtv on wft.id = wtv.workflow_template_id 10 | INNER JOIN cron_workflows cw on wtv.id = cw.workflow_template_version_id) q 11 | WHERE cwfs.workflow_template_version_id = q.wtv_id; 12 | ALTER TABLE cron_workflows ALTER COLUMN namespace SET NOT NULL; 13 | -- +goose Down 14 | -- SQL in this section is executed when the migration is rolled back. 15 | ALTER TABLE cron_workflows DROP COLUMN namespace; 16 | -------------------------------------------------------------------------------- /db/sql/20200421103946_add_back_workflow_template_versions.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE workflow_template_versions 3 | ( 4 | id serial PRIMARY KEY, 5 | uid varchar(30), 6 | workflow_template_id integer NOT NULL REFERENCES workflow_templates ON DELETE CASCADE, 7 | version integer NOT NULL, 8 | is_latest boolean NOT NULL, 9 | manifest text NOT NULL, 10 | 11 | -- auditing info 12 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc') 13 | ); 14 | 15 | ALTER TABLE workflow_templates DROP COLUMN versions; 16 | 17 | -- +goose Down 18 | ALTER TABLE workflow_templates ADD COLUMN versions INTEGER; 19 | UPDATE workflow_templates SET versions = 1; 20 | ALTER TABLE workflow_templates ALTER COLUMN versions SET NOT NULL; 21 | 22 | DROP TABLE workflow_template_versions; 23 | 24 | -------------------------------------------------------------------------------- /manifest/service-minio-onepanel.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "networking.istio.io/v1alpha3", 3 | "kind": "VirtualService", 4 | "metadata": { 5 | "name": "minio", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "spec": { 9 | "hosts": [ 10 | "sys-storage-$(applicationDefaultNamespace).$(applicationDomain)" 11 | ], 12 | "gateways": [ 13 | "istio-system/ingressgateway" 14 | ], 15 | "http": [ 16 | { 17 | "match": [ 18 | { 19 | "uri": { 20 | "prefix": "/" 21 | } 22 | } 23 | ], 24 | "route": [ 25 | { 26 | "destination": { 27 | "port": { 28 | "number": 9000 29 | }, 30 | "host": "minio-gateway.$(applicationDefaultNamespace).svc.cluster.local" 31 | } 32 | } 33 | ] 34 | } 35 | ] 36 | } 37 | } -------------------------------------------------------------------------------- /db/go/20211028205201_cvat_1_6.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20211028205201() { 10 | if _, ok := initializedMigrations[20211028205201]; !ok { 11 | goose.AddMigration(Up20211028205201, Down20211028205201) 12 | initializedMigrations[20211028205201] = true 13 | } 14 | } 15 | 16 | // Up20211028205201 creates the new cvat 1.6.0 workspace template 17 | func Up20211028205201(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return createWorkspaceTemplate( 20 | filepath.Join("workspaces", "cvat_1_6_0", "20211028205201.yaml"), 21 | "CVAT_1.6.0", 22 | "Powerful and efficient Computer Vision Annotation Tool (CVAT)") 23 | } 24 | 25 | // Down20211028205201 archives the new cvat 1.6.0 workspace template 26 | func Down20211028205201(tx *sql.Tx) error { 27 | return archiveWorkspaceTemplate("CVAT_1.6.0") 28 | } 29 | -------------------------------------------------------------------------------- /manifest/networkpolicy-onepanel-defaultnamespace.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "networking.k8s.io/v1", 3 | "kind": "NetworkPolicy", 4 | "metadata": { 5 | "labels": { 6 | "app": "onepanel" 7 | }, 8 | "name": "onepanel", 9 | "namespace": "$(applicationDefaultNamespace)" 10 | }, 11 | "spec": { 12 | "egress": [ 13 | { 14 | "to": [ 15 | { 16 | "ipBlock": { 17 | "cidr": "0.0.0.0/0", 18 | "except": [ 19 | "169.254.169.254/32" 20 | ] 21 | } 22 | } 23 | ] 24 | } 25 | ], 26 | "ingress": [ 27 | { 28 | "from": [ 29 | { 30 | "namespaceSelector": { 31 | "matchLabels": { 32 | "app.kubernetes.io/part-of": "onepanel" 33 | } 34 | } 35 | } 36 | ] 37 | } 38 | ], 39 | "podSelector": {} 40 | } 41 | } -------------------------------------------------------------------------------- /pkg/filter.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import sq "github.com/Masterminds/squirrel" 4 | 5 | // LabelFilter represents a filter that has labels 6 | type LabelFilter interface { 7 | // GetLabels returns the labels to filter by. These are assumed to be ANDed together. 8 | GetLabels() []*Label 9 | } 10 | 11 | // ApplyLabelSelectQuery returns a query builder that adds where statements to filter by labels in the filter, if there are any 12 | // labelSelector is the database column that has the labels, such as "we.labels" for workflowExecutions aliased by "we". 13 | func ApplyLabelSelectQuery(labelSelector string, sb sq.SelectBuilder, filter LabelFilter) (sq.SelectBuilder, error) { 14 | labels := filter.GetLabels() 15 | 16 | if len(labels) == 0 { 17 | return sb, nil 18 | } 19 | 20 | labelsJSON, err := LabelsToJSONString(labels) 21 | if err != nil { 22 | return sb, err 23 | } 24 | 25 | sb = sb.Where(labelSelector+" @> ?", labelsJSON) 26 | 27 | return sb, nil 28 | } 29 | -------------------------------------------------------------------------------- /db/go/20201115145814_add_tensorboard_to_maskrcnn.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201115145814() { 10 | if _, ok := initializedMigrations[20201115145814]; !ok { 11 | goose.AddMigration(Up20201115145814, Down20201115145814) 12 | initializedMigrations[20201115145814] = true 13 | } 14 | } 15 | 16 | // Up20201115145814 add TensorBoard sidecar to TFODs 17 | func Up20201115145814(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "maskrcnn-training", "20201115145814.yaml"), 21 | maskRCNNWorkflowTemplateName, 22 | map[string]string{ 23 | "used-by": "cvat", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201115145814 do nothing 29 | func Down20201115145814(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | **What this PR does**: 8 | 9 | **Which issue(s) this PR fixes**: 10 | 14 | Fixes onepanelio/core# 15 | 16 | **Special notes for your reviewer**: 17 | 18 | **Checklist** 19 | 20 | Please check if applies 21 | 22 | - [ ] I have added/updated relevant unit tests 23 | - [ ] I have added/updated relevant documentation 24 | 25 | Required 26 | 27 | - [ ] I accept to release these changes under the Apache 2.0 License -------------------------------------------------------------------------------- /db/sql/20200518163745_add_conditional_unique_index_to_workflow_templates.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE workflow_templates DROP CONSTRAINT IF EXISTS workflow_templates_uid_key; 3 | ALTER TABLE workflow_templates DROP CONSTRAINT IF EXISTS workflow_templates_uid_namespace_key; 4 | DROP INDEX IF EXISTS workflow_templates_name_namespace_key; 5 | CREATE UNIQUE INDEX workflow_templates_name_namespace_key ON workflow_templates (name, namespace) WHERE is_archived = false; 6 | CREATE UNIQUE INDEX workflow_templates_uid_namespace_key ON workflow_templates (uid, namespace) WHERE is_archived = false; 7 | 8 | -- +goose Down 9 | -- SQL in this section is executed when the migration is rolled back. 10 | DROP INDEX workflow_templates_name_namespace_key; 11 | DROP INDEX workflow_templates_uid_namespace_key; 12 | ALTER TABLE workflow_templates ADD CONSTRAINT workflow_templates_uid_key UNIQUE (uid); 13 | ALTER TABLE workflow_templates ADD CONSTRAINT workflow_templates_uid_namespace_key UNIQUE (uid, namespace); -------------------------------------------------------------------------------- /db/sql/20200425172611_workspace_templates.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE workspace_templates 3 | ( 4 | id serial PRIMARY KEY, 5 | uid varchar(30) NOT NULL CHECK(uid <> ''), 6 | name varchar(30) NOT NULL CHECK(name <> ''), 7 | namespace varchar(30) NOT NULL, 8 | is_archived boolean DEFAULT false, 9 | 10 | workflow_template_id integer NOT NULL REFERENCES workflow_templates ON DELETE CASCADE, 11 | 12 | -- auditing info 13 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 14 | modified_at timestamp 15 | ); 16 | 17 | CREATE UNIQUE INDEX workspace_templates_name_namespace_key ON workspace_templates (name, namespace) WHERE is_archived = false; 18 | CREATE UNIQUE INDEX workspace_templates_uid_namespace_key ON workspace_templates (uid, namespace) WHERE is_archived = false; 19 | 20 | -- +goose Down 21 | DROP TABLE workspace_templates; 22 | -------------------------------------------------------------------------------- /db/go/20201115134934_add_tensorboard_to_tfod.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201115134934() { 10 | if _, ok := initializedMigrations[20201115134934]; !ok { 11 | goose.AddMigration(Up20201115134934, Down20201115134934) 12 | initializedMigrations[20201115134934] = true 13 | } 14 | } 15 | 16 | // Up20201115134934 add TensorBoard sidecar to TFODs 17 | func Up20201115134934(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "tf-object-detection-training", "20201115134934.yaml"), 21 | tensorflowObjectDetectionWorkflowTemplateName, 22 | map[string]string{ 23 | "used-by": "cvat", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201115134934 do nothing 29 | func Down20201115134934(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /db/go/20210107094725_update_cvat_workspace.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210107094725() { 10 | if _, ok := initializedMigrations[20210107094725]; !ok { 11 | goose.AddMigration(Up20210107094725, Down20210107094725) 12 | initializedMigrations[20210107094725] = true 13 | } 14 | } 15 | 16 | //Up20210107094725 updates CVAT to latest image 17 | func Up20210107094725(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20210107094725.yaml"), 21 | cvatTemplateName) 22 | } 23 | 24 | //Down20210107094725 reverts to previous CVAT image 25 | func Down20210107094725(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "cvat", "20201211161117.yaml"), 29 | cvatTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /db/go/20210129094725_update_cvat_workspace.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210129134326() { 10 | if _, ok := initializedMigrations[20210129134326]; !ok { 11 | goose.AddMigration(Up20210129134326, Down20210129134326) 12 | initializedMigrations[20210129134326] = true 13 | } 14 | } 15 | 16 | //Up20210129134326 updates CVAT to latest image 17 | func Up20210129134326(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20210129134326.yaml"), 21 | cvatTemplateName) 22 | } 23 | 24 | //Down20210129134326 reverts to previous CVAT image 25 | func Down20210129134326(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "cvat", "20210107094725.yaml"), 29 | cvatTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **`opctl` version** 14 | ```bash 15 | $ opctl version 16 | 17 | CLI version: v0.16.0-rc.0 18 | Manifest version: v0.16.0-rc.0 19 | API version: v0.16.0-rc.0 20 | Web UI version: v0.16.0-rc.0 21 | ``` 22 | 23 | **`opctl init` command** 24 | You can find this at the top of your `params.yaml` file. [e.g. `opctl init --provider eks --artifact-repository-provider s3 --gpu-device-plugins nvidia`] 25 | 26 | **Kubernetes information** 27 | - Cloud provider: [e.g. AKS, EKS, GKE, Microk8s] 28 | - Kubernetes version: [e.g. 1.17.13] 29 | 30 | **Machine information** 31 | - OS: [e.g. Ubuntu 18.04, Windows 10 19042.631, macOS 10.14.6] 32 | - Browser: [e.g. Chrome, Firefox, Safari] 33 | 34 | **Screenshots** 35 | If applicable, add screenshots to help explain your problem. 36 | -------------------------------------------------------------------------------- /pkg/workspace_types_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/asaskevich/govalidator" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func assertWorkspaceNameInvalid(t *testing.T, name string) { 10 | ws := Workspace{ 11 | UID: "test", 12 | Name: name, 13 | } 14 | 15 | valid, _ := govalidator.ValidateStruct(ws) 16 | 17 | assert.False(t, valid) 18 | } 19 | 20 | func assertWorkspaceNameValid(t *testing.T, name string) { 21 | ws := Workspace{ 22 | UID: "test", 23 | Name: name, 24 | } 25 | 26 | valid, _ := govalidator.ValidateStruct(ws) 27 | 28 | assert.True(t, valid) 29 | } 30 | 31 | func Test_WorkspaceNameValidation_RegexValid(t *testing.T) { 32 | assertWorkspaceNameInvalid(t, "600s") 33 | 34 | assertWorkspaceNameValid(t, "test-5") 35 | assertWorkspaceNameValid(t, "test 5") 36 | assertWorkspaceNameValid(t, "TEst 5") 37 | assertWorkspaceNameValid(t, "CVAT") 38 | assertWorkspaceNameValid(t, "My CVAT Workspace") 39 | assertWorkspaceNameValid(t, "CVAT Workspace 1") 40 | } 41 | -------------------------------------------------------------------------------- /db/go/20201115133046_update_cvat_env_vars.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201115133046() { 10 | if _, ok := initializedMigrations[20201115133046]; !ok { 11 | goose.AddMigration(Up20201115133046, Down20201115133046) 12 | initializedMigrations[20201115133046] = true 13 | } 14 | } 15 | 16 | //Up20201115133046 updates CVAT environment variables 17 | func Up20201115133046(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20201115133046.yaml"), 21 | cvatTemplateName) 22 | } 23 | 24 | //Down20201115133046 reverts latest environment variable updates 25 | func Down20201115133046(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "cvat", "20201113094916.yaml"), 29 | cvatTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /helper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.5 AS builder 2 | 3 | WORKDIR / 4 | 5 | RUN apt-get update 6 | RUN apt-get install -y --no-install-recommends unzip=6.0-23+deb10u1 7 | RUN curl -sL -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/protoc-3.14.0-linux-x86_64.zip 8 | RUN unzip protoc.zip -d proto 9 | RUN curl -sL -o jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 10 | RUN chmod +x jq 11 | 12 | FROM golang:1.15.5 13 | 14 | WORKDIR /root 15 | COPY ./go.* ./ 16 | 17 | RUN go get -u github.com/pressly/goose/cmd/goose 18 | RUN go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \ 19 | github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \ 20 | google.golang.org/protobuf/cmd/protoc-gen-go \ 21 | google.golang.org/grpc/cmd/protoc-gen-go-grpc 22 | 23 | RUN rm go.mod go.sum 24 | 25 | COPY --from=builder /jq /usr/local/bin 26 | COPY --from=builder /proto/bin/protoc /usr/local/bin 27 | COPY --from=builder /proto/include /usr/local/include/ 28 | 29 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /db/sql/20200429141030_add_manifest_to_crons.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | ALTER TABLE cron_workflows ADD COLUMN manifest TEXT; 3 | UPDATE cron_workflows SET manifest = ''; 4 | ALTER TABLE cron_workflows ALTER manifest SET NOT NULL; 5 | 6 | ALTER TABLE cron_workflows 7 | DROP COLUMN schedule, 8 | DROP COLUMN timezone, 9 | DROP COLUMN suspend, 10 | DROP COLUMN concurrency_policy, 11 | DROP COLUMN starting_deadline_seconds, 12 | DROP COLUMN successful_jobs_history_limit, 13 | DROP COLUMN failed_jobs_history_limit, 14 | DROP COLUMN workflow_spec 15 | ; 16 | 17 | -- +goose Down 18 | ALTER TABLE cron_workflows 19 | ADD COLUMN schedule varchar(255), 20 | ADD COLUMN timezone varchar(255), 21 | ADD COLUMN suspend boolean, 22 | ADD COLUMN concurrency_policy varchar(255), 23 | ADD COLUMN starting_deadline_seconds INT, 24 | ADD COLUMN successful_jobs_history_limit INT, 25 | ADD COLUMN failed_jobs_history_limit INT, 26 | ADD COLUMN workflow_spec TEXT 27 | ; 28 | 29 | ALTER TABLE cron_workflows DROP COLUMN manifest; 30 | -------------------------------------------------------------------------------- /db/go/20201214133458_fix_jupyterlab_gpu.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201214133458() { 10 | if _, ok := initializedMigrations[20201214133458]; !ok { 11 | goose.AddMigration(Up20201214133458, Down20201214133458) 12 | initializedMigrations[20201214133458] = true 13 | } 14 | } 15 | 16 | // Up20201214133458 fixes an issue where LD_LIBRARY_PATH is not present for JupyterLab 17 | func Up20201214133458(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "jupyterlab", "20201214133458.yaml"), 21 | jupyterLabTemplateName) 22 | } 23 | 24 | // Down20201214133458 undoes the change 25 | func Down20201214133458(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "jupyterlab", "20201031165106.yaml"), 29 | jupyterLabTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /api/proto/api.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "protoc-gen-openapiv2/options/annotations.proto"; 7 | 8 | option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { 9 | info: { 10 | title: "Onepanel"; 11 | description: "Onepanel API"; 12 | version: "0.10.0"; 13 | contact: { 14 | name: "Onepanel project"; 15 | url: "https://github.com/onepanelio/core"; 16 | }; 17 | }; 18 | host: "localhost:8888"; 19 | schemes: HTTP; 20 | schemes: HTTPS; 21 | consumes: "application/json"; 22 | produces: "application/json"; 23 | produces: "application/octet-stream"; 24 | security_definitions: { 25 | security: { 26 | key: "Bearer"; 27 | value: { 28 | type: TYPE_API_KEY; 29 | in: IN_HEADER; 30 | name: "authorization"; 31 | description: "Authentication token, prefixed by Bearer" 32 | } 33 | } 34 | } 35 | security: { 36 | security_requirement: { 37 | key: "Bearer"; 38 | value: {}; 39 | } 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /db/go/20201211161117_fix_resources_gke.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201211161117() { 10 | if _, ok := initializedMigrations[20201211161117]; !ok { 11 | goose.AddMigration(Up20201211161117, Down20201211161117) 12 | initializedMigrations[20201211161117] = true 13 | } 14 | } 15 | 16 | // Up20201211161117 updated cvat workspace template with a new ONEPANEL_MAIN_CONTAINER environment variable 17 | func Up20201211161117(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20201211161117.yaml"), 21 | cvatTemplateName) 22 | } 23 | 24 | // Down20201211161117 reverts the cvat workspace update 25 | func Down20201211161117(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "cvat", "20201115133046.yaml"), 29 | cvatTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /db/go/20201229205644_fix_jupyter_workspace_yaml.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201229205644() { 10 | if _, ok := initializedMigrations[20201229205644]; !ok { 11 | goose.AddMigration(Up20201229205644, Down20201229205644) 12 | initializedMigrations[20201229205644] = true 13 | } 14 | } 15 | 16 | // Up20201229205644 updates the jupyterlab workspace template 17 | func Up20201229205644(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "jupyterlab", "20201229205644.yaml"), 21 | jupyterLabTemplateName) 22 | } 23 | 24 | // Down20201229205644 rolls back the jupyterab workspace template update 25 | func Down20201229205644(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "jupyterlab", "20201214133458.yaml"), 29 | jupyterLabTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /db/go/20210129142057_update_jupyter_workspace.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210129142057() { 10 | if _, ok := initializedMigrations[20210129142057]; !ok { 11 | goose.AddMigration(Up20210129142057, Down20210129142057) 12 | initializedMigrations[20210129142057] = true 13 | } 14 | } 15 | 16 | // Up20210129142057 updates the jupyterlab workspace template 17 | func Up20210129142057(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "jupyterlab", "20210129142057.yaml"), 21 | jupyterLabTemplateName) 22 | } 23 | 24 | // Down20210129142057 rolls back the jupyterab workspace template update 25 | func Down20210129142057(tx *sql.Tx) error { 26 | // This code is executed when the migration is rolled back. 27 | return updateWorkspaceTemplateManifest( 28 | filepath.Join("workspaces", "jupyterlab", "20201229205644.yaml"), 29 | jupyterLabTemplateName) 30 | } 31 | -------------------------------------------------------------------------------- /db/go/20201028145442_update_jupyter_lab_template.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201028145442() { 10 | if _, ok := initializedMigrations[20201028145442]; !ok { 11 | goose.AddMigration(Up20201028145442, Down20201028145442) 12 | initializedMigrations[20201028145442] = true 13 | } 14 | } 15 | 16 | // Up20201028145442 updates the jupyterlab workspace to include container lifecycle hooks. 17 | // These hooks will attempt to persist conda, pip, and jupyterlab extensions between pause and shut-down. 18 | func Up20201028145442(tx *sql.Tx) error { 19 | // This code is executed when the migration is applied. 20 | return updateWorkspaceTemplateManifest( 21 | filepath.Join("workspaces", "jupyterlab", "20201028145442.yaml"), 22 | jupyterLabTemplateName) 23 | } 24 | 25 | // Down20201028145442 removes the lifecycle hooks from the template. 26 | func Down20201028145442(tx *sql.Tx) error { 27 | // This code is executed when the migration is rolled back. 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/push_tag.yaml: -------------------------------------------------------------------------------- 1 | name: Publish docker image on tag push 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@master 11 | - uses: little-core-labs/get-git-tag@v3.0.1 12 | id: tagName 13 | - name: Publish to Registry 14 | uses: elgohr/Publish-Docker-Github-Action@master 15 | with: 16 | name: onepanel/core 17 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 18 | password: ${{ secrets.DOCKER_HUB_TOKEN }} 19 | tags: "${{ env.GIT_TAG_NAME }}" 20 | - name: Notify Slack Channels 21 | uses: rtCamp/action-slack-notify@v2.1.1 22 | env: 23 | SLACK_CHANNEL: org 24 | SLACK_ICON: https://avatars1.githubusercontent.com/u/30390575?s=48&v=4 25 | SLACK_TITLE: New Core Version 26 | SLACK_USERNAME: opBot 27 | SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} 28 | MSG_MINIMAL: true 29 | SLACK_MESSAGE: "${{ env.GIT_TAG_NAME }}\nDocker Tag: onepanel/core:${{ env.GIT_TAG_NAME }}" -------------------------------------------------------------------------------- /db/go/20201113094916_update_cvat_onepanel_sdk.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201113094916() { 10 | if _, ok := initializedMigrations[20201113094916]; !ok { 11 | goose.AddMigration(Up20201113094916, Down20201113094916) 12 | initializedMigrations[20201113094916] = true 13 | } 14 | } 15 | 16 | //Up20201113094916 updates CVAT with python-sdk 0.15.0 17 | //Of note, this replaces the authentication request endpoint. 18 | func Up20201113094916(tx *sql.Tx) error { 19 | // This code is executed when the migration is applied. 20 | return updateWorkspaceTemplateManifest( 21 | filepath.Join("workspaces", "cvat", "20201113094916.yaml"), 22 | cvatTemplateName) 23 | } 24 | 25 | //Down20201113094916 updates CVAT back to previous python-sdk version of 0.14.0 26 | func Down20201113094916(tx *sql.Tx) error { 27 | // This code is executed when the migration is rolled back. 28 | return updateWorkspaceTemplateManifest( 29 | filepath.Join("workspaces", "cvat", "20201102104048.yaml"), 30 | cvatTemplateName) 31 | } 32 | -------------------------------------------------------------------------------- /db/go/20210414165510_add_deep_learning_desktop_workspace.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | var deepLearningDesktopTemplateName = "Deep Learning Desktop" 10 | 11 | func initialize20210414165510() { 12 | if _, ok := initializedMigrations[20210414165510]; !ok { 13 | goose.AddMigration(Up20210414165510, Down20210414165510) 14 | initializedMigrations[20210414165510] = true 15 | } 16 | } 17 | 18 | // Up20210414165510 creates the Deep Learning Desktop Workspace Template 19 | func Up20210414165510(tx *sql.Tx) error { 20 | // This code is executed when the migration is applied. 21 | return createWorkspaceTemplate( 22 | filepath.Join("workspaces", "vnc", "20210414165510.yaml"), 23 | deepLearningDesktopTemplateName, 24 | "Deep learning desktop with VNC") 25 | } 26 | 27 | // Down20210414165510 removes the Deep Learning Desktop Workspace Template 28 | func Down20210414165510(tx *sql.Tx) error { 29 | // This code is executed when the migration is rolled back. 30 | return archiveWorkspaceTemplate(deepLearningDesktopTemplateName) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/util/router/api.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import "fmt" 4 | 5 | // API provides methods to generate urls for the API 6 | type API interface { 7 | UpdateWorkspaceStatus(namespace, uid string) string 8 | } 9 | 10 | // api is a basic implementation of router.API 11 | type api struct { 12 | protocol string 13 | fqdn string 14 | } 15 | 16 | // UpdateWorkspaceStatus generates a url to update the status of a workspace 17 | func (a *api) UpdateWorkspaceStatus(namespace, uid string) string { 18 | // /apis/v1beta1/{namespace}/workspaces/{uid}/status 19 | return fmt.Sprintf("%v%v/apis/v1beta1/%v/workspaces/%v/status", a.protocol, a.fqdn, namespace, uid) 20 | } 21 | 22 | // NewAPIRouter creates a new api router used to generate urls for the api 23 | func NewAPIRouter(protocol, fqdn string) (API, error) { 24 | return &api{ 25 | protocol: protocol, 26 | fqdn: fqdn, 27 | }, nil 28 | } 29 | 30 | // NewRelativeAPIRouter creates an api router that does relative routes, with no protocol or fqdn 31 | func NewRelativeAPIRouter() (API, error) { 32 | return &api{ 33 | protocol: "", 34 | fqdn: "", 35 | }, nil 36 | } 37 | -------------------------------------------------------------------------------- /api/proto/namespace.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | 8 | service NamespaceService { 9 | rpc ListNamespaces(ListNamespacesRequest) returns (ListNamespacesResponse) { 10 | option (google.api.http) = { 11 | get: "/apis/v1beta1/namespaces" 12 | }; 13 | } 14 | 15 | rpc CreateNamespace(CreateNamespaceRequest) returns (Namespace) { 16 | option (google.api.http) = { 17 | post: "/apis/v1beta1/namespaces" 18 | body: "namespace" 19 | }; 20 | } 21 | } 22 | 23 | message ListNamespacesRequest { 24 | int32 pageSize = 1; 25 | int32 page = 2; 26 | string query = 3; 27 | } 28 | 29 | message ListNamespacesResponse { 30 | int32 count = 1; 31 | repeated Namespace namespaces = 2; 32 | int32 page = 3; 33 | int32 pages = 4; 34 | int32 totalCount = 5; 35 | } 36 | 37 | message CreateNamespaceRequest { 38 | Namespace namespace = 1; 39 | } 40 | 41 | message Namespace { 42 | string name = 1; 43 | string sourceName = 2; 44 | } -------------------------------------------------------------------------------- /pkg/util/s3/s3.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "io" 5 | 6 | minio "github.com/minio/minio-go/v6" 7 | ) 8 | 9 | type Client struct { 10 | *minio.Client 11 | } 12 | 13 | type GetObjectOptions = minio.GetObjectOptions 14 | 15 | type Config struct { 16 | AccessKey string 17 | SecretKey string 18 | Endpoint string 19 | Region string 20 | InSecure bool 21 | } 22 | 23 | func NewClient(config Config) (s3Client *Client, err error) { 24 | var minioClient *minio.Client 25 | if config.Region != "" { 26 | minioClient, err = minio.NewWithRegion(config.Endpoint, config.AccessKey, config.SecretKey, !config.InSecure, config.Region) 27 | } else { 28 | minioClient, err = minio.New(config.Endpoint, config.AccessKey, config.SecretKey, !config.InSecure) 29 | } 30 | if err != nil { 31 | return 32 | } 33 | return &Client{Client: minioClient}, nil 34 | } 35 | 36 | func (c *Client) GetObject(bucket, key string, opts GetObjectOptions) (stream io.ReadCloser, err error) { 37 | stream, err = c.Client.GetObject(bucket, key, opts) 38 | if err != nil { 39 | return 40 | } 41 | if stream == nil { 42 | defer stream.Close() 43 | } 44 | 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /db/go/20201221194344_pytorch_update.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201221194344() { 10 | if _, ok := initializedMigrations[20201221194344]; !ok { 11 | goose.AddMigration(Up20201221194344, Down20201221194344) 12 | initializedMigrations[20201221194344] = true 13 | } 14 | } 15 | 16 | // Up20201221194344 updates pytorch_training with the sys.nodepool changes 17 | func Up20201221194344(tx *sql.Tx) error { 18 | return updateWorkflowTemplateManifest( 19 | filepath.Join("workflows", "pytorch-mnist-training", "20201221194344.yaml"), 20 | pytorchWorkflowTemplateName, 21 | map[string]string{ 22 | "created-by": "system", 23 | }, 24 | ) 25 | } 26 | 27 | // Down20201221194344 undoes the sys-nodepool changes 28 | func Down20201221194344(tx *sql.Tx) error { 29 | // This code is executed when the migration is rolled back. 30 | return updateWorkflowTemplateManifest( 31 | filepath.Join("workflows", "pytorch-mnist-training", "20200605090509.yaml"), 32 | pytorchWorkflowTemplateName, 33 | map[string]string{ 34 | "created-by": "system", 35 | }, 36 | ) 37 | } 38 | -------------------------------------------------------------------------------- /api/proto/config.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | import "google/protobuf/empty.proto"; 8 | 9 | service ConfigService { 10 | rpc GetConfig (google.protobuf.Empty) returns (GetConfigResponse) { 11 | option (google.api.http) = { 12 | get: "/apis/v1beta1/config" 13 | }; 14 | } 15 | 16 | rpc GetNamespaceConfig (GetNamespaceConfigRequest) returns (GetNamespaceConfigResponse) { 17 | option (google.api.http) = { 18 | get: "/apis/v1beta1/{namespace}/config" 19 | }; 20 | } 21 | } 22 | 23 | message GetNamespaceConfigRequest { 24 | string namespace = 1; 25 | } 26 | 27 | message GetNamespaceConfigResponse { 28 | string bucket = 1; 29 | } 30 | 31 | message GetConfigResponse { 32 | string apiUrl = 1; 33 | string domain = 2; 34 | string fqdn = 3; 35 | NodePool nodePool = 4; 36 | } 37 | 38 | message NodePoolOption { 39 | string name = 1; 40 | string value = 2; 41 | } 42 | 43 | message NodePool { 44 | string label = 1; 45 | repeated NodePoolOption options = 2; 46 | } -------------------------------------------------------------------------------- /db/go/20201102104048_update_cvat_reduce_vols.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201102104048() { 10 | if _, ok := initializedMigrations[20201102104048]; !ok { 11 | goose.AddMigration(Up20201102104048, Down20201102104048) 12 | initializedMigrations[20201102104048] = true 13 | } 14 | } 15 | 16 | // Up20201102104048 updates CVAT to use less volumes. 17 | // Through the use of environment variables, various CVAT data directories 18 | // are placed under one path, and that path is on one volume. 19 | func Up20201102104048(tx *sql.Tx) error { 20 | // This code is executed when the migration is applied. 21 | return updateWorkspaceTemplateManifest( 22 | filepath.Join("workspaces", "cvat", "20201102104048.yaml"), 23 | cvatTemplateName) 24 | } 25 | 26 | // Down20201102104048 reverts CVAT back to original amount of volumes. 27 | func Down20201102104048(tx *sql.Tx) error { 28 | // This code is executed when the migration is rolled back. 29 | return updateWorkspaceTemplateManifest( 30 | filepath.Join("workspaces", "cvat", "20201016170415.yaml"), 31 | cvatTemplateName) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/util/request/request.go: -------------------------------------------------------------------------------- 1 | package request 2 | 3 | import ( 4 | "github.com/Masterminds/squirrel" 5 | "github.com/onepanelio/core/pkg/util/request/pagination" 6 | "github.com/onepanelio/core/pkg/util/request/sort" 7 | ) 8 | 9 | // Request creates a new resource request with criteria to pagination, filter, and sort the results. 10 | type Request struct { 11 | Pagination *pagination.PaginationRequest 12 | Filter interface{} 13 | Sort *sort.Criteria 14 | } 15 | 16 | // HasSorting returns true if there are any sorting criteria in the request 17 | func (r *Request) HasSorting() bool { 18 | return r != nil && 19 | r.Sort != nil && 20 | len(r.Sort.Properties) > 0 21 | } 22 | 23 | // HasFilter returns true if there is any filtering criteria in the request 24 | func (r *Request) HasFilter() bool { 25 | return r != nil && 26 | r.Filter != nil 27 | } 28 | 29 | // ApplyPaginationToSelect applies the pagination to the selectBuilder, if there is a pagination. 30 | func (r *Request) ApplyPaginationToSelect(sb *squirrel.SelectBuilder) *squirrel.SelectBuilder { 31 | if r == nil || r.Pagination == nil { 32 | return sb 33 | } 34 | 35 | return r.Pagination.ApplyToSelect(sb) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/util/router/router.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Web provides methods to generate urls for the web client 8 | // this can be used to generate urls for workspaces or workflows when they are ready. 9 | type Web interface { 10 | WorkflowExecution(namespace, uid string) string 11 | } 12 | 13 | // web is a basic implementation of router.Web 14 | type web struct { 15 | protocol string 16 | fqdn string 17 | } 18 | 19 | // WorkflowExecution generates a url to view a specific workflow 20 | func (w *web) WorkflowExecution(namespace, uid string) string { 21 | // //workflows/ 22 | return fmt.Sprintf("%v%v/%v/workflows/%v", w.protocol, w.fqdn, namespace, uid) 23 | } 24 | 25 | // NewWebRouter creates a new web router used to generate urls for the web client 26 | func NewWebRouter(protocol, fqdn string) (Web, error) { 27 | return &web{ 28 | protocol: protocol, 29 | fqdn: fqdn, 30 | }, nil 31 | } 32 | 33 | // NewRelativeWebRouter creates a web router that does relative routes, with no protocol or fqdn 34 | func NewRelativeWebRouter() (Web, error) { 35 | return &web{ 36 | protocol: "", 37 | fqdn: "", 38 | }, nil 39 | } 40 | -------------------------------------------------------------------------------- /api/third_party/google/api/annotations.proto: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } 32 | -------------------------------------------------------------------------------- /db/sql/20200502184729_workspaces.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | CREATE TABLE workspaces 3 | ( 4 | id serial PRIMARY KEY, 5 | uid varchar(30) NOT NULL CHECK(uid <> ''), 6 | name varchar(30) NOT NULL CHECK(name <> ''), 7 | namespace varchar(30) NOT NULL, 8 | phase varchar(50) NOT NULL, 9 | parameters jsonb NOT NULL, 10 | 11 | workspace_template_id integer NOT NULL REFERENCES workspace_templates ON DELETE CASCADE, 12 | workspace_template_version integer NOT NULL, 13 | 14 | started_at timestamp, 15 | paused_at timestamp, 16 | terminated_at timestamp, 17 | 18 | -- auditing info 19 | created_at timestamp NOT NULL DEFAULT (NOW() at time zone 'utc'), 20 | modified_at timestamp 21 | ); 22 | 23 | CREATE UNIQUE INDEX workspaces_name_namespace_key ON workspaces (name, namespace) WHERE phase <> 'Terminated'; 24 | CREATE UNIQUE INDEX workspaces_uid_namespace_key ON workspaces (uid, namespace) WHERE phase <> 'Terminated'; 25 | 26 | -- +goose Down 27 | DROP TABLE workspaces; 28 | -------------------------------------------------------------------------------- /pkg/database.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | sq "github.com/Masterminds/squirrel" 5 | "github.com/jmoiron/sqlx" 6 | ) 7 | 8 | // DB represents a database connection. It wraps a sqlx.DB to provide convenience methods. 9 | type DB struct { 10 | sqlx.DB 11 | } 12 | 13 | // NewDB creates a new DB using an existing sqlx.DB connection. 14 | func NewDB(db *sqlx.DB) *DB { 15 | return &DB{ 16 | *db, 17 | } 18 | } 19 | 20 | // Selectx performs a select query using a squirrel SelectBuilder as an argument. 21 | // 22 | // This is a convenience wrapper. Any errors from squirrel or sqlx are returned as is. 23 | func (db *DB) Selectx(dest interface{}, builder sq.SelectBuilder) error { 24 | sql, args, err := builder.ToSql() 25 | if err != nil { 26 | return err 27 | } 28 | 29 | return db.Select(dest, sql, args...) 30 | } 31 | 32 | // Getx performs a get query using a squirrel SelectBuilder as an argument. 33 | // 34 | // This is a convenience wrapper. Any errors from squirrel or sqlx are returned as is. 35 | func (db *DB) Getx(dest interface{}, builder sq.SelectBuilder) error { 36 | query, args, err := builder.ToSql() 37 | if err != nil { 38 | return err 39 | } 40 | 41 | return db.Get(dest, query, args...) 42 | } 43 | -------------------------------------------------------------------------------- /db/go/20201031165106_add_tensorboard_env_var_to_jupyterlab_template.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201031165106() { 10 | if _, ok := initializedMigrations[20201031165106]; !ok { 11 | goose.AddMigration(Up20201031165106, Down20201031165106) 12 | initializedMigrations[20201031165106] = true 13 | } 14 | } 15 | 16 | // Up20201031165106 updates the jupyterlab workspace to include container lifecycle hooks. 17 | // These hooks will attempt to persist conda, pip, and jupyterlab extensions between pause and shut-down. 18 | func Up20201031165106(tx *sql.Tx) error { 19 | // This code is executed when the migration is applied. 20 | return updateWorkspaceTemplateManifest( 21 | filepath.Join("workspaces", "jupyterlab", "20201031165106.yaml"), 22 | jupyterLabTemplateName) 23 | } 24 | 25 | // Down20201031165106 removes the lifecycle hooks from the template. 26 | func Down20201031165106(tx *sql.Tx) error { 27 | // This code is executed when the migration is rolled back. 28 | return updateWorkspaceTemplateManifest( 29 | filepath.Join("workspaces", "jupyterlab", "20201028145442.yaml"), 30 | jupyterLabTemplateName) 31 | } 32 | -------------------------------------------------------------------------------- /db/go/20201209124226_update_tensorflow_training.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201209124226() { 10 | if _, ok := initializedMigrations[20201209124226]; !ok { 11 | goose.AddMigration(Up20201209124226, Down20201209124226) 12 | initializedMigrations[20201209124226] = true 13 | } 14 | } 15 | 16 | // Up20201209124226 updates the tensorflow workflow 17 | func Up20201209124226(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "tensorflow-mnist-training", "20201209124226.yaml"), 21 | tensorflowWorkflowTemplateName, 22 | map[string]string{ 23 | "framework": "tensorflow", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201209124226 rolls back the tensorflow workflow 29 | func Down20201209124226(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return updateWorkflowTemplateManifest( 32 | filepath.Join("workflows", "tensorflow-mnist-training", "20200605090535.yaml"), 33 | tensorflowWorkflowTemplateName, 34 | map[string]string{ 35 | "framework": "tensorflow", 36 | }, 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /db/go/20201130130433_update_tfod_path.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201130130433() { 10 | if _, ok := initializedMigrations[20201130130433]; !ok { 11 | goose.AddMigration(Up20201130130433, Down20201130130433) 12 | initializedMigrations[20201130130433] = true 13 | } 14 | } 15 | 16 | // Up20201130130433 remove namespace to resolve checkpoint path issue 17 | func Up20201130130433(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "tf-object-detection-training", "20201130130433.yaml"), 21 | tensorflowObjectDetectionWorkflowTemplateName, 22 | map[string]string{ 23 | "used-by": "cvat", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201130130433 do nothing 29 | func Down20201130130433(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return updateWorkflowTemplateManifest( 32 | filepath.Join("workflows", "tf-object-detection-training", "20201115134934.yaml"), 33 | tensorflowObjectDetectionWorkflowTemplateName, 34 | map[string]string{ 35 | "used-by": "cvat", 36 | }, 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /db/go/20201225172926_add_hyperparameter_tuning_example_workflow_template.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | const hyperparameterTuningTemplateName = "Hyperparameter Tuning Example" 10 | 11 | func initialize20201225172926() { 12 | if _, ok := initializedMigrations[20201225172926]; !ok { 13 | goose.AddMigration(Up20201225172926, Down20201225172926) 14 | initializedMigrations[20201225172926] = true 15 | } 16 | } 17 | 18 | // Up20201225172926 adds Hyperparameter Tuning Workflow Template 19 | func Up20201225172926(tx *sql.Tx) error { 20 | // This code is executed when the migration is applied. 21 | return createWorkflowTemplate( 22 | filepath.Join("workflows", "hyperparameter-tuning", "20201225172926.yaml"), 23 | hyperparameterTuningTemplateName, 24 | map[string]string{ 25 | "framework": "tensorflow", 26 | "tuner": "TPE", 27 | "created-by": "system", 28 | }, 29 | ) 30 | } 31 | 32 | // Down20201225172926 archives Hyperparameter Tuning Workflow Template 33 | func Down20201225172926(tx *sql.Tx) error { 34 | // This code is executed when the migration is rolled back. 35 | return archiveWorkflowTemplate(hyperparameterTuningTemplateName) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/util/sql/sql_test.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func Test_FormatSelect_Columns_NoData(t *testing.T) { 9 | result := FormatColumnSelect([]string{}) 10 | 11 | assert.Equal(t, len(result), 0) 12 | } 13 | 14 | func Test_FormatSelect_Columns(t *testing.T) { 15 | result := FormatColumnSelect([]string{"name", "uid"}) 16 | 17 | assert.Equal(t, len(result), 2) 18 | 19 | for _, item := range result { 20 | if item != "name" && item != "uid" { 21 | t.Error("item not in possible list") 22 | } 23 | } 24 | } 25 | 26 | func Test_FormatSelect_Alias(t *testing.T) { 27 | result := FormatColumnSelect([]string{"name", "uid"}, "u") 28 | 29 | assert.Equal(t, len(result), 2) 30 | 31 | for _, item := range result { 32 | if item != "u.name" && item != "u.uid" { 33 | t.Error("item not in possible list") 34 | } 35 | } 36 | } 37 | 38 | func Test_FormatSelect_AliasDestination(t *testing.T) { 39 | result := FormatColumnSelect([]string{"name", "uid"}, "u", "user") 40 | 41 | assert.Equal(t, len(result), 2) 42 | 43 | for _, item := range result { 44 | if item != `u.name "user.name"` && item != `u.uid "user.uid"` { 45 | t.Error("item not in possible list") 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /db/go/20201208155805_replace_tty_with_env_var_for_maskrcnn.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201208155805() { 10 | if _, ok := initializedMigrations[20201208155805]; !ok { 11 | goose.AddMigration(Up20201208155805, Down20201208155805) 12 | initializedMigrations[20201208155805] = true 13 | } 14 | } 15 | 16 | // Up20201208155805 update the maskrcnn workflow template to replace tty with an environment variable 17 | func Up20201208155805(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "maskrcnn-training", "20201208155115.yaml"), 21 | maskRCNNWorkflowTemplateName, 22 | map[string]string{ 23 | "used-by": "cvat", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201208155805 rolls back the environment variable change 29 | func Down20201208155805(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return updateWorkflowTemplateManifest( 32 | filepath.Join("workflows", "maskrcnn-training", "20201115145814.yaml"), 33 | maskRCNNWorkflowTemplateName, 34 | map[string]string{ 35 | "used-by": "cvat", 36 | }, 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /.github/workflows/push_dev_branch.yaml: -------------------------------------------------------------------------------- 1 | name: Publish dev docker image 2 | on: 3 | push: 4 | paths-ignore: 5 | - LICENSE 6 | - ".github/**" 7 | - "*.md" 8 | branches: 9 | - master 10 | jobs: 11 | test-code-job: 12 | runs-on: ubuntu-latest 13 | services: 14 | postgres: 15 | image: postgres:12.3 16 | env: 17 | POSTGRES_DB: onepanel 18 | POSTGRES_USER: admin 19 | POSTGRES_PASSWORD: tester 20 | options: >- 21 | --health-cmd pg_isready 22 | --health-interval 10s 23 | --health-timeout 5s 24 | --health-retries 5 25 | steps: 26 | - uses: actions/checkout@master 27 | - name: Run testing code 28 | uses: cedrickring/golang-action@1.5.2 29 | with: 30 | args: go test github.com/onepanelio/core/pkg -db=postgres 31 | build: 32 | needs: test-code-job 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/checkout@master 36 | - name: Publish to Registry 37 | uses: elgohr/Publish-Docker-Github-Action@master 38 | with: 39 | name: onepanel/core 40 | tags: "latest" 41 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 42 | password: ${{ secrets.DOCKER_HUB_TOKEN }} -------------------------------------------------------------------------------- /pkg/istio.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "github.com/onepanelio/core/pkg/util" 6 | "google.golang.org/grpc/codes" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "k8s.io/client-go/kubernetes/scheme" 9 | "k8s.io/client-go/rest" 10 | "strings" 11 | ) 12 | 13 | const istioVirtualServiceResource = "VirtualServices" 14 | 15 | func istioModelRestClient() (*rest.RESTClient, error) { 16 | config := *NewConfig() 17 | config.GroupVersion = &schema.GroupVersion{Group: "networking.istio.io", Version: "v1alpha3"} 18 | config.APIPath = "/apis" 19 | config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 20 | 21 | return rest.RESTClientFor(&config) 22 | } 23 | 24 | // CreateVirtualService creates an istio virtual service 25 | func (c *Client) CreateVirtualService(namespace string, data interface{}) error { 26 | restClient, err := istioModelRestClient() 27 | if err != nil { 28 | return err 29 | } 30 | 31 | err = restClient.Post(). 32 | Namespace(namespace). 33 | Resource(istioVirtualServiceResource). 34 | Body(data). 35 | Do(). 36 | Error() 37 | 38 | if err != nil && strings.Contains(err.Error(), "already exists") { 39 | return util.NewUserError(codes.AlreadyExists, fmt.Sprintf("VirtualService already exists")) 40 | } 41 | 42 | return err 43 | } 44 | -------------------------------------------------------------------------------- /db/go/20201223062947_tf_training_update_node_pool.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201223062947() { 10 | if _, ok := initializedMigrations[20201223062947]; !ok { 11 | goose.AddMigration(Up20201223062947, Down20201223062947) 12 | initializedMigrations[20201223062947] = true 13 | } 14 | } 15 | 16 | // Up20201223062947 updates tensorflow-mnist-training with sys.nodepool changes 17 | func Up20201223062947(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "tensorflow-mnist-training", "20201223062947.yaml"), 21 | tensorflowWorkflowTemplateName, 22 | map[string]string{ 23 | "created-by": "system", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201223062947 undoes sys.nodepool changes 29 | func Down20201223062947(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return updateWorkflowTemplateManifest( 32 | filepath.Join("workflows", "tensorflow-mnist-training", "20201223062947.yaml"), 33 | tensorflowWorkflowTemplateName, 34 | map[string]string{ 35 | "created-by": "system", 36 | "used-by": "cvat", 37 | }, 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /db/go/20201028145443_update_vscode_template.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201028145443() { 10 | if _, ok := initializedMigrations[20201028145443]; !ok { 11 | goose.AddMigration(Up20201028145443, Down20201028145443) 12 | initializedMigrations[20201028145443] = true 13 | } 14 | } 15 | 16 | // Up20201028145443 migration will add lifecycle hooks to VSCode template. 17 | // These hooks will attempt to export the conda, pip, and vscode packages that are installed, 18 | // to a text file. 19 | // On workspace resume / start, the code then tries to install these packages. 20 | func Up20201028145443(tx *sql.Tx) error { 21 | // This code is executed when the migration is applied. 22 | return updateWorkspaceTemplateManifest( 23 | filepath.Join("workspaces", "vscode", "20201028145443.yaml"), 24 | vscodeWorkspaceTemplateName) 25 | } 26 | 27 | // Down20201028145443 removes the lifecycle hooks from VSCode workspace template. 28 | func Down20201028145443(tx *sql.Tx) error { 29 | // This code is executed when the migration is rolled back. 30 | return updateWorkspaceTemplateManifest( 31 | filepath.Join("workspaces", "vscode", "20201028145443.yaml"), 32 | vscodeWorkspaceTemplateName) 33 | } 34 | -------------------------------------------------------------------------------- /db/go/20210129152427_update_vscode_template.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210129152427() { 10 | if _, ok := initializedMigrations[20210129152427]; !ok { 11 | goose.AddMigration(Up20210129152427, Down20210129152427) 12 | initializedMigrations[20210129152427] = true 13 | } 14 | } 15 | 16 | // Up20210129152427 migration will add lifecycle hooks to VSCode template. 17 | // These hooks will attempt to export the conda, pip, and vscode packages that are installed, 18 | // to a text file. 19 | // On workspace resume / start, the code then tries to install these packages. 20 | func Up20210129152427(tx *sql.Tx) error { 21 | // This code is executed when the migration is applied. 22 | return updateWorkspaceTemplateManifest( 23 | filepath.Join("workspaces", "vscode", "20210129152427.yaml"), 24 | vscodeWorkspaceTemplateName) 25 | } 26 | 27 | // Down20210129152427 removes the lifecycle hooks from VSCode workspace template. 28 | func Down20210129152427(tx *sql.Tx) error { 29 | // This code is executed when the migration is rolled back. 30 | return updateWorkspaceTemplateManifest( 31 | filepath.Join("workspaces", "vscode", "20201028145443.yaml"), 32 | vscodeWorkspaceTemplateName) 33 | } 34 | -------------------------------------------------------------------------------- /db/go/20201221195937_maskrcnn_update_parameters_to_select_nodepool.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201221195937() { 10 | if _, ok := initializedMigrations[20201221195937]; !ok { 11 | goose.AddMigration(Up20201221195937, Down20201221195937) 12 | initializedMigrations[20201221195937] = true 13 | } 14 | } 15 | 16 | // Up20201221195937 updates maskrcnn with sys.nodepool changes 17 | func Up20201221195937(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "maskrcnn-training", "20201221195937.yaml"), 21 | maskRCNNWorkflowTemplateName, 22 | map[string]string{ 23 | "created-by": "system", 24 | "used-by": "cvat", 25 | }, 26 | ) 27 | } 28 | 29 | // Down20201221195937 undoes the sys.nodepool changes 30 | func Down20201221195937(tx *sql.Tx) error { 31 | // This code is executed when the migration is rolled back. 32 | return updateWorkflowTemplateManifest( 33 | filepath.Join("workflows", "maskrcnn-training", "20201208155115.yaml"), 34 | maskRCNNWorkflowTemplateName, 35 | map[string]string{ 36 | "created-by": "system", 37 | "used-by": "cvat", 38 | }, 39 | ) 40 | } 41 | -------------------------------------------------------------------------------- /db/go/20201208155115_replace_tty_with_env_var_for_tfod.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201208155115() { 10 | if _, ok := initializedMigrations[20201208155115]; !ok { 11 | goose.AddMigration(Up20201208155115, Down20201208155115) 12 | initializedMigrations[20201208155115] = true 13 | } 14 | } 15 | 16 | // Up20201208155115 update the tf-object-detection-training workflow template to replace tty with an environment variable 17 | func Up20201208155115(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "tf-object-detection-training", "20201208155115.yaml"), 21 | tensorflowObjectDetectionWorkflowTemplateName, 22 | map[string]string{ 23 | "used-by": "cvat", 24 | }, 25 | ) 26 | } 27 | 28 | // Down20201208155115 rolls back the environment variable change 29 | func Down20201208155115(tx *sql.Tx) error { 30 | // This code is executed when the migration is rolled back. 31 | return updateWorkflowTemplateManifest( 32 | filepath.Join("workflows", "tf-object-detection-training", "20201130130433.yaml"), 33 | tensorflowObjectDetectionWorkflowTemplateName, 34 | map[string]string{ 35 | "used-by": "cvat", 36 | }, 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /db/go/20201223202929_tfod_update_node_pool.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20201223202929() { 10 | if _, ok := initializedMigrations[20201223202929]; !ok { 11 | goose.AddMigration(Up20201223202929, Down20201223202929) 12 | initializedMigrations[20201223202929] = true 13 | } 14 | } 15 | 16 | // Up20201223202929 updates tf-object-detection-training with sys.nodepool 17 | func Up20201223202929(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | return updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "tf-object-detection-training", "20201223202929.yaml"), 21 | tensorflowObjectDetectionWorkflowTemplateName, 22 | map[string]string{ 23 | "created-by": "system", 24 | "used-by": "cvat", 25 | }, 26 | ) 27 | } 28 | 29 | // Down20201223202929 undoes the sys.nodepool changes 30 | func Down20201223202929(tx *sql.Tx) error { 31 | // This code is executed when the migration is rolled back. 32 | return updateWorkflowTemplateManifest( 33 | filepath.Join("workflows", "tf-object-detection-training", "20201208155115.yaml"), 34 | tensorflowObjectDetectionWorkflowTemplateName, 35 | map[string]string{ 36 | "created-by": "system", 37 | "used-by": "cvat", 38 | }, 39 | ) 40 | } 41 | -------------------------------------------------------------------------------- /api/proto/files.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | 8 | service FileService { 9 | rpc GetObjectDownloadPresignedURL (GetObjectPresignedUrlRequest) returns (GetPresignedUrlResponse) { 10 | option (google.api.http) = { 11 | get: "/apis/v1beta1/{namespace}/files/presigned-url/{key=**}" 12 | }; 13 | } 14 | 15 | rpc ListFiles (ListFilesRequest) returns (ListFilesResponse) { 16 | option (google.api.http) = { 17 | get: "/apis/v1beta1/{namespace}/files/list/{path=**}" 18 | }; 19 | } 20 | } 21 | 22 | message File { 23 | string path = 1; 24 | string name = 2; 25 | string extension = 3; 26 | int64 size = 4; 27 | string contentType = 5; 28 | string lastModified = 6; 29 | bool directory = 7; 30 | } 31 | 32 | message ListFilesRequest { 33 | string namespace = 1; 34 | string path = 2; 35 | int32 page = 3; 36 | int32 perPage = 4; 37 | } 38 | 39 | message ListFilesResponse { 40 | int32 count = 1; 41 | int32 totalCount = 2; 42 | int32 page = 3; 43 | int32 pages = 4; 44 | repeated File files = 5; 45 | string parentPath = 6; 46 | } 47 | 48 | message GetObjectPresignedUrlRequest { 49 | string namespace = 1; 50 | string key = 2; 51 | } 52 | 53 | message GetPresignedUrlResponse { 54 | string url = 1; 55 | int64 size = 2; 56 | } -------------------------------------------------------------------------------- /pkg/util/data/migration.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "gopkg.in/yaml.v3" 5 | "io/ioutil" 6 | ) 7 | 8 | // ManifestFile represents a file that contains information about a workflow or workspace template 9 | type ManifestFile struct { 10 | Metadata ManifestFileMetadata `yaml:"metadata"` 11 | Spec interface{} `yaml:"spec"` 12 | } 13 | 14 | // ManifestFileMetadata represents information about the tempalte we are working with 15 | type ManifestFileMetadata struct { 16 | Name string 17 | Kind string // {Workflow, Workspace} 18 | Version uint64 19 | Action string // {create,update} 20 | Description *string 21 | Labels map[string]string 22 | Deprecated *bool 23 | Source *string 24 | } 25 | 26 | // SpecString returns the spec of a manifest file as a string 27 | func (m *ManifestFile) SpecString() (string, error) { 28 | data, err := yaml.Marshal(m.Spec) 29 | if err != nil { 30 | return "", err 31 | } 32 | 33 | return string(data), err 34 | } 35 | 36 | // ManifestFileFromFile loads a manifest from a yaml file. 37 | func ManifestFileFromFile(path string) (*ManifestFile, error) { 38 | fileData, err := ioutil.ReadFile(path) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | manifest := &ManifestFile{} 44 | if err := yaml.Unmarshal(fileData, manifest); err != nil { 45 | return nil, err 46 | } 47 | 48 | return manifest, nil 49 | } 50 | -------------------------------------------------------------------------------- /pkg/util/types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "database/sql/driver" 5 | "encoding/json" 6 | "errors" 7 | ) 8 | 9 | // JSONLabels is a wrapper type to support JSONB database operations. 10 | // Add a JSONLabels type to a class field and use it with a JSONB column 11 | type JSONLabels map[string]string 12 | 13 | // Unmarshal unmarshal's the json in j to v, as in json.Unmarshal. 14 | func (l *JSONLabels) Unmarshal(v interface{}) error { 15 | if len(*l) == 0 { 16 | *l = make(map[string]string) 17 | } 18 | v = l 19 | return nil 20 | } 21 | 22 | // Value returns j as a value. This does a validating unmarshal into another 23 | // RawMessage. If j is invalid json, it returns an error. 24 | // Note that nil values will return "{}" - empty JSON. 25 | func (l JSONLabels) Value() (driver.Value, error) { 26 | if l == nil { 27 | return json.Marshal(make(map[string]string)) 28 | } 29 | 30 | return json.Marshal(l) 31 | } 32 | 33 | // Scan stores the src in *j. No validation is done. 34 | func (l *JSONLabels) Scan(src interface{}) error { 35 | var source []byte 36 | switch t := src.(type) { 37 | case string: 38 | source = []byte(t) 39 | case []byte: 40 | if len(t) == 0 { 41 | source = []byte("{}") 42 | } else { 43 | source = t 44 | } 45 | case nil: 46 | *l = make(map[string]string) 47 | default: 48 | return errors.New("incompatible type for JSONLabels") 49 | } 50 | 51 | return json.Unmarshal(source, l) 52 | } 53 | -------------------------------------------------------------------------------- /api/proto/services.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | 8 | service ServiceService { 9 | rpc GetService(GetServiceRequest) returns (Service) { 10 | option (google.api.http) = { 11 | get: "/apis/v1beta1/{namespace}/service/{name}" 12 | }; 13 | } 14 | 15 | rpc ListServices(ListServicesRequest) returns (ListServicesResponse) { 16 | option (google.api.http) = { 17 | get: "/apis/v1beta1/{namespace}/service" 18 | }; 19 | } 20 | 21 | rpc HasService(HasServiceRequest) returns (HasServiceResponse) { 22 | option (google.api.http) = { 23 | get: "/apis/v1beta/service/{name}" 24 | }; 25 | } 26 | } 27 | 28 | message Service { 29 | string name = 1; 30 | string url = 2; 31 | } 32 | 33 | message GetServiceRequest { 34 | string namespace = 1; 35 | string name = 2; 36 | } 37 | 38 | message HasServiceRequest { 39 | string name = 1; 40 | } 41 | 42 | message HasServiceResponse { 43 | bool hasService= 1; 44 | } 45 | 46 | message ListServicesRequest { 47 | string namespace = 1; 48 | int32 pageSize = 2; 49 | int32 page = 3; 50 | } 51 | 52 | message ListServicesResponse { 53 | int32 count = 1; 54 | repeated Service services = 2; 55 | int32 page = 3; 56 | int32 pages = 4; 57 | int32 totalCount = 5; 58 | } -------------------------------------------------------------------------------- /db/go/20200728190804_update_workflow_template_labels.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | v1 "github.com/onepanelio/core/pkg" 6 | "github.com/pressly/goose" 7 | ) 8 | 9 | func initialize20200728190804() { 10 | if _, ok := initializedMigrations[20200728190804]; !ok { 11 | goose.AddMigration(Up20200728190804, Down20200728190804) 12 | initializedMigrations[20200728190804] = true 13 | } 14 | } 15 | 16 | // Up20200728190804 is a legacy migration. Due to code changes, it no longer does anything. 17 | // It used to update labels so that we keep track of WorkflowTemplate labels. 18 | // Before, only workflow template versions had labels, but to speed up some queries, we now cache the latest version's labels 19 | // for workflow templates themselves. 20 | func Up20200728190804(tx *sql.Tx) error { 21 | // This code is executed when the migration is applied. 22 | if migrationHasAlreadyBeenRun(20200728190804) { 23 | return nil 24 | } 25 | 26 | // Do nothing, be preserve for legacy. 27 | 28 | return nil 29 | } 30 | 31 | // Down20200728190804 rolls down the migration by deleting all workflow template labels, since they did not exist before this 32 | func Down20200728190804(tx *sql.Tx) error { 33 | // This code is executed when the migration is rolled back. 34 | 35 | client, err := getClient() 36 | if err != nil { 37 | return err 38 | } 39 | defer client.DB.Close() 40 | 41 | return client.DeleteResourceLabels(tx, v1.TypeWorkflowTemplate) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/util/env/env.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import ( 4 | "github.com/onepanelio/core/pkg/util/ptr" 5 | corev1 "k8s.io/api/core/v1" 6 | "os" 7 | ) 8 | 9 | const ( 10 | DefaultEnvironmentVariableSecret = "onepanel-default-env" 11 | ) 12 | 13 | // GetEnv gets the environment variable value, or returns fallback if the environment variable does not exist 14 | // Deprecated: use Get instead 15 | func GetEnv(key, fallback string) string { 16 | if value, ok := os.LookupEnv(key); ok { 17 | return value 18 | } 19 | return fallback 20 | } 21 | 22 | // Get gets the environment variable value, or returns fallback if the environment variable does not exist 23 | func Get(key, fallback string) string { 24 | if value, ok := os.LookupEnv(key); ok { 25 | return value 26 | } 27 | return fallback 28 | } 29 | 30 | func PrependEnvVarToContainer(container *corev1.Container, name, value string) { 31 | for _, e := range container.Env { 32 | if e.Name == name { 33 | return 34 | } 35 | } 36 | container.Env = append([]corev1.EnvVar{ 37 | { 38 | Name: name, 39 | Value: value, 40 | }, 41 | }, container.Env...) 42 | } 43 | 44 | func AddDefaultEnvVarsToContainer(container *corev1.Container) { 45 | container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{ 46 | SecretRef: &corev1.SecretEnvSource{ 47 | LocalObjectReference: corev1.LocalObjectReference{ 48 | Name: DefaultEnvironmentVariableSecret, 49 | }, 50 | Optional: ptr.Bool(true), 51 | }, 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/util/sql/sql.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // FormatColumnSelect returns a list of column names to be used in a SQL Select modified with optional alias and destination. 8 | // 9 | // aliasAndDestination supports two arguments, an alias followed by a destination. Any arguments after are ignored. 10 | // 11 | // If an alias is provided, each column is prefixed with it. Otherwise the columns are returned as is. 12 | // 13 | // If a destination is provided, each column will be assigned to it. Otherwise there is no adjustment. 14 | // 15 | // Example - alias, no destination. 16 | // Input: ([id, name], "w") 17 | // Output: [w.id, w.name] 18 | // 19 | // Example - with alias, destination 20 | // Input: ([id, name], "w", "workflow") 21 | // Output: [w.id "workflow.id", w.name "workflow.name"] 22 | func FormatColumnSelect(columns []string, aliasAndDestination ...string) []string { 23 | results := make([]string, 0) 24 | 25 | alias := "" 26 | destination := "" 27 | 28 | if len(aliasAndDestination) > 0 { 29 | alias = aliasAndDestination[0] 30 | } 31 | 32 | if len(aliasAndDestination) > 1 { 33 | destination = aliasAndDestination[1] 34 | } 35 | 36 | for _, str := range columns { 37 | result := str 38 | 39 | if alias != "" { 40 | result = alias + "." + result 41 | } 42 | 43 | if destination != "" { 44 | result += fmt.Sprintf(` "%v.%v"`, destination, str) 45 | } 46 | results = append(results, result) 47 | } 48 | 49 | return results 50 | } 51 | -------------------------------------------------------------------------------- /pkg/types_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | // TestMetrics_Add tests the Add method of the Metrics type 9 | func TestMetrics_Add(t *testing.T) { 10 | var initial Metrics = []*Metric{{ 11 | Name: "accuracy", 12 | Value: 0.98, 13 | Format: "", 14 | }} 15 | 16 | initial.Add(&Metric{ 17 | Name: "success", 18 | Value: 1.0, 19 | Format: "%", 20 | }, false) 21 | 22 | assert.Len(t, initial, 2) 23 | 24 | initial.Add(&Metric{ 25 | Name: "accuracy", 26 | Value: 0.99, 27 | Format: "%", 28 | }, false) 29 | 30 | assert.Len(t, initial, 3) 31 | 32 | initial.Add(&Metric{ 33 | Name: "accuracy", 34 | Value: 0.99, 35 | Format: "%", 36 | }, true) 37 | 38 | assert.Len(t, initial, 3) 39 | assert.True(t, initial[0].Value == 0.99) 40 | } 41 | 42 | // TestMetrics_Merge tests the Merge method of the Metrics Type 43 | func TestMetrics_Merge(t *testing.T) { 44 | var initial Metrics = []*Metric{{ 45 | Name: "accuracy", 46 | Value: 0.98, 47 | Format: "", 48 | }, { 49 | Name: "success", 50 | Value: 1.0, 51 | Format: "%", 52 | }} 53 | 54 | var toMerge Metrics = []*Metric{{ 55 | Name: "accuracy", 56 | Value: 0.00, 57 | Format: "", 58 | }, { 59 | Name: "success", 60 | Value: 1.0, 61 | Format: "%", 62 | }, { 63 | Name: "test", 64 | Value: 0.5, 65 | Format: "", 66 | }} 67 | 68 | initial.Merge(toMerge, true) 69 | 70 | assert.Len(t, initial, 3) 71 | assert.True(t, initial[0].Value == 0.00) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/util/error.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "google.golang.org/grpc/status" 7 | 8 | "github.com/lib/pq" 9 | "google.golang.org/grpc/codes" 10 | ) 11 | 12 | // UserError implements a new error type for user facing errors 13 | type UserError struct { 14 | Code codes.Code 15 | Message string 16 | } 17 | 18 | // Error returns error messages 19 | func (e *UserError) Error() string { 20 | return e.Message 21 | } 22 | 23 | // GRPCStatus is used by gRPC to return the correct gRPC status codes 24 | func (e *UserError) GRPCStatus() *status.Status { 25 | return status.New(e.Code, e.Message) 26 | } 27 | 28 | // NewUserError returns an instance of UserError with the appropriate code and message 29 | func NewUserError(code codes.Code, message string) error { 30 | return &UserError{Code: code, Message: message} 31 | } 32 | 33 | func pqError(err *pq.Error) (code codes.Code) { 34 | switch err.Code { 35 | case "23505": 36 | code = codes.AlreadyExists 37 | default: 38 | code = codes.Unknown 39 | } 40 | return 41 | } 42 | 43 | // NewUserErrorWrap wraps pq errors and returns an instance of UserError 44 | func NewUserErrorWrap(err error, entity string) error { 45 | var ( 46 | code codes.Code 47 | message string 48 | pqErr *pq.Error 49 | userErr *UserError 50 | ) 51 | if errors.As(err, &pqErr) { 52 | code = pqError(pqErr) 53 | message = fmt.Sprintf("%v already exists.", entity) 54 | } else if errors.As(err, &userErr) { 55 | return err 56 | } else { 57 | code = codes.Unknown 58 | message = "Unknown error." 59 | } 60 | 61 | return NewUserError(code, message) 62 | } 63 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vscode/20200929144301.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Visual Studio Code" 3 | kind: Workspace 4 | version: 20200929144301 5 | action: create 6 | description: "Open source code editor" 7 | spec: 8 | # Docker containers that are part of the Workspace 9 | containers: 10 | - name: vscode 11 | image: onepanel/vscode:1.0.0 12 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."] 13 | ports: 14 | - containerPort: 8080 15 | name: vscode 16 | volumeMounts: 17 | - name: data 18 | mountPath: /data 19 | ports: 20 | - name: vscode 21 | port: 8080 22 | protocol: TCP 23 | targetPort: 8080 24 | routes: 25 | - match: 26 | - uri: 27 | prefix: / #vscode runs at the default route 28 | route: 29 | - destination: 30 | port: 31 | number: 8080 32 | # DAG Workflow to be executed once a Workspace action completes (optional) 33 | #postExecutionWorkflow: 34 | # entrypoint: main 35 | # templates: 36 | # - name: main 37 | # dag: 38 | # tasks: 39 | # - name: slack-notify 40 | # template: slack-notify 41 | # - name: slack-notify 42 | # container: 43 | # image: technosophos/slack-notify 44 | # args: 45 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 46 | # command: 47 | # - sh 48 | # - -c -------------------------------------------------------------------------------- /pkg/util/collection/collection.go: -------------------------------------------------------------------------------- 1 | package collection 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | // RepeatSymbol returns symbol times, with between each one. 8 | // if symbol = ?, separator = , and count = 5 9 | // this returns: "?,?,?,?,?" 10 | func RepeatSymbol(count int, symbol, separator string) string { 11 | result := "" 12 | for i := 0; i < count; i++ { 13 | if i != 0 { 14 | result += separator 15 | } 16 | 17 | result += symbol 18 | } 19 | 20 | return result 21 | } 22 | 23 | // RemoveBlanks goes through the data, assumed to be an array or map of some kind, 24 | // and removes any data that is a nil or zero value. Maps with no keys are also removed. 25 | // 26 | // Note that this will not check the data again. So if you have the following 27 | // parent: { 28 | // child: {} 29 | // } 30 | // 31 | // The result will be 32 | // parent: {} 33 | // 34 | // it will not go through it again and remove parent. 35 | func RemoveBlanks(data interface{}) { 36 | if mapping, ok := data.(map[string]interface{}); ok { 37 | keysToDelete := make([]string, 0) 38 | for key, v := range mapping { 39 | rv := reflect.ValueOf(v) 40 | if v == nil || rv.IsZero() { 41 | keysToDelete = append(keysToDelete, key) 42 | } else if vMap, vMapOk := v.(map[string]interface{}); vMapOk && len(vMap) == 0 { 43 | keysToDelete = append(keysToDelete, key) 44 | } else { 45 | RemoveBlanks(v) 46 | } 47 | } 48 | 49 | for _, keyToDelete := range keysToDelete { 50 | delete(mapping, keyToDelete) 51 | } 52 | } else if list, ok := data.([]interface{}); ok { 53 | for _, listItem := range list { 54 | RemoveBlanks(listItem) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /db/go/20200814160856_add_description_to_jupyter_template.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | ) 7 | 8 | func initialize20200814160856() { 9 | if _, ok := initializedMigrations[20200814160856]; !ok { 10 | goose.AddMigration(Up20200814160856, Down20200814160856) 11 | initializedMigrations[20200814160856] = true 12 | } 13 | } 14 | 15 | // Up20200814160856 runs a migration to add description to jupyterlab template 16 | func Up20200814160856(tx *sql.Tx) error { 17 | // This code is executed when the migration is applied. 18 | // This code is executed when the migration is applied. 19 | client, err := getClient() 20 | if err != nil { 21 | return err 22 | } 23 | defer client.DB.Close() 24 | 25 | migrationsRan, err := getRanSQLMigrations(client) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | if _, ok := migrationsRan[20200814160856]; ok { 31 | return nil 32 | } 33 | 34 | namespaces, err := client.ListOnepanelEnabledNamespaces() 35 | if err != nil { 36 | return err 37 | } 38 | 39 | for _, namespace := range namespaces { 40 | workspaceTemplate, err := client.GetWorkspaceTemplate(namespace.Name, "jupyterlab", 0) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | workspaceTemplate.Description = "Interactive development environment for notebooks" 46 | if _, err := client.UpdateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil { 47 | return err 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | 54 | // Down20200814160856 does nothing 55 | func Down20200814160856(tx *sql.Tx) error { 56 | // This code is executed when the migration is rolled back. 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /manifest/abs/deployment.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "apps/v1", 3 | "kind": "Deployment", 4 | "metadata": { 5 | "name": "minio-gateway", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "spec": { 9 | "replicas": 1, 10 | "selector": { 11 | "matchLabels": { 12 | "app": "minio-gateway" 13 | } 14 | }, 15 | "template": { 16 | "metadata": { 17 | "labels": { 18 | "app": "minio-gateway" 19 | }, 20 | "annotations": { 21 | "sidecar.istio.io/inject": "false" 22 | } 23 | }, 24 | "spec": { 25 | "containers": [ 26 | { 27 | "name": "minio-gateway", 28 | "image": "minio/minio:RELEASE.2021-06-17T00-10-46Z.hotfix.49f6035b1", 29 | "args": [ 30 | "gateway", 31 | "azure" 32 | ], 33 | "env": [ 34 | { 35 | "name": "MINIO_ACCESS_KEY", 36 | "valueFrom": { 37 | "secretKeyRef": { 38 | "name": "onepanel", 39 | "key": "artifactRepositoryS3AccessKey" 40 | } 41 | } 42 | }, 43 | { 44 | "name": "MINIO_SECRET_KEY", 45 | "valueFrom": { 46 | "secretKeyRef": { 47 | "name": "onepanel", 48 | "key": "artifactRepositoryS3SecretKey" 49 | } 50 | } 51 | } 52 | ], 53 | "ports": [ 54 | { 55 | "containerPort": 9000 56 | } 57 | ] 58 | } 59 | ] 60 | } 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /pkg/util/request/sort/sort.go: -------------------------------------------------------------------------------- 1 | package sort 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // Order represents a sorting order such as created_at, desc 9 | type Order struct { 10 | Property string 11 | Direction string 12 | } 13 | 14 | // Criteria represents the sorting criteria for a list of resources 15 | type Criteria struct { 16 | Properties []Order 17 | } 18 | 19 | // New parses the properties, represented as comma delimited fields, into a Criteria struct 20 | // The first part is the properties, the second part is the delimiter used between properties. If none is provided, 21 | // a semi-colon (;) is used. 22 | // Each property is assumed to be of the form: propertyName,desc;propertyName2;asc 23 | // example: createdAt,desc;name,asc 24 | func New(parts ...string) (*Criteria, error) { 25 | if len(parts) == 0 { 26 | return nil, fmt.Errorf("no properties provided to create a Criteria") 27 | } 28 | 29 | separator := ";" 30 | if len(parts) > 1 { 31 | separator = parts[1] 32 | } 33 | 34 | criteria := &Criteria{ 35 | Properties: make([]Order, 0), 36 | } 37 | 38 | if parts[0] == "" { 39 | return criteria, nil 40 | } 41 | 42 | items := strings.Split(parts[0], separator) 43 | 44 | for _, item := range items { 45 | parts := strings.Split(item, ",") 46 | if len(parts) != 2 { 47 | return nil, fmt.Errorf("badly formatted sort: '%v'", item) 48 | } 49 | 50 | direction := strings.ToLower(parts[1]) 51 | 52 | if direction != "asc" && direction != "desc" { 53 | return nil, fmt.Errorf("unknown sort '%v'", parts[1]) 54 | } 55 | 56 | newSort := Order{ 57 | Property: parts[0], 58 | Direction: direction, 59 | } 60 | 61 | criteria.Properties = append(criteria.Properties, newSort) 62 | } 63 | 64 | return criteria, nil 65 | } 66 | -------------------------------------------------------------------------------- /cmd/goose/goose.go: -------------------------------------------------------------------------------- 1 | // This is custom goose binary to support .go migration files in ./db dir 2 | 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "github.com/jmoiron/sqlx" 8 | migrations "github.com/onepanelio/core/db/go" 9 | v1 "github.com/onepanelio/core/pkg" 10 | "log" 11 | "os" 12 | "path/filepath" 13 | 14 | "github.com/pressly/goose" 15 | ) 16 | 17 | var ( 18 | flags = flag.NewFlagSet("goose", flag.ExitOnError) 19 | dir = flags.String("dir", ".", "directory with migration files") 20 | ) 21 | 22 | func main() { 23 | flags.Parse(os.Args[1:]) 24 | args := flags.Args() 25 | 26 | if len(args) < 1 { 27 | flags.Usage() 28 | return 29 | } 30 | 31 | kubeConfig := v1.NewConfig() 32 | client, err := v1.NewClient(kubeConfig, nil, nil) 33 | if err != nil { 34 | log.Fatalf("Failed to connect to Kubernetes cluster: %v", err) 35 | } 36 | config, err := client.GetSystemConfig() 37 | if err != nil { 38 | log.Fatalf("Failed to get system config: %v", err) 39 | } 40 | 41 | dbDriverName, dbDataSourceName := config.DatabaseConnection() 42 | db := sqlx.MustConnect(dbDriverName, dbDataSourceName) 43 | defer db.Close() 44 | 45 | command := args[0] 46 | 47 | arguments := []string{} 48 | if len(args) > 2 { 49 | arguments = append(arguments, args[2:]...) 50 | } 51 | 52 | goose.SetTableName("goose_db_version") 53 | if err := goose.Run(command, db.DB, filepath.Join(*dir, "sql"), arguments...); err != nil { 54 | log.Fatalf("Failed to run database sql migrations: %v %v", command, err) 55 | } 56 | 57 | goose.SetTableName("goose_db_go_version") 58 | migrations.Initialize() 59 | if err := goose.Run(command, db.DB, filepath.Join(*dir, "go"), arguments...); err != nil { 60 | log.Fatalf("Failed to run database go migrations: %v %v", command, err) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/label_types_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | // TestLabelFromString tests the LabelFromString function 10 | func TestLabelFromString(t *testing.T) { 11 | // Blank value gives us no label 12 | label, err := LabelFromString("") 13 | assert.NotNil(t, err) 14 | assert.Nil(t, label) 15 | 16 | // Missing value, should give error 17 | label, err = LabelFromString("key=a") 18 | assert.NotNil(t, err) 19 | 20 | // Missing value, but still have comma, should give error 21 | label, err = LabelFromString("key=a,") 22 | assert.NotNil(t, err) 23 | 24 | // Missing key, should give error 25 | label, err = LabelFromString("value=a") 26 | assert.NotNil(t, err) 27 | 28 | // Missing key, still have comma, should give error 29 | label, err = LabelFromString("value=a,") 30 | assert.NotNil(t, err) 31 | 32 | // Correct, should not give an error 33 | label, err = LabelFromString("key=a,value=b") 34 | assert.Nil(t, err) 35 | assert.Equal(t, label.Key, "a") 36 | assert.Equal(t, label.Value, "b") 37 | } 38 | 39 | // TestLabelsFromString tests the LabelsFromString function 40 | func TestLabelsFromString(t *testing.T) { 41 | // Empty should give no error and no labels 42 | labels, err := LabelsFromString("") 43 | assert.Nil(t, err) 44 | assert.Len(t, labels, 0) 45 | 46 | // Bad data, should give no labels 47 | labels, err = LabelsFromString("&&&&") 48 | assert.Nil(t, err) 49 | assert.Len(t, labels, 0) 50 | 51 | // Test just one label 52 | labels, err = LabelsFromString("key=a,value=b") 53 | assert.Nil(t, err) 54 | assert.Len(t, labels, 1) 55 | 56 | // Test many labels 57 | labels, err = LabelsFromString("key=a,value=b&key=c,value=d&key=e,value=f") 58 | assert.Nil(t, err) 59 | assert.Len(t, labels, 3) 60 | } 61 | -------------------------------------------------------------------------------- /db/go/20210329171739_update_workspaces.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210329171739() { 10 | if _, ok := initializedMigrations[20210329171739]; !ok { 11 | goose.AddMigration(Up20210329171739, Down20210329171739) 12 | initializedMigrations[20210329171739] = true 13 | } 14 | } 15 | 16 | // Up20210329171739 updates workspaces to use new images 17 | func Up20210329171739(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | 20 | if err := updateWorkspaceTemplateManifest( 21 | filepath.Join("workspaces", "cvat", "20210323175655.yaml"), 22 | cvatTemplateName); err != nil { 23 | return err 24 | } 25 | 26 | if err := updateWorkspaceTemplateManifest( 27 | filepath.Join("workspaces", "jupyterlab", "20210323175655.yaml"), 28 | jupyterLabTemplateName); err != nil { 29 | return err 30 | } 31 | 32 | return updateWorkspaceTemplateManifest( 33 | filepath.Join("workspaces", "vscode", "20210323175655.yaml"), 34 | vscodeWorkspaceTemplateName) 35 | } 36 | 37 | // Down20210329171739 rolls back image updates for workspaces 38 | func Down20210329171739(tx *sql.Tx) error { 39 | // This code is executed when the migration is rolled back. 40 | if err := updateWorkspaceTemplateManifest( 41 | filepath.Join("workspaces", "cvat", "20210224180017.yaml"), 42 | cvatTemplateName); err != nil { 43 | return err 44 | } 45 | 46 | if err := updateWorkspaceTemplateManifest( 47 | filepath.Join("workspaces", "jupyterlab", "20210224180017.yaml"), 48 | jupyterLabTemplateName); err != nil { 49 | return err 50 | } 51 | 52 | return updateWorkspaceTemplateManifest( 53 | filepath.Join("workspaces", "vscode", "20210224180017.yaml"), 54 | vscodeWorkspaceTemplateName) 55 | } 56 | -------------------------------------------------------------------------------- /db/go/20210224180017_update_workspace_templates.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210224180017() { 10 | if _, ok := initializedMigrations[20210224180017]; !ok { 11 | goose.AddMigration(Up20210224180017, Down20210224180017) 12 | initializedMigrations[20210224180017] = true 13 | } 14 | } 15 | 16 | // Up20210224180017 Updates workspace templates with the latest filesyncer image 17 | func Up20210224180017(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | if err := updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20210224180017.yaml"), 21 | cvatTemplateName); err != nil { 22 | return err 23 | } 24 | 25 | if err := updateWorkspaceTemplateManifest( 26 | filepath.Join("workspaces", "jupyterlab", "20210224180017.yaml"), 27 | jupyterLabTemplateName); err != nil { 28 | return err 29 | } 30 | 31 | return updateWorkspaceTemplateManifest( 32 | filepath.Join("workspaces", "vscode", "20210224180017.yaml"), 33 | vscodeWorkspaceTemplateName) 34 | } 35 | 36 | // Down20210224180017 Rolls back the filesyncer image updates 37 | func Down20210224180017(tx *sql.Tx) error { 38 | // This code is executed when the migration is rolled back. 39 | if err := updateWorkspaceTemplateManifest( 40 | filepath.Join("workspaces", "vscode", "20210224180017.yaml"), 41 | vscodeWorkspaceTemplateName); err != nil { 42 | return err 43 | } 44 | 45 | if err := updateWorkspaceTemplateManifest( 46 | filepath.Join("workspaces", "jupyterlab", "20210224180017.yaml"), 47 | jupyterLabTemplateName); err != nil { 48 | return err 49 | } 50 | 51 | return updateWorkspaceTemplateManifest( 52 | filepath.Join("workspaces", "cvat", "20210224180017.yaml"), 53 | cvatTemplateName) 54 | } 55 | -------------------------------------------------------------------------------- /pkg/util/gcs/gcs.go: -------------------------------------------------------------------------------- 1 | package gcs 2 | 3 | import ( 4 | "cloud.google.com/go/storage" 5 | log "github.com/sirupsen/logrus" 6 | "golang.org/x/net/context" 7 | "golang.org/x/oauth2/google" 8 | "google.golang.org/api/option" 9 | "io" 10 | ) 11 | 12 | // Client is a struct used for accessing Google Cloud Storage. 13 | type Client struct { 14 | *storage.Client 15 | } 16 | 17 | // NewClient handles the details of initializing the connection to Google Cloud Storage. 18 | // - Note that the permissions are set to ReadWrite. 19 | func NewClient(namespace string, serviceAccountJSON string) (gcsClient *Client, err error) { 20 | ctx := context.Background() 21 | creds, err := google.CredentialsFromJSON(ctx, []byte(serviceAccountJSON), storage.ScopeReadWrite) 22 | if err != nil { 23 | log.WithFields(log.Fields{ 24 | "Namespace": namespace, 25 | "JSON": serviceAccountJSON, 26 | "Error": err.Error(), 27 | }).Error("GetGCSClient failed when initializing a new GCS client.") 28 | return 29 | } 30 | client, err := storage.NewClient(ctx, option.WithCredentials(creds)) 31 | if err != nil { 32 | log.WithFields(log.Fields{ 33 | "Namespace": namespace, 34 | "JSON": serviceAccountJSON, 35 | "Error": err.Error(), 36 | }).Error("GetGCSClient failed when initializing a new GCS client.") 37 | return 38 | } 39 | 40 | return &Client{Client: client}, nil 41 | } 42 | 43 | /* GetObject retrieves a specific object from Google Cloud Storage. 44 | - Function Name is meant to be consistent with S3's. 45 | */ 46 | func (c *Client) GetObject(bucket, key string) (stream io.ReadCloser, err error) { 47 | ctx := context.Background() 48 | stream, err = c.Client.Bucket(bucket).Object(key).NewReader(ctx) 49 | if err != nil { 50 | return 51 | } 52 | if stream == nil { 53 | defer stream.Close() 54 | } 55 | 56 | return 57 | } 58 | -------------------------------------------------------------------------------- /db/go/20200922103448_add_jupyter_lab_description.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | uid2 "github.com/onepanelio/core/pkg/util/uid" 6 | "github.com/pressly/goose" 7 | ) 8 | 9 | func initialize20200922103448() { 10 | if _, ok := initializedMigrations[20200922103448]; !ok { 11 | goose.AddMigration(Up20200922103448, Down20200922103448) 12 | initializedMigrations[20200922103448] = true 13 | } 14 | } 15 | 16 | // Up20200922103448 adds a description to the jupyterlab workspace template 17 | func Up20200922103448(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | client, err := getClient() 20 | if err != nil { 21 | return err 22 | } 23 | defer client.DB.Close() 24 | 25 | migrationsRan, err := getRanSQLMigrations(client) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | if _, ok := migrationsRan[20200922103448]; ok { 31 | return nil 32 | } 33 | 34 | namespaces, err := client.ListOnepanelEnabledNamespaces() 35 | if err != nil { 36 | return err 37 | } 38 | 39 | uid, err := uid2.GenerateUID(jupyterLabTemplateName, 30) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | for _, namespace := range namespaces { 45 | workspaceTemplate, err := client.GetWorkspaceTemplate(namespace.Name, uid, 0) 46 | if err != nil { 47 | return err 48 | } 49 | if workspaceTemplate == nil { 50 | continue 51 | } 52 | 53 | // Adding description 54 | workspaceTemplate.Description = "Interactive development environment for notebooks" 55 | 56 | if _, err := client.UpdateWorkspaceTemplate(namespace.Name, workspaceTemplate); err != nil { 57 | return err 58 | } 59 | } 60 | 61 | return nil 62 | } 63 | 64 | // Down20200922103448 does nothing 65 | func Down20200922103448(tx *sql.Tx) error { 66 | // This code is executed when the migration is rolled back. 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /db/go/20210323175655_update_templates_for_pns.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210323175655() { 10 | if _, ok := initializedMigrations[20210323175655]; !ok { 11 | goose.AddMigration(Up20210323175655, Down20210323175655) 12 | initializedMigrations[20210323175655] = true 13 | } 14 | } 15 | 16 | // Up20210323175655 update workflows to support new PNS mode 17 | func Up20210323175655(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | if err := updateWorkflowTemplateManifest( 20 | filepath.Join("workflows", "pytorch-mnist-training", "20210323175655.yaml"), 21 | pytorchWorkflowTemplateName, 22 | map[string]string{ 23 | "created-by": "system", 24 | "framework": "pytorch", 25 | }); err != nil { 26 | return err 27 | } 28 | 29 | return updateWorkflowTemplateManifest( 30 | filepath.Join("workflows", "tensorflow-mnist-training", "20210323175655.yaml"), 31 | tensorflowWorkflowTemplateName, 32 | map[string]string{ 33 | "created-by": "system", 34 | "framework": "tensorflow", 35 | }) 36 | } 37 | 38 | // Down20210323175655 reverts updating workflows to support PNS 39 | func Down20210323175655(tx *sql.Tx) error { 40 | // This code is executed when the migration is rolled back. 41 | if err := updateWorkflowTemplateManifest( 42 | filepath.Join("workflows", "tensorflow-mnist-training", "20210118175809.yaml"), 43 | tensorflowWorkflowTemplateName, 44 | map[string]string{ 45 | "created-by": "system", 46 | "framework": "tensorflow", 47 | }); err != nil { 48 | return err 49 | } 50 | 51 | return updateWorkflowTemplateManifest( 52 | filepath.Join("workflows", "pytorch-mnist-training", "20210118175809.yaml"), 53 | pytorchWorkflowTemplateName, 54 | map[string]string{ 55 | "created-by": "system", 56 | "framework": "pytorch", 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /api/proto/auth.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | import "protoc-gen-openapiv2/options/annotations.proto"; 8 | 9 | service AuthService { 10 | rpc IsValidToken(IsValidTokenRequest) returns (IsValidTokenResponse) { 11 | option (google.api.http) = { 12 | post: "/apis/v1beta1/auth/token" 13 | body: "*" 14 | }; 15 | option deprecated = true; 16 | } 17 | 18 | rpc GetAccessToken(GetAccessTokenRequest) returns (GetAccessTokenResponse) { 19 | option (google.api.http) = { 20 | post: "/apis/v1beta1/auth/get_access_token" 21 | body: "*" 22 | }; 23 | 24 | option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = { 25 | security: { 26 | } 27 | }; 28 | } 29 | 30 | rpc IsAuthorized(IsAuthorizedRequest) returns (IsAuthorizedResponse) { 31 | option (google.api.http) = { 32 | post: "/apis/v1beta1/auth" 33 | body: "isAuthorized" 34 | }; 35 | } 36 | } 37 | 38 | message IsValidTokenRequest { 39 | string username = 1; 40 | string token = 2; 41 | } 42 | 43 | message IsValidTokenResponse { 44 | string domain = 1; 45 | string token = 2; 46 | string username = 3; 47 | } 48 | 49 | message IsAuthorized { 50 | string namespace = 1; 51 | string verb = 2; 52 | string group = 3; 53 | string resource = 4; 54 | string resourceName = 5; 55 | } 56 | 57 | message IsAuthorizedRequest { 58 | IsAuthorized isAuthorized = 1; 59 | } 60 | 61 | message IsAuthorizedResponse { 62 | bool authorized = 1; 63 | } 64 | 65 | message GetAccessTokenRequest { 66 | string username = 1; 67 | string token = 2; 68 | } 69 | 70 | message GetAccessTokenResponse { 71 | string domain = 1; 72 | string accessToken = 2; 73 | string username = 3; 74 | } -------------------------------------------------------------------------------- /api/third_party/protoc-gen-openapiv2/options/annotations.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package grpc.gateway.protoc_gen_openapiv2.options; 4 | 5 | option go_package = "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options"; 6 | 7 | import "google/protobuf/descriptor.proto"; 8 | import "protoc-gen-openapiv2/options/openapiv2.proto"; 9 | 10 | extend google.protobuf.FileOptions { 11 | // ID assigned by protobuf-global-extension-registry@google.com for gRPC-Gateway project. 12 | // 13 | // All IDs are the same, as assigned. It is okay that they are the same, as they extend 14 | // different descriptor messages. 15 | Swagger openapiv2_swagger = 1042; 16 | } 17 | extend google.protobuf.MethodOptions { 18 | // ID assigned by protobuf-global-extension-registry@google.com for gRPC-Gateway project. 19 | // 20 | // All IDs are the same, as assigned. It is okay that they are the same, as they extend 21 | // different descriptor messages. 22 | Operation openapiv2_operation = 1042; 23 | } 24 | extend google.protobuf.MessageOptions { 25 | // ID assigned by protobuf-global-extension-registry@google.com for gRPC-Gateway project. 26 | // 27 | // All IDs are the same, as assigned. It is okay that they are the same, as they extend 28 | // different descriptor messages. 29 | Schema openapiv2_schema = 1042; 30 | } 31 | extend google.protobuf.ServiceOptions { 32 | // ID assigned by protobuf-global-extension-registry@google.com for gRPC-Gateway project. 33 | // 34 | // All IDs are the same, as assigned. It is okay that they are the same, as they extend 35 | // different descriptor messages. 36 | Tag openapiv2_tag = 1042; 37 | } 38 | extend google.protobuf.FieldOptions { 39 | // ID assigned by protobuf-global-extension-registry@google.com for gRPC-Gateway project. 40 | // 41 | // All IDs are the same, as assigned. It is okay that they are the same, as they extend 42 | // different descriptor messages. 43 | JSONSchema openapiv2_field = 1042; 44 | } 45 | -------------------------------------------------------------------------------- /pkg/file_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | ) 7 | 8 | // File represents a system file. 9 | type File struct { 10 | Path string 11 | Name string 12 | Size int64 13 | Extension string 14 | ContentType string 15 | LastModified time.Time 16 | Directory bool 17 | } 18 | 19 | // FilePathToParentPath given a path, returns the parent path, assuming a '/' delimiter 20 | // Result does not have a trailing slash. 21 | // -> a/b/c/d would return a/b/c 22 | // -> a/b/c/d/ would return a/b/c 23 | // If path is empty string, it is returned. 24 | // If path is '/' (root) it is returned as is. 25 | // If there is no '/', '/' is returned. 26 | func FilePathToParentPath(path string) string { 27 | separator := "/" 28 | if path == "" || path == separator { 29 | return path 30 | } 31 | 32 | if strings.HasSuffix(path, "/") { 33 | path = path[0 : len(path)-1] 34 | } 35 | 36 | lastIndexOfForwardSlash := strings.LastIndex(path, separator) 37 | if lastIndexOfForwardSlash <= 0 { 38 | return separator 39 | } 40 | 41 | return path[0:lastIndexOfForwardSlash] 42 | } 43 | 44 | // FilePathToExtension returns the file's extension if it uses a dot "." to denote it. 45 | // otherwise it returns the text following the last dot in the path. 46 | func FilePathToExtension(path string) string { 47 | dotIndex := strings.LastIndex(path, ".") 48 | 49 | if dotIndex == -1 { 50 | return "" 51 | } 52 | 53 | if dotIndex == (len(path) - 1) { 54 | return "" 55 | } 56 | 57 | return path[dotIndex+1:] 58 | } 59 | 60 | // FilePathToName returns the name of the file, assuming that "/" denote directories and that the 61 | // file name is after the last "/" 62 | func FilePathToName(path string) string { 63 | if strings.HasSuffix(path, "/") { 64 | path = path[:len(path)-1] 65 | } 66 | 67 | lastSlashIndex := strings.LastIndex(path, "/") 68 | if lastSlashIndex < 0 { 69 | return path 70 | } 71 | 72 | return path[lastSlashIndex+1:] 73 | } 74 | -------------------------------------------------------------------------------- /pkg/util/request/pagination/pagination.go: -------------------------------------------------------------------------------- 1 | package pagination 2 | 3 | import ( 4 | "github.com/Masterminds/squirrel" 5 | "math" 6 | ) 7 | 8 | type PaginationRequest struct { 9 | Page uint64 10 | PageSize uint64 11 | } 12 | 13 | // New creates a new pagination request from the page and page size. 14 | func New(page, pageSize int32) *PaginationRequest { 15 | pr := NewRequest(page, pageSize) 16 | 17 | return &pr 18 | } 19 | 20 | // NewRequest creates a new pagination request (not pointer) from the page and page size 21 | func NewRequest(page, pageSize int32) PaginationRequest { 22 | if page == 0 { 23 | page = 1 24 | } 25 | 26 | if pageSize == 0 { 27 | pageSize = 15 28 | } 29 | 30 | return PaginationRequest{ 31 | Page: uint64(page), 32 | PageSize: uint64(pageSize), 33 | } 34 | } 35 | 36 | // Start creates a new PaginationRequest that starts at the first page. 37 | // You can provide an optional pageSize argument. If none is provided, 15 is used. 38 | // All arguments apart from the first one are ignored. 39 | func Start(pageSize ...int32) *PaginationRequest { 40 | if len(pageSize) > 0 { 41 | pr := NewRequest(1, pageSize[0]) 42 | return &pr 43 | } 44 | 45 | pr := NewRequest(1, 15) 46 | 47 | return &pr 48 | } 49 | 50 | func (pr *PaginationRequest) Offset() uint64 { 51 | // start at page 1. 52 | return (pr.Page - 1) * pr.PageSize 53 | } 54 | 55 | func (pr *PaginationRequest) CalculatePages(count int) int32 { 56 | return int32(math.Ceil(float64(count) / float64(pr.PageSize))) 57 | } 58 | 59 | func (pr *PaginationRequest) ApplyToSelect(sb *squirrel.SelectBuilder) *squirrel.SelectBuilder { 60 | if pr == nil { 61 | return sb 62 | } 63 | 64 | result := sb.Limit(pr.PageSize). 65 | Offset(pr.Offset()) 66 | 67 | return &result 68 | } 69 | 70 | // Advance returns a new pagination request with the page incremented by 1 71 | func (pr *PaginationRequest) Advance() *PaginationRequest { 72 | return &PaginationRequest{ 73 | Page: pr.Page + 1, 74 | PageSize: pr.PageSize, 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vnc/20210414165510.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Deep Learning Desktop" 3 | kind: Workspace 4 | version: 20210414165510 5 | action: create 6 | description: "Deep learning desktop with VNC" 7 | spec: 8 | arguments: 9 | parameters: 10 | # parameter screen-resolution allows users to select screen resolution 11 | - name: screen-resolution 12 | value: 1680x1050 13 | type: select.select 14 | displayName: Screen Resolution 15 | options: 16 | - name: 1280x1024 17 | value: 1280x1024 18 | - name: 1680x1050 19 | value: 1680x1050 20 | - name: 2880x1800 21 | value: 2880x1800 22 | containers: 23 | - name: ubuntu 24 | image: onepanel/vnc:dl-vnc 25 | env: 26 | - name: VNC_PASSWORDLESS 27 | value: true 28 | - name: VNC_RESOLUTION 29 | value: '{{workflow.parameters.screen-resolution}}' 30 | ports: 31 | - containerPort: 6901 32 | name: vnc 33 | volumeMounts: 34 | - name: data 35 | mountPath: /data 36 | ports: 37 | - name: vnc 38 | port: 80 39 | protocol: TCP 40 | targetPort: 6901 41 | routes: 42 | - match: 43 | - uri: 44 | prefix: / 45 | route: 46 | - destination: 47 | port: 48 | number: 80 49 | # DAG Workflow to be executed once a Workspace action completes (optional) 50 | #postExecutionWorkflow: 51 | # entrypoint: main 52 | # templates: 53 | # - name: main 54 | # dag: 55 | # tasks: 56 | # - name: slack-notify 57 | # template: slack-notify 58 | # - name: slack-notify 59 | # container: 60 | # image: technosophos/slack-notify 61 | # args: 62 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 63 | # command: 64 | # - sh -------------------------------------------------------------------------------- /design-docs/Persist Packages during Workspace Switch or Update.md: -------------------------------------------------------------------------------- 1 | # Background 2 | A user may install pip packages, conda packages, jupyterlab extensions, 3 | or vscode extensions. 4 | 5 | To improve the user experience, we should save these packages 6 | when the user pauses and resumes the workspace. 7 | - And when the user changes the workspace machine. 8 | 9 | To achieve this end, we’re using lifecycle hooks for containers, via Kubernetes. 10 | 11 | We’ll add these hooks to workspace templates, to VSCode and JupyterLab. 12 | 13 | Note: We're not supporting CVAT Workspace persistence. 14 | - User does not have access to the terminal through the UI. 15 | 16 | # Problem(s) 17 | Originally, we attempted to tar up the packages of conda, vscode, and jupyter. 18 | This worked, but ate non-trivial amount of space. Roughly ~2 GB with a clean workspace, 19 | using our dockerfile image. 20 | - So we'd need to warn the user to have enough space for their volume mounts 21 | - We'd also have to warn the user that if there is not enough space, the packages 22 | may not persist. 23 | 24 | The other problem was we needed to extend the terminationGracePeriodSeconds of the 25 | pod. 26 | - K8s sets 30 seconds by default 27 | 28 | We use the preStop hook to do the back-up. 29 | - This operation runs for 2-3 minutes on a clean workspace image 30 | - For a user with more packages, it would take longer. 31 | - k8s will grant a 2 second increase once, if preStop is not done. 32 | 33 | Otherwise, k8s kills the preStop hook and the container. 34 | 35 | We thought about exposing the terminationGracePeriodSeconds to the user, 36 | allow them to change this value as needed. 37 | 38 | And lastly, k8s will wait the entire grace period. 39 | - Even if the preStop hook finishes early, k8s will wait to kill the container. 40 | - In testing, k8s waited 20 minutes before terminating the container 41 | 42 | So we decided to export the list of installed packages, as a text file. 43 | - Then, we re-install on workspace start-up with postStart hook. 44 | 45 | # References 46 | - https://github.com/onepanelio/core/issues/623 -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | COMMIT_HASH=$(shell git rev-parse --short HEAD) 2 | 3 | .PHONY: init 4 | 5 | init: 6 | ifndef version 7 | $(error version is undefined) 8 | endif 9 | 10 | jq: 11 | cat api/apidocs.swagger.json \ 12 | | jq 'walk( if type == "object" then with_entries( .key |= sub( "api\\."; "") ) else . end )' \ 13 | | jq 'walk( if type == "string" then gsub( "api\\."; "") else . end )' \ 14 | | jq '.info.version = "$(version)"' \ 15 | > api/api.swagger.json 16 | rm api/apidocs.swagger.json 17 | 18 | protoc: 19 | protoc -I/usr/local/include \ 20 | -Iapi/third_party/ \ 21 | -Iapi/proto \ 22 | --go_out ./api/gen --go_opt paths=source_relative \ 23 | --go-grpc_out ./api/gen --go-grpc_opt paths=source_relative \ 24 | --go-grpc_opt paths=source_relative \ 25 | --grpc-gateway_out ./api/gen \ 26 | --grpc-gateway_opt logtostderr=true \ 27 | --grpc-gateway_opt allow_delete_body=true \ 28 | --grpc-gateway_opt paths=source_relative \ 29 | --grpc-gateway_opt generate_unbound_methods=true \ 30 | --openapiv2_out ./api \ 31 | --openapiv2_opt allow_merge=true \ 32 | --openapiv2_opt fqn_for_openapi_name=true \ 33 | --openapiv2_opt allow_delete_body=true \ 34 | --openapiv2_opt logtostderr=true \ 35 | --openapiv2_opt simple_operation_ids=true \ 36 | api/proto/*.proto 37 | 38 | api-internal: init protoc jq 39 | 40 | api: init 41 | docker run --rm --mount type=bind,source="${PWD}",target=/root onepanel/helper:v1.0.0 make api-internal version=$(version) 42 | 43 | docker-build: 44 | docker build -t onepanel-core . 45 | docker tag onepanel-core:latest onepanel/core:$(COMMIT_HASH) 46 | 47 | docker-push: 48 | docker push onepanel/core:$(COMMIT_HASH) 49 | 50 | docker-custom: 51 | docker build -t onepanel-core . 52 | docker tag onepanel-core:latest onepanel/core:$(TAG) 53 | docker push onepanel/core:$(TAG) 54 | 55 | docker: docker-build docker-push 56 | 57 | run-tests: 58 | docker run --rm --name test-onepanel-postgres -p 5432:5432 -e POSTGRES_USER=admin -e POSTGRES_PASSWORD=tester -e POSTGRES_DB=onepanel -d postgres:12.3 59 | go test github.com/onepanelio/core/pkg -count=1 ||: 60 | docker stop test-onepanel-postgres -------------------------------------------------------------------------------- /db/go/20210719190719_update_filesyncer.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/pressly/goose" 6 | "path/filepath" 7 | ) 8 | 9 | func initialize20210719190719() { 10 | if _, ok := initializedMigrations[20210719190719]; !ok { 11 | goose.AddMigration(Up20210719190719, Down20210719190719) 12 | initializedMigrations[20210719190719] = true 13 | } 14 | } 15 | 16 | // Up20210719190719 updates the workspace templates to use new v1.0.0 of filesyncer 17 | func Up20210719190719(tx *sql.Tx) error { 18 | // This code is executed when the migration is applied. 19 | if err := updateWorkspaceTemplateManifest( 20 | filepath.Join("workspaces", "cvat", "20210719190719.yaml"), 21 | cvatTemplateName); err != nil { 22 | return err 23 | } 24 | 25 | if err := updateWorkspaceTemplateManifest( 26 | filepath.Join("workspaces", "jupyterlab", "20210719190719.yaml"), 27 | jupyterLabTemplateName); err != nil { 28 | return err 29 | } 30 | 31 | if err := updateWorkspaceTemplateManifest( 32 | filepath.Join("workspaces", "vnc", "20210719190719.yaml"), 33 | deepLearningDesktopTemplateName); err != nil { 34 | return err 35 | } 36 | 37 | return updateWorkspaceTemplateManifest( 38 | filepath.Join("workspaces", "vscode", "20210719190719.yaml"), 39 | vscodeWorkspaceTemplateName) 40 | } 41 | 42 | // Down20210719190719 rolls back the change to update filesyncer 43 | func Down20210719190719(tx *sql.Tx) error { 44 | // This code is executed when the migration is rolled back. 45 | if err := updateWorkspaceTemplateManifest( 46 | filepath.Join("workspaces", "cvat", "20210323175655.yaml"), 47 | cvatTemplateName); err != nil { 48 | return err 49 | } 50 | 51 | if err := updateWorkspaceTemplateManifest( 52 | filepath.Join("workspaces", "jupyterlab", "20210323175655.yaml"), 53 | jupyterLabTemplateName); err != nil { 54 | return err 55 | } 56 | 57 | if err := updateWorkspaceTemplateManifest( 58 | filepath.Join("workspaces", "vnc", "20210414165510.yaml"), 59 | deepLearningDesktopTemplateName); err != nil { 60 | return err 61 | } 62 | 63 | return updateWorkspaceTemplateManifest( 64 | filepath.Join("workspaces", "vscode", "20210323175655.yaml"), 65 | vscodeWorkspaceTemplateName) 66 | } 67 | -------------------------------------------------------------------------------- /db/go/20200727144157_add_parameters_to_workflow_template_version.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "database/sql" 5 | v1 "github.com/onepanelio/core/pkg" 6 | "github.com/onepanelio/core/pkg/util/request/pagination" 7 | "github.com/pressly/goose" 8 | ) 9 | 10 | func initialize20200727144157() { 11 | if _, ok := initializedMigrations[20200727144157]; !ok { 12 | goose.AddMigration(Up20200727144157, Down20200727144157) 13 | initializedMigrations[20200727144157] = true 14 | } 15 | } 16 | 17 | // Up20200727144157 will go through all WorkflowTemplateVersion database entries 18 | // and attempt to generate the "parameters" column from the "manifests" column. 19 | func Up20200727144157(tx *sql.Tx) error { 20 | // This code is executed when the migration is applied. 21 | 22 | client, err := getClient() 23 | if err != nil { 24 | return err 25 | } 26 | defer client.DB.Close() 27 | 28 | migrationsRan, err := getRanSQLMigrations(client) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | if _, ok := migrationsRan[20200727144157]; ok { 34 | return nil 35 | } 36 | 37 | pageSize := int32(100) 38 | page := int32(0) 39 | paginator := pagination.NewRequest(page, pageSize) 40 | wtvsResults := -1 41 | for wtvsResults != 0 { 42 | wtvs, err := client.ListWorkflowTemplateVersionsAll(&paginator) 43 | if err != nil { 44 | return err 45 | } 46 | //Exit condition; Check for more results 47 | wtvsResults = len(wtvs) 48 | if wtvsResults > 0 { 49 | page++ 50 | paginator = pagination.NewRequest(page, pageSize) 51 | } 52 | 53 | for _, wtv := range wtvs { 54 | params, err := v1.ParseParametersFromManifest([]byte(wtv.Manifest)) 55 | if err != nil { 56 | return err 57 | } 58 | wtv.Parameters = params 59 | err = client.UpdateWorkflowTemplateVersion(wtv) 60 | if err != nil { 61 | return err 62 | } 63 | } 64 | } 65 | 66 | return nil 67 | } 68 | 69 | // Down20200727144157 can be run before 20200727155027_add_parameters_col_to_workflow_template_version.sql 70 | // Nothing happens because the referenced SQL file will drop the "parameters" column. 71 | func Down20200727144157(tx *sql.Tx) error { 72 | // This code is executed when the migration is rolled back. 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/onepanelio/core 2 | 3 | go 1.15 4 | 5 | require ( 6 | cloud.google.com/go/storage v1.10.0 7 | github.com/Azure/go-autorest v14.0.0+incompatible // indirect 8 | github.com/Azure/go-autorest/autorest/adal v0.8.2 // indirect 9 | github.com/Masterminds/squirrel v1.1.0 10 | github.com/argoproj/argo v0.0.0-20210112203504-f97bef5d0036 11 | github.com/argoproj/pkg v0.2.0 12 | github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 13 | github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect 14 | github.com/ghodss/yaml v1.0.0 15 | github.com/go-sql-driver/mysql v1.5.0 // indirect 16 | github.com/golang/protobuf v1.4.3 17 | github.com/google/uuid v1.1.2 18 | github.com/gorilla/handlers v1.4.2 19 | github.com/grpc-ecosystem/go-grpc-middleware v1.2.0 20 | github.com/grpc-ecosystem/grpc-gateway v1.14.6 21 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.0.1 22 | github.com/hashicorp/go-uuid v1.0.2 // indirect 23 | github.com/hashicorp/golang-lru v0.5.4 // indirect 24 | github.com/jmoiron/sqlx v1.2.0 25 | github.com/lib/pq v1.3.0 26 | github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect 27 | github.com/minio/minio-go/v6 v6.0.45 28 | github.com/pkg/errors v0.9.1 29 | github.com/pressly/goose v2.6.0+incompatible 30 | github.com/sirupsen/logrus v1.6.0 31 | github.com/stretchr/testify v1.6.1 32 | github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc 33 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b 34 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 35 | google.golang.org/api v0.30.0 36 | google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 37 | google.golang.org/grpc v1.33.1 38 | google.golang.org/protobuf v1.25.0 39 | gopkg.in/yaml.v2 v2.2.8 40 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 41 | istio.io/api v0.0.0-20200107183329-ed4b507c54e1 42 | k8s.io/api v0.18.2 43 | k8s.io/apimachinery v0.18.2 44 | k8s.io/client-go v0.18.2 45 | sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e // indirect 46 | sigs.k8s.io/yaml v1.2.0 47 | ) 48 | 49 | replace ( 50 | k8s.io/api => k8s.io/api v0.17.8 51 | k8s.io/apimachinery => k8s.io/apimachinery v0.17.8 52 | k8s.io/client-go => k8s.io/client-go v0.17.8 53 | ) 54 | -------------------------------------------------------------------------------- /db/yaml/workspaces/jupyterlab/20200525160514.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: JupyterLab 3 | kind: Workspace 4 | version: 20200525160514 5 | action: create 6 | description: "Interactive development environment for notebooks" 7 | spec: 8 | # Docker containers that are part of the Workspace 9 | containers: 10 | - name: jupyterlab-tensorflow 11 | image: jupyter/tensorflow-notebook 12 | command: [start.sh, jupyter] 13 | env: 14 | - name: tornado 15 | value: "{ 'headers': { 'Content-Security-Policy': \"frame-ancestors * 'self'\" } }" 16 | args: 17 | - lab 18 | - --LabApp.token='' 19 | - --LabApp.allow_remote_access=True 20 | - --LabApp.allow_origin="*" 21 | - --LabApp.disable_check_xsrf=True 22 | - --LabApp.trust_xheaders=True 23 | - --LabApp.tornado_settings=$(tornado) 24 | - --notebook-dir='/data' 25 | ports: 26 | - containerPort: 8888 27 | name: jupyterlab 28 | # Volumes to be mounted in this container 29 | # Onepanel will automatically create these volumes and mount them to the container 30 | volumeMounts: 31 | - name: data 32 | mountPath: /data 33 | # Ports that need to be exposed 34 | ports: 35 | - name: jupyterlab 36 | port: 80 37 | protocol: TCP 38 | targetPort: 8888 39 | # Routes that will map to ports 40 | routes: 41 | - match: 42 | - uri: 43 | prefix: / 44 | route: 45 | - destination: 46 | port: 47 | number: 80 48 | # DAG Workflow to be executed once a Workspace action completes 49 | # postExecutionWorkflow: 50 | # entrypoint: main 51 | # templates: 52 | # - name: main 53 | # dag: 54 | # tasks: 55 | # - name: slack-notify 56 | # template: slack-notify 57 | # - name: slack-notify 58 | # container: 59 | # image: technosophos/slack-notify 60 | # args: 61 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 62 | # command: 63 | # - sh 64 | # - -c -------------------------------------------------------------------------------- /pkg/common_types_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | // TestParseParametersFromManifest makes sure that we have correct parsing of parameters from a manifest 9 | func TestParseParametersFromManifest(t *testing.T) { 10 | manifest := `arguments: 11 | parameters: 12 | - name: source 13 | value: https://github.com/onepanelio/Mask_RCNN.git 14 | - name: dataset-path 15 | value: datasets/test_05142020170720 16 | visibility: public 17 | - name: model-path 18 | value: models/rush/cvat6-20 19 | - name: extras 20 | value: none 21 | - name: task-name 22 | value: test 23 | - name: num-classes 24 | value: 2 25 | - name: tf-image 26 | value: tensorflow/tensorflow:1.13.1-py3 27 | - displayName: Node pool 28 | hint: Name of node pool or group 29 | type: select.select 30 | name: sys-node-pool 31 | required: true 32 | options: 33 | - name: 'CPU: 2, RAM: 8GB' 34 | value: Standard_D2s_v3 35 | - name: 'CPU: 4, RAM: 16GB' 36 | value: Standard_D4s_v3 37 | - name: 'GPU: 1xK80, CPU: 6, RAM: 56GB' 38 | value: Standard_NC6 39 | ` 40 | 41 | parameters, err := ParseParametersFromManifest([]byte(manifest)) 42 | assert.Nil(t, err) 43 | assert.NotNil(t, parameters) 44 | assert.Len(t, parameters, 8) 45 | 46 | keyedParameters := MapParametersByName(parameters) 47 | 48 | // Make sure visibility is set 49 | assert.Equal(t, *keyedParameters["dataset-path"].Visibility, "public") 50 | 51 | // Make sure visibility is public if omitted 52 | assert.Equal(t, *keyedParameters["tf-image"].Visibility, "public") 53 | 54 | // Make sure numbers, slashes, dashes, and letters are parsed correctly 55 | assert.Equal(t, *keyedParameters["tf-image"].Value, "tensorflow/tensorflow:1.13.1-py3") 56 | 57 | // Make sure integers are parsed as strings and not ignored or omitted 58 | assert.Equal(t, *keyedParameters["num-classes"].Value, "2") 59 | 60 | // Make sure missing values have a nil value to show they are not there 61 | assert.Nil(t, keyedParameters["sys-node-pool"].Value, nil) 62 | 63 | // Make sure options are parsed 64 | assert.Len(t, keyedParameters["sys-node-pool"].Options, 3) 65 | 66 | // Make sure string values are correctly parsed 67 | assert.Equal(t, *keyedParameters["extras"].Value, "none") 68 | } 69 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vscode/20210129152427.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Visual Studio Code" 3 | kind: Workspace 4 | version: 20210129152427 5 | action: update 6 | spec: 7 | containers: 8 | - name: vscode 9 | image: onepanel/vscode:1.0.0 10 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."] 11 | env: 12 | - name: ONEPANEL_MAIN_CONTAINER 13 | value: 'true' 14 | ports: 15 | - containerPort: 8080 16 | name: vscode 17 | volumeMounts: 18 | - name: data 19 | mountPath: /data 20 | lifecycle: 21 | postStart: 22 | exec: 23 | command: 24 | - /bin/sh 25 | - -c 26 | - > 27 | condayml="/data/.environment.yml"; 28 | vscodetxt="/data/.vscode-extensions.txt"; 29 | if [ -f "$condayml" ]; then conda env update -f $condayml; fi; 30 | if [ -f "$vscodetxt" ]; then cat $vscodetxt | xargs -n 1 code-server --install-extension; fi; 31 | preStop: 32 | exec: 33 | command: 34 | - /bin/sh 35 | - -c 36 | - > 37 | conda env export > /data/.environment.yml -n base; 38 | code-server --list-extensions | tail -n +2 > /data/.vscode-extensions.txt; 39 | - name: sys-filesyncer 40 | image: onepanel/filesyncer:v0.18.0 41 | imagePullPolicy: Always 42 | args: 43 | - server 44 | - -server-prefix=/sys/filesyncer 45 | volumeMounts: 46 | - name: data 47 | mountPath: /data 48 | - name: sys-namespace-config 49 | mountPath: /etc/onepanel 50 | readOnly: true 51 | ports: 52 | - name: vscode 53 | port: 8080 54 | protocol: TCP 55 | targetPort: 8080 56 | - name: fs 57 | port: 8888 58 | protocol: TCP 59 | targetPort: 8888 60 | routes: 61 | - match: 62 | - uri: 63 | prefix: /sys/filesyncer 64 | route: 65 | - destination: 66 | port: 67 | number: 8888 68 | - match: 69 | - uri: 70 | prefix: / 71 | route: 72 | - destination: 73 | port: 74 | number: 8080 75 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vscode/20210224180017.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Visual Studio Code" 3 | kind: Workspace 4 | version: 20210224180017 5 | action: update 6 | spec: 7 | containers: 8 | - name: vscode 9 | image: onepanel/vscode:1.0.0 10 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."] 11 | env: 12 | - name: ONEPANEL_MAIN_CONTAINER 13 | value: 'true' 14 | ports: 15 | - containerPort: 8080 16 | name: vscode 17 | volumeMounts: 18 | - name: data 19 | mountPath: /data 20 | lifecycle: 21 | postStart: 22 | exec: 23 | command: 24 | - /bin/sh 25 | - -c 26 | - > 27 | condayml="/data/.environment.yml"; 28 | vscodetxt="/data/.vscode-extensions.txt"; 29 | if [ -f "$condayml" ]; then conda env update -f $condayml; fi; 30 | if [ -f "$vscodetxt" ]; then cat $vscodetxt | xargs -n 1 code-server --install-extension; fi; 31 | preStop: 32 | exec: 33 | command: 34 | - /bin/sh 35 | - -c 36 | - > 37 | conda env export > /data/.environment.yml -n base; 38 | code-server --list-extensions | tail -n +2 > /data/.vscode-extensions.txt; 39 | - name: sys-filesyncer 40 | image: onepanel/filesyncer:v0.19.0 41 | imagePullPolicy: Always 42 | args: 43 | - server 44 | - -server-prefix=/sys/filesyncer 45 | volumeMounts: 46 | - name: data 47 | mountPath: /data 48 | - name: sys-namespace-config 49 | mountPath: /etc/onepanel 50 | readOnly: true 51 | ports: 52 | - name: vscode 53 | port: 8080 54 | protocol: TCP 55 | targetPort: 8080 56 | - name: fs 57 | port: 8888 58 | protocol: TCP 59 | targetPort: 8888 60 | routes: 61 | - match: 62 | - uri: 63 | prefix: /sys/filesyncer 64 | route: 65 | - destination: 66 | port: 67 | number: 8888 68 | - match: 69 | - uri: 70 | prefix: / 71 | route: 72 | - destination: 73 | port: 74 | number: 8080 75 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vscode/20210719190719.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Visual Studio Code" 3 | kind: Workspace 4 | version: 20210719190719 5 | action: update 6 | spec: 7 | containers: 8 | - name: vscode 9 | image: onepanel/vscode:v0.20.0_code-server.3.9.1 10 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."] 11 | env: 12 | - name: ONEPANEL_MAIN_CONTAINER 13 | value: 'true' 14 | ports: 15 | - containerPort: 8080 16 | name: vscode 17 | volumeMounts: 18 | - name: data 19 | mountPath: /data 20 | lifecycle: 21 | postStart: 22 | exec: 23 | command: 24 | - /bin/sh 25 | - -c 26 | - > 27 | condayml="/data/.environment.yml"; 28 | vscodetxt="/data/.vscode-extensions.txt"; 29 | if [ -f "$condayml" ]; then conda env update -f $condayml; fi; 30 | if [ -f "$vscodetxt" ]; then cat $vscodetxt | xargs -n 1 code-server --install-extension; fi; 31 | preStop: 32 | exec: 33 | command: 34 | - /bin/sh 35 | - -c 36 | - > 37 | conda env export > /data/.environment.yml -n base; 38 | code-server --list-extensions | tail -n +2 > /data/.vscode-extensions.txt; 39 | - name: sys-filesyncer 40 | image: onepanel/filesyncer:v1.0.0 41 | imagePullPolicy: Always 42 | args: 43 | - server 44 | - -server-prefix=/sys/filesyncer 45 | volumeMounts: 46 | - name: data 47 | mountPath: /data 48 | - name: sys-namespace-config 49 | mountPath: /etc/onepanel 50 | readOnly: true 51 | ports: 52 | - name: vscode 53 | port: 8080 54 | protocol: TCP 55 | targetPort: 8080 56 | - name: fs 57 | port: 8888 58 | protocol: TCP 59 | targetPort: 8888 60 | routes: 61 | - match: 62 | - uri: 63 | prefix: /sys/filesyncer 64 | route: 65 | - destination: 66 | port: 67 | number: 8888 68 | - match: 69 | - uri: 70 | prefix: / 71 | route: 72 | - destination: 73 | port: 74 | number: 8080 75 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vscode/20210323175655.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Visual Studio Code" 3 | kind: Workspace 4 | version: 20210323175655 5 | action: update 6 | spec: 7 | containers: 8 | - name: vscode 9 | image: onepanel/vscode:v0.20.0_code-server.3.9.1 10 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."] 11 | env: 12 | - name: ONEPANEL_MAIN_CONTAINER 13 | value: 'true' 14 | ports: 15 | - containerPort: 8080 16 | name: vscode 17 | volumeMounts: 18 | - name: data 19 | mountPath: /data 20 | lifecycle: 21 | postStart: 22 | exec: 23 | command: 24 | - /bin/sh 25 | - -c 26 | - > 27 | condayml="/data/.environment.yml"; 28 | vscodetxt="/data/.vscode-extensions.txt"; 29 | if [ -f "$condayml" ]; then conda env update -f $condayml; fi; 30 | if [ -f "$vscodetxt" ]; then cat $vscodetxt | xargs -n 1 code-server --install-extension; fi; 31 | preStop: 32 | exec: 33 | command: 34 | - /bin/sh 35 | - -c 36 | - > 37 | conda env export > /data/.environment.yml -n base; 38 | code-server --list-extensions | tail -n +2 > /data/.vscode-extensions.txt; 39 | - name: sys-filesyncer 40 | image: onepanel/filesyncer:v0.20.0 41 | imagePullPolicy: Always 42 | args: 43 | - server 44 | - -server-prefix=/sys/filesyncer 45 | volumeMounts: 46 | - name: data 47 | mountPath: /data 48 | - name: sys-namespace-config 49 | mountPath: /etc/onepanel 50 | readOnly: true 51 | ports: 52 | - name: vscode 53 | port: 8080 54 | protocol: TCP 55 | targetPort: 8080 56 | - name: fs 57 | port: 8888 58 | protocol: TCP 59 | targetPort: 8888 60 | routes: 61 | - match: 62 | - uri: 63 | prefix: /sys/filesyncer 64 | route: 65 | - destination: 66 | port: 67 | number: 8888 68 | - match: 69 | - uri: 70 | prefix: / 71 | route: 72 | - destination: 73 | port: 74 | number: 8080 75 | -------------------------------------------------------------------------------- /api/proto/label.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | 8 | service LabelService { 9 | rpc GetAvailableLabels (GetAvailableLabelsRequest) returns (GetLabelsResponse) { 10 | option (google.api.http) = { 11 | get: "/apis/v1beta1/{namespace}/{resource}/labels" 12 | }; 13 | } 14 | 15 | rpc GetLabels (GetLabelsRequest) returns (GetLabelsResponse) { 16 | option (google.api.http) = { 17 | get: "/apis/v1beta1/{namespace}/{resource}/{uid}/labels" 18 | }; 19 | } 20 | 21 | rpc AddLabels (AddLabelsRequest) returns (GetLabelsResponse) { 22 | option (google.api.http) = { 23 | post: "/apis/v1beta1/{namespace}/{resource}/{uid}/labels" 24 | body: "labels" 25 | }; 26 | } 27 | 28 | rpc ReplaceLabels (ReplaceLabelsRequest) returns (GetLabelsResponse) { 29 | option (google.api.http) = { 30 | put: "/apis/v1beta1/{namespace}/{resource}/{uid}/labels" 31 | body: "labels" 32 | }; 33 | } 34 | 35 | rpc DeleteLabel (DeleteLabelRequest) returns (GetLabelsResponse) { 36 | option (google.api.http) = { 37 | delete: "/apis/v1beta1/{namespace}/{resource}/{uid}/labels/{key}" 38 | }; 39 | } 40 | } 41 | 42 | message KeyValue { 43 | string key = 1; 44 | string value = 2; 45 | } 46 | 47 | message Labels { 48 | repeated KeyValue items = 1; 49 | } 50 | 51 | message AddLabelsRequest { 52 | string namespace = 1; 53 | string resource = 2; 54 | string uid = 3; 55 | Labels labels = 4; 56 | } 57 | 58 | message ReplaceLabelsRequest { 59 | string namespace = 1; 60 | string resource = 2; 61 | string uid = 3; 62 | Labels labels = 4; 63 | } 64 | 65 | message GetLabelsRequest { 66 | string namespace = 1; 67 | string resource = 2; 68 | string uid = 3; 69 | } 70 | 71 | message GetAvailableLabelsRequest { 72 | string namespace = 1; 73 | string resource = 2; 74 | string keyLike = 3; 75 | string skipKeys = 4; 76 | } 77 | 78 | message GetLabelsResponse { 79 | repeated KeyValue labels = 1; 80 | } 81 | 82 | message DeleteLabelRequest { 83 | string namespace = 1; 84 | string resource = 2; 85 | string uid = 3; 86 | string key = 4; 87 | } -------------------------------------------------------------------------------- /db/yaml/workspaces/vscode/20201028145443.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Visual Studio Code" 3 | kind: Workspace 4 | version: 20201028145443 5 | action: update 6 | spec: 7 | # Docker containers that are part of the Workspace 8 | containers: 9 | - name: vscode 10 | image: onepanel/vscode:1.0.0 11 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && /usr/bin/entrypoint.sh --bind-addr 0.0.0.0:8080 --auth none ."] 12 | ports: 13 | - containerPort: 8080 14 | name: vscode 15 | volumeMounts: 16 | - name: data 17 | mountPath: /data 18 | lifecycle: 19 | postStart: 20 | exec: 21 | command: 22 | - /bin/sh 23 | - -c 24 | - > 25 | condayml="/data/.environment.yml"; 26 | vscodetxt="/data/.vscode-extensions.txt"; 27 | if [ -f "$condayml" ]; then conda env update -f $condayml; fi; 28 | if [ -f "$vscodetxt" ]; then cat $vscodetxt | xargs -n 1 code-server --install-extension; fi; 29 | preStop: 30 | exec: 31 | command: 32 | - /bin/sh 33 | - -c 34 | - > 35 | conda env export > /data/.environment.yml -n base; 36 | code-server --list-extensions | tail -n +2 > /data/.vscode-extensions.txt; 37 | ports: 38 | - name: vscode 39 | port: 8080 40 | protocol: TCP 41 | targetPort: 8080 42 | routes: 43 | - match: 44 | - uri: 45 | prefix: / #vscode runs at the default route 46 | route: 47 | - destination: 48 | port: 49 | number: 8080 50 | # DAG Workflow to be executed once a Workspace action completes (optional) 51 | #postExecutionWorkflow: 52 | # entrypoint: main 53 | # templates: 54 | # - name: main 55 | # dag: 56 | # tasks: 57 | # - name: slack-notify 58 | # template: slack-notify 59 | # - name: slack-notify 60 | # container: 61 | # image: technosophos/slack-notify 62 | # args: 63 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 64 | # command: 65 | # - sh 66 | # - -c -------------------------------------------------------------------------------- /manifest/gcs/deployment.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "apps/v1", 3 | "kind": "Deployment", 4 | "metadata": { 5 | "name": "minio-gateway", 6 | "namespace": "$(applicationDefaultNamespace)" 7 | }, 8 | "spec": { 9 | "replicas": 1, 10 | "selector": { 11 | "matchLabels": { 12 | "app": "minio-gateway" 13 | } 14 | }, 15 | "template": { 16 | "metadata": { 17 | "labels": { 18 | "app": "minio-gateway" 19 | }, 20 | "annotations": { 21 | "sidecar.istio.io/inject": "false" 22 | } 23 | }, 24 | "spec": { 25 | "containers": [ 26 | { 27 | "name": "minio-gateway", 28 | "image": "minio/minio:RELEASE.2021-06-17T00-10-46Z.hotfix.49f6035b1", 29 | "volumeMounts": [ 30 | { 31 | "name": "gcs-credentials", 32 | "mountPath": "/etc/gcs", 33 | "readOnly": true 34 | } 35 | ], 36 | "args": [ 37 | "gateway", 38 | "gcs" 39 | ], 40 | "env": [ 41 | { 42 | "name": "MINIO_ACCESS_KEY", 43 | "valueFrom": { 44 | "secretKeyRef": { 45 | "name": "onepanel", 46 | "key": "artifactRepositoryS3AccessKey" 47 | } 48 | } 49 | }, 50 | { 51 | "name": "MINIO_SECRET_KEY", 52 | "valueFrom": { 53 | "secretKeyRef": { 54 | "name": "onepanel", 55 | "key": "artifactRepositoryS3SecretKey" 56 | } 57 | } 58 | }, 59 | { 60 | "name": "GOOGLE_APPLICATION_CREDENTIALS", 61 | "value": "/etc/gcs/credentials.json" 62 | } 63 | ] 64 | } 65 | ], 66 | "volumes": [ 67 | { 68 | "name": "gcs-credentials", 69 | "projected": { 70 | "sources": [ 71 | { 72 | "secret": { 73 | "name": "artifact-repository-gcs-credentials" 74 | } 75 | } 76 | ] 77 | } 78 | } 79 | ] 80 | } 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /db/yaml/workspaces/jupyterlab/20200821162630.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: JupyterLab 3 | kind: Workspace 4 | version: 20200821162630 5 | action: update 6 | description: "Interactive development environment for notebooks" 7 | spec: 8 | # Docker containers that are part of the Workspace 9 | containers: 10 | - name: jupyterlab-tensorflow 11 | image: onepanel/jupyterlab:1.0.1 12 | command: ["/bin/bash", "-c", "start.sh jupyter lab --LabApp.token='' --LabApp.allow_remote_access=True --LabApp.allow_origin=\"*\" --LabApp.disable_check_xsrf=True --LabApp.trust_xheaders=True --LabApp.base_url=/ --LabApp.tornado_settings='{\"headers\":{\"Content-Security-Policy\":\"frame-ancestors * \'self\'\"}}' --notebook-dir='/data' --allow-root"] 13 | env: 14 | - name: tornado 15 | value: "'{'headers':{'Content-Security-Policy':\"frame-ancestors\ *\ \'self'\"}}'" 16 | args: 17 | ports: 18 | - containerPort: 8888 19 | name: jupyterlab 20 | - containerPort: 6006 21 | name: tensorboard 22 | volumeMounts: 23 | - name: data 24 | mountPath: /data 25 | ports: 26 | - name: jupyterlab 27 | port: 80 28 | protocol: TCP 29 | targetPort: 8888 30 | - name: tensorboard 31 | port: 6006 32 | protocol: TCP 33 | targetPort: 6006 34 | routes: 35 | - match: 36 | - uri: 37 | prefix: /tensorboard 38 | route: 39 | - destination: 40 | port: 41 | number: 6006 42 | - match: 43 | - uri: 44 | prefix: / #jupyter runs at the default route 45 | route: 46 | - destination: 47 | port: 48 | number: 80 49 | # DAG Workflow to be executed once a Workspace action completes (optional) 50 | #postExecutionWorkflow: 51 | # entrypoint: main 52 | # templates: 53 | # - name: main 54 | # dag: 55 | # tasks: 56 | # - name: slack-notify 57 | # template: slack-notify 58 | # - name: slack-notify 59 | # container: 60 | # image: technosophos/slack-notify 61 | # args: 62 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 63 | # command: 64 | # - sh 65 | # - -c -------------------------------------------------------------------------------- /db/yaml/workspaces/jupyterlab/20200929153931.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: JupyterLab 3 | kind: Workspace 4 | version: 20200929153931 5 | action: update 6 | description: "Interactive development environment for notebooks" 7 | spec: 8 | # Docker containers that are part of the Workspace 9 | containers: 10 | - name: jupyterlab-tensorflow 11 | image: onepanel/jupyterlab:1.0.1 12 | command: ["/bin/bash", "-c", "pip install onepanel-sdk && start.sh jupyter lab --LabApp.token='' --LabApp.allow_remote_access=True --LabApp.allow_origin=\"*\" --LabApp.disable_check_xsrf=True --LabApp.trust_xheaders=True --LabApp.base_url=/ --LabApp.tornado_settings='{\"headers\":{\"Content-Security-Policy\":\"frame-ancestors * \'self\'\"}}' --notebook-dir='/data' --allow-root"] 13 | env: 14 | - name: tornado 15 | value: "'{'headers':{'Content-Security-Policy':\"frame-ancestors\ *\ \'self'\"}}'" 16 | args: 17 | ports: 18 | - containerPort: 8888 19 | name: jupyterlab 20 | - containerPort: 6006 21 | name: tensorboard 22 | volumeMounts: 23 | - name: data 24 | mountPath: /data 25 | ports: 26 | - name: jupyterlab 27 | port: 80 28 | protocol: TCP 29 | targetPort: 8888 30 | - name: tensorboard 31 | port: 6006 32 | protocol: TCP 33 | targetPort: 6006 34 | routes: 35 | - match: 36 | - uri: 37 | prefix: /tensorboard 38 | route: 39 | - destination: 40 | port: 41 | number: 6006 42 | - match: 43 | - uri: 44 | prefix: / #jupyter runs at the default route 45 | route: 46 | - destination: 47 | port: 48 | number: 80 49 | # DAG Workflow to be executed once a Workspace action completes (optional) 50 | #postExecutionWorkflow: 51 | # entrypoint: main 52 | # templates: 53 | # - name: main 54 | # dag: 55 | # tasks: 56 | # - name: slack-notify 57 | # template: slack-notify 58 | # - name: slack-notify 59 | # container: 60 | # image: technosophos/slack-notify 61 | # args: 62 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 63 | # command: 64 | # - sh 65 | # - -c -------------------------------------------------------------------------------- /api/proto/inference_service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | import "google/protobuf/empty.proto"; 8 | 9 | service InferenceService { 10 | rpc CreateInferenceService (CreateInferenceServiceRequest) returns (GetInferenceServiceResponse) { 11 | option (google.api.http) = { 12 | post: "/apis/v1beta1/{namespace}/inferenceservice" 13 | body: "*" 14 | }; 15 | } 16 | 17 | rpc GetInferenceService(InferenceServiceIdentifier) returns (GetInferenceServiceResponse) { 18 | option (google.api.http) = { 19 | get: "/apis/v1beta1/{namespace}/inferenceservice/{name}" 20 | }; 21 | } 22 | 23 | rpc DeleteInferenceService (InferenceServiceIdentifier) returns (google.protobuf.Empty) { 24 | option (google.api.http) = { 25 | delete: "/apis/v1beta1/{namespace}/inferenceservice/{name}" 26 | }; 27 | } 28 | } 29 | 30 | message InferenceServiceIdentifier { 31 | string namespace = 1; 32 | string name = 2; 33 | } 34 | 35 | message Env { 36 | string name = 1; 37 | string value = 2; 38 | } 39 | 40 | message Container { 41 | string image = 1; 42 | string name = 2; 43 | repeated Env env = 3; 44 | } 45 | 46 | message InferenceServiceTransformer { 47 | repeated Container containers = 1; 48 | string minCpu = 2; 49 | string minMemory = 3; 50 | string maxCpu = 4; 51 | string maxMemory = 5; 52 | } 53 | 54 | message InferenceServicePredictor { 55 | string name = 1; 56 | string runtimeVersion = 2; 57 | string storageUri = 3; 58 | string nodeSelector = 4; 59 | string minCpu = 5; 60 | string minMemory = 6; 61 | string maxCpu = 7; 62 | string maxMemory = 8; 63 | } 64 | 65 | message CreateInferenceServiceRequest { 66 | string namespace = 1; 67 | string name = 2; 68 | string defaultTransformerImage = 3; 69 | 70 | InferenceServicePredictor predictor = 4; 71 | InferenceServiceTransformer transformer = 5; 72 | } 73 | 74 | message DeployModelResponse { 75 | string status = 1; 76 | } 77 | 78 | message InferenceServiceCondition { 79 | string lastTransitionTime = 1; 80 | string status = 2; 81 | string type = 3; 82 | } 83 | 84 | message GetInferenceServiceResponse { 85 | bool ready = 1; 86 | repeated InferenceServiceCondition conditions = 2; 87 | string predictUrl = 3; 88 | } 89 | 90 | message InferenceServiceEndpoints { 91 | string predict = 1; 92 | } -------------------------------------------------------------------------------- /pkg/workflow_template_version_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/onepanelio/core/pkg/util/sql" 8 | "github.com/onepanelio/core/pkg/util/types" 9 | ) 10 | 11 | // WorkflowTemplateVersion represents a different version of a WorkflowTemplate 12 | // each version can have a different manifest and labels. 13 | // This is used to version control the template 14 | type WorkflowTemplateVersion struct { 15 | ID uint64 16 | UID string 17 | Version int64 18 | IsLatest bool `db:"is_latest"` 19 | Manifest string 20 | CreatedAt time.Time `db:"created_at"` 21 | WorkflowTemplate *WorkflowTemplate `db:"workflow_template"` 22 | Labels types.JSONLabels 23 | Parameters []Parameter 24 | ParametersBytes []byte `db:"parameters"` // to load from database 25 | Description string `db:"description"` 26 | } 27 | 28 | // WorkflowTemplateVersionsToIDs returns an array of ids from the input WorkflowTemplateVersion with no duplicates. 29 | func WorkflowTemplateVersionsToIDs(resources []*WorkflowTemplateVersion) (ids []uint64) { 30 | mappedIds := make(map[uint64]bool) 31 | 32 | // This is to make sure we don't have duplicates 33 | for _, resource := range resources { 34 | mappedIds[resource.ID] = true 35 | } 36 | 37 | for id := range mappedIds { 38 | ids = append(ids, id) 39 | } 40 | 41 | return 42 | } 43 | 44 | // LoadParametersFromBytes loads Parameters from the WorkflowTemplateVersion's ParameterBytes field. 45 | func (wtv *WorkflowTemplateVersion) LoadParametersFromBytes() ([]Parameter, error) { 46 | loadedParameters := make([]Parameter, 0) 47 | 48 | err := json.Unmarshal(wtv.ParametersBytes, &loadedParameters) 49 | if err != nil { 50 | return wtv.Parameters, err 51 | } 52 | 53 | // It might be nil because the value "null" is stored in db if there are no parameters. 54 | // for consistency, we return an empty array. 55 | if loadedParameters == nil { 56 | loadedParameters = make([]Parameter, 0) 57 | } 58 | 59 | wtv.Parameters = loadedParameters 60 | 61 | return wtv.Parameters, err 62 | } 63 | 64 | // getWorkflowTemplateVersionColumns returns all of the columns for workflow template versions modified by alias, destination. 65 | // see formatColumnSelect 66 | func getWorkflowTemplateVersionColumns(aliasAndDestination ...string) []string { 67 | columns := []string{"id", "created_at", "version", "is_latest", "manifest", "parameters", "labels", "description"} 68 | return sql.FormatColumnSelect(columns, aliasAndDestination...) 69 | } 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ![build](https://img.shields.io/github/workflow/status/onepanelio/onepanel/Publish%20dev%20docker%20image/master?color=01579b) 4 | ![code](https://img.shields.io/codacy/grade/d060fc4d1ac64b85b78f85c691ead86a?color=01579b) 5 | [![release](https://img.shields.io/github/v/release/onepanelio/core?color=01579b)](https://github.com/onepanelio/core/releases) 6 | [![sdk](https://img.shields.io/pypi/v/onepanel-sdk?color=01579b&label=sdk)](https://pypi.org/project/onepanel-sdk/) 7 | [![docs](https://img.shields.io/github/v/release/onepanelio/core?color=01579b&label=docs)](https://docs.onepanel.io) 8 | [![issues](https://img.shields.io/github/issues-raw/onepanelio/core?color=01579b&label=issues)](https://github.com/onepanelio/core/issues) 9 | [![lfai](https://img.shields.io/badge/link-LFAI-01579b)](https://landscape.lfai.foundation/?selected=onepanel) 10 | [![license](https://img.shields.io/github/license/onepanelio/core?color=01579b)](https://opensource.org/licenses/Apache-2.0) 11 | 12 | ## End-to-end computer vision platform 13 | Label, build, train, tune, deploy and automate in a unified platform that runs on any cloud and on-premises. 14 | 15 | https://user-images.githubusercontent.com/1211823/116489376-afc60000-a849-11eb-8e8b-b0c64c07c144.mp4 16 | 17 | ## Why Onepanel? 18 | 19 | 20 | ## Quick start 21 | See [quick start guide](https://docs.onepanel.ai/docs/getting-started/quickstart) to get started. 22 | 23 | ## Community 24 | To submit a feature request, report a bug or documentation issue, please open a GitHub [pull request](https://github.com/onepanelio/core/pulls) or [issue](https://github.com/onepanelio/core/issues). 25 | 26 | ## Contributing 27 | Onepanel is modular and consists of [multiple repositories](https://docs.onepanel.ai/docs/getting-started/contributing/#project-repositories). 28 | 29 | See [contribution guide](https://docs.onepanel.ai/docs/getting-started/contributing) and `CONTRIBUTING.md` in each repository for additional contribution guidelines. 30 | 31 | ## Acknowledgments 32 | Onepanel seamlessly integrates the following open source projects under the hood: 33 | 34 | [Argo](https://github.com/argoproj/argo-workflows) | [Couler](https://github.com/couler-proj/couler) | [CVAT](https://github.com/opencv/cvat) | [JupyterLab](https://github.com/jupyterlab/jupyterlab) | [NNI](https://github.com/microsoft/nni) 35 | 36 | We are grateful for the support these communities provide and do our best to contribute back as much as possible. 37 | 38 | ## License 39 | Onepanel is licensed under [Apache 2.0](https://github.com/onepanelio/core/blob/master/LICENSE). -------------------------------------------------------------------------------- /server/service_server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | api "github.com/onepanelio/core/api/gen" 6 | v1 "github.com/onepanelio/core/pkg" 7 | "github.com/onepanelio/core/server/auth" 8 | ) 9 | 10 | // ServiceServer contains actions for installed services 11 | type ServiceServer struct { 12 | api.UnimplementedServiceServiceServer 13 | } 14 | 15 | // NewServiceServer creates a new ServiceServer 16 | func NewServiceServer() *ServiceServer { 17 | return &ServiceServer{} 18 | } 19 | 20 | func apiService(service *v1.Service) (apiService *api.Service) { 21 | return &api.Service{ 22 | Name: service.Name, 23 | Url: service.URL, 24 | } 25 | } 26 | 27 | // ListServices returns all of the services in the system 28 | func (c *ServiceServer) ListServices(ctx context.Context, req *api.ListServicesRequest) (*api.ListServicesResponse, error) { 29 | client := getClient(ctx) 30 | allowed, err := auth.IsAuthorized(client, "", "list", "", "onepanel-service", "") 31 | if err != nil || !allowed { 32 | return nil, err 33 | } 34 | 35 | services, err := client.ListServices(req.Namespace) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | apiServices := make([]*api.Service, len(services)) 41 | for i, service := range services { 42 | apiServices[i] = apiService(service) 43 | } 44 | 45 | return &api.ListServicesResponse{ 46 | Count: int32(len(services)), 47 | Services: apiServices, 48 | Page: 1, 49 | Pages: 1, 50 | TotalCount: int32(len(services)), 51 | }, nil 52 | } 53 | 54 | // GetService returns a particular service identified by name 55 | func (c *ServiceServer) GetService(ctx context.Context, req *api.GetServiceRequest) (*api.Service, error) { 56 | client := getClient(ctx) 57 | allowed, err := auth.IsAuthorized(client, "", "get", "", "onepanel-service", "") 58 | if err != nil || !allowed { 59 | return nil, err 60 | } 61 | 62 | service, err := client.GetService(req.Namespace, req.Name) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | apiService := apiService(service) 68 | 69 | return apiService, nil 70 | } 71 | 72 | // HasService checks if the cluster has a service set up and enabled 73 | func (c *ServiceServer) HasService(ctx context.Context, req *api.HasServiceRequest) (*api.HasServiceResponse, error) { 74 | client := getClient(ctx) 75 | allowed, err := auth.IsAuthorized(client, "", "get", "", "onepanel-service", "") 76 | if err != nil || !allowed { 77 | return nil, err 78 | } 79 | 80 | hasService, err := client.HasService(req.Name) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | return &api.HasServiceResponse{ 86 | HasService: hasService, 87 | }, nil 88 | } 89 | -------------------------------------------------------------------------------- /db/yaml/workspaces/vnc/20210719190719.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "Deep Learning Desktop" 3 | kind: Workspace 4 | version: 20210719190719 5 | action: update 6 | description: "Deep learning desktop with VNC" 7 | spec: 8 | arguments: 9 | parameters: 10 | # parameter screen-resolution allows users to select screen resolution 11 | - name: screen-resolution 12 | value: 1680x1050 13 | type: select.select 14 | displayName: Screen Resolution 15 | options: 16 | - name: 1280x1024 17 | value: 1280x1024 18 | - name: 1680x1050 19 | value: 1680x1050 20 | - name: 2880x1800 21 | value: 2880x1800 22 | containers: 23 | - name: ubuntu 24 | image: onepanel/vnc:dl-vnc 25 | env: 26 | - name: VNC_PASSWORDLESS 27 | value: true 28 | - name: VNC_RESOLUTION 29 | value: '{{workflow.parameters.screen-resolution}}' 30 | ports: 31 | - containerPort: 6901 32 | name: vnc 33 | volumeMounts: 34 | - name: data 35 | mountPath: /data 36 | - name: sys-filesyncer 37 | image: onepanel/filesyncer:v1.0.0 38 | imagePullPolicy: Always 39 | args: 40 | - server 41 | - -host=localhost:8889 42 | - -server-prefix=/sys/filesyncer 43 | volumeMounts: 44 | - name: data 45 | mountPath: /data 46 | - name: sys-namespace-config 47 | mountPath: /etc/onepanel 48 | readOnly: true 49 | ports: 50 | - name: vnc 51 | port: 80 52 | protocol: TCP 53 | targetPort: 6901 54 | - name: fs 55 | port: 8889 56 | protocol: TCP 57 | targetPort: 8889 58 | routes: 59 | - match: 60 | - uri: 61 | prefix: /sys/filesyncer 62 | route: 63 | - destination: 64 | port: 65 | number: 8889 66 | - match: 67 | - uri: 68 | prefix: / 69 | route: 70 | - destination: 71 | port: 72 | number: 80 73 | # DAG Workflow to be executed once a Workspace action completes (optional) 74 | #postExecutionWorkflow: 75 | # entrypoint: main 76 | # templates: 77 | # - name: main 78 | # dag: 79 | # tasks: 80 | # - name: slack-notify 81 | # template: slack-notify 82 | # - name: slack-notify 83 | # container: 84 | # image: technosophos/slack-notify 85 | # args: 86 | # - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify 87 | # command: 88 | # - sh -------------------------------------------------------------------------------- /api/proto/cron_workflow.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package api; 4 | option go_package = "github.com/onepanelio/core/api/gen"; 5 | 6 | import "google/api/annotations.proto"; 7 | import "google/protobuf/empty.proto"; 8 | import "workflow.proto"; 9 | import "label.proto"; 10 | 11 | service CronWorkflowService { 12 | rpc CreateCronWorkflow (CreateCronWorkflowRequest) returns (CronWorkflow) { 13 | option (google.api.http) = { 14 | post: "/apis/v1beta1/{namespace}/cron_workflow" 15 | body: "cronWorkflow" 16 | }; 17 | } 18 | 19 | rpc UpdateCronWorkflow(UpdateCronWorkflowRequest) returns (CronWorkflow) { 20 | option (google.api.http) = { 21 | put: "/apis/v1beta1/{namespace}/cron_workflow/{uid}" 22 | body: "cronWorkflow" 23 | }; 24 | } 25 | 26 | rpc GetCronWorkflow(GetCronWorkflowRequest) returns (CronWorkflow) { 27 | option (google.api.http) = { 28 | get: "/apis/v1beta1/{namespace}/cron_workflow/{uid}" 29 | }; 30 | } 31 | 32 | rpc ListCronWorkflows(ListCronWorkflowRequest) returns (ListCronWorkflowsResponse) { 33 | option (google.api.http) = { 34 | get: "/apis/v1beta1/{namespace}/cron_workflows" 35 | additional_bindings { 36 | get: "/apis/v1beta1/{namespace}/cron_workflows/{workflow_template_name}" 37 | } 38 | }; 39 | } 40 | 41 | rpc DeleteCronWorkflow (DeleteCronWorkflowRequest) returns (google.protobuf.Empty) { 42 | option (google.api.http) = { 43 | delete: "/apis/v1beta1/{namespace}/cron_workflows/{uid}" 44 | }; 45 | } 46 | } 47 | 48 | message CronWorkflow { 49 | string name = 1; 50 | string uid = 2; 51 | string manifest = 3; 52 | 53 | WorkflowExecution workflowExecution = 4; 54 | 55 | repeated KeyValue labels = 5; 56 | string namespace = 6; 57 | } 58 | 59 | message CreateCronWorkflowRequest { 60 | string namespace = 1; 61 | CronWorkflow cronWorkflow = 2; 62 | } 63 | 64 | message GetCronWorkflowRequest { 65 | string namespace = 1; 66 | string uid = 2; 67 | } 68 | 69 | message UpdateCronWorkflowRequest { 70 | string namespace = 1 ; 71 | string uid = 2; 72 | CronWorkflow cronWorkflow = 3; 73 | } 74 | 75 | message DeleteCronWorkflowRequest { 76 | string namespace = 1; 77 | string uid = 2; 78 | } 79 | 80 | message ListCronWorkflowRequest { 81 | string namespace = 1; 82 | string workflow_template_name = 2; 83 | int32 pageSize = 3; 84 | int32 page = 4; 85 | } 86 | 87 | message ListCronWorkflowsResponse { 88 | int32 count = 1; 89 | repeated CronWorkflow cronWorkflows = 2; 90 | int32 page = 3; 91 | int32 pages = 4; 92 | int32 totalCount = 5; 93 | } -------------------------------------------------------------------------------- /db/yaml/workflows/pytorch-mnist-training/20200605090509.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "PyTorch Training" 3 | kind: Workflow 4 | version: 20200605090509 5 | action: create 6 | labels: 7 | "created-by": "system" 8 | framework: pytorch 9 | spec: 10 | entrypoint: main 11 | arguments: 12 | parameters: 13 | - name: source 14 | value: https://github.com/onepanelio/pytorch-examples.git 15 | - name: command 16 | value: "python mnist/main.py --epochs=1" 17 | volumeClaimTemplates: 18 | - metadata: 19 | name: data 20 | spec: 21 | accessModes: [ "ReadWriteOnce" ] 22 | resources: 23 | requests: 24 | storage: 2Gi 25 | - metadata: 26 | name: output 27 | spec: 28 | accessModes: [ "ReadWriteOnce" ] 29 | resources: 30 | requests: 31 | storage: 2Gi 32 | templates: 33 | - name: main 34 | dag: 35 | tasks: 36 | - name: train-model 37 | template: pytorch 38 | # Uncomment section below to send metrics to Slack 39 | # - name: notify-in-slack 40 | # dependencies: [train-model] 41 | # template: slack-notify-success 42 | # arguments: 43 | # parameters: 44 | # - name: status 45 | # value: "{{tasks.train-model.status}}" 46 | # artifacts: 47 | # - name: metrics 48 | # from: "{{tasks.train-model.outputs.artifacts.sys-metrics}}" 49 | - name: pytorch 50 | inputs: 51 | artifacts: 52 | - name: src 53 | path: /mnt/src 54 | git: 55 | repo: "{{workflow.parameters.source}}" 56 | outputs: 57 | artifacts: 58 | - name: model 59 | path: /mnt/output 60 | optional: true 61 | archive: 62 | none: {} 63 | container: 64 | image: pytorch/pytorch:latest 65 | command: [sh,-c] 66 | args: ["{{workflow.parameters.command}}"] 67 | workingDir: /mnt/src 68 | volumeMounts: 69 | - name: data 70 | mountPath: /mnt/data 71 | - name: output 72 | mountPath: /mnt/output 73 | - name: slack-notify-success 74 | container: 75 | image: technosophos/slack-notify 76 | command: [sh,-c] 77 | args: ['SLACK_USERNAME=Worker SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify'] 78 | inputs: 79 | parameters: 80 | - name: status 81 | artifacts: 82 | - name: metrics 83 | path: /tmp/metrics.json 84 | optional: true -------------------------------------------------------------------------------- /server/config_server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/golang/protobuf/ptypes/empty" 7 | api "github.com/onepanelio/core/api/gen" 8 | v1 "github.com/onepanelio/core/pkg" 9 | "github.com/onepanelio/core/server/auth" 10 | ) 11 | 12 | // ConfigServer contains actions for system configuration related items 13 | type ConfigServer struct { 14 | api.UnimplementedConfigServiceServer 15 | } 16 | 17 | // NewConfigServer creates a new ConfigServer 18 | func NewConfigServer() *ConfigServer { 19 | return &ConfigServer{} 20 | } 21 | 22 | func getArtifactRepositoryBucket(client *v1.Client, namespace string) (string, error) { 23 | if namespace == "" { 24 | return "", nil 25 | } 26 | 27 | namespaceConfig, err := client.GetNamespaceConfig(namespace) 28 | if err != nil { 29 | return "", err 30 | } 31 | 32 | if namespaceConfig.ArtifactRepository.S3 != nil { 33 | return namespaceConfig.ArtifactRepository.S3.Bucket, nil 34 | } 35 | 36 | return "", fmt.Errorf("unknown artifact repository") 37 | } 38 | 39 | // GetConfig returns the system configuration options 40 | func (c *ConfigServer) GetConfig(ctx context.Context, req *empty.Empty) (*api.GetConfigResponse, error) { 41 | client := getClient(ctx) 42 | allowed, err := auth.IsAuthorized(client, "", "list", "", "namespaces", "") 43 | if err != nil || !allowed { 44 | return nil, err 45 | } 46 | 47 | sysConfig, err := client.GetSystemConfig() 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | nodePool := &api.NodePool{ 53 | Label: *sysConfig.GetValue("applicationNodePoolLabel"), 54 | Options: make([]*api.NodePoolOption, 0), 55 | } 56 | 57 | nodePoolOptions, err := sysConfig.NodePoolOptions() 58 | if err != nil { 59 | return nil, err 60 | } 61 | type ConfigServer struct{} 62 | for _, option := range nodePoolOptions { 63 | nodePool.Options = append(nodePool.Options, &api.NodePoolOption{ 64 | Name: option.Name, 65 | Value: option.Value, 66 | }) 67 | } 68 | 69 | return &api.GetConfigResponse{ 70 | ApiUrl: sysConfig["ONEPANEL_API_URL"], 71 | Domain: sysConfig["ONEPANEL_DOMAIN"], 72 | Fqdn: sysConfig["ONEPANEL_FQDN"], 73 | NodePool: nodePool, 74 | }, err 75 | } 76 | 77 | // GetNamespaceConfig returns the namespace configuration 78 | func (c *ConfigServer) GetNamespaceConfig(ctx context.Context, req *api.GetNamespaceConfigRequest) (*api.GetNamespaceConfigResponse, error) { 79 | client := getClient(ctx) 80 | allowed, err := auth.IsAuthorized(client, "", "get", "", "namespaces", "") 81 | if err != nil || !allowed { 82 | return nil, err 83 | } 84 | 85 | bucket, err := getArtifactRepositoryBucket(client, req.Namespace) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return &api.GetNamespaceConfigResponse{ 91 | Bucket: bucket, 92 | }, err 93 | } 94 | -------------------------------------------------------------------------------- /server/namespace_server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "math" 6 | "strings" 7 | 8 | api "github.com/onepanelio/core/api/gen" 9 | v1 "github.com/onepanelio/core/pkg" 10 | "github.com/onepanelio/core/server/auth" 11 | ) 12 | 13 | // NamespaceServer is an implementation of the grpc NamespaceServer 14 | type NamespaceServer struct { 15 | api.UnimplementedNamespaceServiceServer 16 | } 17 | 18 | // NewNamespaceServer creates a new NamespaceServer 19 | func NewNamespaceServer() *NamespaceServer { 20 | return &NamespaceServer{} 21 | } 22 | 23 | func apiNamespace(ns *v1.Namespace) (namespace *api.Namespace) { 24 | namespace = &api.Namespace{ 25 | Name: ns.Name, 26 | } 27 | 28 | return 29 | } 30 | 31 | // ListNamespaces returns a list of all namespaces available in the system 32 | func (s *NamespaceServer) ListNamespaces(ctx context.Context, req *api.ListNamespacesRequest) (*api.ListNamespacesResponse, error) { 33 | client := getClient(ctx) 34 | allowed, err := auth.IsAuthorized(client, "", "list", "", "namespaces", "") 35 | if err != nil || !allowed { 36 | return nil, err 37 | } 38 | 39 | if req.PageSize <= 0 { 40 | req.PageSize = 15 41 | } 42 | 43 | namespaces, err := client.ListNamespaces() 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | var apiNamespaces []*api.Namespace 49 | for _, ns := range namespaces { 50 | if req.Query == "" || (req.Query != "" && strings.Contains(ns.Name, req.Query)) { 51 | apiNamespaces = append(apiNamespaces, apiNamespace(ns)) 52 | } 53 | } 54 | 55 | pages := int32(math.Ceil(float64(len(apiNamespaces)) / float64(req.PageSize))) 56 | if req.Page > pages { 57 | req.Page = pages 58 | } 59 | 60 | if req.Page <= 0 { 61 | req.Page = 1 62 | } 63 | 64 | start := (req.Page - 1) * req.PageSize 65 | end := start + req.PageSize 66 | if end >= int32(len(apiNamespaces)) { 67 | end = int32(len(apiNamespaces)) 68 | } 69 | 70 | return &api.ListNamespacesResponse{ 71 | Count: end - start, 72 | Namespaces: apiNamespaces[start:end], 73 | Page: req.Page, 74 | Pages: pages, 75 | TotalCount: int32(len(apiNamespaces)), 76 | }, nil 77 | } 78 | 79 | // CreateNamespace creates a new namespace in the system 80 | func (s *NamespaceServer) CreateNamespace(ctx context.Context, createNamespace *api.CreateNamespaceRequest) (*api.Namespace, error) { 81 | client := getClient(ctx) 82 | allowed, err := auth.IsAuthorized(client, "", "create", "", "namespaces", "") 83 | if err != nil || !allowed { 84 | return nil, err 85 | } 86 | 87 | namespace, err := client.CreateNamespace(createNamespace.Namespace.SourceName, createNamespace.Namespace.Name) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | return &api.Namespace{ 93 | Name: namespace.Name, 94 | SourceName: createNamespace.Namespace.SourceName, 95 | }, nil 96 | } 97 | -------------------------------------------------------------------------------- /pkg/util/label/label.go: -------------------------------------------------------------------------------- 1 | package label 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | const ( 8 | OnepanelPrefix = "onepanel.io/" 9 | TagPrefix = "tags.onepanel.io/" 10 | WorkflowTemplate = OnepanelPrefix + "workflow-template" 11 | WorkflowTemplateUid = OnepanelPrefix + "workflow-template-uid" 12 | WorkflowTemplateVersionUid = OnepanelPrefix + "workflow-template-version-uid" 13 | WorkspaceTemplateVersionUid = OnepanelPrefix + "workspace-template-version-uid" 14 | WorkflowUid = OnepanelPrefix + "workflow-uid" 15 | CronWorkflowUid = OnepanelPrefix + "cron-workflow-uid" 16 | Version = OnepanelPrefix + "version" 17 | VersionLatest = OnepanelPrefix + "version-latest" 18 | ) 19 | 20 | // Label represents a Key/Value pair label 21 | type Label struct { 22 | Key string 23 | Value string 24 | } 25 | 26 | // Function that modifies an input string 27 | type StringModifier func(string) string 28 | 29 | // Returns a map where only the keys that have the input prefix are kept. 30 | func FilterByPrefix(prefix string, inputLabels map[string]string) (labels map[string]string) { 31 | labels = make(map[string]string) 32 | 33 | for key, value := range inputLabels { 34 | if strings.HasPrefix(key, prefix) { 35 | labels[key] = value 36 | } 37 | } 38 | 39 | return 40 | } 41 | 42 | func RemovePrefix(prefix string, inputLabels map[string]string) (labels map[string]string) { 43 | labels = make(map[string]string) 44 | 45 | prefixLen := len(prefix) 46 | for key, value := range inputLabels { 47 | formattedKey := key[prefixLen:] 48 | labels[formattedKey] = value 49 | } 50 | 51 | return 52 | } 53 | 54 | // Delete all of the keys in the inputLabels 55 | func Delete(inputLabels map[string]string, keys ...string) { 56 | for _, key := range keys { 57 | delete(inputLabels, key) 58 | } 59 | 60 | return 61 | } 62 | 63 | // Delete all keys that have the passed in prefix. 64 | func DeleteWithPrefix(inputLabels map[string]string, prefix string) { 65 | for key := range inputLabels { 66 | if strings.HasPrefix(key, prefix) { 67 | delete(inputLabels, key) 68 | } 69 | } 70 | } 71 | 72 | // Adds all of the key/values in additions to destination. 73 | // Each key is formatted according to the modifier function. 74 | func MergeLabels(destination, additions map[string]string, modifier StringModifier) { 75 | for key, value := range additions { 76 | formattedKey := modifier(key) 77 | destination[formattedKey] = value 78 | } 79 | } 80 | 81 | // Adds all of the keys/values in additions to destination 82 | // Each key in addition will be modified by having prefix prepended to it. 83 | func MergeLabelsPrefix(destination, additions map[string]string, prefix string) { 84 | MergeLabels(destination, additions, func(s string) string { 85 | return prefix + s 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /pkg/workflow_execution_test.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | // TestClient_CreateWorkflowExecution tests creating a workflow execution 9 | func TestClient_CreateWorkflowExecution(t *testing.T) { 10 | c := DefaultTestClient() 11 | clearDatabase(t) 12 | 13 | namespace := "onepanel" 14 | 15 | wt := &WorkflowTemplate{ 16 | Name: "test", 17 | Manifest: defaultWorkflowTemplate, 18 | } 19 | wt, _ = c.CreateWorkflowTemplate(namespace, wt) 20 | 21 | we := &WorkflowExecution{ 22 | Name: "test", 23 | } 24 | 25 | we, err := c.CreateWorkflowExecution(namespace, we, wt) 26 | assert.Nil(t, err) 27 | } 28 | 29 | // TestClient_GetWorkflowExecution tests getting a workflow execution that exists 30 | func TestClient_GetWorkflowExecution(t *testing.T) { 31 | c := DefaultTestClient() 32 | clearDatabase(t) 33 | 34 | namespace := "onepanel" 35 | 36 | wt := &WorkflowTemplate{ 37 | Name: "test", 38 | Manifest: defaultWorkflowTemplate, 39 | } 40 | wt, _ = c.CreateWorkflowTemplate(namespace, wt) 41 | 42 | we := &WorkflowExecution{ 43 | Name: "test", 44 | } 45 | 46 | we, _ = c.CreateWorkflowExecution(namespace, we, wt) 47 | 48 | getWe, err := c.GetWorkflowExecution(namespace, we.UID) 49 | assert.Nil(t, err) 50 | 51 | assert.Equal(t, we.Name, getWe.Name) 52 | assert.Equal(t, we.UID, getWe.UID) 53 | } 54 | 55 | // TestClient_GetWorkflowExecution tests getting a workflow execution that doesn't exist 56 | func TestClient_GetWorkflowExecution_NotExists(t *testing.T) { 57 | c := DefaultTestClient() 58 | clearDatabase(t) 59 | 60 | namespace := "onepanel" 61 | 62 | getWe, err := c.GetWorkflowExecution(namespace, "not-exist") 63 | assert.Nil(t, getWe) 64 | assert.Nil(t, err) 65 | } 66 | 67 | // TestClient_ArchiveWorkflowExecution_NotExist makes sure there is no error if the workflow 68 | // execution does not exist 69 | func TestClient_ArchiveWorkflowExecution_NotExist(t *testing.T) { 70 | c := DefaultTestClient() 71 | clearDatabase(t) 72 | 73 | err := c.ArchiveWorkflowExecution("onepanel-no-exist", "test-no-exist") 74 | assert.Nil(t, err) 75 | } 76 | 77 | // TestClient_ArchiveWorkflowExecution_Exist makes sure we archive an existing workflow execution correctly 78 | func TestClient_ArchiveWorkflowExecution_Exist(t *testing.T) { 79 | c := DefaultTestClient() 80 | clearDatabase(t) 81 | 82 | namespace := "onepanel" 83 | weName := "test" 84 | 85 | wt := &WorkflowTemplate{ 86 | Name: "test", 87 | Manifest: defaultWorkflowTemplate, 88 | } 89 | wt, _ = c.CreateWorkflowTemplate(namespace, wt) 90 | 91 | we := &WorkflowExecution{ 92 | Name: weName, 93 | } 94 | 95 | we, err := c.CreateWorkflowExecution(namespace, we, wt) 96 | 97 | err = c.ArchiveWorkflowExecution(namespace, weName) 98 | assert.Nil(t, err) 99 | } 100 | -------------------------------------------------------------------------------- /db/yaml/workflows/tensorflow-mnist-training/20200605090535.yaml: -------------------------------------------------------------------------------- 1 | metadata: 2 | name: "TensorFlow Training" 3 | kind: Workflow 4 | version: 20200605090535 5 | action: create 6 | source: "https://github.com/onepanelio/templates/blob/master/workflows/tensorflow-mnist-training/template.yaml" 7 | labels: 8 | "created-by": "system" 9 | framework: tensorflow 10 | spec: 11 | entrypoint: main 12 | arguments: 13 | parameters: 14 | - name: source 15 | value: https://github.com/onepanelio/tensorflow-examples.git 16 | - name: command 17 | value: "python mnist/main.py --epochs=5" 18 | volumeClaimTemplates: 19 | - metadata: 20 | name: data 21 | spec: 22 | accessModes: [ "ReadWriteOnce" ] 23 | resources: 24 | requests: 25 | storage: 2Gi 26 | - metadata: 27 | name: output 28 | spec: 29 | accessModes: [ "ReadWriteOnce" ] 30 | resources: 31 | requests: 32 | storage: 2Gi 33 | templates: 34 | - name: main 35 | dag: 36 | tasks: 37 | - name: train-model 38 | template: pytorch 39 | # Uncomment section below to send metrics to Slack 40 | # - name: notify-in-slack 41 | # dependencies: [train-model] 42 | # template: slack-notify-success 43 | # arguments: 44 | # parameters: 45 | # - name: status 46 | # value: "{{tasks.train-model.status}}" 47 | # artifacts: 48 | # - name: metrics 49 | # from: "{{tasks.train-model.outputs.artifacts.sys-metrics}}" 50 | - name: pytorch 51 | inputs: 52 | artifacts: 53 | - name: src 54 | path: /mnt/src 55 | git: 56 | repo: "{{workflow.parameters.source}}" 57 | outputs: 58 | artifacts: 59 | - name: model 60 | path: /mnt/output 61 | optional: true 62 | archive: 63 | none: {} 64 | container: 65 | image: tensorflow/tensorflow:latest 66 | command: [sh,-c] 67 | args: ["{{workflow.parameters.command}}"] 68 | workingDir: /mnt/src 69 | volumeMounts: 70 | - name: data 71 | mountPath: /mnt/data 72 | - name: output 73 | mountPath: /mnt/output 74 | - name: slack-notify-success 75 | container: 76 | image: technosophos/slack-notify 77 | command: [sh,-c] 78 | args: ['SLACK_USERNAME=Worker SLACK_TITLE="{{workflow.name}} {{inputs.parameters.status}}" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE=$(cat /tmp/metrics.json)} ./slack-notify'] 79 | inputs: 80 | parameters: 81 | - name: status 82 | artifacts: 83 | - name: metrics 84 | path: /tmp/metrics.json 85 | optional: true -------------------------------------------------------------------------------- /pkg/inference_service.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | "github.com/onepanelio/core/pkg/util" 6 | "google.golang.org/grpc/codes" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "k8s.io/apimachinery/pkg/util/json" 9 | "k8s.io/client-go/kubernetes/scheme" 10 | "k8s.io/client-go/rest" 11 | "strings" 12 | ) 13 | 14 | func modelRestClient() (*rest.RESTClient, error) { 15 | config := *NewConfig() 16 | config.GroupVersion = &schema.GroupVersion{Group: "serving.kubeflow.org", Version: "v1beta1"} 17 | config.APIPath = "/apis" 18 | config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() 19 | 20 | return rest.RESTClientFor(&config) 21 | } 22 | 23 | // CreateInferenceService creates an InferenceService with KFServing 24 | func (c *Client) CreateInferenceService(deployment *InferenceService) error { 25 | resource := deployment.ToResource() 26 | 27 | data, err := json.Marshal(resource) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | restClient, err := modelRestClient() 33 | if err != nil { 34 | return err 35 | } 36 | 37 | err = restClient.Post(). 38 | Namespace(deployment.Namespace). 39 | Name(deployment.Name). 40 | Resource(inferenceServiceResource). 41 | Body(data). 42 | Do(). 43 | Error() 44 | 45 | if err != nil && strings.Contains(err.Error(), "already exists") { 46 | return util.NewUserError(codes.AlreadyExists, fmt.Sprintf("InferenceService with name '%v' already exists", deployment.Name)) 47 | } 48 | 49 | return err 50 | } 51 | 52 | // GetModelStatus returns the model's status 53 | func (c *Client) GetModelStatus(namespace, name string) (*InferenceServiceStatus, error) { 54 | restClient, err := modelRestClient() 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | result := &k8sModel{} 60 | 61 | err = restClient.Get(). 62 | Namespace(namespace). 63 | Name(name). 64 | Resource(inferenceServiceResource). 65 | Do(). 66 | Into(result) 67 | 68 | if err != nil && strings.Contains(err.Error(), "not found") { 69 | return nil, util.NewUserError(codes.NotFound, "not found") 70 | } 71 | 72 | predictURL := result.Status.URL 73 | suffixIndex := strings.LastIndex(result.Status.Address.URL, "cluster.local") 74 | if suffixIndex >= 0 { 75 | predictURL += result.Status.Address.URL[suffixIndex+13:] 76 | } 77 | 78 | status := &InferenceServiceStatus{ 79 | Conditions: result.Status.Conditions, 80 | Ready: result.Status.Ready(), 81 | PredictURL: predictURL, 82 | } 83 | 84 | return status, err 85 | } 86 | 87 | // DeleteModel deletes the model 88 | func (c *Client) DeleteModel(namespace, name string) error { 89 | restClient, err := modelRestClient() 90 | if err != nil { 91 | return err 92 | } 93 | 94 | return restClient.Delete(). 95 | Namespace(namespace). 96 | Name(name). 97 | Resource(inferenceServiceResource). 98 | Do(). 99 | Error() 100 | } 101 | -------------------------------------------------------------------------------- /pkg/label_types.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | // Label represents a database-backed label row. 11 | type Label struct { 12 | ID uint64 13 | CreatedAt time.Time `db:"created_at"` 14 | Key string 15 | Value string 16 | Resource string 17 | ResourceID uint64 `db:"resource_id"` 18 | } 19 | 20 | // LabelsToMapping converts Label structs to a map of key:value 21 | func LabelsToMapping(labels ...*Label) map[string]string { 22 | result := make(map[string]string) 23 | 24 | for _, label := range labels { 25 | result[label.Key] = label.Value 26 | } 27 | 28 | return result 29 | } 30 | 31 | // LabelsFromString parses a string into labels 32 | // Format: key=,value=&key2=,value2= 33 | func LabelsFromString(value string) (labels []*Label, err error) { 34 | labels = make([]*Label, 0) 35 | 36 | if value == "" { 37 | return 38 | } 39 | 40 | labelParts := strings.Split(value, "&") 41 | if len(labelParts) == 0 { 42 | return 43 | } 44 | 45 | for _, part := range labelParts { 46 | if part == "" { 47 | continue 48 | } 49 | 50 | newLabel, err := LabelFromString(part) 51 | if err != nil { 52 | return labels, err 53 | } 54 | if newLabel == nil { 55 | continue 56 | } 57 | 58 | labels = append(labels, newLabel) 59 | } 60 | 61 | return 62 | } 63 | 64 | // LabelFromString converts a parses into a label 65 | // Format: key=,value= 66 | func LabelFromString(value string) (label *Label, err error) { 67 | parts := strings.Split(value, ",") 68 | if len(parts) != 2 { 69 | return nil, fmt.Errorf("label does not have two parts, key/value") 70 | } 71 | 72 | label = &Label{} 73 | 74 | first := parts[0] 75 | firstItems := strings.Split(first, "=") 76 | if len(firstItems) != 2 { 77 | return nil, fmt.Errorf(`incorrectly formatted label "%v"`, first) 78 | } 79 | 80 | if firstItems[0] == "key" { 81 | label.Key = firstItems[1] 82 | } else if firstItems[0] == "value" { 83 | label.Value = firstItems[1] 84 | } 85 | 86 | second := parts[1] 87 | secondItems := strings.Split(second, "=") 88 | if len(secondItems) != 2 { 89 | return nil, fmt.Errorf(`incorrectly formatted label "%v"`, second) 90 | } 91 | 92 | if secondItems[0] == "key" { 93 | label.Key = secondItems[1] 94 | } else if secondItems[0] == "value" { 95 | label.Value = secondItems[1] 96 | } 97 | 98 | return label, nil 99 | } 100 | 101 | // LabelsToJSONString converts an array of labels to a json string representing an object 102 | // where the keys are the label keys and the values are the label values 103 | func LabelsToJSONString(labels []*Label) (string, error) { 104 | labelMap := LabelsToMapping(labels...) 105 | 106 | resultBytes, err := json.Marshal(labelMap) 107 | if err != nil { 108 | return "", err 109 | } 110 | 111 | return string(resultBytes), nil 112 | } 113 | -------------------------------------------------------------------------------- /api/third_party/google/api/httpbody.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | 16 | syntax = "proto3"; 17 | 18 | package google.api; 19 | 20 | import "google/protobuf/any.proto"; 21 | 22 | option cc_enable_arenas = true; 23 | option go_package = "google.golang.org/genproto/googleapis/api/httpbody;httpbody"; 24 | option java_multiple_files = true; 25 | option java_outer_classname = "HttpBodyProto"; 26 | option java_package = "com.google.api"; 27 | option objc_class_prefix = "GAPI"; 28 | 29 | // Message that represents an arbitrary HTTP body. It should only be used for 30 | // payload formats that can't be represented as JSON, such as raw binary or 31 | // an HTML page. 32 | // 33 | // 34 | // This message can be used both in streaming and non-streaming API methods in 35 | // the request as well as the response. 36 | // 37 | // It can be used as a top-level request field, which is convenient if one 38 | // wants to extract parameters from either the URL or HTTP template into the 39 | // request fields and also want access to the raw HTTP body. 40 | // 41 | // Example: 42 | // 43 | // message GetResourceRequest { 44 | // // A unique request id. 45 | // string request_id = 1; 46 | // 47 | // // The raw HTTP body is bound to this field. 48 | // google.api.HttpBody http_body = 2; 49 | // } 50 | // 51 | // service ResourceService { 52 | // rpc GetResource(GetResourceRequest) returns (google.api.HttpBody); 53 | // rpc UpdateResource(google.api.HttpBody) returns 54 | // (google.protobuf.Empty); 55 | // } 56 | // 57 | // Example with streaming methods: 58 | // 59 | // service CaldavService { 60 | // rpc GetCalendar(stream google.api.HttpBody) 61 | // returns (stream google.api.HttpBody); 62 | // rpc UpdateCalendar(stream google.api.HttpBody) 63 | // returns (stream google.api.HttpBody); 64 | // } 65 | // 66 | // Use of this type only changes how the request and response bodies are 67 | // handled, all other features will continue to work unchanged. 68 | message HttpBody { 69 | // The HTTP Content-Type header value specifying the content type of the body. 70 | string content_type = 1; 71 | 72 | // The HTTP request/response body as raw binary. 73 | bytes data = 2; 74 | 75 | // Application specific response metadata. Must be set in the first response 76 | // for streaming APIs. 77 | repeated google.protobuf.Any extensions = 3; 78 | } --------------------------------------------------------------------------------