├── .github ├── FUNDING.yml └── workflows │ └── stale.yml ├── src ├── integrations │ ├── nats_manager_proxy.py │ ├── nats_cron_job.py │ └── nats_cron_jobs.py ├── k8s │ └── k8s_watcher_proxy.py ├── constants │ ├── velero.py │ ├── response.py │ ├── watchdog.py │ ├── k8s.py │ └── resources.py ├── schemas │ ├── request │ │ ├── apprise_test_service.py │ │ ├── create_user_service.py │ │ ├── default_bsl.py │ │ ├── storage_class_map.py │ │ ├── create_backup_from_schedule.py │ │ ├── delete_resource.py │ │ ├── pause_schedule.py │ │ ├── unlock_restic_repo.py │ │ ├── create_cloud_credentials.py │ │ ├── update_backup_expiration.py │ │ ├── create_vsl.py │ │ ├── update_vsl.py │ │ ├── update_user_config.py │ │ ├── create_restore.py │ │ ├── create_bsl.py │ │ ├── update_bsl.py │ │ ├── create_backup.py │ │ ├── create_schedule.py │ │ └── update_schedule.py │ ├── velero_log.py │ ├── velero_describe.py │ ├── velero_storage_class.py │ ├── velero_storage_location_response.py │ └── response │ │ ├── successful_backups.py │ │ ├── successful_restores.py │ │ ├── successful_schedules.py │ │ └── successful_bsl.py ├── api │ ├── common │ │ ├── app_health.py │ │ └── routers │ │ │ └── health.py │ └── v1 │ │ ├── api_v1.py │ │ └── routers │ │ ├── stats.py │ │ ├── inspect.py │ │ ├── pvb.py │ │ ├── vsl.py │ │ └── sc_mapping.py ├── __main__.py ├── controllers │ ├── agent.py │ ├── common.py │ ├── stats.py │ ├── pvb.py │ ├── inspect.py │ ├── sc_mapping.py │ ├── restore.py │ ├── vsl.py │ ├── setup.py │ ├── requests.py │ ├── bsl.py │ ├── schedule.py │ ├── repo.py │ ├── watchdog.py │ ├── k8s.py │ └── backup.py ├── main.py ├── models │ └── k8s │ │ ├── vsl.py │ │ ├── repo.py │ │ ├── bsl.py │ │ ├── restore.py │ │ ├── schedule.py │ │ └── backup.py ├── service │ ├── resource.py │ ├── utils │ │ └── cleanup_requests.py │ ├── describe.py │ ├── requests.py │ ├── logs.py │ ├── backup_storage_class.py │ ├── inspect_download_backup.py │ ├── location_credentials.py │ ├── pvb.py │ ├── inspect.py │ ├── restore.py │ ├── repo.py │ ├── vsl.py │ ├── velero.py │ ├── sc_mapping.py │ └── k8s_secret.py ├── startup_watchers.py ├── .env.template └── utils │ └── process.py ├── .gitignore ├── docker ├── .env.template ├── docker-compose.yaml ├── Dockerfile ├── Dockerfile-old ├── jenkins-amd64 │ ├── Jenkins-dev │ └── Jenkins ├── Jenkins-dev ├── Jenkins-dev-arm └── Jenkins ├── README.md ├── requirements.txt └── tests └── test_connection.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: davideserio 2 | -------------------------------------------------------------------------------- /src/integrations/nats_manager_proxy.py: -------------------------------------------------------------------------------- 1 | nat_manager = None 2 | -------------------------------------------------------------------------------- /src/k8s/k8s_watcher_proxy.py: -------------------------------------------------------------------------------- 1 | k8s_watcher_manager = None 2 | -------------------------------------------------------------------------------- /src/constants/velero.py: -------------------------------------------------------------------------------- 1 | VELERO = { 2 | "GROUP": "velero.io", 3 | "VERSION": "v1", 4 | } 5 | -------------------------------------------------------------------------------- /src/schemas/request/apprise_test_service.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class AppriseTestServiceRequestSchema(BaseModel): 5 | config: str 6 | -------------------------------------------------------------------------------- /src/schemas/request/create_user_service.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class CreateUserServiceRequestSchema(BaseModel): 5 | config: str 6 | -------------------------------------------------------------------------------- /src/schemas/request/default_bsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class DefaultBslRequestSchema(BaseModel): 5 | name: str 6 | default: bool = True 7 | -------------------------------------------------------------------------------- /src/schemas/request/storage_class_map.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class StorageClassMapRequestSchema(BaseModel): 5 | storageClassMapping: dict 6 | -------------------------------------------------------------------------------- /src/schemas/request/create_backup_from_schedule.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class CreateBackupFromScheduleRequestSchema(BaseModel): 5 | scheduleName: str 6 | -------------------------------------------------------------------------------- /src/schemas/velero_log.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | from pydantic import BaseModel 3 | 4 | 5 | class VeleroLog(BaseModel): 6 | logs: Optional[List[str]] = None 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | __pycache__/ 3 | *.log 4 | .env 5 | *.sh 6 | src/tmp/ 7 | tmp/ 8 | download/ 9 | **/dl/**/*.tar.gz 10 | /db/ 11 | yarn.lock 12 | package.json 13 | node_modules/ -------------------------------------------------------------------------------- /src/api/common/app_health.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | from api.common.routers import health 3 | 4 | appAgentHealth = APIRouter() 5 | appAgentHealth.include_router(health.router) 6 | -------------------------------------------------------------------------------- /src/schemas/velero_describe.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional 2 | from pydantic import BaseModel 3 | 4 | 5 | class VeleroDescribe(BaseModel): 6 | details: Optional[Dict] = None 7 | -------------------------------------------------------------------------------- /src/schemas/request/delete_resource.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | 3 | class DeleteResourceRequestSchema(BaseModel): 4 | name: str = Field(..., description="The name of the resource.") 5 | -------------------------------------------------------------------------------- /src/schemas/request/pause_schedule.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | 3 | class PauseScheduleRequestSchema(BaseModel): 4 | name: str = Field(..., description="The name of the resource.") 5 | -------------------------------------------------------------------------------- /src/schemas/request/unlock_restic_repo.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class UnlockResticRepoRequestSchema(BaseModel): 5 | bsl: str 6 | repositoryUrl: str 7 | removeAll: bool 8 | -------------------------------------------------------------------------------- /src/schemas/velero_storage_class.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict, Optional 2 | from pydantic import BaseModel 3 | 4 | 5 | class VeleroStorageClass(BaseModel): 6 | storage_classes: Optional[List[Dict]] = None 7 | -------------------------------------------------------------------------------- /docker/.env.template: -------------------------------------------------------------------------------- 1 | KUBE_CONFIG_FILE= 2 | CONTAINER_MODE= 3 | K8S_IN_CLUSTER_MODE= 4 | K8S_VELERO_NAMESPACE= 5 | ORIGINS=
6 | SECURITY_TOKEN_KEY= -------------------------------------------------------------------------------- /src/schemas/request/create_cloud_credentials.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class CreateCloudCredentialsRequestSchema(BaseModel): 5 | newSecretName: str 6 | newSecretKey: str 7 | awsAccessKeyId: str 8 | awsSecretAccessKey: str 9 | -------------------------------------------------------------------------------- /src/schemas/velero_storage_location_response.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict 2 | from pydantic import BaseModel 3 | 4 | 5 | class VeleroStorageLocation(BaseModel): 6 | success: bool 7 | storage_info: Optional[Dict] = None 8 | error: Optional[str] = None 9 | -------------------------------------------------------------------------------- /src/schemas/response/successful_backups.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from models.k8s.backup import BackupResponseSchema 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | 6 | class SuccessfulBackupResponse(SuccessfulRequest[List[BackupResponseSchema]]): 7 | pass 8 | -------------------------------------------------------------------------------- /src/schemas/response/successful_restores.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from models.k8s.restore import RestoreResponseSchema 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | 6 | class SuccessfulRestoreResponse(SuccessfulRequest[List[RestoreResponseSchema]]): 7 | pass 8 | -------------------------------------------------------------------------------- /src/schemas/response/successful_schedules.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from models.k8s.schedule import ScheduleResponseSchema 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | 6 | class SuccessfulScheduleResponse(SuccessfulRequest[List[ScheduleResponseSchema]]): 7 | pass 8 | -------------------------------------------------------------------------------- /src/schemas/response/successful_bsl.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from models.k8s.bsl import BackupStorageLocationResponseSchema 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | 6 | 7 | class SuccessfulBslResponse(SuccessfulRequest[List[BackupStorageLocationResponseSchema]]): 8 | pass 9 | -------------------------------------------------------------------------------- /src/schemas/request/update_backup_expiration.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | 3 | class UpdateBackupExpirationRequestSchema(BaseModel): 4 | backupName: str 5 | expiration: str = Field( 6 | ..., 7 | pattern=r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$', 8 | description="Expiration date must follow the format YYYY-MM-DDTHH:MM:SSZ" 9 | ) 10 | -------------------------------------------------------------------------------- /src/__main__.py: -------------------------------------------------------------------------------- 1 | from vui_common.main import run_api 2 | from vui_common.ws import ws_manager_proxy 3 | from vui_common import app_data 4 | 5 | from ws.ws_manager import WebSocketManager 6 | 7 | app_data.__app_name__ = "VUI-API" 8 | app_data.__app_summary__ = "VUI-API is part of VUI-Project" 9 | 10 | ws_manager_proxy.ws_manager = WebSocketManager() 11 | 12 | if __name__ == "__main__": 13 | run_api(app_module="main:app") 14 | -------------------------------------------------------------------------------- /src/controllers/agent.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | from service.watchdog import check_watchdog_online_service 6 | 7 | 8 | async def watchdog_online_handler(): 9 | payload = await check_watchdog_online_service() 10 | 11 | response = SuccessfulRequest(payload=payload) 12 | return JSONResponse(content=response.model_dump(), status_code=200) 13 | -------------------------------------------------------------------------------- /src/constants/response.py: -------------------------------------------------------------------------------- 1 | common_error_authenticated_response = { 2 | 400: { 3 | "description": "Bad Request - Invalid parameters", 4 | "content": {"application/json": { 5 | "example": {"detail": {"title": "title error....", "description": "title description...."}}}} 6 | }, 7 | 401: { 8 | "description": "Unauthorized - Authentication required", 9 | "content": {"application/json": {"example": {"detail": "Not authenticated"}}} 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from contextlib import asynccontextmanager 3 | 4 | from vui_common.app import create_base_app 5 | 6 | from api.common.app_health import appAgentHealth 7 | from api.v1.api_v1 import v1 8 | 9 | from startup_watchers import init_watchers 10 | 11 | 12 | @asynccontextmanager 13 | async def lifespan(app: FastAPI): 14 | init_watchers(app) 15 | yield 16 | 17 | 18 | app = create_base_app(component='agent', lifespan=lifespan) 19 | 20 | app.include_router(appAgentHealth, prefix="/health") 21 | app.include_router(v1, prefix="/v1") 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Velero-API 2 | 3 | Velero-API is part of the VUI project. 4 | 5 | [Read VUI project documentation](https://vui.seriohub.com/) 6 | 7 | ## Acknowledgements 8 | 9 | A huge thank you to everyone who: 10 | 11 | - **Contributes** to the project with code, documentation, or feedback. Your contributions help make this project better. 12 | - **Stars** the project. Your support by starring helps others discover it and is greatly appreciated. 13 | 14 | ## Special Thanks to Our Sponsors 15 | 16 | [![Mercedes-Benz](https://avatars.githubusercontent.com/mercedes-benz?s=50)](https://github.com/mercedes-benz) 17 | -------------------------------------------------------------------------------- /src/schemas/request/create_vsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Optional, Dict 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class CreateVslRequestSchema(BaseModel): 7 | name: str = Field(..., description="The name of the VSL.") 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | provider: str = Field(..., description="The name of the provider.") 10 | 11 | config: Optional[Dict[str, str]] = Field(None, description="Configuration fields.") 12 | 13 | credentialName: Optional[str] = Field(None, description="The name of the existing secret containing credentials.") 14 | credentialKey: Optional[str] = Field(None, description="The key within the secret for the credentials.") 15 | -------------------------------------------------------------------------------- /src/schemas/request/update_vsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Optional, Dict 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class UpdateVslRequestSchema(BaseModel): 7 | name: str = Field(..., description="The name of the VSL.") 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | provider: str = Field(..., description="The name of the provider.") 10 | 11 | config: Optional[Dict[str, str]] = Field(None, description="Configuration fields.") 12 | 13 | credentialName: Optional[str] = Field(None, description="The name of the existing secret containing credentials.") 14 | credentialKey: Optional[str] = Field(None, description="The key within the secret for the credentials.") 15 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: 'Mark stale issues and pull requests' 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | stale: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/stale@v9 13 | with: 14 | repo-token: ${{ secrets.GITHUB_TOKEN }} 15 | stale-issue-message: > 16 | This issue has been automatically marked as stale due to inactivity for 14 days. 17 | It will be closed in 7 days if no further activity occurs. 18 | close-issue-message: > 19 | Closing this issue due to prolonged inactivity. Feel free to reopen if needed. 20 | days-before-stale: 14 21 | days-before-close: 7 22 | stale-issue-label: 'stale' 23 | exempt-issue-labels: 'enhancement,bug' 24 | -------------------------------------------------------------------------------- /src/constants/watchdog.py: -------------------------------------------------------------------------------- 1 | ENVIRONMENT = [ 2 | "K8S_VELERO_NAMESPACE", 3 | "K8S_VELERO_UI_NAMESPACE", 4 | "PROCESS_KUBE_CONFIG", 5 | "CLUSTER_ID", 6 | "K8S_IN_CLUSTER_MODE", 7 | "IGNORE_NM_", 8 | "BUILD_VERSION", 9 | "BUILD_DATE", 10 | "API_ENDPOINT_URL", 11 | "API_ENDPOINT_PORT", 12 | "DEBUG_LEVEL", 13 | "UVICORN_RELOAD", 14 | "EXPIRES_DAYS_WARNING", 15 | "BACKUP_ENABLED", 16 | "SCHEDULE_ENABLED", 17 | "REPORT_SCHEDULE_ITEM_PREFIX", 18 | "REPORT_BACKUP_ITEM_PREFIX", 19 | "NOTIFICATION_SKIP_COMPLETED", 20 | "NOTIFICATION_SKIP_DELETING", 21 | "NOTIFICATION_SKIP_INPROGRESS", 22 | "NOTIFICATION_SKIP_REMOVED", 23 | "FORCE_LOCAL_PROCESS_CYCLE", 24 | "FORCE_LOCAL_PROCESS_VALUE", 25 | "PROCESS_CYCLE_SEC", 26 | "APPRISE", 27 | "SEND_START_MESSAGE", 28 | "SEND_REPORT_AT_STARTUP", 29 | ] 30 | -------------------------------------------------------------------------------- /src/constants/k8s.py: -------------------------------------------------------------------------------- 1 | K8S_PLURALS = { 2 | "Pod": "pods", 3 | "Service": "services", 4 | "ConfigMap": "configmaps", 5 | "Secret": "secrets", 6 | "Node": "nodes", 7 | "Namespace": "namespaces", 8 | "Event": "events", 9 | "Endpoints": "endpoints", 10 | "Deployment": "deployments", 11 | "StatefulSet": "statefulsets", 12 | "DaemonSet": "daemonsets", 13 | "ReplicaSet": "replicasets", 14 | "Job": "jobs", 15 | "CronJob": "cronjobs", 16 | "Ingress": "ingresses", 17 | "StorageClass": "storageclasses", 18 | "EndpointSlice": "endpointslices", 19 | "NetworkPolicy": "networkpolicies", 20 | "PersistentVolumeClaim": "persistentvolumeclaims", 21 | "ClusterRoleBinding": "clusterrolebindings", 22 | "ControllerRevision": "controllerrevisions", 23 | "RoleBinding": "rolebindings", 24 | "ServiceAccount": "serviceaccounts", 25 | "ClusterRole": "clusterroles", 26 | "Role": "roles", 27 | "CustomResourceDefinition": "customresourcedefinitions", 28 | } 29 | -------------------------------------------------------------------------------- /docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.0' 2 | services: 3 | velero-api-stack: 4 | container_name: velero-api 5 | image: velero-api:1.3 6 | #restart: always 7 | network_mode: host 8 | command: ["python3", "-u", "main.py"] 9 | working_dir: /app 10 | volumes: 11 | - velero_api_config:/root/.kube 12 | - /etc/localtime:/etc/localtime:ro 13 | environment: 14 | - CONTAINER_MODE=True 15 | - VELERO_CLI_PATH=./velero-client 16 | - VELERO_CLI_PATH_CUSTOM=./velero-client-binary 17 | - VELERO_CLI_VERSION=v1.12.2 18 | - VELERO_CLI_DEST_PATH=/bin 19 | - API_ENDPOINT_PORT=8001 20 | - VELERO_NAMESPACE=velero 21 | - ORIGINS=["http://localhost:3000", "http://127.0.0.1:3000"] 22 | - SECURITY_TOKEN_KEY= 23 | - SECURITY_REFRESH_TOKEN_KEY= 24 | - SECURITY_PATH_DATABASE=./data 25 | - DEFAULT_ADMIN_USERNAME=admin 26 | - DEFAULT_ADMIN_PASSWORD=admin 27 | ports: 28 | - "8001:8001" 29 | volumes: 30 | velero_api_config: 31 | external: true 32 | -------------------------------------------------------------------------------- /src/schemas/request/update_user_config.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, field_validator 2 | 3 | 4 | class UpdateUserConfigRequestSchema(BaseModel): 5 | backupEnabled: bool 6 | scheduleEnabled: bool 7 | notificationSkipCompleted: bool 8 | notificationSkipDeleting: bool 9 | notificationSkipInProgress: bool 10 | notificationSkipRemoved: bool 11 | processCycleSeconds: int 12 | expireDaysWarning: int 13 | reportBackupItemPrefix: str 14 | reportScheduleItemPrefix: str 15 | 16 | @field_validator( 17 | "backupEnabled", "scheduleEnabled", "notificationSkipCompleted", 18 | "notificationSkipDeleting", "notificationSkipInProgress", "notificationSkipRemoved", mode="before" 19 | ) 20 | @classmethod 21 | def normalize_bool(cls, value): 22 | if isinstance(value, str): 23 | lower_value = value.lower() 24 | if lower_value == "true": 25 | return True 26 | elif lower_value == "false": 27 | return False 28 | return value 29 | -------------------------------------------------------------------------------- /src/controllers/common.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | # from utils.commons import logs_string_to_list 6 | 7 | from service.logs import get_velero_logs_service 8 | from service.describe import get_velero_resource_details_service 9 | 10 | 11 | async def get_resource_describe_handler(resource_name: str, resource_type: str): 12 | payload = await get_velero_resource_details_service(resource_name, resource_type) 13 | 14 | response = SuccessfulRequest(payload=payload.details) 15 | return JSONResponse(content=response.model_dump(), status_code=200) 16 | 17 | 18 | async def get_resource_logs_handler(resource_name: str, resource_type: str): 19 | payload = await get_velero_logs_service(resource_name, resource_type) 20 | 21 | # logs = payload.logs 22 | 23 | # data = logs_string_to_list('\n'.join(logs)) 24 | 25 | response = SuccessfulRequest(payload=payload) 26 | return JSONResponse(content=response.model_dump(), status_code=200) 27 | -------------------------------------------------------------------------------- /src/schemas/request/create_restore.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Optional, Dict, List 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class CreateRestoreRequestSchema(BaseModel): 7 | name: str 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | 10 | # spec 11 | backupName: Optional[str] 12 | scheduleName: Optional[str] 13 | itemOperationTimeout: Optional[str] = "4h" 14 | namespaceMapping: Optional[Dict[str, str]] = None 15 | includedNamespaces: Optional[List[str]] = None 16 | excludedNamespaces: Optional[List[str]] = None 17 | includedResources: Optional[List[str]] = None 18 | excludedResources: Optional[List[str]] = None 19 | 20 | includeClusterResources: Optional[bool] = None 21 | restorePVs: Optional[bool] = True 22 | preserveNodePorts: Optional[bool] = True 23 | # existingResourcePolicy: Optional[bool] = True 24 | 25 | # spec child 26 | labelSelector: Optional[Dict[str, str]] = None 27 | writeSparseFiles: Optional[bool] = True 28 | -------------------------------------------------------------------------------- /src/controllers/stats.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | from service.backup import get_backups_service 6 | from service.restore import get_restores_service 7 | from service.stats import get_stats_service 8 | from service.schedule_heatmap import get_schedules_heatmap_service 9 | 10 | 11 | async def get_stats_handler(): 12 | payload = await get_stats_service() 13 | 14 | response = SuccessfulRequest(payload=payload) 15 | return JSONResponse(content=response.model_dump(), status_code=200) 16 | 17 | 18 | async def get_in_progress_task_handler(): 19 | backups = await get_backups_service(in_progress=True) 20 | restores = await get_restores_service(in_progress=True) 21 | payload = [*backups, *restores] 22 | 23 | response = SuccessfulRequest(payload=payload) 24 | return JSONResponse(content=response.model_dump(), status_code=200) 25 | 26 | 27 | async def get_schedules_heatmap_handler(): 28 | payload = await get_schedules_heatmap_service() 29 | 30 | response = SuccessfulRequest(payload=payload) 31 | return JSONResponse(content=response.model_dump(), status_code=200) 32 | -------------------------------------------------------------------------------- /src/models/k8s/vsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Optional, Dict 3 | 4 | 5 | class VolumeSnapshotLocationMetadataResponse(BaseModel): 6 | """Response metadata for VolumeSnapshotLocation""" 7 | name: str 8 | namespace: Optional[str] 9 | uid: Optional[str] = None 10 | resourceVersion: Optional[str] = None 11 | creationTimestamp: Optional[str] = None 12 | labels: Optional[Dict[str, str]] = None 13 | annotations: Optional[Dict[str, str]] = None 14 | 15 | class Config: 16 | extra = "allow" 17 | 18 | 19 | class VolumeSnapshotLocationSpecResponse(BaseModel): 20 | """Response specification for VolumeSnapshotLocation""" 21 | provider: str 22 | credentialName: Optional[str] = None 23 | credentialKey: Optional[str] = None 24 | config: Optional[Dict[str, str]] = None 25 | 26 | class Config: 27 | extra = "allow" 28 | 29 | 30 | class VolumeSnapshotLocationResponseSchema(BaseModel): 31 | """Full response schema for VolumeSnapshotLocation""" 32 | apiVersion: str 33 | kind: str 34 | 35 | metadata: Optional[VolumeSnapshotLocationMetadataResponse] 36 | spec: Optional[VolumeSnapshotLocationSpecResponse] 37 | 38 | class Config: 39 | extra = "allow" 40 | -------------------------------------------------------------------------------- /src/schemas/request/create_bsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Optional, Dict 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class CreateBslRequestSchema(BaseModel): 7 | name: str = Field(..., description="The name of the BSL.") 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | 10 | provider: str = Field(..., description="The name of the provider.") 11 | bucket: str = Field(..., description="The S3 bucket name.") 12 | prefix: Optional[str] = Field(None, description="The S3 prefix name.") 13 | accessMode: str = Field(..., description="The access mode (e.g., read, write).") 14 | 15 | config: Optional[Dict[str, str]] = Field(None, description="Configuration fields.") 16 | 17 | credentialName: Optional[str] = Field(None, description="The name of the existing secret containing credentials.") 18 | credentialKey: Optional[str] = Field(None, description="The key within the secret for the credentials.") 19 | 20 | backupSyncPeriod: str = Field("2m0s", description="The synchronization period (e.g., '15m', '1h').") 21 | validationFrequency: str = Field("1m", description="The validation frequency (e.g., '1h', '24h').") 22 | 23 | default: bool = Field(False, description="Whether this BSL is the default.") 24 | -------------------------------------------------------------------------------- /src/schemas/request/update_bsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import Optional, Dict 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class UpdateBslRequestSchema(BaseModel): 7 | name: str = Field(..., description="The name of the BSL.") 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | 10 | provider: str = Field(..., description="The name of the provider.") 11 | bucket: str = Field(..., description="The S3 bucket name.") 12 | prefix: Optional[str] = Field(None, description="The S3 prefix name.") 13 | accessMode: str = Field(..., description="The access mode (e.g., read, write).") 14 | 15 | config: Optional[Dict[str, str]] = Field(None, description="Configuration fields.") 16 | 17 | credentialName: Optional[str] = Field(None, description="The name of the existing secret containing credentials.") 18 | credentialKey: Optional[str] = Field(None, description="The key within the secret for the credentials.") 19 | 20 | backupSyncPeriod: str = Field("2m0s", description="The synchronization period (e.g., '15m', '1h').") 21 | validationFrequency: str = Field("1m", description="The validation frequency (e.g., '1h', '24h').") 22 | 23 | default: bool = Field(False, description="Whether this BSL is the default.") 24 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/seriohub/vui-common.git@v0.1.3#egg=vui_common 2 | aiohappyeyeballs==2.6.1 3 | aiohttp==3.12.12 4 | aiosignal==1.3.2 5 | annotated-types==0.7.0 6 | anyio==4.8.0 7 | argon2-cffi==23.1.0 8 | argon2-cffi-bindings==21.2.0 9 | async-timeout==5.0.1 10 | attrs==25.1.0 11 | cachetools==5.5.1 12 | certifi==2025.1.31 13 | cffi==1.17.1 14 | charset-normalizer==3.4.1 15 | click==8.1.8 16 | croniter==6.0.0 17 | durationpy==0.9 18 | ecdsa==0.19.0 19 | exceptiongroup==1.2.2 20 | fastapi==0.115.8 21 | frozenlist==1.5.0 22 | google-auth==2.38.0 23 | greenlet==3.1.1 24 | h11==0.16.0 25 | idna==3.10 26 | kubernetes==32.0.0 27 | minio==7.2.15 28 | multidict==6.1.0 29 | nats-py==2.10.0 30 | oauthlib==3.2.2 31 | passlib==1.7.4 32 | propcache==0.2.1 33 | pyasn1==0.4.6 34 | pyasn1_modules==0.4.1 35 | pycparser==2.22 36 | pycryptodome==3.21.0 37 | pydantic==2.10.6 38 | pydantic_core==2.27.2 39 | python-dateutil==2.9.0.post0 40 | python-dotenv==1.0.1 41 | python-jose==3.4.0 42 | python-multipart==0.0.19 43 | pytz==2025.1 44 | PyYAML==6.0.2 45 | requests==2.32.4 46 | requests-oauthlib==2.0.0 47 | rsa==4.9 48 | six==1.17.0 49 | sniffio==1.3.1 50 | SQLAlchemy==2.0.37 51 | starlette==0.45.3 52 | typing_extensions==4.12.2 53 | urllib3==2.3.0 54 | uvicorn==0.34.0 55 | websocket-client==1.8.0 56 | wsproto==1.2.0 57 | yarl==1.18.3 58 | ldap3~=2.9.1 59 | kubernetes_asyncio==32.3.2 60 | aiofiles~=24.1.0 61 | -------------------------------------------------------------------------------- /src/service/resource.py: -------------------------------------------------------------------------------- 1 | from service.bsl import get_bsls_service 2 | from service.k8s import get_namespaces_service, get_resources_service 3 | from service.k8s_configmap import list_configmaps_service 4 | from service.vsl import get_vsls_service 5 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 6 | 7 | 8 | @trace_k8s_async_method(description="Get backup/schedule creation settings") 9 | async def get_resource_creation_settings_service(): 10 | namespaces = await get_namespaces_service() 11 | bsls = await get_bsls_service() 12 | vsls = await get_vsls_service() 13 | 14 | bsls = [bsl.model_dump() for bsl in bsls] 15 | vsls = [vsl.model_dump() for vsl in vsls] 16 | 17 | resource_policy = list_configmaps_service() 18 | 19 | backup_location_list = [item['metadata']['name'] for item in bsls if 20 | 'metadata' in item and 'name' in item['metadata']] 21 | snapshot_location_list = [item['metadata']['name'] for item in vsls if 22 | 'metadata' in item and 'name' in item['metadata']] 23 | valid_resources = (await get_resources_service()) 24 | 25 | payload = {'namespaces': namespaces, 26 | 'backup_location': backup_location_list, 27 | 'snapshot_location': snapshot_location_list, 28 | 'resources': valid_resources, 29 | 'resource_policy': resource_policy 30 | } 31 | 32 | return payload 33 | -------------------------------------------------------------------------------- /src/startup_watchers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from ws.ws_manager import WebSocketManager 3 | # from ws import ws_manager_proxy 4 | from vui_common.ws import ws_manager_proxy 5 | 6 | from integrations.nats_manager import NatsManager 7 | from integrations import nats_manager_proxy 8 | 9 | from k8s.k8s_watch_manager import K8sWatchManager 10 | from k8s import k8s_watcher_proxy 11 | from vui_common.configs.config_proxy import config_app 12 | 13 | def init_watchers(app): 14 | ws_manager_proxy.ws_manager = WebSocketManager() 15 | if config_app.nats.enable: 16 | nats_manager_proxy.nat_manager = NatsManager(app) 17 | 18 | async def send_global_to_all(message: str): 19 | await ws_manager_proxy.ws_manager.broadcast(message) 20 | if config_app.nats.enable: 21 | await nats_manager_proxy.nat_manager.publish_global_event(message) 22 | 23 | async def send_user_to_all(user_id: str, message: str): 24 | await ws_manager_proxy.ws_manager.send_personal_message(user_id, message) 25 | if config_app.nats.enable: 26 | await nats_manager_proxy.nat_manager.publish_user_event(user_id, message) 27 | 28 | k8s_watcher_proxy.k8s_watcher_manager = K8sWatchManager( 29 | send_global_callback=send_global_to_all, 30 | send_user_callback=send_user_to_all 31 | ) 32 | 33 | if config_app.nats.enable: 34 | asyncio.create_task(nats_manager_proxy.nat_manager.run()) 35 | asyncio.create_task(k8s_watcher_proxy.k8s_watcher_manager.start_global_watch_tasks()) 36 | -------------------------------------------------------------------------------- /src/controllers/pvb.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | 5 | from service.pvb import (get_pod_volume_backups_service, 6 | get_pod_volume_backup_details_service, 7 | get_pod_volume_restore_service, 8 | get_pod_volume_restore_details_service) 9 | 10 | 11 | async def get_pod_volume_backups_handler(): 12 | payload = await get_pod_volume_backups_service() 13 | items = payload['items'] 14 | response = SuccessfulRequest(payload=items) 15 | return JSONResponse(content=response.model_dump(), status_code=200) 16 | 17 | 18 | async def get_pod_volume_backup_details_handler(backup_name: str): 19 | payload = await get_pod_volume_backup_details_service(backup_name) 20 | 21 | response = SuccessfulRequest(payload=payload) 22 | return JSONResponse(content=response.model_dump(), status_code=200) 23 | 24 | async def get_pod_volume_restore_handler(): 25 | payload = await get_pod_volume_restore_service() 26 | items = payload['items'] 27 | response = SuccessfulRequest(payload=items) 28 | return JSONResponse(content=response.model_dump(), status_code=200) 29 | 30 | async def get_pod_volume_restore_details_handler(backup_name: str): 31 | payload = await get_pod_volume_restore_details_service(backup_name) 32 | 33 | response = SuccessfulRequest(payload=payload) 34 | return JSONResponse(content=response.model_dump(), status_code=200) -------------------------------------------------------------------------------- /src/integrations/nats_cron_job.py: -------------------------------------------------------------------------------- 1 | class NatsCronJob: 2 | def __init__(self, endpoint: str, 3 | credential_required: bool, 4 | interval: int): 5 | self._endpoint = endpoint 6 | self._cr = credential_required 7 | self._interval = interval 8 | self.__from_last_publish_sec = interval + 1 9 | 10 | @property 11 | def endpoint(self): 12 | return self._endpoint 13 | 14 | @endpoint.setter 15 | def endpoint(self, value: str): 16 | self._endpoint = value 17 | 18 | @property 19 | def credential(self): 20 | return self._cr 21 | 22 | @credential.setter 23 | def credential(self, value: bool): 24 | self._cr = value 25 | 26 | @property 27 | def interval(self): 28 | return self._interval 29 | 30 | @property 31 | def ky_key(self): 32 | return self._endpoint.replace("/", "_") 33 | 34 | @property 35 | def get_data(self): 36 | return {'endpoint': self._endpoint, 37 | 'credential': self._cr, 38 | 'intervals': self._interval, 39 | 'kv_key': self.ky_key} 40 | 41 | @property 42 | def time_elapsed(self): 43 | return self.__from_last_publish_sec 44 | 45 | @time_elapsed.setter 46 | def time_elapsed(self, value: int): 47 | self.__from_last_publish_sec += value 48 | 49 | @property 50 | def is_elapsed(self): 51 | return self.__from_last_publish_sec > self._interval 52 | 53 | def reset_timer(self): 54 | self.__from_last_publish_sec = 0 55 | -------------------------------------------------------------------------------- /src/controllers/inspect.py: -------------------------------------------------------------------------------- 1 | import os 2 | from fastapi.responses import JSONResponse 3 | 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | from vui_common.configs.config_proxy import config_app 6 | from service.inspect import (get_folders_list, 7 | # get_directory_contents, 8 | read_json_file, 9 | get_recursive_directory_contents) 10 | 11 | 12 | async def get_backups_handler(): 13 | payload = await get_folders_list(config_app.app.inspect_folder) 14 | 15 | response = SuccessfulRequest(payload=payload) 16 | return JSONResponse(content=response.model_dump(), status_code=200) 17 | 18 | 19 | # async def get_folders_handler(path: str): 20 | # payload = await get_directory_contents(os.path.join(config_app.app.inspect_folder, path)) 21 | # 22 | # response = SuccessfulRequest(payload=payload) 23 | # return JSONResponse(content=response.model_dump(), status_code=200) 24 | # 25 | 26 | async def get_file_content_handler(path: str): 27 | payload = await read_json_file(os.path.join(config_app.app.inspect_folder, path)) 28 | 29 | response = SuccessfulRequest(payload=payload) 30 | return JSONResponse(content=response.model_dump(), status_code=200) 31 | 32 | 33 | async def get_recursive_directory_contents_handler(backup: str): 34 | payload = await get_recursive_directory_contents(os.path.join(config_app.app.inspect_folder, backup)) 35 | 36 | response = SuccessfulRequest(payload=payload) 37 | return JSONResponse(content=response.model_dump(), status_code=200) 38 | -------------------------------------------------------------------------------- /src/schemas/request/create_backup.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Optional, Dict, List 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class CreateBackupRequestSchema(BaseModel): 7 | name: str 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | 10 | # spec 11 | csiSnapshotTimeout: Optional[str] = "10m" 12 | itemOperationTimeout: Optional[str] = "4h" 13 | includedNamespaces: Optional[List[str]] = None 14 | excludedNamespaces: Optional[List[str]] = None 15 | includedResources: Optional[List[str]] = None 16 | excludedResources: Optional[List[str]] = None 17 | orderedResources: Optional[Dict[str, List[str]]] = None 18 | includeClusterResources: Optional[bool | None] = None 19 | excludedClusterScopedResources: Optional[List[str]] = None 20 | includedClusterScopedResources: Optional[List[str]] = None 21 | excludedNamespaceScopedResources: Optional[List[str]] = None 22 | includedNamespaceScopedResources: Optional[List[str]] = None 23 | snapshotVolumes: Optional[bool | None] = None 24 | storageLocation: Optional[str] = None 25 | volumeSnapshotLocations: Optional[List[str]] = None 26 | ttl: Optional[str] = "24h" 27 | defaultVolumesToFsBackup: Optional[bool] = None 28 | snapshotMoveData: Optional[bool] = None 29 | datamover: Optional[str] = None 30 | 31 | # spec childs 32 | labelSelector: Optional[Dict[str, str]] = None # "matchLabels": {"app": "velero", "component": "server"}, 33 | parallelFilesUpload: Optional[int] = 10 34 | resourcePolicy: Optional[str] = None 35 | -------------------------------------------------------------------------------- /src/controllers/sc_mapping.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.schemas.response.successful_request import SuccessfulRequest 4 | from vui_common.schemas.notification import Notification 5 | from schemas.request.storage_class_map import StorageClassMapRequestSchema 6 | 7 | from service.sc_mapping import (get_storages_classes_map_service, 8 | update_storages_classes_mapping_service, 9 | delete_storages_classes_mapping_service) 10 | 11 | 12 | async def get_storages_classes_map_handler(): 13 | payload = await get_storages_classes_map_service() 14 | 15 | response = SuccessfulRequest(payload=payload) 16 | return JSONResponse(content=response.model_dump(), status_code=200) 17 | 18 | 19 | async def update_storages_classes_mapping_handler(maps: StorageClassMapRequestSchema): 20 | payload = await update_storages_classes_mapping_service(data_list=maps.storageClassMapping) 21 | 22 | msg = Notification(title='Storage Class Map', description=f"Done!", type_='INFO') 23 | response = SuccessfulRequest(notifications=[msg], payload=payload) 24 | return JSONResponse(content=response.model_dump(), status_code=200) 25 | 26 | 27 | async def delete_storages_classes_mapping_handler(data_list=None): 28 | payload = await delete_storages_classes_mapping_service(data_list=data_list) 29 | 30 | msg = Notification(title='Storage Class Map', description=f"Deleted!", type_='INFO') 31 | response = SuccessfulRequest(notifications=[msg], payload=payload) 32 | return JSONResponse(content=response.model_dump(), status_code=200) 33 | -------------------------------------------------------------------------------- /src/models/k8s/repo.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Optional, Dict, List 3 | # from datetime import datetime 4 | 5 | 6 | class BackupRepositoryMetadata(BaseModel): 7 | """Represents the metadata section of a BackupRepository.""" 8 | name: str 9 | generateName: Optional[str] = None 10 | namespace: str 11 | uid: str 12 | resourceVersion: Optional[str] 13 | generation: Optional[int] 14 | creationTimestamp: Optional[str] 15 | labels: Optional[Dict[str, str]] 16 | managedFields: Optional[List[Dict]] 17 | 18 | class Config: 19 | extra = "allow" 20 | 21 | 22 | class BackupRepositorySpec(BaseModel): 23 | """Represents the spec section of a BackupRepository.""" 24 | volumeNamespace: Optional[str] 25 | backupStorageLocation: Optional[str] 26 | repositoryType: Optional[str] 27 | resticIdentifier: Optional[str] 28 | maintenanceFrequency: Optional[str] 29 | 30 | class Config: 31 | extra = "allow" 32 | 33 | 34 | class BackupRepositoryStatus(BaseModel): 35 | """Represents the status section of a BackupRepository.""" 36 | phase: Optional[str] = None 37 | message: Optional[str] = None 38 | lastMaintenanceTime: Optional[str] = None 39 | 40 | class Config: 41 | extra = "allow" 42 | 43 | 44 | class BackupRepositoryResponseSchema(BaseModel): 45 | """Full response schema for a BackupRepository object.""" 46 | kind: str 47 | apiVersion: str 48 | metadata: BackupRepositoryMetadata 49 | spec: Optional[BackupRepositorySpec] = None 50 | status: Optional[BackupRepositoryStatus] = None 51 | 52 | class Config: 53 | extra = "allow" 54 | -------------------------------------------------------------------------------- /src/schemas/request/create_schedule.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, List 2 | from pydantic import BaseModel 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class CreateScheduleRequestSchema(BaseModel): 7 | name: str 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | 10 | # spec 11 | schedule: str 12 | paused: Optional[bool] = False 13 | useOwnerReferencesInBackup: Optional[bool] = False 14 | 15 | # spec.template 16 | csiSnapshotTimeout: Optional[str] = "10m" 17 | includedNamespaces: Optional[List[str]] = None 18 | excludedNamespaces: Optional[List[str]] = None 19 | includedResources: Optional[List[str]] = None 20 | excludedResources: Optional[List[str]] = None 21 | orderedResources: Optional[Dict[str, List[str]]] = None 22 | includeClusterResources: Optional[bool] = None 23 | excludedClusterScopedResources: Optional[List[str]] = None 24 | includedClusterScopedResources: Optional[List[str]] = None 25 | excludedNamespaceScopedResources: Optional[List[str]] = None 26 | includedNamespaceScopedResources: Optional[List[str]] = None 27 | snapshotVolumes: Optional[bool | None] = None 28 | storageLocation: Optional[str] = None 29 | volumeSnapshotLocations: Optional[List[str]] = None 30 | ttl: Optional[str] = "24h" 31 | defaultVolumesToFsBackup: Optional[bool] = None 32 | snapshotMoveData: Optional[bool] = None 33 | datamover: Optional[str] = None 34 | 35 | # spec childs 36 | labelSelector: Optional[Dict[str, str]] = None # "matchLabels": {"app": "velero", "component": "server"}, 37 | parallelFilesUpload: Optional[int] = 10 38 | resourcePolicy: Optional[str] = None 39 | -------------------------------------------------------------------------------- /src/schemas/request/update_schedule.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, List 2 | from pydantic import BaseModel 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | class UpdateScheduleRequestSchema(BaseModel): 7 | name: str 8 | namespace: Optional[str] = config_app.k8s.velero_namespace 9 | 10 | # spec 11 | schedule: str 12 | paused: Optional[bool] = False 13 | useOwnerReferencesInBackup: Optional[bool] = False 14 | 15 | # spec.template 16 | csiSnapshotTimeout: Optional[str] = "10m" 17 | includedNamespaces: Optional[List[str]] = None 18 | excludedNamespaces: Optional[List[str]] = None 19 | includedResources: Optional[List[str]] = None 20 | excludedResources: Optional[List[str]] = None 21 | orderedResources: Optional[Dict[str, List[str]]] = None 22 | includeClusterResources: Optional[bool] = None 23 | excludedClusterScopedResources: Optional[List[str]] = None 24 | includedClusterScopedResources: Optional[List[str]] = None 25 | excludedNamespaceScopedResources: Optional[List[str]] = None 26 | includedNamespaceScopedResources: Optional[List[str]] = None 27 | snapshotVolumes: Optional[bool | None] = None 28 | storageLocation: Optional[str] = None 29 | volumeSnapshotLocations: Optional[List[str]] = None 30 | ttl: Optional[str] = "24h" 31 | defaultVolumesToFsBackup: Optional[bool] = None 32 | snapshotMoveData: Optional[bool] = None 33 | datamover: Optional[str] = None 34 | 35 | # spec childs 36 | labelSelector: Optional[Dict[str, str]] = None # "matchLabels": {"app": "velero", "component": "server"}, 37 | parallelFilesUpload: Optional[int] = 10 38 | resourcePolicy: Optional[str] = None 39 | -------------------------------------------------------------------------------- /src/service/utils/cleanup_requests.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | from kubernetes import client 3 | from vui_common.configs.config_proxy import config_app 4 | from vui_common.logger.logger_proxy import logger 5 | 6 | from constants.velero import VELERO 7 | 8 | custom_objects = client.CustomObjectsApi() 9 | 10 | 11 | def cleanup_server_request(resource_name: str, plural: str): 12 | """ 13 | Deletes the resource after use to avoid accumulation in the cluster. 14 | 15 | :param resource_name: Name of the resource associated plural. 16 | :param plural: plural associated with the resource name. 17 | """ 18 | logger.info(f"Cleanup {plural}:{resource_name}") 19 | try: 20 | custom_objects.delete_namespaced_custom_object( 21 | group=VELERO["GROUP"], 22 | version=VELERO["VERSION"], 23 | namespace=config_app.k8s.velero_namespace, 24 | plural=plural, 25 | name=resource_name 26 | ) 27 | logger.info(f"{plural} '{resource_name}' successfully deleted.") 28 | except client.exceptions.ApiException as e: 29 | if e.status == 404: 30 | logger.error(f"DownloadRequest '{resource_name}' does not exist, no deletion required.") 31 | raise HTTPException(status_code=400, 32 | detail=f"DownloadRequest '{resource_name}' does not exist, no deletion " 33 | f"required.") 34 | 35 | else: 36 | logger.error(f"Error while deleting DownloadRequest '{resource_name}': {e}") 37 | raise HTTPException(status_code=400, 38 | detail=f"Error while deleting DownloadRequest '{resource_name}': {e}") 39 | -------------------------------------------------------------------------------- /src/controllers/restore.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from schemas.request.create_restore import CreateRestoreRequestSchema 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | from schemas.response.successful_restores import SuccessfulRestoreResponse 6 | from vui_common.schemas.notification import Notification 7 | 8 | from service.restore import get_restores_service, create_restore_service, delete_restore_service 9 | 10 | 11 | async def get_restores_handler(in_progress=False): 12 | payload = await get_restores_service(in_progress=in_progress) 13 | 14 | response = SuccessfulRestoreResponse(payload=payload) 15 | return JSONResponse(content=response.model_dump(), status_code=200) 16 | 17 | 18 | async def create_restore_handler(restore: CreateRestoreRequestSchema): 19 | payload = await create_restore_service(restore_data=restore) 20 | 21 | msg = Notification(title='Create Restore', 22 | description=f"Restore from {restore.backupName} " 23 | f"{restore.name} created successfully", 24 | type_='INFO') 25 | response = SuccessfulRequest(notifications=[msg], payload=payload) 26 | return JSONResponse(content=response.model_dump(), status_code=201) 27 | 28 | 29 | async def delete_restore_handler(restore_name: str): 30 | payload = await delete_restore_service(restore_name=restore_name) 31 | 32 | msg = Notification(title='Delete restore', 33 | description=f"Restore {restore_name} deleted request done!", 34 | type_='INFO') 35 | response = SuccessfulRequest(notifications=[msg], payload=payload) 36 | return JSONResponse(content=response.model_dump(), status_code=200) 37 | -------------------------------------------------------------------------------- /src/models/k8s/bsl.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Optional, Dict 3 | 4 | 5 | class BSLObjectStorage(BaseModel): 6 | """Definisce la configurazione per l'Object Storage""" 7 | bucket: str 8 | prefix: Optional[str] = None 9 | caCert: Optional[str] = None 10 | 11 | class Config: 12 | extra = "allow" 13 | 14 | class BSLCredential(BaseModel): 15 | """Definisce le credenziali del provider di storage""" 16 | name: str 17 | key: str 18 | 19 | 20 | class BackupStorageLocationSpec(BaseModel): 21 | """Definisce la configurazione dello Storage Location""" 22 | provider: str 23 | objectStorage: Optional[BSLObjectStorage] = None 24 | credential: Optional[BSLCredential] = None 25 | backupSyncPeriod: Optional[str] = "2m0s" 26 | validationFrequency: Optional[str] = None 27 | config: Optional[Dict[str, str]] = None 28 | 29 | class Config: 30 | extra = "allow" 31 | 32 | 33 | class BackupStorageLocationMetadata(BaseModel): 34 | """Definisce i metadati dello Storage Location""" 35 | name: str 36 | namespace: Optional[str] = "velero" 37 | uid: Optional[str] 38 | 39 | class Config: 40 | extra = "allow" 41 | 42 | 43 | class BackupStorageLocationStatus(BaseModel): 44 | """Definisce lo stato dello Storage Location""" 45 | phase: Optional[str] = None 46 | lastSyncedTime: Optional[str] = None 47 | errors: Optional[int] = 0 48 | 49 | 50 | class BackupStorageLocationResponseSchema(BaseModel): 51 | """Schema completo per un Backup Storage Location""" 52 | apiVersion: str = "velero.io/v1" 53 | kind: str = "BackupStorageLocation" 54 | metadata: BackupStorageLocationMetadata 55 | spec: BackupStorageLocationSpec 56 | status: Optional[BackupStorageLocationStatus] = None 57 | -------------------------------------------------------------------------------- /src/service/describe.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | from kubernetes import client 3 | from schemas.velero_describe import VeleroDescribe 4 | from constants.velero import VELERO 5 | from constants.resources import RESOURCES, ResourcesNames 6 | from vui_common.configs.config_proxy import config_app 7 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 8 | 9 | custom_objects = client.CustomObjectsApi() 10 | 11 | 12 | @trace_k8s_async_method(description="Get velero resource details") 13 | async def get_velero_resource_details_service(resource_name: str, resource_type: str) -> VeleroDescribe: 14 | """Gets the details of a Velero resource (Backup, Restore, etc.) directly from the Kubernetes API""" 15 | # if resource_type not in VELERO_RESOURCES: 16 | if resource_type.upper() not in ResourcesNames.__members__: 17 | # return VeleroDescribe(success=False, error=f"Unsupported resource type: {resource_type}") 18 | raise HTTPException(status_code=400, detail=f"Unsupported resource type: {resource_type}") 19 | 20 | try: 21 | resource_enum = ResourcesNames[resource_type.upper()] 22 | 23 | # Retrieve resource details directly from Kubernetes 24 | resource = custom_objects.get_namespaced_custom_object( 25 | group=VELERO['GROUP'], 26 | version=VELERO['VERSION'], 27 | namespace=config_app.k8s.velero_namespace, 28 | plural=RESOURCES[resource_enum].plural, 29 | name=resource_name 30 | ) 31 | 32 | return VeleroDescribe(details=resource) 33 | 34 | except client.exceptions.ApiException as e: 35 | if e.status == 404: 36 | raise HTTPException(status_code=404, detail=f"{resource_type.capitalize()} '{resource_name}' not found.") 37 | raise HTTPException(status_code=400, detail=f"{str(e)}") 38 | -------------------------------------------------------------------------------- /src/api/common/routers/health.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, status 2 | 3 | from controllers.agent import watchdog_online_handler 4 | 5 | from vui_common.utils.swagger import route_description 6 | from vui_common.utils.exceptions import handle_exceptions_endpoint 7 | from vui_common.security.helpers.rate_limiter import LimiterRequests 8 | from vui_common.security.helpers.rate_limiter import RateLimiter 9 | from vui_common.schemas.response.successful_request import SuccessfulRequest 10 | 11 | 12 | router = APIRouter() 13 | 14 | 15 | tag_name = 'Health' 16 | 17 | endpoint_limiter = LimiterRequests(tags=tag_name, 18 | default_key='L1') 19 | 20 | 21 | # ------------------------------------------------------------------------------------------------ 22 | # GET WATCHDOG INFO 23 | # ------------------------------------------------------------------------------------------------ 24 | 25 | 26 | limiter_watchdog = endpoint_limiter.get_limiter_cust('info_watchdog') 27 | route = '/watchdog' 28 | 29 | 30 | @router.get(path=route, 31 | tags=[tag_name], 32 | summary='Get info watchdog', 33 | description=route_description(tag=tag_name, 34 | route=route, 35 | limiter_calls=limiter_watchdog.max_request, 36 | limiter_seconds=limiter_watchdog.seconds), 37 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_watchdog.seconds, 38 | max_requests=limiter_watchdog.max_request))], 39 | response_model=SuccessfulRequest, 40 | status_code=status.HTTP_200_OK) 41 | @handle_exceptions_endpoint 42 | async def watchdog_config(): 43 | return await watchdog_online_handler() 44 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim as velero-api-base 2 | LABEL maintainer="DS" 3 | 4 | # Update packages and install basics 5 | RUN apt-get update && apt-get install --no-install-recommends -y \ 6 | curl \ 7 | python3-pip \ 8 | git \ 9 | restic \ 10 | && rm -rf /var/lib/apt/lists/* 11 | 12 | RUN pip install --upgrade pip 13 | 14 | COPY requirements.txt requirements.txt 15 | 16 | RUN pip3 install --no-cache-dir -r requirements.txt 17 | 18 | # prepare dir 19 | RUN mkdir /app \ 20 | && mkdir /app/data 21 | # && mkdir /app/velero-client \ 22 | # && mkdir /app/velero-client-binary 23 | 24 | # Set the working directory inside the container 25 | WORKDIR /app 26 | 27 | # Copy the project files to the container 28 | COPY ./src /app 29 | # COPY ../velero-client /app/velero-client 30 | 31 | # Exclude specific files or directories from the project 32 | RUN find . -type f \( -name "*.yaml" -o -name "*.db" -o -name ".env" \) -delete \ 33 | && find . -type d -name "__pycache__" -exec rm -rf {} + \ 34 | && find . -type f -name "tmp" -delete 35 | 36 | # install kubectl 37 | RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \ 38 | && chmod +x ./kubectl \ 39 | && mv ./kubectl /usr/local/bin 40 | 41 | # expose port 42 | EXPOSE 8001 43 | 44 | # Expose a volume the database 45 | VOLUME /app/data 46 | 47 | # Expose a volume for custom binary download 48 | # VOLUME /app/velero-client-binary 49 | 50 | WORKDIR /app/ 51 | 52 | RUN echo "Ready!" 53 | CMD ["python3", "-u" , "__main__.py"] 54 | 55 | # ##################################################################################################### 56 | 57 | FROM velero-api-base as velero-api 58 | ARG VERSION 59 | ARG BUILD_DATE 60 | 61 | ENV BUILD_VERSION=$VERSION 62 | ENV BUILD_DATE=$BUILD_DATE 63 | -------------------------------------------------------------------------------- /src/controllers/vsl.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from schemas.request.update_vsl import UpdateVslRequestSchema 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | from vui_common.schemas.notification import Notification 6 | from schemas.request.create_vsl import CreateVslRequestSchema 7 | 8 | from service.vsl import get_vsls_service, create_vsl_service, delete_vsl_service, update_vsl_service 9 | 10 | 11 | async def get_vsl_handler(): 12 | payload = await get_vsls_service() 13 | 14 | response = SuccessfulRequest(payload=payload) 15 | return JSONResponse(content=response.model_dump(), status_code=200) 16 | 17 | 18 | async def create_vsl_handler(create_bsl: CreateVslRequestSchema): 19 | payload = await create_vsl_service(create_bsl) 20 | 21 | msg = Notification(title='Create bsl', description=f"BSL {create_bsl.name} created!", type_='INFO') 22 | response = SuccessfulRequest(notifications=[msg], payload=payload) 23 | return JSONResponse(content=response.model_dump(), status_code=201) 24 | 25 | 26 | async def delete_vsl_handler(bsl_delete: str): 27 | payload = await delete_vsl_service(vsl_name=bsl_delete) 28 | msg = Notification(title='Delete bsl', description=f'Bsl {bsl_delete} deleted request done!', type_='INFO') 29 | response = SuccessfulRequest(notifications=[msg], payload=payload) 30 | return JSONResponse(content=response.model_dump(), status_code=200) 31 | 32 | async def update_vsl_handler(vsl: UpdateVslRequestSchema): 33 | payload = await update_vsl_service(vsl_data=vsl) 34 | 35 | msg = Notification(title='Vsl', 36 | description=f"Vsl '{vsl.name}' successfully updated.", 37 | type_='INFO') 38 | 39 | response = SuccessfulRequest(payload=payload, notifications=[msg]) 40 | return JSONResponse(content=response.model_dump(), status_code=200) 41 | -------------------------------------------------------------------------------- /docker/Dockerfile-old: -------------------------------------------------------------------------------- 1 | FROM python:3.10 as velero-api-base 2 | LABEL maintainer="DS" 3 | 4 | # ARG VERSION 5 | # ARG BUILD_DATE 6 | # 7 | # ENV BUILD_VERSION=$VERSION 8 | # ENV BUILD_DATE=$BUILD_DATE 9 | 10 | # Update packages and install basics 11 | RUN apt-get update && apt-get install -y \ 12 | wget \ 13 | unzip \ 14 | git \ 15 | nano 16 | 17 | RUN apt-get -y install python3-pip 18 | 19 | RUN pip install --upgrade pip 20 | 21 | COPY requirements.txt requirements.txt 22 | RUN pip3 install -r requirements.txt 23 | 24 | # prepare dir 25 | RUN mkdir /app 26 | 27 | # Set the working directory inside the container 28 | WORKDIR /app 29 | 30 | # Copy the project files to the container 31 | COPY /src /app 32 | 33 | # Exclude specific files or directories from the project 34 | # Add more `--exclude` arguments as needed 35 | RUN find . -type f -name "tmp" -exec rm -rf {} + 36 | RUN find . -type f -name "*.yaml" -exec rm -f {} + 37 | RUN find . -type f -name "*.db" -exec rm -f {} + 38 | RUN find . -type f -name "__pycache__" -exec rm -rf {} + 39 | RUN find . -type f -name "*.env" -exec rm -f {} + 40 | 41 | # RUN rm -rf /app/tmp /app/__pycache__ 42 | 43 | # install kubectl 44 | RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl 45 | RUN chmod +x ./kubectl 46 | RUN mv ./kubectl /usr/local/bin 47 | 48 | RUN mkdir /app/data 49 | 50 | RUN mkdir /app/velero-client 51 | COPY ../velero-client /app/velero-client 52 | 53 | RUN mkdir /app/velero-client-binary 54 | 55 | # expose port 56 | EXPOSE 8001 57 | # Expose a volume the database 58 | VOLUME /app/data 59 | 60 | # Expose a volume for custom binary download 61 | VOLUME /app/velero-client-binary 62 | 63 | WORKDIR /app/ 64 | RUN echo "Ready!" 65 | CMD ["python3", "-u" , "main.py"] 66 | 67 | FROM velero-api-base as velero-api 68 | ARG VERSION 69 | ARG BUILD_DATE 70 | 71 | ENV BUILD_VERSION=$VERSION 72 | ENV BUILD_DATE=$BUILD_DATE 73 | -------------------------------------------------------------------------------- /src/service/requests.py: -------------------------------------------------------------------------------- 1 | from kubernetes import client 2 | 3 | from vui_common.configs.config_proxy import config_app 4 | from constants.velero import VELERO 5 | from constants.resources import RESOURCES, ResourcesNames 6 | from schemas.request.delete_resource import DeleteResourceRequestSchema 7 | from service.utils.cleanup_requests import cleanup_server_request 8 | 9 | custom_objects = client.CustomObjectsApi() 10 | 11 | 12 | async def get_server_status_requests_service(): 13 | ssr = custom_objects.list_namespaced_custom_object( 14 | group=VELERO["GROUP"], 15 | version=VELERO["VERSION"], 16 | namespace=config_app.k8s.velero_namespace, 17 | plural=RESOURCES[ResourcesNames.SERVER_STATUS_REQUEST].plural 18 | ) 19 | 20 | return ssr 21 | 22 | 23 | async def get_download_requests_service(): 24 | dr = custom_objects.list_namespaced_custom_object( 25 | group=VELERO["GROUP"], 26 | version=VELERO["VERSION"], 27 | namespace=config_app.k8s.velero_namespace, 28 | plural=RESOURCES[ResourcesNames.DOWNLOAD_REQUEST].plural 29 | ) 30 | 31 | return dr 32 | 33 | 34 | async def get_delete_backup_requests_service(): 35 | dbr = custom_objects.list_namespaced_custom_object( 36 | group=VELERO["GROUP"], 37 | version=VELERO["VERSION"], 38 | namespace=config_app.k8s.velero_namespace, 39 | plural=RESOURCES[ResourcesNames.DELETE_BACKUP_REQUEST].plural 40 | ) 41 | 42 | return dbr 43 | 44 | async def delete_download_requests_service(request: DeleteResourceRequestSchema): 45 | cleanup_server_request(request.name, RESOURCES[ResourcesNames.DOWNLOAD_REQUEST].plural) 46 | 47 | async def delete_delete_download_requests_service(request: DeleteResourceRequestSchema): 48 | cleanup_server_request(request.name, RESOURCES[ResourcesNames.DELETE_BACKUP_REQUEST].plural) 49 | 50 | async def delete_server_status_requests_service(request: DeleteResourceRequestSchema): 51 | cleanup_server_request(request.name, RESOURCES[ResourcesNames.SERVER_STATUS_REQUEST].plural) 52 | -------------------------------------------------------------------------------- /src/controllers/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from fastapi.responses import JSONResponse 3 | 4 | from vui_common.configs.config_proxy import config_app 5 | 6 | from vui_common.schemas.response.successful_request import SuccessfulRequest 7 | 8 | from vui_common.service.k8s import get_config_map_service 9 | from service.velero import get_velero_version_service, get_pods_service 10 | 11 | 12 | async def get_env_handler(): 13 | if os.getenv('K8S_IN_CLUSTER_MODE').lower() == 'true': 14 | env_data = await get_config_map_service(namespace=config_app.k8s.vui_namespace, 15 | configmap_name=f"{config_app.helm.release_name}-api-config") 16 | 17 | else: 18 | env_data = config_app.get_env_variables() 19 | 20 | response = SuccessfulRequest(payload=env_data) 21 | return JSONResponse(content=response.model_dump(), status_code=200) 22 | 23 | 24 | async def get_velero_version_handler(): 25 | payload = await get_velero_version_service() 26 | 27 | response = SuccessfulRequest(payload=payload) 28 | return JSONResponse(content=response.model_dump(), status_code=200) 29 | 30 | 31 | async def get_velero_pods_handler(): 32 | label_selectors_by_type = { 33 | "velero": "name=velero", 34 | "node-agent": "name=node-agent" 35 | } 36 | payload = await get_pods_service(label_selectors_by_type=label_selectors_by_type, 37 | namespace=config_app.k8s.velero_namespace) 38 | 39 | response = SuccessfulRequest(payload=payload) 40 | return JSONResponse(content=response.model_dump(), status_code=200) 41 | 42 | 43 | async def get_vui_pods_handler(): 44 | label_selectors_by_type = { 45 | "CORE": "component=core", 46 | "API": "component=api", 47 | "BROKER": "component=broker", 48 | "UI": "component=ui", 49 | "WATCHDOG": "component=watchdog", 50 | } 51 | payload = await get_pods_service(label_selectors_by_type, namespace=config_app.k8s.vui_namespace) 52 | 53 | response = SuccessfulRequest(payload=payload) 54 | return JSONResponse(content=response.model_dump(), status_code=200) 55 | -------------------------------------------------------------------------------- /src/controllers/requests.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from schemas.request.delete_resource import DeleteResourceRequestSchema 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | from schemas.response.successful_restores import SuccessfulRestoreResponse 6 | from service.requests import (get_server_status_requests_service, 7 | get_download_requests_service, 8 | get_delete_backup_requests_service, 9 | delete_download_requests_service, 10 | delete_delete_download_requests_service, 11 | delete_server_status_requests_service) 12 | 13 | 14 | async def get_server_status_requests_handler(): 15 | payload = await get_server_status_requests_service() 16 | 17 | response = SuccessfulRestoreResponse(payload=payload) 18 | return JSONResponse(content=response.model_dump(), status_code=200) 19 | 20 | 21 | async def get_download_requests_handler(): 22 | payload = await get_download_requests_service() 23 | 24 | response = SuccessfulRequest(payload=payload) 25 | return JSONResponse(content=response.model_dump(), status_code=201) 26 | 27 | 28 | async def get_delete_backup_requests_handler(): 29 | payload = await get_delete_backup_requests_service() 30 | 31 | response = SuccessfulRequest(payload=payload) 32 | return JSONResponse(content=response.model_dump(), status_code=200) 33 | 34 | async def delete_download_request_handler(request: DeleteResourceRequestSchema): 35 | payload = await delete_download_requests_service(request) 36 | 37 | response = SuccessfulRequest(payload=payload) 38 | return JSONResponse(content=response.model_dump(), status_code=200) 39 | 40 | async def delete_delete_backup_request_handler(request: DeleteResourceRequestSchema): 41 | payload = await delete_delete_download_requests_service(request) 42 | 43 | response = SuccessfulRequest(payload=payload) 44 | return JSONResponse(content=response.model_dump(), status_code=200) 45 | 46 | async def delete_server_status_request_handler(request: DeleteResourceRequestSchema): 47 | payload = await delete_server_status_requests_service(request) 48 | 49 | response = SuccessfulRequest(payload=payload) 50 | return JSONResponse(content=response.model_dump(), status_code=200) -------------------------------------------------------------------------------- /tests/test_connection.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from datetime import datetime 3 | 4 | import requests 5 | import json 6 | 7 | prefix = "api/v1/" 8 | backend_url = "http://127.0.0.1:8001/" 9 | login_data = {'username': 'admin', 'password': 'admin'} 10 | 11 | print(f"connect to:{backend_url}") 12 | print(f"request session") 13 | session = requests.Session() 14 | response = session.post(backend_url + prefix + "token", login_data) 15 | response = json.loads(response.content.decode('utf-8')) 16 | print(f"response: {response}") 17 | if 'access_token' in response: 18 | session.headers.update({"Authorization": 'Bearer ' + response['access_token']}) 19 | else: 20 | print(f"error") 21 | # try to read the information about user logged 22 | a = session.get(backend_url + prefix + "users/me/info") 23 | response2 = json.loads(a.content.decode('utf-8')) 24 | print(f"me:{response2}") 25 | 26 | # try to read the info endpoint (no login is required) 27 | b = session.get(backend_url + "api/v1/info") 28 | response3 = json.loads(b.content.decode('utf-8')) 29 | print(f"info:{response3}") 30 | nLoop = 0 31 | while True: 32 | nLoop += 1 33 | b = session.get(backend_url + prefix + "users/me/info") 34 | response3 = json.loads(b.content.decode('utf-8')) 35 | auth = False 36 | print(f"===>{b.text}") 37 | print(f"===>{b.headers}") 38 | print(f"===>{b.content}") 39 | print(f"===>{b.ok}") 40 | print(f"===>{b.reason}") 41 | print(f"===>{b.status_code}") 42 | 43 | print(f"[{nLoop}] {datetime.now()} {response3} {b} {auth}") 44 | 45 | if b is not None: 46 | if b.status_code == 200: 47 | auth = True 48 | elif b.status_code == 401: 49 | print(f"Try to renew the token") 50 | response = session.post(backend_url + prefix + "token", login_data) 51 | print(f"response:{response}") 52 | response4 = json.loads(response.content.decode('utf-8')) 53 | print(f"response decode=>:{response4}") 54 | if 'access_token' in response4: 55 | session.headers.update({"Authorization": 'Bearer ' + response4['access_token']}) 56 | else: 57 | print(f"error") 58 | 59 | sleep(10) 60 | 61 | c = session.get(backend_url + "api/v1/utils/health/") 62 | response4 = json.loads(c.content.decode('utf-8')) 63 | print(f"[{nLoop}] {datetime.now()} {response4} {c}") 64 | print(f"*****") 65 | sleep(10) 66 | -------------------------------------------------------------------------------- /src/.env.template: -------------------------------------------------------------------------------- 1 | CONTAINER_MODE=False 2 | DEBUG_LEVEL=info 3 | K8S_IN_CLUSTER_MODE=False 4 | K8S_VELERO_NAMESPACE=velero 5 | K8S_VELERO_UI_NAMESPACE=velero-ui 6 | ORIGINS_1=http://localhost:3000 7 | ORIGINS_2=http://127.0.0.1:3000 8 | ORIGINS_3=http://10.10.0.10 9 | # ORIGINS_4=* 10 | API_ENDPOINT_URL=0.0.0.0 11 | API_ENDPOINT_PORT=8001 12 | VELERO_CLI_VERSION=v1.12.2 13 | VELERO_CLI_PATH=../velero-client 14 | VELERO_CLI_PATH_CUSTOM=./velero-client-binary 15 | VELERO_CLI_DEST_PATH=/usr/local/bin 16 | API_ENABLE_DOCUMENTATION=1 17 | API_TOKEN_EXPIRATION_MIN=60 18 | API_TOKEN_REFRESH_EXPIRATION_DAYS=7 19 | SECURITY_PATH_DATABASE=./data 20 | SECURITY_TOKEN_KEY= 21 | SECURITY_REFRESH_TOKEN_KEY= 22 | SECURITY_DISABLE_USERS_PWD_RATE=1 23 | API_RATE_LIMITER_L1=60:20 24 | API_RATE_LIMITER_CUSTOM_1=Security:xxx:60:20 25 | DOWNLOAD_TMP_FOLDER=/tmp/velero-api 26 | DEFAULT_ADMIN_USERNAME=admin 27 | DEFAULT_ADMIN_PASSWORD=admin 28 | 29 | # RESTIC 30 | RESTIC_PASSWORD=static-passw0rd 31 | 32 | # AUTHENTICATION 33 | # AUTH_ENABLED=False 34 | # AUTH_TYPE=LDAP # BUILT-IN or LDAP, BUILT-IN is default 35 | # LDAP_URI=ldaps://ldap.example.com:636 36 | # LDAP_USE_SSL=True 37 | # LDAP_BASE_DN=dc=example,dc=com 38 | 39 | # LDAP SERVICE ACCOUNT: is recommended to be read-only 40 | # LDAP_BIND_DN=cn=admin-readonly,dc=example,dc=com 41 | # LDAP_BIND_PASSWORD= 42 | # LDAP_USER_SEARCH_FILTER=(&(objectClass=person)(uid={username})) 43 | 44 | # Authorization disabled, access allowed to all authenticated users 45 | # LDAP_AUTHZ_ENABLED=True 46 | # LDAP_AUTHZ_STRATEGY=GROUP # GROUP or ATTRIBUTE 47 | 48 | # AUTHZ through group membership 49 | # LDAP_AUTHZ_BASE_DN=cn=admins,ou=groups,dc=example,dc=com 50 | # LDAP_AUTHZ_FILTER=(&(objectClass=groupOfNames)(member={user_dn})) 51 | 52 | # AUTHZ through attribute value 53 | # LDAP_AUTHZ_ATTRIBUTE=department 54 | # LDAP_AUTHZ_VALUE=IT 55 | 56 | NATS_ENABLE=false 57 | # NATS_ENDPOINT_URL=127.0.0.1 58 | # NATS_PORT_CLIENT=4222 59 | # NATS_USERNAME=user 60 | # NATS_PASSWORD=password 61 | # NATS_ALIVE_SEC=60 62 | # NATS_REQUEST_TIMEOUT_SEC=2 63 | # NATS_RETRY_REG_SEC=30 64 | # NATS_RETRY_CONN_SEC=20 65 | # NATS_PORT_MONITORING=30 66 | # NATS_CRON_UPDATE_K8S_HEALTH=300 67 | # NATS_CRON_UPDATE_STATS_GET=300 68 | # NATS_CRON_UPDATE_BACKUP=300 69 | # NATS_CRON_UPDATE_RESTORE=300 70 | # NATS_CRON_UPDATE_BACKUP_LOCATION=300 71 | # NATS_CRON_UPDATE_STORAGE_LOCATION=300 72 | # NATS_CRON_UPDATE_REPOSITORIES=300 73 | # NATS_CRON_UPDATE_SC_MAPPING=300 -------------------------------------------------------------------------------- /src/constants/resources.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import Dict 4 | 5 | 6 | @dataclass(frozen=True) 7 | class Resource: 8 | name: str 9 | kind: str 10 | plural: str 11 | 12 | 13 | class ResourcesNames(Enum): 14 | BACKUP = "BACKUP" 15 | RESTORE = "RESTORE" 16 | SCHEDULE = "SCHEDULE" 17 | DOWNLOAD_REQUEST = "DOWNLOAD_REQUEST" 18 | DELETE_BACKUP_REQUEST = "DELETE_BACKUP_REQUEST" 19 | POD_VOLUME_BACKUP = "POD_VOLUME_BACKUP" 20 | POD_VOLUME_RESTORE = "POD_VOLUME_RESTORE" 21 | BACKUP_REPOSITORY = "BACKUP_REPOSITORY" 22 | RESTIC_REPOSITORY = "RESTIC_REPOSITORY" 23 | BACKUP_STORAGE_LOCATION = "BACKUP_STORAGE_LOCATION" 24 | VOLUME_SNAPSHOT_LOCATION = "VOLUME_SNAPSHOT_LOCATION" 25 | SERVER_STATUS_REQUEST = "SERVER_STATUS_REQUEST" 26 | 27 | 28 | PLURALS = [ 29 | "backups", 30 | "restores", 31 | "schedules", 32 | "downloadrequests", 33 | "deletebackuprequests", 34 | "podvolumebackups", 35 | "podvolumerestores", 36 | "resticrepositories", 37 | "backuprepositories", 38 | "backupstoragelocations", 39 | "volumesnapshotlocations", 40 | "serverstatusrequests", 41 | ] 42 | 43 | RESOURCES: Dict[ResourcesNames, Resource] = { 44 | ResourcesNames.BACKUP: Resource("Backup", "Backup", "backups"), 45 | ResourcesNames.RESTORE: Resource("Restore", "Restore", "restores"), 46 | ResourcesNames.SCHEDULE: Resource("Schedule", "Schedule", "schedules"), 47 | ResourcesNames.DOWNLOAD_REQUEST: Resource("Download Request", "DownloadRequest", "downloadrequests"), 48 | ResourcesNames.DELETE_BACKUP_REQUEST: Resource("Delete Backup Request", "DeleteBackupRequest", "deletebackuprequests"), 49 | ResourcesNames.POD_VOLUME_BACKUP: Resource("Pod Volume Backup", "PodVolumeBackup", "podvolumebackups"), 50 | ResourcesNames.POD_VOLUME_RESTORE: Resource("Pod Volume Restore", "PodVolumeRestore", "podvolumerestores"), 51 | ResourcesNames.RESTIC_REPOSITORY: Resource("Restic Repository", "ResticRepository", "resticrepositories"), 52 | ResourcesNames.BACKUP_REPOSITORY: Resource("Backup Repository", "BackupRepository", "backuprepositories"), 53 | ResourcesNames.BACKUP_STORAGE_LOCATION: Resource("Backup Storage Location", "BackupStorageLocation", "backupstoragelocations"), 54 | ResourcesNames.VOLUME_SNAPSHOT_LOCATION: Resource("Volume Snapshot Location", "VolumeSnapshotLocation", "volumesnapshotlocations"), 55 | ResourcesNames.SERVER_STATUS_REQUEST: Resource("Server Status Request", "ServerStatusRequest", "serverstatusrequests"), 56 | } 57 | -------------------------------------------------------------------------------- /src/service/logs.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | 3 | import requests 4 | import gzip 5 | import tempfile 6 | 7 | from schemas.velero_log import VeleroLog 8 | from service.utils.download_request import create_download_request 9 | from service.utils.cleanup_requests import cleanup_server_request 10 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 11 | 12 | VELERO_LOG_TYPES = { 13 | "backup": "BackupLog", 14 | "restore": "RestoreLog" 15 | } 16 | 17 | ACCEPTED_MIME_TYPES = [ 18 | "application/gzip", 19 | "binary/octet-stream", 20 | "application/octet-stream" 21 | ] 22 | 23 | @trace_k8s_async_method(description="Get velero resource logs") 24 | async def get_velero_logs_service(resource_name: str, resource_type: str) -> VeleroLog: 25 | """Retrieve logs from a Velero resource (Backup, Restore, etc.) using DownloadRequest""" 26 | if resource_type not in VELERO_LOG_TYPES: 27 | raise HTTPException(status_code=400, detail=f"Unsupported resource type: {resource_type}") 28 | 29 | log_kind = VELERO_LOG_TYPES[resource_type] 30 | 31 | try: 32 | # Creation of the DownloadRequest or retrieval of the URL if already available 33 | log_url = await create_download_request(resource_name, log_kind) 34 | if not log_url: 35 | raise HTTPException(status_code=408, detail=f"Unable to retrieve log download URL") 36 | 37 | # Download and extract logs 38 | logs = await _download_and_extract_logs(log_url) 39 | 40 | # DownloadRequest cleanup to avoid buildup 41 | # cleanup_download_request(resource_name) 42 | return logs 43 | 44 | except Exception as e: 45 | raise HTTPException(status_code=400, detail=f"Error {str(e)}") 46 | 47 | 48 | async def _download_and_extract_logs(log_url: str) -> VeleroLog: 49 | """Download the .gz file, check the format, and return logs""" 50 | try: 51 | response = requests.get(log_url, stream=True) 52 | if response.status_code != 200: 53 | raise HTTPException(status_code=400, detail=f"Download error: {response.status_code}") 54 | 55 | # Check the type of content 56 | mime_type = response.headers.get("Content-Type", "").split(";")[0] 57 | if mime_type not in ACCEPTED_MIME_TYPES: 58 | raise HTTPException(status_code=400, detail=f"Invalid response: Unsupported mime type '{mime_type}'") 59 | 60 | with tempfile.NamedTemporaryFile(delete=True) as temp_file: 61 | temp_file.write(response.content) 62 | temp_file.flush() 63 | 64 | # Open the .gz file and read the contents 65 | with gzip.open(temp_file.name, "rb") as gz_file: 66 | log_content = gz_file.read().decode("utf-8").split("\n") 67 | 68 | return VeleroLog(success=True, logs=log_content) 69 | 70 | except Exception as e: 71 | raise HTTPException(status_code=408, detail=f"Request Timeout {str(e)}") 72 | -------------------------------------------------------------------------------- /src/controllers/bsl.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from schemas.request.update_bsl import UpdateBslRequestSchema 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | from vui_common.schemas.notification import Notification 6 | from schemas.request.create_bsl import CreateBslRequestSchema 7 | from schemas.request.default_bsl import DefaultBslRequestSchema 8 | 9 | from service.bsl import (remove_default_bsl_service, 10 | set_default_bsl_service, 11 | create_bsl_service, 12 | delete_bsl_service, 13 | get_bsls_service, 14 | update_bsl_service) 15 | 16 | 17 | async def get_bsls_handler(): 18 | payload = await get_bsls_service() 19 | 20 | response = SuccessfulRequest(payload=payload) 21 | return JSONResponse(content=response.model_dump(), status_code=200) 22 | 23 | 24 | async def create_bsl_handler(bsl: CreateBslRequestSchema): 25 | payload = await create_bsl_service(bsl) 26 | 27 | msg = Notification(title='Create bsl', description=f"BSL {bsl.name} created!", type_='INFO') 28 | response = SuccessfulRequest(notifications=[msg], payload=payload) 29 | return JSONResponse(content=response.model_dump(), status_code=201) 30 | 31 | 32 | async def set_default_bsl_handler(default_bsl: DefaultBslRequestSchema): 33 | payload = await set_default_bsl_service(default_bsl.name) 34 | 35 | msg = Notification(title='Default bsl', description=f"BSL {default_bsl.name} set as default!", type_='INFO') 36 | response = SuccessfulRequest(notifications=[msg], payload=payload) 37 | return JSONResponse(content=response.model_dump(), status_code=201) 38 | 39 | 40 | async def set_remove_default_bsl_handler(default_bsl: DefaultBslRequestSchema): 41 | payload = await remove_default_bsl_service(default_bsl.name) 42 | 43 | msg = Notification(title='Default bsl', description=f"BSL {default_bsl.name} removed as default!", type_='INFO') 44 | response = SuccessfulRequest(notifications=[msg], payload=payload) 45 | return JSONResponse(content=response.model_dump(), status_code=201) 46 | 47 | 48 | async def delete_bsl_handler(bsl_name: str): 49 | payload = await delete_bsl_service(bsl_name=bsl_name) 50 | 51 | msg = Notification(title='Delete bsl', 52 | description=f'Bsl {bsl_name} deleted request done!', 53 | type_='INFO') 54 | 55 | response = SuccessfulRequest(payload=payload) 56 | response.notifications = [msg] 57 | return JSONResponse(content=response.model_dump(), status_code=200) 58 | 59 | 60 | async def update_bsl_handler(bsl: UpdateBslRequestSchema): 61 | payload = await update_bsl_service(bsl_data=bsl) 62 | 63 | msg = Notification(title='Bsl', 64 | description=f"Bsl '{bsl.name}' successfully updated.", 65 | type_='INFO') 66 | 67 | response = SuccessfulRequest(payload=payload, notifications=[msg]) 68 | return JSONResponse(content=response.model_dump(), status_code=200) 69 | -------------------------------------------------------------------------------- /src/service/backup_storage_class.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from typing import List, Dict 4 | 5 | from fastapi import HTTPException 6 | 7 | from service.utils.download_request import (create_download_request, 8 | download_and_extract_backup) 9 | 10 | from schemas.velero_storage_class import VeleroStorageClass 11 | 12 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 13 | 14 | 15 | @trace_k8s_async_method(description="Get backup storage classes") 16 | async def get_backup_storage_classes_service(backup_name: str) -> VeleroStorageClass: 17 | """ 18 | Retrieve the StorageClasses used in a Velero backup using a DownloadRequest. 19 | """ 20 | 21 | # Create a DownloadRequest to retrieve backup data 22 | download_url = await create_download_request(backup_name, "BackupContents") 23 | if not download_url: 24 | raise HTTPException(status_code=400, detail=f"Create a DownloadRequest to retrieve backup data") 25 | 26 | # Download and extract the file containing the Kubernetes manifest 27 | extracted_path = await download_and_extract_backup(download_url) 28 | if not extracted_path: 29 | raise HTTPException(status_code=400, detail=f"Error while extracting backup '{backup_name}'") 30 | 31 | # Extracting StorageClasses from PVCs 32 | storage_classes = await _extract_storage_classes_from_pvc_service(extracted_path) 33 | 34 | # Cleaning the DownloadRequest after use 35 | # cleanup_download_request(backup_name) 36 | 37 | if storage_classes: 38 | return VeleroStorageClass(storage_classes=storage_classes) 39 | 40 | return VeleroStorageClass(storage_classes=[]) 41 | 42 | 43 | @trace_k8s_async_method(description="Extract storage classes from pvc") 44 | async def _extract_storage_classes_from_pvc_service(extracted_path: str) -> List[Dict]: 45 | """ 46 | Extracts StorageClasses from the manifest of PersistentVolumeClaims (PVCs). 47 | """ 48 | storage_classes = [] 49 | 50 | pvc_path = os.path.join(extracted_path, "resources", "persistentvolumeclaims", "namespaces") 51 | if not os.path.exists(pvc_path): 52 | return [] 53 | 54 | for namespace in os.listdir(pvc_path): 55 | namespace_path = os.path.join(pvc_path, namespace) 56 | if not os.path.isdir(namespace_path): 57 | continue 58 | 59 | for pvc_file in os.listdir(namespace_path): 60 | pvc_file_path = os.path.join(namespace_path, pvc_file) 61 | try: 62 | with open(pvc_file_path, "r") as f: 63 | pvc_data = json.load(f) 64 | 65 | if "spec" in pvc_data and "storageClassName" in pvc_data["spec"]: 66 | # print(pvc_data) 67 | storage_classes.append( 68 | {"name": pvc_data["metadata"]["name"], "storageClass": pvc_data["spec"]["storageClassName"]}) 69 | 70 | except Exception as e: 71 | raise HTTPException(status_code=400, detail=f"Error reading PVC {pvc_file_path}: {e}") 72 | 73 | return storage_classes 74 | -------------------------------------------------------------------------------- /src/models/k8s/restore.py: -------------------------------------------------------------------------------- 1 | from typing import Literal, Optional, List, Dict, Any 2 | from pydantic import BaseModel 3 | 4 | 5 | # RESTORE STATUS 6 | class RestoreStatus(str): 7 | NEW = "New" 8 | FAILED_VALIDATION = "FailedValidation" 9 | IN_PROGRESS = "InProgress" 10 | WAITING_FOR_PLUGIN_OPERATIONS = "WaitingForPluginOperations" 11 | WAITING_FOR_PLUGIN_OPERATIONS_PARTIALLY_FAILED = "WaitingForPluginOperationsPartiallyFailed" 12 | COMPLETED = "Completed" 13 | PARTIALLY_FAILED = "PartiallyFailed" 14 | FAILED = "Failed" 15 | 16 | 17 | # UPLOADER 18 | class UploaderConfig(BaseModel): 19 | writeSparseFiles: Optional[bool] = True 20 | parallelFilesDownload: Optional[int] = 10 21 | 22 | 23 | # RESTORE SPEC 24 | class RestoreSpec(BaseModel): 25 | backupName: Optional[str] = None 26 | scheduleName: Optional[str] = None 27 | itemOperationTimeout: Optional[str] = "4h" 28 | uploaderConfig: Optional[UploaderConfig] = None 29 | includedNamespaces: Optional[List[str]] = None 30 | excludedNamespaces: Optional[List[str]] = None 31 | includedResources: Optional[List[str]] = None 32 | excludedResources: Optional[List[str]] = None 33 | restoreStatus: Optional[Dict[str, List[str]]] = None 34 | includeClusterResources: Optional[bool] = None 35 | labelSelector: Optional[Dict[str, Dict[str, str]]] = None 36 | orLabelSelectors: Optional[List[Dict[str, Dict[str, str]]]] = None 37 | namespaceMapping: Optional[Dict[str, str]] = None 38 | restorePVs: Optional[bool] = None 39 | preserveNodePorts: Optional[bool] = None 40 | existingResourcePolicy: Optional[str] = None 41 | resourceModifier: Optional[Dict[str, str]] = None 42 | hooks: Optional[Dict[str, List[Dict[str, str]]]] = None 43 | 44 | 45 | # RESTORE STATUS 46 | class RestoreStatusDetails(BaseModel): 47 | phase: Optional[Literal[ 48 | "New", "FailedValidation", "InProgress", "WaitingForPluginOperations", 49 | "WaitingForPluginOperationsPartiallyFailed", "Completed", "PartiallyFailed", "Failed" 50 | ]] = None 51 | validationErrors: Optional[List[str]] = None 52 | restoreItemOperationsAttempted: Optional[int] = 0 53 | restoreItemOperationsCompleted: Optional[int] = 0 54 | restoreItemOperationsFailed: Optional[int] = 0 55 | warnings: Optional[int] = 0 56 | errors: Optional[int] = 0 57 | failureReason: Optional[str] = None 58 | 59 | class Config: 60 | extra = "allow" 61 | 62 | # RESTORE METADATA 63 | class RestoreMetadata(BaseModel): 64 | name: str 65 | namespace: str 66 | uid: Optional[str] = None # for read 67 | resourceVersion: Optional[str] = None # for read 68 | generation: Optional[int] = None # for read 69 | creationTimestamp: Optional[str] = None # for read 70 | labels: Optional[Dict[str, str]] = None # for read 71 | annotations: Optional[Dict[str, str]] = None # for read 72 | managedFields: Optional[List[Dict[str, Any]]] = None # for read 73 | 74 | class Config: 75 | extra = "allow" 76 | 77 | # RESTORE RESPONSE 78 | class RestoreResponseSchema(BaseModel): 79 | apiVersion: str = "velero.io/v1" 80 | kind: str = "Restore" 81 | metadata: RestoreMetadata 82 | spec: Optional[RestoreSpec] = None 83 | status: Optional[RestoreStatusDetails] = None 84 | -------------------------------------------------------------------------------- /src/controllers/schedule.py: -------------------------------------------------------------------------------- 1 | # import json 2 | 3 | from fastapi.responses import JSONResponse 4 | 5 | from vui_common.schemas.response.successful_request import SuccessfulRequest 6 | from vui_common.schemas.notification import Notification 7 | from schemas.request.create_schedule import CreateScheduleRequestSchema 8 | from schemas.request.update_schedule import UpdateScheduleRequestSchema 9 | from schemas.response.successful_schedules import SuccessfulScheduleResponse 10 | 11 | from service.schedule import (get_schedules_service, 12 | delete_schedule_service, 13 | create_schedule_service, 14 | pause_schedule_service, 15 | update_schedule_service) 16 | from vui_common.logger.logger_proxy import logger 17 | 18 | 19 | async def get_schedules_handler(): 20 | payload = await get_schedules_service() 21 | 22 | response = SuccessfulScheduleResponse(payload=payload) 23 | return JSONResponse(content=response.model_dump(), status_code=200) 24 | 25 | 26 | async def pause_schedule_handler(schedule: str): 27 | logger.info(f"Sert schedule pause=True") 28 | payload = await pause_schedule_service(schedule) 29 | msg = Notification(title='Pause schedule', 30 | description=f"Schedule {schedule} pause request done!", 31 | type_='INFO') 32 | response = SuccessfulRequest(notifications=[msg], payload=payload) 33 | return JSONResponse(content=response.model_dump(), status_code=200) 34 | 35 | 36 | async def unpause_schedule_handler(schedule: str): 37 | payload = await pause_schedule_service(schedule_name=schedule, paused=False) 38 | 39 | msg = Notification(title='Pause schedule', 40 | description=f"Schedule {schedule} start request done!", 41 | type_='INFO') 42 | response = SuccessfulRequest(notifications=[msg], payload=payload) 43 | return JSONResponse(content=response.model_dump(), status_code=200) 44 | 45 | 46 | async def create_schedule_handler(info: CreateScheduleRequestSchema): 47 | payload = await create_schedule_service(schedule_data=info) 48 | 49 | msg = Notification(title='Create schedule', 50 | description=f"Schedule {info.name} created!", 51 | type_='INFO') 52 | response = SuccessfulRequest(notifications=[msg], payload=payload) 53 | return JSONResponse(content=response.model_dump(), status_code=200) 54 | 55 | 56 | async def delete_schedule_handler(schedule_name: str): 57 | payload = await delete_schedule_service(schedule_name=schedule_name) 58 | 59 | msg = Notification(title='Delete schedule', 60 | description=f"Schedule {schedule_name} deleted request done!", 61 | type_='INFO') 62 | response = SuccessfulRequest(notifications=[msg], payload=payload) 63 | return JSONResponse(content=response.model_dump(), status_code=200) 64 | 65 | 66 | async def update_schedule_handler(schedule: UpdateScheduleRequestSchema): 67 | payload = await update_schedule_service(schedule_data=schedule) 68 | 69 | msg = Notification(title='Schedule', 70 | description=f"Schedule '{schedule.name}' successfully updated.", 71 | type_='INFO') 72 | response = SuccessfulRequest(notifications=[msg], payload=payload) 73 | return JSONResponse(content=response.model_dump(), status_code=200) 74 | -------------------------------------------------------------------------------- /src/api/v1/api_v1.py: -------------------------------------------------------------------------------- 1 | from fastapi import Depends, APIRouter 2 | 3 | from api.v1.routers import (sc_mapping, 4 | repo, 5 | restore, 6 | stats, 7 | k8s, 8 | vsl, 9 | backup, 10 | schedule, 11 | bsl, 12 | setup, 13 | watchdog, 14 | pvb, 15 | location, 16 | inspect, 17 | requests) 18 | # from vui_common.security.routers import authentication, user 19 | from vui_common.security.authentication.auth_service import get_current_active_user 20 | 21 | from vui_common.configs.config_proxy import config_app 22 | 23 | 24 | v1 = APIRouter() 25 | 26 | if config_app.app.auth_enabled: 27 | 28 | # v1.include_router(authentication.router) 29 | # v1.include_router(user.router, 30 | # dependencies=[Depends(get_current_active_user)]) 31 | 32 | v1.include_router(backup.router, 33 | dependencies=[Depends(get_current_active_user)]) 34 | v1.include_router(restore.router, 35 | dependencies=[Depends(get_current_active_user)]) 36 | v1.include_router(schedule.router, 37 | dependencies=[Depends(get_current_active_user)]) 38 | v1.include_router(setup.router, 39 | dependencies=[Depends(get_current_active_user)]) 40 | v1.include_router(k8s.router, 41 | dependencies=[Depends(get_current_active_user)]) 42 | v1.include_router(repo.router, 43 | dependencies=[Depends(get_current_active_user)]) 44 | v1.include_router(bsl.router, 45 | dependencies=[Depends(get_current_active_user)]) 46 | v1.include_router(vsl.router, 47 | dependencies=[Depends(get_current_active_user)]) 48 | v1.include_router(pvb.router, 49 | dependencies=[Depends(get_current_active_user)]) 50 | v1.include_router(sc_mapping.router, 51 | dependencies=[Depends(get_current_active_user)]) 52 | v1.include_router(stats.router, 53 | dependencies=[Depends(get_current_active_user)]) 54 | v1.include_router(watchdog.router, 55 | dependencies=[Depends(get_current_active_user)]) 56 | v1.include_router(location.router, 57 | dependencies=[Depends(get_current_active_user)]) 58 | v1.include_router(inspect.router, 59 | dependencies=[Depends(get_current_active_user)]) 60 | v1.include_router(requests.router, 61 | dependencies=[Depends(get_current_active_user)]) 62 | 63 | else: 64 | v1.include_router(backup.router) 65 | v1.include_router(restore.router) 66 | v1.include_router(schedule.router) 67 | v1.include_router(setup.router) 68 | v1.include_router(k8s.router) 69 | v1.include_router(repo.router) 70 | v1.include_router(bsl.router) 71 | v1.include_router(vsl.router) 72 | v1.include_router(pvb.router) 73 | v1.include_router(sc_mapping.router) 74 | v1.include_router(stats.router) 75 | v1.include_router(watchdog.router) 76 | v1.include_router(location.router) 77 | v1.include_router(inspect.router) 78 | v1.include_router(requests.router) 79 | -------------------------------------------------------------------------------- /src/service/inspect_download_backup.py: -------------------------------------------------------------------------------- 1 | # from fastapi import HTTPException 2 | from vui_common.configs.config_proxy import config_app 3 | # from service.logs import _download_and_extract_logs 4 | from service.utils.download_request import create_download_request 5 | from service.utils.cleanup_requests import cleanup_server_request 6 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 7 | # import os 8 | # import tempfile 9 | # import requests 10 | # import gzip 11 | # import shutil 12 | # from fastapi import HTTPException 13 | 14 | import os 15 | import requests 16 | import shutil 17 | import tarfile 18 | from fastapi import HTTPException 19 | 20 | 21 | @trace_k8s_async_method(description="Download backup") 22 | async def inspect_download_backup_service(backup_name: str) -> bool: 23 | try: 24 | # Creation of the DownloadRequest or retrieval of the URL if already available 25 | backup_url = await create_download_request(backup_name, 'BackupContents') 26 | if not backup_url: 27 | raise HTTPException(status_code=408, detail=f"Unable to retrieve log download URL") 28 | 29 | # Download and extract logs 30 | await _download_and_extract_contents(backup_name, backup_url) 31 | 32 | # DownloadRequest cleanup to avoid buildup 33 | # cleanup_download_request(backup_name) 34 | return True 35 | 36 | except Exception as e: 37 | raise HTTPException(status_code=400, detail=f"Error {str(e)}") 38 | 39 | 40 | async def _download_and_extract_contents(backup_name: str, log_url: str, 41 | extract_dir: str = config_app.app.inspect_folder) -> str: 42 | """ 43 | Scarica un file .tar.gz, lo estrae in una cartella specifica e restituisce il percorso della cartella estratta. 44 | 45 | :param backup_name: Nome del backup per creare una sottocartella. 46 | :param log_url: URL del file .tar.gz da scaricare. 47 | :param extract_dir: Directory base in cui estrarre i file (di default ../tmp/velero_logs). 48 | :return: Percorso della cartella contenente i file estratti. 49 | """ 50 | try: 51 | # Creazione della cartella specifica per il backup 52 | backup_dir = os.path.join(extract_dir, backup_name) 53 | os.makedirs(backup_dir, exist_ok=True) 54 | 55 | # Percorso dove salvare il file tar.gz 56 | tar_gz_path = os.path.join(backup_dir, f"{backup_name}.tar.gz") 57 | 58 | # Scaricare il file tar.gz 59 | response = requests.get(log_url, stream=True) 60 | if response.status_code != 200: 61 | raise HTTPException(status_code=400, detail=f"Download error: {response.status_code}") 62 | 63 | # Salvare il file tar.gz nella cartella specificata 64 | with open(tar_gz_path, "wb") as file: 65 | shutil.copyfileobj(response.raw, file) 66 | 67 | # Creazione della cartella di estrazione 68 | extracted_dir = os.path.join(backup_dir, "./") 69 | os.makedirs(extracted_dir, exist_ok=True) 70 | 71 | # Estrarre il contenuto del file .tar.gz 72 | with tarfile.open(tar_gz_path, "r:gz") as tar: 73 | tar.extractall(path=extracted_dir) 74 | 75 | # Opzionale: rimuovere il file tar.gz dopo l'estrazione per risparmiare spazio 76 | os.remove(tar_gz_path) 77 | 78 | return extracted_dir 79 | 80 | except Exception as e: 81 | raise HTTPException(status_code=408, detail=f"Request Timeout {str(e)}") 82 | -------------------------------------------------------------------------------- /src/models/k8s/schedule.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, Any, List 2 | from pydantic import BaseModel 3 | from enum import Enum 4 | 5 | # ------------------- SCHEDULE STATUS ------------------- 6 | class SchedulePhase(str, Enum): 7 | NEW = "New" 8 | ENABLED = "Enabled" 9 | FAILED_VALIDATION = "FailedValidation" 10 | 11 | class ScheduleStatus(BaseModel): 12 | phase: Optional[SchedulePhase] = None 13 | lastBackup: Optional[str] = None 14 | validationErrors: Optional[List[str]] = None 15 | 16 | # ------------------- HOOKS & RESOURCE HOOKS ------------------- 17 | class HookExec(BaseModel): 18 | container: Optional[str] = None 19 | command: List[str] 20 | onError: Optional[str] = "Fail" 21 | timeout: Optional[str] = "30s" 22 | 23 | class HookResource(BaseModel): 24 | name: str 25 | includedNamespaces: Optional[List[str]] = None 26 | excludedNamespaces: Optional[List[str]] = None 27 | includedResources: List[str] 28 | excludedResources: Optional[List[str]] = None 29 | labelSelector: Optional[Dict[str, Dict[str, str]]] = None 30 | pre: Optional[List[Dict[str, HookExec]]] = None 31 | post: Optional[List[Dict[str, HookExec]]] = None 32 | 33 | class Hooks(BaseModel): 34 | resources: Optional[List[HookResource]] = None 35 | 36 | # ------------------- BACKUP TEMPLATE ------------------- 37 | class BackupTemplate(BaseModel): 38 | csiSnapshotTimeout: Optional[str] = "10m" 39 | resourcePolicy: Optional[Dict[str, str]] = None 40 | includedNamespaces: Optional[List[str]] = None 41 | excludedNamespaces: Optional[List[str]] = None 42 | includedResources: Optional[List[str]] = None 43 | excludedResources: Optional[List[str]] = None 44 | orderedResources: Optional[Dict[str, List[str]]] = None 45 | includeClusterResources: Optional[bool] = None 46 | excludedClusterScopedResources: Optional[List[str]] = None 47 | includedClusterScopedResources: Optional[List[str]] = None 48 | excludedNamespaceScopedResources: Optional[List[str]] = None 49 | includedNamespaceScopedResources: Optional[List[str]] = None 50 | labelSelector: Optional[Dict[str, Dict[str, str]]] = None 51 | orLabelSelectors: Optional[List[Dict[str, Dict[str, str]]]] = None 52 | snapshotVolumes: Optional[bool] = None 53 | storageLocation: Optional[str] = None 54 | volumeSnapshotLocations: Optional[List[str]] = None 55 | ttl: Optional[str] = "24h" 56 | defaultVolumesToFsBackup: Optional[bool] = None 57 | snapshotMoveData: Optional[bool] = None 58 | datamover: Optional[str] = None 59 | uploaderConfig: Optional[Dict[str, int]] = None 60 | metadata: Optional[Dict[str, Any]] = None 61 | hooks: Optional[Hooks] = None 62 | 63 | # ------------------- SCHEDULE SPEC ------------------- 64 | class ScheduleSpec(BaseModel): 65 | paused: Optional[bool] = False 66 | schedule: str 67 | useOwnerReferencesInBackup: Optional[bool] = False 68 | template: BackupTemplate 69 | 70 | # ------------------- METADATA ------------------- 71 | class ScheduleMetadata(BaseModel): 72 | name: str 73 | namespace: str 74 | labels: Optional[Dict[str, str]] = None 75 | annotations: Optional[Dict[str, str]] = None 76 | 77 | class Config: 78 | extra = "allow" 79 | 80 | # ------------------- SCHEDULE RESPONSE ------------------- 81 | class ScheduleResponseSchema(BaseModel): 82 | apiVersion: str = "velero.io/v1" 83 | kind: str = "Schedule" 84 | metadata: ScheduleMetadata 85 | spec: ScheduleSpec 86 | status: Optional[ScheduleStatus] = None 87 | -------------------------------------------------------------------------------- /src/service/location_credentials.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import configparser 3 | import os 4 | 5 | from fastapi import HTTPException 6 | from kubernetes import client 7 | 8 | from vui_common.configs.config_proxy import config_app 9 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 10 | 11 | 12 | @trace_k8s_async_method(description="get s3 credential") 13 | async def get_credential_service(secret_name, secret_key): 14 | api_instance = client.CoreV1Api() 15 | 16 | # LS 2024.20.22 use env variable 17 | # secret = api_instance.read_namespaced_secret(name=secret_name, namespace='velero') 18 | secret = api_instance.read_namespaced_secret(name=secret_name, 19 | namespace=os.getenv('K8S_VELERO_NAMESPACE', 'velero')) 20 | if secret.data and secret_key in secret.data: 21 | value = secret.data[secret_key] 22 | decoded_value = base64.b64decode(value) 23 | payload = _parse_config_string(decoded_value.decode('utf-8')) 24 | return payload 25 | else: 26 | raise HTTPException(status_code=400, detail=f"Secret key not found") 27 | 28 | 29 | @trace_k8s_async_method(description="get default s3 credential") 30 | async def get_default_credential_service(): 31 | label_selector = 'app.kubernetes.io/name=velero' 32 | api_instance = client.CoreV1Api() 33 | 34 | secret = api_instance.list_namespaced_secret(namespace=os.getenv('K8S_VELERO_NAMESPACE', 'velero'), 35 | label_selector=label_selector) 36 | 37 | if secret.items[0].data: 38 | value = secret.items[0].data['cloud'] 39 | decoded_value = base64.b64decode(value) 40 | 41 | payload = _parse_config_string(decoded_value.decode('utf-8')) 42 | 43 | return payload 44 | else: 45 | raise HTTPException(status_code=400, detail=f"Secret key not found") 46 | 47 | 48 | @trace_k8s_async_method(description="create cloud credentials") 49 | async def create_cloud_credentials_secret_service(secret_name: str, secret_key: str, aws_access_key_id: str, 50 | aws_secret_access_key: str): 51 | namespace = config_app.k8s.velero_namespace 52 | 53 | # Base64 content encode 54 | credentials_content = f""" 55 | [default] 56 | aws_access_key_id={aws_access_key_id} 57 | aws_secret_access_key={aws_secret_access_key} 58 | """ 59 | credentials_base64 = base64.b64encode(credentials_content.encode("utf-8")).decode("utf-8") 60 | 61 | # Create Secret 62 | secret = client.V1Secret(metadata=client.V1ObjectMeta(name=secret_name, namespace=namespace), 63 | data={f"""{secret_key}""": credentials_base64}, type="Opaque") 64 | 65 | # API client 4 Secrets 66 | api_instance = client.CoreV1Api() 67 | 68 | try: 69 | # Create Secret 70 | api_instance.create_namespaced_secret(namespace=namespace, body=secret) 71 | print(f"Secret '{secret_name}' create in '{namespace}' namespace.") 72 | return True 73 | 74 | except client.exceptions.ApiException as e: 75 | raise HTTPException(status_code=400, detail=f"Exception when create cloud credentials: {e}") 76 | 77 | 78 | def _parse_config_string(config_string): 79 | # Create a ConfigParser object 80 | config_parser = configparser.ConfigParser() 81 | 82 | # read string 83 | config_parser.read_string(config_string) 84 | 85 | # extract values 86 | aws_access_key_id = config_parser.get('default', 'aws_access_key_id', fallback=None) 87 | aws_secret_access_key = config_parser.get('default', 'aws_secret_access_key', fallback=None) 88 | 89 | # crete dict 90 | result = {'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key} 91 | 92 | return result 93 | -------------------------------------------------------------------------------- /src/integrations/nats_cron_jobs.py: -------------------------------------------------------------------------------- 1 | from vui_common.configs.config_proxy import config_app 2 | from integrations.nats_cron_job import NatsCronJob 3 | from vui_common.logger.logger_proxy import logger 4 | 5 | 6 | class NatsCronJobs: 7 | def __init__(self): 8 | self.jobs = {} 9 | self.__init_default_api() 10 | 11 | def __init_default_api(self): 12 | # logger.debug(f"__init_default_api") 13 | self.add_job(endpoint="/v1/stats", 14 | credential=True, 15 | interval=config_app.nats.cron_get_stats_update) 16 | 17 | self.add_job(endpoint="/health/k8s", 18 | credential=False, 19 | interval=config_app.nats.cron_k8s_health_update) 20 | 21 | self.add_job(endpoint="/v1/backups", 22 | credential=True, 23 | interval=config_app.nats.cron_backup_update) 24 | 25 | self.add_job(endpoint="/v1/restores", 26 | credential=True, 27 | interval=config_app.nats.cron_restore_update) 28 | 29 | self.add_job(endpoint="/v1/schedules", 30 | credential=True, 31 | interval=config_app.nats.cron_schedules_update) 32 | 33 | self.add_job(endpoint="/v1/bsl", 34 | credential=True, 35 | interval=config_app.nats.cron_backup_location_update) 36 | 37 | self.add_job(endpoint="/v1/vsl", 38 | credential=True, 39 | interval=config_app.nats.cron_locations_update) 40 | 41 | self.add_job(endpoint="/v1/repos", 42 | credential=True, 43 | interval=config_app.nats.cron_repository_update) 44 | 45 | self.add_job(endpoint="/v1/sc-mapping", 46 | credential=True, 47 | interval=config_app.nats.cron_storage_classes_mapping_update) 48 | 49 | self.add_job(endpoint="/v1/pod-volume-backups", 50 | credential=True, 51 | interval=config_app.nats.cron_storage_classes_mapping_update) 52 | 53 | self.add_job(endpoint="/v1/pod-volume-restores", 54 | credential=True, 55 | interval=config_app.nats.cron_storage_classes_mapping_update) 56 | 57 | def add_job(self, endpoint: str, credential: bool, interval: int): 58 | if len(endpoint) and interval > 0: 59 | jobs = NatsCronJob(endpoint=endpoint, 60 | credential_required=credential, 61 | interval=interval) 62 | self.jobs[jobs.endpoint] = jobs 63 | logger.debug(f"add_job. jobs added with success, endpoint:{endpoint}") 64 | return True 65 | else: 66 | logger.warning(f"add_job. parameters are invalid. " 67 | f"endpoint:{endpoint} interval:{interval}") 68 | return False 69 | 70 | def get_jobs(self, name: str): 71 | if name in self.jobs: 72 | return self.jobs[name] 73 | else: 74 | raise KeyError(f"No timer found with the name: {name}") 75 | 76 | def add_tick_to_interval(self, interval: int): 77 | # self.print_ls.debug(f"add_tick_to_interval") 78 | for key, job in self.jobs.items(): 79 | job.time_elapsed = interval 80 | 81 | def print_info(self): 82 | # self.print_ls.debug(f"add_tick_to_interval") 83 | if not self.jobs: 84 | logger.debug(f"add_tick_to_interval. No cron job to display.") 85 | else: 86 | for name, job in self.jobs.items(): 87 | logger.debug(f"api: {job.endpoint} " 88 | f"interval sec: {job.interval} " 89 | f"key: {job.ky_key} ") 90 | -------------------------------------------------------------------------------- /src/service/pvb.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | from kubernetes import client 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | 6 | async def get_pod_volume_backups_service(): 7 | # Create an instance of the API client 8 | api_instance = client.CustomObjectsApi() 9 | 10 | # Namespace in which Velero is operating 11 | namespace = config_app.k8s.velero_namespace 12 | 13 | # Get Velero's backup items 14 | group = "velero.io" 15 | version = "v1" 16 | plural = "podvolumebackups" 17 | 18 | try: 19 | # API call to get backups 20 | pvb = api_instance.list_namespaced_custom_object(group, version, namespace, plural) 21 | 22 | return pvb 23 | except Exception as e: 24 | print(f"Error: {str(e)}") 25 | raise HTTPException(status_code=400, detail=f"Error {str(e)}") 26 | 27 | 28 | async def get_pod_volume_backup_details_service(backup_name=None): 29 | # Create an instance of the API client 30 | api_instance = client.CustomObjectsApi() 31 | 32 | # Namespace in which Velero is operating 33 | namespace = config_app.k8s.velero_namespace 34 | 35 | # Group, version and plural to access Velero backups 36 | group = "velero.io" 37 | version = "v1" 38 | plural = "podvolumebackups" 39 | 40 | try: 41 | # API call to get backups 42 | pvb = api_instance.list_namespaced_custom_object(group, version, namespace, plural) 43 | 44 | # Filter objects by label velero.io/backup-uid 45 | filtered_items = [ 46 | item for item in pvb.get('items', []) 47 | if item.get('metadata', {}).get('labels', {}).get('velero.io/backup-name') == backup_name 48 | ] if backup_name else pvb.get('items', []) 49 | 50 | # Return only filtered objects 51 | return filtered_items 52 | except Exception as e: 53 | print(f"Error: {str(e)}") 54 | raise HTTPException(status_code=400, detail=f"Error {str(e)}") 55 | 56 | async def get_pod_volume_restore_service(): 57 | # Create an instance of the API client 58 | api_instance = client.CustomObjectsApi() 59 | 60 | # Namespace in which Velero is operating 61 | namespace = config_app.k8s.velero_namespace 62 | 63 | # Get Velero's backup items 64 | group = "velero.io" 65 | version = "v1" 66 | plural = "podvolumerestores" 67 | 68 | try: 69 | # API call to get backups 70 | pvb = api_instance.list_namespaced_custom_object(group, version, namespace, plural) 71 | 72 | return pvb 73 | except Exception as e: 74 | print(f"Error: {str(e)}") 75 | raise HTTPException(status_code=400, detail=f"Error {str(e)}") 76 | 77 | async def get_pod_volume_restore_details_service(restore_name=None): 78 | # Create an instance of the API client 79 | api_instance = client.CustomObjectsApi() 80 | 81 | # Namespace in which Velero is operating 82 | namespace = config_app.k8s.velero_namespace 83 | 84 | # Group, version and plural to access Velero backups 85 | group = "velero.io" 86 | version = "v1" 87 | plural = "podvolumerestores" 88 | 89 | try: 90 | # API call to get backups 91 | pvb = api_instance.list_namespaced_custom_object(group, version, namespace, plural) 92 | 93 | # Filter objects by label velero.io/backup-uid 94 | filtered_items = [ 95 | item for item in pvb.get('items', []) 96 | if item.get('metadata', {}).get('labels', {}).get('velero.io/restore-name') == restore_name 97 | ] if restore_name else pvb.get('items', []) 98 | 99 | # Return only filtered objects 100 | return filtered_items 101 | except Exception as e: 102 | print(f"Error: {str(e)}") 103 | raise HTTPException(status_code=400, detail=f"Error {str(e)}") 104 | -------------------------------------------------------------------------------- /src/controllers/repo.py: -------------------------------------------------------------------------------- 1 | import os 2 | from fastapi.responses import JSONResponse 3 | 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | from vui_common.schemas.notification import Notification 6 | from vui_common.schemas.message import Message 7 | from schemas.request.unlock_restic_repo import UnlockResticRepoRequestSchema 8 | 9 | from service.bsl import get_bsl_credentials_service 10 | from service.repo import (get_repos_service, 11 | get_repo_backup_size_service, 12 | get_restic_repo_locks_service, 13 | unlock_restic_repo_service, 14 | check_restic_repo_service) 15 | 16 | 17 | async def get_repos_handler(): 18 | payload = await get_repos_service() 19 | 20 | response = SuccessfulRequest(payload=payload) 21 | return JSONResponse(content=response.model_dump(), status_code=200) 22 | 23 | 24 | async def get_backup_size_handler(repository_url: str = None, 25 | backup_storage_location: str = None, 26 | repository_name: str = None, 27 | repository_type: str = None, 28 | volume_namespace: str = None 29 | ): 30 | payload = await get_repo_backup_size_service(repository_url=repository_url, 31 | backup_storage_location=backup_storage_location, 32 | repository_name=repository_name, 33 | repository_type=repository_type, 34 | volume_namespace=volume_namespace) 35 | 36 | response = SuccessfulRequest(payload=payload) 37 | return JSONResponse(content=response.model_dump(), status_code=200) 38 | 39 | 40 | async def get_restic_repo_locks_handler(bsl, repository_url): 41 | aws_access_key_id, aws_secret_access_key = await get_bsl_credentials_service(backup_storage_location=bsl) 42 | env = { 43 | **os.environ, 44 | "AWS_ACCESS_KEY_ID": aws_access_key_id, 45 | "AWS_SECRET_ACCESS_KEY": aws_secret_access_key 46 | } 47 | payload = await get_restic_repo_locks_service(env, repository_url) 48 | 49 | msg = Notification(title='Get locks', 50 | description=f"{len(payload[repository_url])} locks found", 51 | type_='INFO') 52 | 53 | response = SuccessfulRequest(payload=payload, notifications=[msg]) 54 | return JSONResponse(content=response.model_dump(), status_code=200) 55 | 56 | 57 | async def unlock_restic_repo_handler(repo: UnlockResticRepoRequestSchema): 58 | aws_access_key_id, aws_secret_access_key = await get_bsl_credentials_service(backup_storage_location=repo.bsl) 59 | env = { 60 | **os.environ, 61 | "AWS_ACCESS_KEY_ID": aws_access_key_id, 62 | "AWS_SECRET_ACCESS_KEY": aws_secret_access_key 63 | } 64 | payload = await unlock_restic_repo_service(env, repo.bsl, repo.repositoryUrl, repo.removeAll) 65 | 66 | response = SuccessfulRequest(payload=payload) 67 | return JSONResponse(content=response.model_dump(), status_code=200) 68 | 69 | 70 | async def check_restic_repo_handler(bsl, repository_url): 71 | aws_access_key_id, aws_secret_access_key = await get_bsl_credentials_service(backup_storage_location=bsl) 72 | env = { 73 | **os.environ, 74 | "AWS_ACCESS_KEY_ID": aws_access_key_id, 75 | "AWS_SECRET_ACCESS_KEY": aws_secret_access_key 76 | } 77 | payload = await check_restic_repo_service(env, repository_url) 78 | 79 | msg = Message(title=f"Check {repository_url}", 80 | description=payload[repository_url], 81 | type_='INFO') 82 | 83 | response = SuccessfulRequest(messages=[msg]) 84 | return JSONResponse(content=response.model_dump(), status_code=200) 85 | -------------------------------------------------------------------------------- /src/service/inspect.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | # from fastapi import HTTPException 5 | 6 | from vui_common.logger.logger_proxy import logger 7 | 8 | 9 | async def get_folders_list(directory: str): 10 | """ 11 | Returns a list of folders present in a given directory. 12 | 13 | :param directory: Directory path to be parsed 14 | :return: List of folders in the directory 15 | """ 16 | try: 17 | # Get the list of directories in the given path 18 | return [{'name': f} for f in os.listdir(directory) if os.path.isdir(os.path.join(directory, f))] 19 | except FileNotFoundError: 20 | # Handle the case where the directory does not exist 21 | logger.error(f"Error: The directory '{directory}' does not exist.") 22 | # raise HTTPException(status_code=404, detail=f"Error: The directory '{directory}' does not exist.") 23 | return [] 24 | except PermissionError: 25 | # Handle the case where access to the directory is denied 26 | logger.error(f"Error: Permission denied to access '{directory}'.") 27 | return [] 28 | 29 | 30 | async def get_directory_contents(directory: str): 31 | """ 32 | Returns a dictionary containing lists of folders and files in a given directory. 33 | 34 | :param directory: Path of the directory to analyze 35 | :return: Dictionary with 'folders' and 'files' as keys 36 | """ 37 | try: 38 | # Get the list of directories in the given path 39 | folders = [f for f in os.listdir(directory) if os.path.isdir(os.path.join(directory, f))] 40 | 41 | # Get the list of files in the given path 42 | files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] 43 | 44 | return {"folders": folders, "files": files} 45 | except FileNotFoundError: 46 | # Handle the case where the directory does not exist 47 | logger.error(f"Error: The directory '{directory}' does not exist.") 48 | return {"folders": [], "files": []} 49 | except PermissionError: 50 | # Handle the case where access to the directory is denied 51 | logger.error(f"Error: Permission denied to access '{directory}'.") 52 | return {"folders": [], "files": []} 53 | 54 | 55 | async def read_json_file(file_path: str): 56 | """ 57 | Reads the contents of a JSON file and returns the parsed data. 58 | 59 | :param file_path: Path to the JSON file 60 | :return: Parsed JSON data or None if an error occurs 61 | """ 62 | try: 63 | with open(file_path, 'r', encoding='utf-8') as file: 64 | return json.load(file) 65 | except FileNotFoundError: 66 | print(f"Error: The file '{file_path}' does not exist.") 67 | except json.JSONDecodeError: 68 | print(f"Error: The file '{file_path}' is not a valid JSON file.") 69 | except Exception as e: 70 | print(f"Error: An unexpected error occurred - {e}") 71 | return None 72 | 73 | 74 | async def get_recursive_directory_contents(directory: str): 75 | """ 76 | Recursively retrieves the contents of a directory and formats them in the specified structure. 77 | 78 | :param directory: Path of the directory to analyze 79 | :return: List of dictionaries formatted as required 80 | """ 81 | 82 | def build_tree(root_path): 83 | tree = [] 84 | try: 85 | for item in os.listdir(root_path): 86 | item_path = os.path.join(root_path, item) 87 | relative_path = os.path.relpath(item_path, directory).replace("\\", "/") 88 | 89 | if os.path.isdir(item_path): 90 | tree.append({"value": relative_path, "label": item, "children": build_tree(item_path)}) 91 | else: 92 | tree.append({"value": relative_path, "label": item}) 93 | except PermissionError: 94 | print(f"Error: Permission denied to access '{root_path}'.") 95 | return tree 96 | 97 | return build_tree(directory) 98 | -------------------------------------------------------------------------------- /src/models/k8s/backup.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, Any, List 2 | from pydantic import BaseModel 3 | from enum import Enum 4 | 5 | 6 | # BACKUP STATUS 7 | class BackupPhase(str, Enum): 8 | NEW = "New" 9 | FAILED_VALIDATION = "FailedValidation" 10 | IN_PROGRESS = "InProgress" 11 | WAITING_FOR_PLUGIN_OPERATIONS = "WaitingForPluginOperations" 12 | WAITING_FOR_PLUGIN_OPERATIONS_PARTIALLY_FAILED = "WaitingForPluginOperationsPartiallyFailed" 13 | FINALIZING_AFTER_PLUGIN_OPERATIONS = "FinalizingafterPluginOperations" 14 | FINALIZING_PARTIALLY_FAILED = "FinalizingPartiallyFailed" 15 | COMPLETED = "Completed" 16 | PARTIALLY_FAILED = "PartiallyFailed" 17 | FAILED = "Failed" 18 | DELETING = "Deleting" 19 | FINALIZING = "Finalizing" 20 | 21 | class Config: 22 | extra = "allow" 23 | 24 | # BACKUP RESOURCE POLICY 25 | class ResourcePolicy(BaseModel): 26 | kind: str 27 | name: str 28 | 29 | 30 | # LABEL SELECTOR 31 | class LabelSelector(BaseModel): 32 | matchLabels: Optional[Dict[str, str]] = None 33 | 34 | 35 | # BACKUP SPEC 36 | class BackupSpec(BaseModel): 37 | csiSnapshotTimeout: Optional[str] = "10m" 38 | itemOperationTimeout: Optional[str] = "4h" 39 | resourcePolicy: Optional[ResourcePolicy] = None 40 | includedNamespaces: Optional[List[str]] = None 41 | excludedNamespaces: Optional[List[str]] = None 42 | includedResources: Optional[List[str]] = None 43 | excludedResources: Optional[List[str]] = None 44 | orderedResources: Optional[Dict[str, List[str]]] = None 45 | includeClusterResources: Optional[bool] = None 46 | excludedClusterScopedResources: Optional[List[str]] = None 47 | includedClusterScopedResources: Optional[List[str]] = None 48 | excludedNamespaceScopedResources: Optional[List[str]] = None 49 | includedNamespaceScopedResources: Optional[List[str]] = None 50 | labelSelector: Optional[LabelSelector] = None 51 | orLabelSelectors: Optional[List[LabelSelector]] = None 52 | snapshotVolumes: Optional[bool] = None 53 | storageLocation: Optional[str] = None 54 | volumeSnapshotLocations: Optional[List[str]] = None 55 | ttl: Optional[str] = "24h0m0s" 56 | defaultVolumesToFsBackup: Optional[bool] = None 57 | snapshotMoveData: Optional[bool] = None 58 | datamover: Optional[str] = "velero" 59 | uploaderConfig: Optional[Dict[str, int]] = None 60 | hooks: Optional[Dict[str, Any]] = None 61 | 62 | 63 | # BACKUP METADATA 64 | class BackupMetadata(BaseModel): 65 | name: str 66 | namespace: str 67 | uid: Optional[str] = None 68 | resourceVersion: Optional[str] = None 69 | generation: Optional[int] = None 70 | creationTimestamp: Optional[str] = None 71 | labels: Optional[Dict[str, str]] = None 72 | annotations: Optional[Dict[str, str]] = None 73 | managedFields: Optional[List[Dict[str, Any]]] = None 74 | 75 | class Config: 76 | extra = "allow" 77 | 78 | 79 | # BACKUP STATUS 80 | class BackupStatus(BaseModel): 81 | version: Optional[int] = 1 82 | expiration: Optional[str] = None 83 | phase: Optional[BackupPhase] = None 84 | validationErrors: Optional[List[str]] = None 85 | startTimestamp: Optional[str] = None 86 | completionTimestamp: Optional[str] = None 87 | volumeSnapshotsAttempted: Optional[int] = 0 88 | volumeSnapshotsCompleted: Optional[int] = 0 89 | backupItemOperationsAttempted: Optional[int] = 0 90 | backupItemOperationsCompleted: Optional[int] = 0 91 | backupItemOperationsFailed: Optional[int] = 0 92 | warnings: Optional[int] = 0 93 | errors: Optional[int] = 0 94 | failureReason: Optional[str] = None 95 | resourceList: Optional[Dict] = None 96 | 97 | class Config: 98 | extra = "allow" 99 | 100 | 101 | # BACKUP RESPONSE 102 | class BackupResponseSchema(BaseModel): 103 | apiVersion: str = "velero.io/v1" 104 | kind: str = "Backup" 105 | metadata: BackupMetadata 106 | spec: Optional[BackupSpec] = None 107 | status: Optional[BackupStatus] = None 108 | -------------------------------------------------------------------------------- /src/api/v1/routers/stats.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, status, Depends 2 | 3 | from constants.response import common_error_authenticated_response 4 | 5 | from vui_common.security.helpers.rate_limiter import RateLimiter, LimiterRequests 6 | 7 | from vui_common.utils.swagger import route_description 8 | from vui_common.utils.exceptions import handle_exceptions_endpoint 9 | 10 | from vui_common.schemas.response.successful_request import SuccessfulRequest 11 | 12 | from controllers.stats import (get_stats_handler, 13 | get_in_progress_task_handler, 14 | get_schedules_heatmap_handler) 15 | 16 | router = APIRouter() 17 | rate_limiter = RateLimiter() 18 | 19 | 20 | tag_name = 'Statistics' 21 | endpoint_limiter = LimiterRequests(tags=tag_name, 22 | default_key='L1') 23 | 24 | # ------------------------------------------------------------------------------------------------ 25 | # GET VELERO STATS 26 | # ------------------------------------------------------------------------------------------------ 27 | 28 | 29 | limiter = endpoint_limiter.get_limiter_cust('utilis_stats') 30 | route = '/stats' 31 | 32 | 33 | @router.get( 34 | path=route, 35 | tags=[tag_name], 36 | summary='Get backups repository', 37 | description=route_description(tag=tag_name, 38 | route=route, 39 | limiter_calls=limiter.max_request, 40 | limiter_seconds=limiter.seconds), 41 | dependencies=[Depends(RateLimiter(interval_seconds=limiter.seconds, 42 | max_requests=limiter.max_request))], 43 | response_model=SuccessfulRequest, 44 | responses=common_error_authenticated_response, 45 | status_code=status.HTTP_200_OK) 46 | @handle_exceptions_endpoint 47 | async def get_velero_stats(): 48 | return await get_stats_handler() 49 | 50 | 51 | # ------------------------------------------------------------------------------------------------ 52 | # GET TASK IN PROGRESS 53 | # ------------------------------------------------------------------------------------------------ 54 | 55 | 56 | limiter_inprog = endpoint_limiter.get_limiter_cust('utilis_in_progress') 57 | route = '/stats/in-progress' 58 | 59 | 60 | @router.get( 61 | path=route, 62 | tags=[tag_name], 63 | summary='Get operations in progress', 64 | description=route_description(tag=tag_name, 65 | route=route, 66 | limiter_calls=limiter_inprog.max_request, 67 | limiter_seconds=limiter_inprog.seconds), 68 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_inprog.seconds, 69 | max_requests=limiter_inprog.max_request))], 70 | response_model=SuccessfulRequest, 71 | responses=common_error_authenticated_response, 72 | status_code=status.HTTP_200_OK) 73 | @handle_exceptions_endpoint 74 | async def get_in_progress_task(): 75 | return await get_in_progress_task_handler() 76 | 77 | 78 | # ------------------------------------------------------------------------------------------------ 79 | # GET SCHEDULE STATS 80 | # ------------------------------------------------------------------------------------------------ 81 | 82 | 83 | limiter_schedules = endpoint_limiter.get_limiter_cust('stats_schedules') 84 | route = '/stats/schedules' 85 | 86 | 87 | @router.get( 88 | path=route, 89 | tags=[tag_name], 90 | summary='Get schedules stats', 91 | description=route_description(tag=tag_name, 92 | route=route, 93 | limiter_calls=limiter_inprog.max_request, 94 | limiter_seconds=limiter_inprog.seconds), 95 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_inprog.seconds, 96 | max_requests=limiter_inprog.max_request))], 97 | response_model=SuccessfulRequest, 98 | responses=common_error_authenticated_response, 99 | status_code=status.HTTP_200_OK) 100 | #@handle_exceptions_endpoint 101 | async def get_schedules_heatmap(): 102 | return await get_schedules_heatmap_handler() 103 | -------------------------------------------------------------------------------- /src/controllers/watchdog.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | from vui_common.schemas.response.successful_request import SuccessfulRequest 6 | from vui_common.schemas.notification import Notification 7 | from schemas.request.apprise_test_service import AppriseTestServiceRequestSchema 8 | from schemas.request.create_user_service import CreateUserServiceRequestSchema 9 | from schemas.request.update_user_config import UpdateUserConfigRequestSchema 10 | 11 | from service.watchdog import (get_watchdog_version_service, 12 | send_watchdog_report, 13 | get_watchdog_env_services, 14 | get_watchdog_report_cron_service, 15 | send_watchdog_test_notification_service, 16 | restart_watchdog_service, 17 | get_watchdog_user_configs_service, 18 | update_watchdog_user_configs_service, 19 | get_apprise_services, 20 | create_apparise_services, 21 | delete_apprise_services) 22 | 23 | 24 | async def version_handler(): 25 | payload = await get_watchdog_version_service() 26 | 27 | response = SuccessfulRequest(payload=payload) 28 | return JSONResponse(content=response.model_dump(), status_code=200) 29 | 30 | 31 | async def send_report_handler(): 32 | payload = await send_watchdog_report() 33 | 34 | response = SuccessfulRequest(payload=payload) 35 | return JSONResponse(content=response.model_dump(), status_code=200) 36 | 37 | 38 | async def get_env_handler(): 39 | payload = await get_watchdog_env_services() 40 | 41 | response = SuccessfulRequest(payload=payload) 42 | return JSONResponse(content=response.model_dump(), status_code=200) 43 | 44 | 45 | async def get_cron_handler(): 46 | payload = await get_watchdog_report_cron_service(job_name=config_app.watchdog.report_cronjob_name) 47 | 48 | response = SuccessfulRequest(payload=payload) 49 | return JSONResponse(content=response.model_dump(), status_code=200) 50 | 51 | 52 | async def send_test_notification_handler(provider: AppriseTestServiceRequestSchema): 53 | payload = await send_watchdog_test_notification_service(provider.config) 54 | 55 | msg = Notification(title='Send Notification', description=f"Test notification done!", type_='Success') 56 | response = SuccessfulRequest(notifications=[msg], payload=payload) 57 | return JSONResponse(content=response.model_dump(), status_code=200) 58 | 59 | 60 | async def restart_handler(): 61 | payload = await restart_watchdog_service() 62 | 63 | response = SuccessfulRequest(payload=payload) 64 | return JSONResponse(content=response.model_dump(), status_code=200) 65 | 66 | 67 | async def user_configs_handler(): 68 | payload = await get_watchdog_user_configs_service() 69 | secrets = await get_apprise_services() 70 | 71 | payload['APPRISE'] = ';'.join(secrets) 72 | response = SuccessfulRequest(payload=payload) 73 | return JSONResponse(content=response.model_dump(), status_code=200) 74 | 75 | 76 | async def update_user_configs_handler(user_configs: UpdateUserConfigRequestSchema): 77 | payload = await update_watchdog_user_configs_service(user_configs) 78 | 79 | response = SuccessfulRequest(payload=payload) 80 | return JSONResponse(content=response.model_dump(), status_code=200) 81 | 82 | 83 | async def get_apprise_services_handler(): 84 | payload = await get_apprise_services() 85 | 86 | response = SuccessfulRequest(payload=payload) 87 | return JSONResponse(content=response.model_dump(), status_code=200) 88 | 89 | 90 | async def create_apprise_service_handler(service: CreateUserServiceRequestSchema): 91 | payload = await create_apparise_services(service.config) 92 | 93 | msg = Notification(title='Watchdog', 94 | description=f"New service added!", 95 | type_='INFO') 96 | response = SuccessfulRequest(notifications=[msg], payload=payload) 97 | return JSONResponse(content=response.model_dump(), status_code=201) 98 | 99 | 100 | async def delete_apprise_service_handler(service: str): 101 | payload = await delete_apprise_services(service) 102 | 103 | msg = Notification(title='Watchdog', 104 | description=f'Service deleted!', 105 | type_='INFO') 106 | 107 | response = SuccessfulRequest(payload=payload) 108 | response.notifications = [msg] 109 | return JSONResponse(content=response.model_dump(), status_code=200) 110 | -------------------------------------------------------------------------------- /src/service/restore.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from kubernetes import client 4 | 5 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 6 | from constants.velero import VELERO 7 | from constants.resources import RESOURCES, ResourcesNames 8 | from datetime import datetime 9 | from schemas.request.create_restore import CreateRestoreRequestSchema 10 | from models.k8s.restore import RestoreResponseSchema 11 | from vui_common.configs.config_proxy import config_app 12 | 13 | custom_objects = client.CustomObjectsApi() 14 | 15 | 16 | # @trace_k8s_async_method(description="get a restores list") 17 | async def get_restores_service(in_progress: bool = False) -> List[RestoreResponseSchema]: 18 | """Retrieve all Velero schedules""" 19 | 20 | restores = custom_objects.list_namespaced_custom_object( 21 | group=VELERO["GROUP"], 22 | version=VELERO["VERSION"], 23 | namespace=config_app.k8s.velero_namespace, 24 | plural=RESOURCES[ResourcesNames.RESTORE].plural 25 | ) 26 | 27 | filtered_restores = {} 28 | now = datetime.utcnow() 29 | 30 | for item in restores.get("items", []): 31 | metadata = item["metadata"] 32 | status = item.get("status", {}) 33 | phase = status.get("phase", "").lower() 34 | completion_timestamp = status.get("completionTimestamp") 35 | 36 | if in_progress: 37 | has_completion_timestamp = completion_timestamp is not None 38 | diff_in_seconds = None 39 | 40 | if has_completion_timestamp: 41 | datetime_completion_timestamp = datetime.strptime(completion_timestamp, '%Y-%m-%dT%H:%M:%SZ') 42 | diff_in_seconds = (now - datetime_completion_timestamp).total_seconds() 43 | 44 | if not ( 45 | phase.endswith("ing") or 46 | phase == "inprogress" or 47 | (has_completion_timestamp and diff_in_seconds is not None and diff_in_seconds < 180) 48 | ): 49 | continue 50 | 51 | filtered_restores[metadata["uid"]] = item 52 | 53 | restore_list = [RestoreResponseSchema(**item) for item in filtered_restores.values()] 54 | 55 | return restore_list 56 | 57 | 58 | @trace_k8s_async_method(description="Get a restore details") 59 | async def get_restore_details_service(restore_name: str) -> RestoreResponseSchema: 60 | """Retrieve details of a single schedule""" 61 | restore = custom_objects.get_namespaced_custom_object( 62 | group=VELERO["GROUP"], 63 | version=VELERO["VERSION"], 64 | namespace=config_app.k8s.velero_namespace, 65 | plural=RESOURCES[ResourcesNames.RESTORE].plural, 66 | name=restore_name 67 | ) 68 | return RestoreResponseSchema(**restore) 69 | 70 | 71 | @trace_k8s_async_method(description="Create a restore") 72 | async def create_restore_service(restore_data: CreateRestoreRequestSchema): 73 | """Create a Velero restore on Kubernetes""" 74 | spec = restore_data.model_dump(exclude_unset=True) 75 | spec.pop("name", None) 76 | spec.pop("namespace", None) 77 | spec.pop("labelSelector", None) 78 | spec.pop("orLabelSelectors", None) 79 | spec.pop("writeSparseFiles", None) 80 | 81 | restore_body = { 82 | "apiVersion": f"{VELERO['GROUP']}/{VELERO['VERSION']}", 83 | "kind": RESOURCES[ResourcesNames.RESTORE].name, 84 | "metadata": { 85 | "name": restore_data.name, 86 | "namespace": restore_data.namespace 87 | }, 88 | "spec": spec 89 | } 90 | 91 | if restore_data.labelSelector: 92 | restore_body['spec']["labelSelector"] = {'matchLabels': restore_data.labelSelector} 93 | 94 | restore_body['spec']['uploaderConfig'] = { 95 | 'writeSparseFiles': restore_data.writeSparseFiles, 96 | } 97 | 98 | response = custom_objects.create_namespaced_custom_object( 99 | group=VELERO["GROUP"], 100 | version=VELERO["VERSION"], 101 | namespace=restore_data.namespace, 102 | plural=RESOURCES[ResourcesNames.RESTORE].plural, 103 | body=restore_body 104 | ) 105 | 106 | return response 107 | 108 | 109 | @trace_k8s_async_method(description="Delete a restore") 110 | async def delete_restore_service(restore_name: str): 111 | """ 112 | Delete an existing Restore from Kubernetes. 113 | """ 114 | 115 | response = custom_objects.delete_namespaced_custom_object( 116 | group=VELERO["GROUP"], 117 | version=VELERO["VERSION"], 118 | namespace=config_app.k8s.velero_namespace, 119 | plural=RESOURCES[ResourcesNames.RESTORE].plural, 120 | name=restore_name 121 | ) 122 | return response 123 | -------------------------------------------------------------------------------- /src/utils/process.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import asyncio 3 | import json 4 | from fastapi import WebSocketDisconnect 5 | import os 6 | 7 | from vui_common.configs.config_proxy import config_app 8 | from vui_common.contexts.context import current_user_var, cp_user 9 | # from ws.websocket_manager import manager 10 | from integrations import nats_manager_proxy 11 | from vui_common.ws import ws_manager_proxy 12 | 13 | from vui_common.logger.logger_proxy import logger 14 | 15 | # if config_app.nats.enable: 16 | # from integrations.nats_manager import get_nats_manager_instance 17 | 18 | 19 | async def _send_message(message): 20 | try: 21 | logger.debug(message) 22 | # await manager.broadcast(message) 23 | user = None 24 | try: 25 | user = current_user_var.get() 26 | except Exception as Ex: 27 | logger.error(f"send message failed {str(Ex)}") 28 | finally: 29 | if config_app.nats.enable and user.is_nats: 30 | nats_manager = nats_manager_proxy.nat_manager 31 | control_plane_user = cp_user.get() 32 | data = {"user": control_plane_user, "msg": message} 33 | await nats_manager.publish("socket." + config_app.k8s.cluster_id, json.dumps(data).encode()) 34 | pass 35 | elif user is not None: 36 | response = {'type': 'process', 'message': message} 37 | await ws_manager_proxy.ws_manager.send_personal_message(str(user.id), json.dumps(response)) 38 | 39 | except WebSocketDisconnect: 40 | logger.error('send message error') 41 | 42 | 43 | async def run_check_output_process(cmd, publish_message=True, cwd='./', env=None): 44 | output = '' 45 | try: 46 | if publish_message: 47 | await _send_message('check output: ' + ' '.join(cmd)) 48 | 49 | # Starts the secondary process asynchronously 50 | process = await asyncio.create_subprocess_exec(*cmd, 51 | stdout=asyncio.subprocess.PIPE, 52 | stderr=asyncio.subprocess.STDOUT, 53 | cwd=cwd, 54 | env={**os.environ} if not env else env) 55 | 56 | # Wait for the process to complete and capture the output 57 | stdout, stderr = await process.communicate() 58 | 59 | # Check for errors in the output 60 | if stderr: 61 | # example: await publish_message_function(stderr.decode()) 62 | pass 63 | 64 | # Decode the output and return a string 65 | output = stdout.decode('utf-8') 66 | 67 | if output.startswith('An error occurred'): 68 | raise Exception('Error') 69 | 70 | return {'success': True, 'data': output} 71 | except subprocess.CalledProcessError as e: 72 | # print("Error", e) 73 | logger.error("Error" + str(e)) 74 | error = {'success': False, 'error': {'title': 'Run Process Check Output Error', 75 | 'description': str(' '.join(cmd)) + '\n' + str( 76 | e.stderr.decode('utf-8').strip()) 77 | } 78 | } 79 | return error 80 | except Exception as e: 81 | # print("Error", e) 82 | logger.error("Error" + str(e)) 83 | error = {'success': False, 'error': {'title': 'Run Process Check Output Error', 84 | 'description': str(e) + ' \n' + str(output) 85 | } 86 | } 87 | return error 88 | 89 | 90 | # async def run_check_call_process(cmd, publish_message=True): 91 | # try: 92 | # if publish_message: 93 | # await _send_message('check call: ' + ' '.join(cmd)) 94 | # 95 | # # Starts the secondary process asynchronously 96 | # process = await asyncio.create_subprocess_exec(*cmd, 97 | # stdout=asyncio.subprocess.PIPE, 98 | # stderr=asyncio.subprocess.STDOUT, ) 99 | # 100 | # # Wait for the completion of the process 101 | # await process.wait() 102 | # 103 | # return {'success': True} 104 | # except Exception as e: 105 | # error = {'success': False, 'error': {'title': 'Run Process Check Output Error', 106 | # 'description': str(' '.join(cmd)) + '\n' + str(e) 107 | # } 108 | # } 109 | # return error 110 | -------------------------------------------------------------------------------- /src/service/repo.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from fastapi import HTTPException 4 | 5 | from models.k8s.repo import BackupRepositoryResponseSchema 6 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 7 | 8 | from utils.minio_wrapper import MinioInterface 9 | from utils.process import run_check_output_process 10 | 11 | from kubernetes import client 12 | 13 | from vui_common.configs.config_proxy import config_app 14 | from constants.velero import VELERO 15 | from constants.resources import RESOURCES, ResourcesNames 16 | 17 | custom_objects = client.CustomObjectsApi() 18 | 19 | 20 | @trace_k8s_async_method(description="Get repositories list") 21 | async def get_repos_service(): 22 | repos = custom_objects.list_namespaced_custom_object( 23 | group=VELERO["GROUP"], 24 | version=VELERO["VERSION"], 25 | namespace=config_app.k8s.velero_namespace, 26 | plural=RESOURCES[ResourcesNames.BACKUP_REPOSITORY].plural 27 | ) 28 | 29 | bsl_list = [BackupRepositoryResponseSchema(**item) for item in repos.get("items", [])] 30 | 31 | return bsl_list 32 | 33 | 34 | @trace_k8s_async_method(description="Get repository backup size from s3") 35 | async def get_repo_backup_size_service(repository_url: str = None, 36 | backup_storage_location: str = None, 37 | repository_name: str = None, 38 | repository_type: str = None, 39 | volume_namespace: str = None): 40 | minio_interface = MinioInterface() 41 | 42 | # temporary fix url 43 | if repository_type.lower() == 'kopia': 44 | repository_url = repository_url.replace('/restic/', '/kopia/') 45 | 46 | # endpoint match 47 | endpoint_match = re.search(r's3:(http://[^/]+)', repository_url) 48 | if endpoint_match: 49 | endpoint_with_protocol = endpoint_match.group(1) 50 | # remove protocol 51 | endpoint = re.sub(r'https?://', '', endpoint_with_protocol) 52 | print("Endpoint:", endpoint) 53 | else: 54 | endpoint = 'default' 55 | # TODO raise error 56 | print("No endpoint found.") 57 | 58 | # extract bucket name 59 | match = re.search(r's3:http://[^/]+/([^/]+)/', repository_url) 60 | if match: 61 | bucket_name = match.group(1) 62 | # print(bucket_name) 63 | else: 64 | bucket_name = 'default' 65 | # TODO raise error 66 | print("No bucket name found.") 67 | 68 | return await minio_interface.get_backup_size(repository_url=repository_url, 69 | endpoint=endpoint, 70 | backup_storage_location=backup_storage_location, 71 | bucket_name=bucket_name, 72 | repository_name=repository_name, 73 | repository_type=repository_type, 74 | volume_namespace=volume_namespace) 75 | 76 | 77 | @trace_k8s_async_method(description="Get restic repository locks") 78 | async def get_restic_repo_locks_service(env, repository_url): 79 | cmd = ['restic', '-q', '--no-lock', 'list', 'locks', '-r', repository_url] 80 | 81 | output = await run_check_output_process(cmd=cmd, env=env) 82 | 83 | if not output['success']: 84 | raise HTTPException(status_code=400, detail=f'Error get repo locks service') 85 | 86 | locks = output['data'] 87 | 88 | return {str(repository_url): list(filter(None, locks.split('\n')))} 89 | 90 | 91 | @trace_k8s_async_method(description="Unlock restic repository") 92 | async def unlock_restic_repo_service(env, bsl, repository_url, remove_all=False): 93 | cmd = ['restic', 'unlock', '-r', repository_url] 94 | if remove_all: 95 | cmd.append('--remove-all') 96 | 97 | output = await run_check_output_process(cmd=cmd, env=env) 98 | 99 | if not output['success']: 100 | raise HTTPException(status_code=400, detail=f'Error unlock repo locks service') 101 | 102 | locks = output['data'] 103 | 104 | return {'bsl': bsl, 105 | 'repositoryUrl': str(repository_url), 106 | 'locks': list(filter(None, locks.split('\n')))} 107 | 108 | 109 | @trace_k8s_async_method(description="Check restic repository") 110 | async def check_restic_repo_service(env, repository_url): 111 | cmd = ['restic', 'check', '-r', repository_url] 112 | 113 | output = await run_check_output_process(cmd=cmd, env=env) 114 | 115 | if not output['success']: 116 | raise HTTPException(status_code=400, detail=f'Error check repo locks service') 117 | 118 | check = output['data'] 119 | 120 | return {str(repository_url): list(filter(None, check.split('\n')))} 121 | -------------------------------------------------------------------------------- /docker/jenkins-amd64/Jenkins-dev: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | 4 | environment { 5 | // Environment variables 6 | 7 | // Source code repository 8 | evGit_Source_Code = 'https://github.com/seriohub/velero-api' 9 | 10 | // Default docker registry (Global environment variable setting) 11 | evDocker_HUB = "${env.DEFAULT_DOCKER_REGISTRY}" 12 | 13 | // Author name 14 | evDocker_Project = "${env.DOCKER_AUTHOR}" 15 | 16 | // Credential stored in Jenkins Credential for logging in the HUB 17 | evDocker_HUB_Credentials = credentials('DEFAULT_DOCKER_CREDENTIALS') 18 | 19 | evTarget = 'velero-api' 20 | evLatestTag = '' 21 | evCommitChangeset = '' 22 | } 23 | stages { 24 | stage('Clone Git Repository') { 25 | steps { 26 | script { 27 | echo 'Clone Git repository...' 28 | 29 | // Clone the GitHub repository 30 | checkout([$class: "GitSCM", branches: [[name: "*/dev"]], userRemoteConfigs: [[url: evGit_Source_Code]]]) 31 | 32 | // Get commit changes 33 | evCommitChangeset = sh(returnStdout: true, script: "git log -1 --pretty='%s'").trim() 34 | } 35 | } 36 | } 37 | 38 | stage('Build Docker Image') { 39 | steps { 40 | script { 41 | echo 'Build Docker Image...' 42 | 43 | // Path to your Dockerfile within the project directory 44 | def dockerfilePath = 'docker/Dockerfile' 45 | def dockerHubProject = "${evDocker_Project}/${evTarget}" 46 | 47 | GIT_TAG = 'dev' 48 | def now=new Date() 49 | 50 | // Build the Docker image, specifying the Dockerfile path 51 | sh "docker build --build-arg VERSION=dev --build-arg BUILD_DATE=${now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))} --target ${evTarget} -t ${dockerHubProject}:${GIT_TAG} -f ${dockerfilePath} ." 52 | } 53 | } 54 | } 55 | 56 | stage('Push Docker Image to Registry') { 57 | steps { 58 | script { 59 | echo "Docker registry: ${evDocker_HUB}" 60 | def pushTo = '' 61 | if (evDocker_HUB != null) { 62 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 63 | pushTo = "$evDocker_HUB" 64 | } 65 | } 66 | 67 | sh "echo ${evDocker_HUB_Credentials_PSW} | docker login -u ${evDocker_HUB_Credentials_USR} --password-stdin ${pushTo}" 68 | 69 | def dockerImageName = "$evDocker_Project/$evTarget" 70 | 71 | if (evDocker_HUB != null) { 72 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 73 | dockerImageName = "$evDocker_HUB/$evDocker_Project/$evTarget" 74 | } 75 | } 76 | 77 | echo "Docker push to:- ${dockerImageName} -" 78 | sh "docker push ${dockerImageName}:${GIT_TAG}" 79 | // sh "docker push ${dockerImageName}:latest" 80 | sh 'docker logout ' 81 | } 82 | } 83 | } 84 | } 85 | post { 86 | always { 87 | script { 88 | echo 'Post always...' 89 | def pipelineName = currentBuild.fullProjectName 90 | def message = '' 91 | 92 | if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { 93 | message = "Jenkins Pipeline ${pipelineName}: Your developer image version has been successfully pushed to the Docker Registry.\nCommit changes:\n ${evCommitChangeset}" 94 | } else { 95 | message = "Jenkins Pipeline ${pipelineName}: Build failed" 96 | } 97 | 98 | def url = "https://api.telegram.org/bot${env.TELEGRAM_TOKEN}/sendMessage" 99 | def payload = [ 100 | chat_id: "${env.TELEGRAM_CHAT_ID}", 101 | text: message 102 | ] 103 | 104 | def response = httpRequest( 105 | url: url, 106 | httpMode: 'POST', 107 | contentType: 'APPLICATION_JSON', 108 | requestBody: groovy.json.JsonOutput.toJson(payload) 109 | ) 110 | 111 | if (response.status == 200) { 112 | echo 'Telegram message sent successfully' 113 | } else { 114 | error "Failed to send Telegram message. Status code: ${response.status}" 115 | } 116 | } 117 | } 118 | } 119 | } -------------------------------------------------------------------------------- /src/service/vsl.py: -------------------------------------------------------------------------------- 1 | from models.k8s.vsl import VolumeSnapshotLocationResponseSchema 2 | from schemas.request.create_vsl import CreateVslRequestSchema 3 | 4 | from kubernetes import client 5 | 6 | from schemas.request.update_vsl import UpdateVslRequestSchema 7 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 8 | 9 | from vui_common.configs.config_proxy import config_app 10 | from constants.velero import VELERO 11 | from constants.resources import RESOURCES, ResourcesNames 12 | 13 | custom_objects = client.CustomObjectsApi() 14 | 15 | 16 | @trace_k8s_async_method(description="Get Volume Snapshot Locations") 17 | async def get_vsls_service(): 18 | vsl = custom_objects.list_namespaced_custom_object( 19 | group=VELERO["GROUP"], 20 | version=VELERO["VERSION"], 21 | namespace=config_app.k8s.velero_namespace, 22 | plural=RESOURCES[ResourcesNames.VOLUME_SNAPSHOT_LOCATION].plural, 23 | ) 24 | vsl_list = [VolumeSnapshotLocationResponseSchema(**item) for item in vsl.get("items", [])] 25 | return vsl_list 26 | 27 | 28 | @trace_k8s_async_method(description="Create Volume Snapshot Locations") 29 | async def create_vsl_service(vsl_data: CreateVslRequestSchema): 30 | """ 31 | Create a new VolumeSnapshotLocation in Kubernetes via the Velero API. 32 | """ 33 | vsl_body = { 34 | "apiVersion": "velero.io/v1", 35 | "kind": "VolumeSnapshotLocation", 36 | "metadata": { 37 | "name": vsl_data.name, 38 | "namespace": vsl_data.namespace 39 | }, 40 | "spec": { 41 | "provider": vsl_data.provider 42 | } 43 | } 44 | 45 | if hasattr(vsl_data, "config") and len(vsl_data.config) > 0: 46 | vsl_body["spec"]["config"] = vsl_data.config 47 | 48 | if (hasattr(vsl_data, "credentialName") and 49 | hasattr(vsl_data, "credentialKey") and 50 | vsl_data.credentialName and vsl_data.credentialName != '' and 51 | vsl_data.credentialKey and vsl_data.credentialKey != ''): 52 | vsl_body['spec']["credential"] = { 53 | "name": vsl_data.credentialName, 54 | "key": vsl_data.credentialKey 55 | } 56 | 57 | response = custom_objects.create_namespaced_custom_object( 58 | group=VELERO["GROUP"], 59 | version=VELERO["VERSION"], 60 | namespace=config_app.k8s.velero_namespace, 61 | plural=RESOURCES[ResourcesNames.VOLUME_SNAPSHOT_LOCATION].plural, 62 | body=vsl_body 63 | ) 64 | return response 65 | 66 | 67 | @trace_k8s_async_method(description="Delete Volume Snapshot Locations") 68 | async def delete_vsl_service(vsl_name: str): 69 | """Delete a Velero BSL""" 70 | response = custom_objects.delete_namespaced_custom_object( 71 | group=VELERO["GROUP"], 72 | version=VELERO["VERSION"], 73 | namespace=config_app.k8s.velero_namespace, 74 | plural=RESOURCES[ResourcesNames.VOLUME_SNAPSHOT_LOCATION].plural, 75 | name=vsl_name 76 | ) 77 | return response 78 | 79 | @trace_k8s_async_method(description="Create bsl") 80 | async def update_vsl_service(vsl_data: UpdateVslRequestSchema): 81 | """ 82 | Update a Backup Storage Location (BSL) in Kubernetes 83 | """ 84 | 85 | existing_vsl = custom_objects.get_namespaced_custom_object( 86 | group=VELERO["GROUP"], 87 | version=VELERO["VERSION"], 88 | namespace=config_app.k8s.velero_namespace, 89 | plural=RESOURCES[ResourcesNames.VOLUME_SNAPSHOT_LOCATION].plural, 90 | name=vsl_data.name 91 | ) 92 | 93 | # Update the necessary fields 94 | 95 | if hasattr(vsl_data, "provider") and vsl_data.provider != '': 96 | existing_vsl["spec"]["provider"] = vsl_data.provider.strip() 97 | 98 | if hasattr(vsl_data, "config") and len(vsl_data.config) > 0: 99 | existing_vsl["spec"]["config"] = vsl_data.config 100 | elif 'config' in existing_vsl["spec"]: 101 | existing_vsl["spec"].pop('config') 102 | 103 | if (hasattr(vsl_data, "credentialName") and 104 | hasattr(vsl_data, "credentialKey") and 105 | vsl_data.credentialName and vsl_data.credentialName != '' and 106 | vsl_data.credentialKey and vsl_data.credentialKey != ''): 107 | existing_vsl['spec']["credential"] = { 108 | "name": vsl_data.credentialName, 109 | "key": vsl_data.credentialKey 110 | } 111 | else: 112 | if "spec" in existing_vsl and isinstance(existing_vsl["spec"], dict) and 'credential' in existing_vsl["spec"]: 113 | existing_vsl['spec'].pop("credential") 114 | 115 | response = custom_objects.replace_namespaced_custom_object( 116 | group=VELERO["GROUP"], 117 | version=VELERO["VERSION"], 118 | namespace=config_app.k8s.velero_namespace, 119 | plural=RESOURCES[ResourcesNames.VOLUME_SNAPSHOT_LOCATION].plural, 120 | name=vsl_data.name, 121 | body=existing_vsl 122 | ) 123 | 124 | return response 125 | -------------------------------------------------------------------------------- /docker/Jenkins-dev: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | 4 | environment { 5 | // Environment variables 6 | 7 | // Source code repository 8 | evGit_Source_Code = 'https://github.com/seriohub/velero-api' 9 | 10 | // Default docker registry (Global environment variable setting) 11 | evDocker_HUB = "${env.DEFAULT_DOCKER_REGISTRY}" 12 | 13 | // Author name 14 | evDocker_Project = "${env.DOCKER_AUTHOR}" 15 | 16 | // Credential stored in Jenkins Credential for logging in the HUB 17 | evDocker_HUB_Credentials = credentials('DEFAULT_DOCKER_CREDENTIALS') 18 | 19 | evTarget = 'velero-api' 20 | evTargetBase = 'velero-api-base' 21 | evLatestTag = '' 22 | evCommitChangeset = '' 23 | } 24 | stages { 25 | stage('Clone Git Repository') { 26 | steps { 27 | script { 28 | echo 'Clone Git repository...' 29 | 30 | // Clone the GitHub repository 31 | checkout([$class: "GitSCM", branches: [[name: "*/dev"]], userRemoteConfigs: [[url: evGit_Source_Code]]]) 32 | 33 | // Get commit changes 34 | evCommitChangeset = sh(returnStdout: true, script: "git log -1 --pretty='%s'").trim() 35 | } 36 | } 37 | } 38 | 39 | stage('Build Docker Image and Push Image to Registry') { 40 | steps { 41 | script { 42 | echo 'Build Docker Image and Push Image to Registry' 43 | 44 | def dockerfilePath = 'docker/Dockerfile' 45 | def dockerHubProject = "${evDocker_Project}/${evTarget}" 46 | 47 | GIT_TAG = 'dev' 48 | def now=new Date() 49 | 50 | echo "Docker registry: ${evDocker_HUB}" 51 | def pushTo = '' 52 | if (evDocker_HUB != null) { 53 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 54 | pushTo = "$evDocker_HUB" 55 | } 56 | } 57 | 58 | sh "echo ${evDocker_HUB_Credentials_PSW} | docker login -u ${evDocker_HUB_Credentials_USR} --password-stdin ${pushTo}" 59 | 60 | def dockerImageName = "$evDocker_Project/$evTarget" 61 | 62 | if (evDocker_HUB != null) { 63 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 64 | dockerImageName = "$evDocker_HUB/$evDocker_Project/$evTarget" 65 | } 66 | } 67 | 68 | echo "Build Docker Image ${dockerImageName} -" 69 | sh "docker buildx use multi-arch-2" 70 | sh "docker buildx build --platform linux/amd64 --cache-from=type=local,src=/buildx-cache/api --cache-to=type=local,mode=max,dest=/buildx-cache/api --target ${evTargetBase} -t velero-api-base -f ${dockerfilePath} ." 71 | sh "docker buildx build --platform linux/amd64 --cache-from=type=local,src=/buildx-cache/api --cache-to=type=local,mode=max,dest=/buildx-cache/api --build-arg VERSION=dev --build-arg BUILD_DATE=${now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))} --target ${evTarget} -t ${dockerHubProject}:${GIT_TAG} -f ${dockerfilePath} --push ." 72 | sh 'docker logout' 73 | } 74 | } 75 | } 76 | } 77 | post { 78 | always { 79 | script { 80 | echo 'Post always...' 81 | def pipelineName = currentBuild.fullProjectName 82 | def message = '' 83 | 84 | if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { 85 | message = "Jenkins Pipeline ${pipelineName}: Your developer image version has been successfully pushed to the Docker Registry.\nCommit changes:\n ${evCommitChangeset}" 86 | } else { 87 | message = "Jenkins Pipeline ${pipelineName}: Build failed" 88 | } 89 | 90 | def url = "https://api.telegram.org/bot${env.TELEGRAM_TOKEN}/sendMessage" 91 | def payload = [ 92 | chat_id: "${env.TELEGRAM_CHAT_ID}", 93 | text: message 94 | ] 95 | 96 | def response = httpRequest( 97 | url: url, 98 | httpMode: 'POST', 99 | contentType: 'APPLICATION_JSON', 100 | requestBody: groovy.json.JsonOutput.toJson(payload) 101 | ) 102 | 103 | if (response.status == 200) { 104 | echo 'Telegram message sent successfully' 105 | } else { 106 | error "Failed to send Telegram message. Status code: ${response.status}" 107 | } 108 | } 109 | } 110 | } 111 | } 112 | 113 | -------------------------------------------------------------------------------- /docker/Jenkins-dev-arm: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | 4 | environment { 5 | // Environment variables 6 | 7 | // Source code repository 8 | evGit_Source_Code = 'https://github.com/seriohub/velero-api' 9 | 10 | // Default docker registry (Global environment variable setting) 11 | evDocker_HUB = "${env.DEFAULT_DOCKER_REGISTRY}" 12 | 13 | // Author name 14 | evDocker_Project = "${env.DOCKER_AUTHOR}" 15 | 16 | // Credential stored in Jenkins Credential for logging in the HUB 17 | evDocker_HUB_Credentials = credentials('DEFAULT_DOCKER_CREDENTIALS') 18 | 19 | evTarget = 'velero-api' 20 | evTargetBase = 'velero-api-base' 21 | evLatestTag = '' 22 | evCommitChangeset = '' 23 | } 24 | stages { 25 | stage('Clone Git Repository') { 26 | steps { 27 | script { 28 | echo 'Clone Git repository...' 29 | 30 | // Clone the GitHub repository 31 | checkout([$class: "GitSCM", branches: [[name: "*/dev"]], userRemoteConfigs: [[url: evGit_Source_Code]]]) 32 | 33 | // Get commit changes 34 | evCommitChangeset = sh(returnStdout: true, script: "git log -1 --pretty='%s'").trim() 35 | } 36 | } 37 | } 38 | 39 | stage('Build Docker Image and Push Image to Registry') { 40 | steps { 41 | script { 42 | echo 'Build Docker Image and Push Image to Registry' 43 | 44 | def dockerfilePath = 'docker/Dockerfile' 45 | def dockerHubProject = "${evDocker_Project}/${evTarget}" 46 | 47 | GIT_TAG = 'dev' 48 | def now=new Date() 49 | 50 | echo "Docker registry: ${evDocker_HUB}" 51 | def pushTo = '' 52 | if (evDocker_HUB != null) { 53 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 54 | pushTo = "$evDocker_HUB" 55 | } 56 | } 57 | 58 | sh "echo ${evDocker_HUB_Credentials_PSW} | docker login -u ${evDocker_HUB_Credentials_USR} --password-stdin ${pushTo}" 59 | 60 | def dockerImageName = "$evDocker_Project/$evTarget" 61 | 62 | if (evDocker_HUB != null) { 63 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 64 | dockerImageName = "$evDocker_HUB/$evDocker_Project/$evTarget" 65 | } 66 | } 67 | 68 | echo "Build Docker Image ${dockerImageName} -" 69 | sh "docker buildx use multi-arch-2" 70 | sh "docker buildx build --platform linux/amd64,linux/arm64 --cache-from=type=local,src=/buildx-cache/api --cache-to=type=local,mode=max,dest=/buildx-cache/api --target ${evTargetBase} -t velero-api-base -f ${dockerfilePath} ." 71 | sh "docker buildx build --platform linux/amd64,linux/arm64 --cache-from=type=local,src=/buildx-cache/api --cache-to=type=local,mode=max,dest=/buildx-cache/api --build-arg VERSION=dev --build-arg BUILD_DATE=${now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))} --target ${evTarget} -t ${dockerHubProject}:${GIT_TAG} -f ${dockerfilePath} --push ." 72 | sh 'docker logout' 73 | } 74 | } 75 | } 76 | } 77 | post { 78 | always { 79 | script { 80 | echo 'Post always...' 81 | def pipelineName = currentBuild.fullProjectName 82 | def message = '' 83 | 84 | if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { 85 | message = "Jenkins Pipeline ${pipelineName}: Your developer image version has been successfully pushed to the Docker Registry.\nCommit changes:\n ${evCommitChangeset}" 86 | } else { 87 | message = "Jenkins Pipeline ${pipelineName}: Build failed" 88 | } 89 | 90 | def url = "https://api.telegram.org/bot${env.TELEGRAM_TOKEN}/sendMessage" 91 | def payload = [ 92 | chat_id: "${env.TELEGRAM_CHAT_ID}", 93 | text: message 94 | ] 95 | 96 | def response = httpRequest( 97 | url: url, 98 | httpMode: 'POST', 99 | contentType: 'APPLICATION_JSON', 100 | requestBody: groovy.json.JsonOutput.toJson(payload) 101 | ) 102 | 103 | if (response.status == 200) { 104 | echo 'Telegram message sent successfully' 105 | } else { 106 | error "Failed to send Telegram message. Status code: ${response.status}" 107 | } 108 | } 109 | } 110 | } 111 | } 112 | 113 | -------------------------------------------------------------------------------- /src/service/velero.py: -------------------------------------------------------------------------------- 1 | import re 2 | import asyncio 3 | from kubernetes import client 4 | from kubernetes.client import ApiException 5 | 6 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 7 | from vui_common.configs.config_proxy import config_app 8 | from datetime import timezone 9 | 10 | coreV1 = client.CoreV1Api() 11 | 12 | 13 | def _parse_version_output(output): 14 | # Initialize the result dictionary 15 | result = {'client': {}, 'server': {}, 'warning': None} 16 | 17 | # Find client information 18 | client_match = re.search(r'Client:\n\tVersion:\s+(?P[\w.-]+)\n\tGit commit:\s+(?P\w+)', 19 | output) 20 | if client_match: 21 | result['client']['version'] = client_match.group('version') 22 | result['client']['GitCommit'] = client_match.group('git_commit') 23 | 24 | # Finds server information 25 | server_match = re.search(r'Server:\n\tVersion:\s+(?P[\w.-]+)', output) 26 | if server_match: 27 | result['server']['version'] = server_match.group('version') 28 | 29 | # Finds warning, if any 30 | warning_match = re.search(r'# WARNING:\s+(?P.+)', output) 31 | if warning_match: 32 | result['warning'] = warning_match.group('warning') 33 | 34 | return result 35 | 36 | 37 | @trace_k8s_async_method(description="Get velero Version") 38 | async def get_velero_version_service(): 39 | namespace = config_app.k8s.velero_namespace 40 | label_selectors = [ 41 | "name=velero", 42 | "app.kubernetes.io/name=velero", 43 | "component=velero", 44 | "k8s-app=velero" 45 | ] 46 | 47 | try: 48 | for label_selector in label_selectors: 49 | pods = coreV1.list_namespaced_pod(namespace=namespace, label_selector=label_selector) 50 | 51 | if pods.items: 52 | pod = pods.items[0] 53 | container_image = pod.spec.containers[0].image 54 | 55 | if ':' in container_image: 56 | _, version = container_image.split(':') 57 | return version 58 | else: 59 | print("Unable to determine version from container image.") 60 | return None 61 | 62 | print("No pods found with any of the specified labels.") 63 | return None 64 | 65 | except ApiException as e: 66 | print(f"Error while accessing pods: {e}") 67 | return None 68 | 69 | 70 | @trace_k8s_async_method(description="Get velero Pods") 71 | async def get_pods_service(label_selectors_by_type, namespace): 72 | coreV1 = client.CoreV1Api() 73 | 74 | pods_info = [] 75 | seen_pods = set() 76 | 77 | # label_selectors_by_type = { 78 | # "velero": "name=velero", 79 | # "node-agent": "name=node-agent" 80 | # } 81 | 82 | for pod_type, label_selector in label_selectors_by_type.items(): 83 | try: 84 | pods = coreV1.list_namespaced_pod(namespace=namespace, label_selector=label_selector) 85 | for pod in pods.items: 86 | pod_name = pod.metadata.name 87 | if pod_name in seen_pods: 88 | continue 89 | 90 | seen_pods.add(pod_name) 91 | 92 | # Info generali 93 | image = pod.spec.containers[0].image if pod.spec.containers else "unknown" 94 | version = image.split(":")[-1] if ":" in image else "unknown" 95 | status = pod.status.phase 96 | restarts = sum(c.restart_count for c in pod.status.container_statuses or []) 97 | created = pod.metadata.creation_timestamp.astimezone( 98 | timezone.utc).isoformat() if pod.metadata.creation_timestamp else "unknown" 99 | 100 | pods_info.append({ 101 | "podName": pod_name, 102 | "type": pod_type, 103 | "nodeName": pod.spec.node_name, 104 | "ip": pod.status.pod_ip, 105 | "status": status, 106 | "restarts": restarts, 107 | "created": created, 108 | "version": version, 109 | }) 110 | 111 | except client.exceptions.ApiException as e: 112 | print(f"Errore durante la richiesta con selector '{label_selector}': {e}") 113 | 114 | return pods_info 115 | 116 | 117 | async def get_pod_logs_service(pod, namespace="velero", lines=100): 118 | coreV1 = client.CoreV1Api() 119 | 120 | def _get_logs(): 121 | try: 122 | return coreV1.read_namespaced_pod_log( 123 | name=pod, 124 | namespace=namespace, 125 | tail_lines=lines, 126 | timestamps=False, 127 | ) 128 | except client.exceptions.ApiException as e: 129 | return f"error while fetching logs for '{pod}': {e}" 130 | 131 | logs = (await asyncio.to_thread(_get_logs)).split("\n") 132 | return logs 133 | -------------------------------------------------------------------------------- /src/controllers/k8s.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | 3 | from vui_common.configs.config_proxy import config_app 4 | 5 | from vui_common.schemas.response.successful_request import SuccessfulRequest 6 | from schemas.request.create_cloud_credentials import CreateCloudCredentialsRequestSchema 7 | 8 | from service.k8s import (get_namespaces_service, 9 | get_storage_classes_service, 10 | get_velero_resource_manifest_service) 11 | from service.k8s_manifest import get_k8s_resource_manifest_service 12 | from service.k8s_secret import get_velero_secret_service, get_secret_keys_service 13 | from service.location_credentials import (get_credential_service, get_default_credential_service, 14 | create_cloud_credentials_secret_service) 15 | from service.velero import get_pod_logs_service 16 | 17 | 18 | async def get_ns_handler(): 19 | payload = await get_namespaces_service() 20 | 21 | response = SuccessfulRequest(payload=payload) 22 | return JSONResponse(content=response.model_dump(), status_code=200) 23 | 24 | 25 | async def get_resources_handler(): 26 | payload = await get_namespaces_service() 27 | 28 | response = SuccessfulRequest(payload=payload) 29 | return JSONResponse(content=response.model_dump(), status_code=200) 30 | 31 | 32 | async def get_credential_handler(secret_name, secret_key): 33 | payload = await get_credential_service(secret_name, secret_key) 34 | 35 | response = SuccessfulRequest(payload=payload) 36 | return JSONResponse(content=response.model_dump(), status_code=200) 37 | 38 | 39 | async def get_default_credential_handler(): 40 | payload = await get_default_credential_service() 41 | 42 | response = SuccessfulRequest(payload=payload) 43 | return JSONResponse(content=response.model_dump(), status_code=200) 44 | 45 | 46 | async def get_k8s_storage_classes_handler(): 47 | payload = await get_storage_classes_service() 48 | 49 | response = SuccessfulRequest(payload=payload) 50 | return JSONResponse(content=response.model_dump(), status_code=200) 51 | 52 | 53 | async def get_pod_logs_handler(pod, target='velero', lines=100): 54 | namespace = config_app.k8s.velero_namespace if target == 'velero' else config_app.k8s.vui_namespace 55 | payload = await get_pod_logs_service(namespace=namespace, pod=pod, lines=lines) 56 | 57 | response = SuccessfulRequest(payload={'logs': payload}) 58 | return JSONResponse(content=response.model_dump(), status_code=200) 59 | 60 | 61 | async def create_cloud_credentials_handler(cloud_credentials: CreateCloudCredentialsRequestSchema): 62 | payload = await create_cloud_credentials_secret_service(cloud_credentials.newSecretName, 63 | cloud_credentials.newSecretKey, 64 | cloud_credentials.awsAccessKeyId, 65 | cloud_credentials.awsSecretAccessKey) 66 | 67 | response = SuccessfulRequest(payload=payload) 68 | return JSONResponse(content=response.model_dump(), status_code=200) 69 | 70 | 71 | async def get_velero_secret_handler(): 72 | payload = await get_velero_secret_service() 73 | 74 | response = SuccessfulRequest(payload=payload) 75 | return JSONResponse(content=response.model_dump(), status_code=200) 76 | 77 | 78 | async def get_velero_secret_key_handler(secret_name): 79 | payload = await get_secret_keys_service(config_app.k8s.velero_namespace, secret_name) 80 | 81 | response = SuccessfulRequest(payload=payload) 82 | return JSONResponse(content=response.model_dump(), status_code=200) 83 | 84 | 85 | async def get_velero_manifest_handler(resource_type: str, resource_name: str, neat: bool): 86 | payload = await get_velero_resource_manifest_service(resource_type=resource_type, 87 | resource_name=resource_name, 88 | neat=neat) 89 | 90 | response = SuccessfulRequest(payload=payload) 91 | return JSONResponse(content=response.model_dump(), status_code=200) 92 | 93 | 94 | async def get_k8s_manifest_handler(kind: str, 95 | name: str, 96 | namespace: str = None, 97 | api_version: str = "v1", 98 | is_cluster_resource: bool = False, 99 | neat=False): 100 | payload = await get_k8s_resource_manifest_service(kind=kind, 101 | name=name, 102 | namespace=namespace, 103 | api_version=api_version, 104 | is_cluster_resource=is_cluster_resource, 105 | neat=neat) 106 | 107 | response = SuccessfulRequest(payload=payload) 108 | return JSONResponse(content=response.model_dump(), status_code=200) 109 | -------------------------------------------------------------------------------- /src/controllers/backup.py: -------------------------------------------------------------------------------- 1 | import json 2 | from fastapi.responses import JSONResponse 3 | 4 | from schemas.request.create_backup import CreateBackupRequestSchema 5 | from schemas.request.create_backup_from_schedule import CreateBackupFromScheduleRequestSchema 6 | from schemas.request.update_backup_expiration import UpdateBackupExpirationRequestSchema 7 | from vui_common.schemas.response.successful_request import SuccessfulRequest 8 | from schemas.response.successful_backups import SuccessfulBackupResponse 9 | from vui_common.schemas.notification import Notification 10 | 11 | from service.backup_storage_class import get_backup_storage_classes_service 12 | from service.resource import get_resource_creation_settings_service 13 | from service.backup import (get_backups_service, 14 | delete_backup_service, 15 | create_backup_service, 16 | create_backup_from_schedule_service, 17 | get_backup_expiration_service, 18 | update_backup_expiration_service, 19 | download_backup_service) 20 | from service.inspect_download_backup import inspect_download_backup_service 21 | 22 | 23 | async def get_creation_settings_handler(): 24 | payload = await get_resource_creation_settings_service() 25 | 26 | response = SuccessfulRequest(payload=payload) 27 | return JSONResponse(content=response.model_dump(), status_code=200) 28 | 29 | 30 | async def get_backups_handler(schedule_name: str | None = None, latest_per_schedule: bool = False, 31 | in_progress: bool = False): 32 | payload = await get_backups_service(schedule_name=schedule_name, 33 | latest_per_schedule=latest_per_schedule, 34 | in_progress=in_progress) 35 | 36 | response = SuccessfulBackupResponse(payload=payload) 37 | return JSONResponse(content=response.model_dump(), status_code=200) 38 | 39 | 40 | async def delete_backup_handler(backup_name: str): 41 | payload = await delete_backup_service(backup_name=backup_name) 42 | 43 | msg = Notification(title='Delete backup', 44 | description=f'Backup {backup_name} deleted request done!', 45 | type_='INFO') 46 | 47 | response = SuccessfulRequest(notifications=[msg], payload=payload) 48 | return JSONResponse(content=response.model_dump(), status_code=200) 49 | 50 | 51 | async def create_backup_handler(backup: CreateBackupRequestSchema): 52 | payload = await create_backup_service(backup_data=backup) 53 | 54 | msg = Notification(title='Create backup', 55 | description=f"Backup {backup.name} created!", 56 | type_='INFO') 57 | response = SuccessfulRequest(notifications=[msg], payload=payload) 58 | return JSONResponse(content=response.model_dump(), status_code=201) 59 | 60 | 61 | async def create_backup_from_schedule_handler(backup: CreateBackupFromScheduleRequestSchema): 62 | payload = await create_backup_from_schedule_service(schedule_name=backup.scheduleName) 63 | 64 | msg = Notification(title='Create backup from schedule', 65 | description=f"Backup created!", 66 | type_='INFO') 67 | response = SuccessfulRequest(notifications=[msg], payload=payload) 68 | return JSONResponse(content=response.model_dump(), status_code=201) 69 | 70 | 71 | async def update_backup_expiration_handler(ttl: UpdateBackupExpirationRequestSchema): 72 | payload = await update_backup_expiration_service(backup_name=ttl.backupName, 73 | expiration=ttl.expiration) 74 | 75 | msg = Notification(title='TTL Updated', 76 | description=f"Backup {ttl.backupName} expiration updated!", 77 | type_='INFO') 78 | response = SuccessfulRequest(notifications=[msg], payload=payload) 79 | return JSONResponse(content=response.model_dump(), status_code=201) 80 | 81 | 82 | async def get_backup_expiration_handler(backup_name: str): 83 | payload = await get_backup_expiration_service(backup_name=backup_name) 84 | 85 | response = SuccessfulRequest(payload=payload) 86 | return JSONResponse(content=response.model_dump(), status_code=200) 87 | 88 | 89 | async def get_backup_storage_classes_handler(backup_name: str): 90 | payload = await get_backup_storage_classes_service(backup_name=backup_name) 91 | 92 | response = SuccessfulRequest(payload=json.loads(payload.model_dump_json())['storage_classes']) 93 | return JSONResponse(content=response.model_dump(), status_code=200) 94 | 95 | 96 | async def download_backup_handler(backup_name: str): 97 | payload = await download_backup_service(backup_name=backup_name) 98 | 99 | response = SuccessfulRequest(payload=payload) 100 | return JSONResponse(content=response.model_dump(), status_code=200) 101 | 102 | 103 | async def inspect_download_backup_handler(backup_name: str): 104 | payload = await inspect_download_backup_service(backup_name=backup_name) 105 | 106 | response = SuccessfulRequest(payload=payload) 107 | return JSONResponse(content=response.model_dump(), status_code=200) 108 | -------------------------------------------------------------------------------- /docker/jenkins-amd64/Jenkins: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | 4 | environment { 5 | // Environment variables 6 | 7 | // Source code repository 8 | evGit_Source_Code = 'https://github.com/seriohub/velero-api' 9 | 10 | // Default docker registry (Global environment variable setting) 11 | evDocker_HUB = "${env.DEFAULT_DOCKER_REGISTRY}" 12 | 13 | // Author name 14 | evDocker_Project = "${env.DOCKER_AUTHOR}" 15 | 16 | // Credential stored in Jenkins Credential for logging in the HUB 17 | evDocker_HUB_Credentials = credentials('DEFAULT_DOCKER_CREDENTIALS') 18 | 19 | evTarget = 'velero-api' 20 | evLatestTag = '' 21 | evCommitChangeset = '' 22 | } 23 | stages { 24 | stage('Clone Git Repository') { 25 | steps { 26 | script { 27 | echo 'Clone Git repository...' 28 | 29 | // Clone the GitHub repository 30 | checkout([$class: "GitSCM", branches: [[name: "*/main"]], userRemoteConfigs: [[url: evGit_Source_Code]]]) 31 | 32 | // Get the latest tag in the main branch 33 | evLatestTag = sh(script: 'git describe --tags --abbrev=0', returnStdout: true).trim() 34 | 35 | // Remove the "v" prefix if it exists 36 | evLatestTag = evLatestTag.startsWith('v') ? evLatestTag.substring(1) : evLatestTag 37 | 38 | echo "Latest Tag: ${evLatestTag}" 39 | 40 | // Set the latest tag as an environment variable for later use 41 | GIT_TAG = evLatestTag 42 | 43 | //Get commit changes 44 | evCommitChangeset = sh(returnStdout: true, script: "git log -1 --pretty='%s'").trim() 45 | } 46 | } 47 | } 48 | 49 | stage('Build Docker Image') { 50 | steps { 51 | script { 52 | echo 'Build Docker Image...' 53 | 54 | // Path to your Dockerfile within the project directory 55 | def dockerfilePath = 'docker/Dockerfile' 56 | def dockerHubProject = "${evDocker_Project}/${evTarget}" 57 | 58 | def now=new Date() 59 | 60 | // Build the Docker image, specifying the Dockerfile path 61 | sh "docker build --build-arg VERSION=${GIT_TAG} --build-arg BUILD_DATE=${now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))} --target ${evTarget} -t ${dockerHubProject}:${GIT_TAG} -t ${dockerHubProject}:latest -f ${dockerfilePath} ." 62 | } 63 | } 64 | } 65 | 66 | stage('Push Docker Image to Registry') { 67 | steps { 68 | script { 69 | echo "Docker registry: ${evDocker_HUB}" 70 | def pushTo = '' 71 | if (evDocker_HUB != null) { 72 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 73 | pushTo = "$evDocker_HUB" 74 | } 75 | } 76 | 77 | sh "echo ${evDocker_HUB_Credentials_PSW} | docker login -u ${evDocker_HUB_Credentials_USR} --password-stdin ${pushTo}" 78 | 79 | def dockerImageName = "$evDocker_Project/$evTarget" 80 | 81 | if (evDocker_HUB != null) { 82 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 83 | dockerImageName = "$evDocker_HUB/$evDocker_Project/$evTarget" 84 | } 85 | } 86 | 87 | echo "Docker push to:- ${dockerImageName} -" 88 | sh "docker push ${dockerImageName}:${GIT_TAG}" 89 | sh "docker push ${dockerImageName}:latest" 90 | sh 'docker logout ' 91 | } 92 | } 93 | } 94 | } 95 | post { 96 | always { 97 | script { 98 | echo 'Post always...' 99 | def pipelineName = currentBuild.fullProjectName 100 | def message = '' 101 | 102 | if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { 103 | message = "Jenkins Pipeline ${pipelineName}: Your developer image version has been successfully pushed to the Docker Registry.\nCommit changes:\n ${evCommitChangeset}" 104 | } else { 105 | message = "Jenkins Pipeline ${pipelineName}: Build failed" 106 | } 107 | 108 | def url = "https://api.telegram.org/bot${env.TELEGRAM_TOKEN}/sendMessage" 109 | def payload = [ 110 | chat_id: "${env.TELEGRAM_CHAT_ID}", 111 | text: message 112 | ] 113 | 114 | def response = httpRequest( 115 | url: url, 116 | httpMode: 'POST', 117 | contentType: 'APPLICATION_JSON', 118 | requestBody: groovy.json.JsonOutput.toJson(payload) 119 | ) 120 | 121 | if (response.status == 200) { 122 | echo 'Telegram message sent successfully' 123 | } else { 124 | error "Failed to send Telegram message. Status code: ${response.status}" 125 | } 126 | } 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /docker/Jenkins: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | 4 | environment { 5 | // Environment variables 6 | 7 | // Source code repository 8 | evGit_Source_Code = 'https://github.com/seriohub/velero-api' 9 | 10 | // Default docker registry (Global environment variable setting) 11 | evDocker_HUB = "${env.DEFAULT_DOCKER_REGISTRY}" 12 | 13 | // Author name 14 | evDocker_Project = "${env.DOCKER_AUTHOR}" 15 | 16 | // Credential stored in Jenkins Credential for logging in the HUB 17 | evDocker_HUB_Credentials = credentials('DEFAULT_DOCKER_CREDENTIALS') 18 | 19 | evTarget = 'velero-api' 20 | evTargetBase = 'velero-api-base' 21 | evLatestTag = '' 22 | evCommitChangeset = '' 23 | } 24 | stages { 25 | stage('Clone Git Repository') { 26 | steps { 27 | script { 28 | echo 'Clone Git repository...' 29 | 30 | // Clone the GitHub repository 31 | checkout([$class: "GitSCM", branches: [[name: "*/main"]], userRemoteConfigs: [[url: evGit_Source_Code]]]) 32 | 33 | // Get the latest tag in the main branch 34 | evLatestTag = sh(script: 'git describe --tags --abbrev=0', returnStdout: true).trim() 35 | 36 | // Remove the "v" prefix if it exists 37 | evLatestTag = evLatestTag.startsWith('v') ? evLatestTag.substring(1) : evLatestTag 38 | 39 | echo "Latest Tag: ${evLatestTag}" 40 | 41 | // Set the latest tag as an environment variable for later use 42 | GIT_TAG = evLatestTag 43 | 44 | //Get commit changes 45 | evCommitChangeset = sh(returnStdout: true, script: "git log -1 --pretty='%s'").trim() 46 | } 47 | } 48 | } 49 | 50 | stage('Build Docker Image and Push Image to Registry') { 51 | steps { 52 | script { 53 | echo 'Build Docker Image and Push Image to Registry' 54 | 55 | // Path to your Dockerfile within the project directory 56 | def dockerfilePath = 'docker/Dockerfile' 57 | def dockerHubProject = "${evDocker_Project}/${evTarget}" 58 | 59 | def now=new Date() 60 | 61 | echo "Docker registry: ${evDocker_HUB}" 62 | def pushTo = '' 63 | if (evDocker_HUB != null) { 64 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 65 | pushTo = "$evDocker_HUB" 66 | } 67 | } 68 | 69 | sh "echo ${evDocker_HUB_Credentials_PSW} | docker login -u ${evDocker_HUB_Credentials_USR} --password-stdin ${pushTo}" 70 | 71 | def dockerImageName = "$evDocker_Project/$evTarget" 72 | 73 | if (evDocker_HUB != null) { 74 | if (evDocker_HUB != '' && !evDocker_HUB.contains('null')) { 75 | dockerImageName = "$evDocker_HUB/$evDocker_Project/$evTarget" 76 | } 77 | } 78 | 79 | echo "Build Docker Image ${dockerImageName} -" 80 | sh "docker buildx use multi-arch-2" 81 | sh "docker buildx build --platform linux/amd64,linux/arm64 --cache-from=type=local,src=/buildx-cache/api --cache-to=type=local,mode=max,dest=/buildx-cache/api --target ${evTargetBase} -t velero-api-base -f ${dockerfilePath} ." 82 | sh "docker buildx build --platform linux/amd64,linux/arm64 --cache-from=type=local,src=/buildx-cache/api --cache-to=type=local,mode=max,dest=/buildx-cache/api --build-arg VERSION=${GIT_TAG} --build-arg BUILD_DATE=${now.format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))} --target ${evTarget} -t ${dockerHubProject}:${GIT_TAG} -t ${dockerImageName}:latest -f ${dockerfilePath} --push ." 83 | sh 'docker logout' 84 | } 85 | } 86 | } 87 | } 88 | post { 89 | always { 90 | script { 91 | echo 'Post always...' 92 | def pipelineName = currentBuild.fullProjectName 93 | def message = '' 94 | 95 | if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { 96 | message = "Jenkins Pipeline ${pipelineName}: Your developer image version has been successfully pushed to the Docker Registry.\nCommit changes:\n ${evCommitChangeset}" 97 | } else { 98 | message = "Jenkins Pipeline ${pipelineName}: Build failed" 99 | } 100 | 101 | def url = "https://api.telegram.org/bot${env.TELEGRAM_TOKEN}/sendMessage" 102 | def payload = [ 103 | chat_id: "${env.TELEGRAM_CHAT_ID}", 104 | text: message 105 | ] 106 | 107 | def response = httpRequest( 108 | url: url, 109 | httpMode: 'POST', 110 | contentType: 'APPLICATION_JSON', 111 | requestBody: groovy.json.JsonOutput.toJson(payload) 112 | ) 113 | 114 | if (response.status == 200) { 115 | echo 'Telegram message sent successfully' 116 | } else { 117 | error "Failed to send Telegram message. Status code: ${response.status}" 118 | } 119 | } 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/api/v1/routers/inspect.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, status 2 | 3 | from constants.response import common_error_authenticated_response 4 | from vui_common.schemas.response.successful_request import SuccessfulRequest 5 | 6 | from vui_common.security.helpers.rate_limiter import RateLimiter, LimiterRequests 7 | 8 | from vui_common.utils.swagger import route_description 9 | from vui_common.utils.exceptions import handle_exceptions_endpoint 10 | 11 | from controllers.inspect import (get_backups_handler, 12 | # get_folders_handler, 13 | get_file_content_handler, 14 | get_recursive_directory_contents_handler) 15 | 16 | router = APIRouter() 17 | 18 | tag_name = "Inspect" 19 | endpoint_limiter = LimiterRequests(tags=tag_name, 20 | default_key='L1') 21 | 22 | # ------------------------------------------------------------------------------------------------ 23 | # GET AVAILABLE BACKUP FOR INSPECT 24 | # ------------------------------------------------------------------------------------------------ 25 | 26 | 27 | limiter_backups = endpoint_limiter.get_limiter_cust('inspect_backups') 28 | route = '/inspect/backups' 29 | 30 | 31 | @router.get( 32 | path=route, 33 | tags=[tag_name], 34 | summary='Get list folder available', 35 | description=route_description(tag=tag_name, 36 | route=route, 37 | limiter_calls=limiter_backups.max_request, 38 | limiter_seconds=limiter_backups.seconds), 39 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backups.seconds, 40 | max_requests=limiter_backups.max_request))], 41 | response_model=SuccessfulRequest, 42 | responses=common_error_authenticated_response, 43 | status_code=status.HTTP_200_OK 44 | ) 45 | @handle_exceptions_endpoint 46 | async def get_backups(): 47 | return await get_backups_handler() 48 | 49 | 50 | # ------------------------------------------------------------------------------------------------ 51 | # GET FOLDERS 52 | # ------------------------------------------------------------------------------------------------ 53 | 54 | # limiter_backups = endpoint_limiter.get_limiter_cust('inspect_folders') 55 | # route = '/inspect/folders' 56 | # 57 | # 58 | # @router.get( 59 | # path=route, 60 | # tags=[tag_name], 61 | # summary='Get list folders', 62 | # description=route_description(tag=tag_name, 63 | # route=route, 64 | # limiter_calls=limiter_backups.max_request, 65 | # limiter_seconds=limiter_backups.seconds), 66 | # dependencies=[Depends(RateLimiter(interval_seconds=limiter_backups.seconds, 67 | # max_requests=limiter_backups.max_request))], 68 | # response_model=SuccessfulRequest, 69 | # responses=common_error_authenticated_response, 70 | # status_code=status.HTTP_200_OK 71 | # ) 72 | # @handle_exceptions_endpoint 73 | # async def get_folders(path: str): 74 | # return await get_folders_handler(path=path) 75 | 76 | 77 | # ------------------------------------------------------------------------------------------------ 78 | # GET FILE CONTENT 79 | # ------------------------------------------------------------------------------------------------ 80 | 81 | limiter_backups = endpoint_limiter.get_limiter_cust('inspect_folders') 82 | route = '/inspect/file' 83 | 84 | 85 | @router.get( 86 | path=route, 87 | tags=[tag_name], 88 | summary='Get list folders', 89 | description=route_description(tag=tag_name, 90 | route=route, 91 | limiter_calls=limiter_backups.max_request, 92 | limiter_seconds=limiter_backups.seconds), 93 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backups.seconds, 94 | max_requests=limiter_backups.max_request))], 95 | response_model=SuccessfulRequest, 96 | responses=common_error_authenticated_response, 97 | status_code=status.HTTP_200_OK 98 | ) 99 | @handle_exceptions_endpoint 100 | async def get_file_content(path: str): 101 | return await get_file_content_handler(path=path) 102 | 103 | # ------------------------------------------------------------------------------------------------ 104 | # GET RECURSIVE FOLDER CONTENT 105 | # ------------------------------------------------------------------------------------------------ 106 | 107 | limiter_backups = endpoint_limiter.get_limiter_cust('inspect_folder_content') 108 | route = '/inspect/folder/content' 109 | 110 | 111 | @router.get( 112 | path=route, 113 | tags=[tag_name], 114 | summary='Get folder content', 115 | description=route_description(tag=tag_name, 116 | route=route, 117 | limiter_calls=limiter_backups.max_request, 118 | limiter_seconds=limiter_backups.seconds), 119 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backups.seconds, 120 | max_requests=limiter_backups.max_request))], 121 | response_model=SuccessfulRequest, 122 | responses=common_error_authenticated_response, 123 | status_code=status.HTTP_200_OK 124 | ) 125 | @handle_exceptions_endpoint 126 | async def get_folder_content(backup: str): 127 | return await get_recursive_directory_contents_handler(backup=backup) 128 | 129 | 130 | -------------------------------------------------------------------------------- /src/api/v1/routers/pvb.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, status, Depends 2 | 3 | from constants.response import common_error_authenticated_response 4 | 5 | from vui_common.security.helpers.rate_limiter import RateLimiter, LimiterRequests 6 | 7 | from vui_common.utils.swagger import route_description 8 | from vui_common.utils.exceptions import handle_exceptions_endpoint 9 | 10 | from vui_common.schemas.response.successful_request import SuccessfulRequest 11 | 12 | from controllers.pvb import (get_pod_volume_backups_handler, 13 | get_pod_volume_restore_handler, 14 | get_pod_volume_backup_details_handler, 15 | get_pod_volume_restore_details_handler) 16 | 17 | router = APIRouter() 18 | 19 | tag_name = "Pod Volume Backups" 20 | 21 | endpoint_limiter = LimiterRequests(tags=tag_name, 22 | default_key='L1') 23 | 24 | # ------------------------------------------------------------------------------------------------ 25 | # GET PODS VOLUME BACKUP 26 | # ------------------------------------------------------------------------------------------------ 27 | 28 | limiter_backups = endpoint_limiter.get_limiter_cust('pod_volume_backups') 29 | route = '/pod-volume-backups' 30 | 31 | 32 | @router.get( 33 | path=route, 34 | tags=[tag_name], 35 | summary='Get pod volume backups', 36 | description=route_description(tag=tag_name, 37 | route=route, 38 | limiter_calls=limiter_backups.max_request, 39 | limiter_seconds=limiter_backups.seconds), 40 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backups.seconds, 41 | max_requests=limiter_backups.max_request))], 42 | response_model=SuccessfulRequest, 43 | responses=common_error_authenticated_response, 44 | status_code=status.HTTP_200_OK) 45 | @handle_exceptions_endpoint 46 | async def get_pvbs_backup(): 47 | return await get_pod_volume_backups_handler() 48 | 49 | 50 | # ------------------------------------------------------------------------------------------------ 51 | # GET POD VOLUME BACKUP DETAILS 52 | # ------------------------------------------------------------------------------------------------ 53 | 54 | 55 | limiter_backup = endpoint_limiter.get_limiter_cust('pod_volume_backup') 56 | route = '/pod-volume-backup' 57 | 58 | 59 | @router.get( 60 | path=route, 61 | tags=[tag_name], 62 | summary='Get pod volume', 63 | description=route_description(tag=tag_name, 64 | route=route, 65 | limiter_calls=limiter_backup.max_request, 66 | limiter_seconds=limiter_backup.seconds), 67 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backup.seconds, 68 | max_requests=limiter_backup.max_request))], 69 | response_model=SuccessfulRequest, 70 | responses=common_error_authenticated_response, 71 | status_code=status.HTTP_200_OK) 72 | @handle_exceptions_endpoint 73 | async def get_pvb_backup_details(backup_name: str): 74 | return await get_pod_volume_backup_details_handler(backup_name) 75 | 76 | 77 | # ------------------------------------------------------------------------------------------------ 78 | # GET PODS VOLUME RESTORE 79 | # ------------------------------------------------------------------------------------------------ 80 | 81 | limiter_backups = endpoint_limiter.get_limiter_cust('pod_volume_restores') 82 | route = '/pod-volume-restores' 83 | 84 | 85 | @router.get( 86 | path=route, 87 | tags=[tag_name], 88 | summary='Get pod volume backups', 89 | description=route_description(tag=tag_name, 90 | route=route, 91 | limiter_calls=limiter_backups.max_request, 92 | limiter_seconds=limiter_backups.seconds), 93 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backups.seconds, 94 | max_requests=limiter_backups.max_request))], 95 | response_model=SuccessfulRequest, 96 | responses=common_error_authenticated_response, 97 | status_code=status.HTTP_200_OK) 98 | @handle_exceptions_endpoint 99 | async def get_pvbs_restore(): 100 | return await get_pod_volume_restore_handler() 101 | 102 | 103 | # ------------------------------------------------------------------------------------------------ 104 | # GET POD VOLUME BACKUP DETAILS 105 | # ------------------------------------------------------------------------------------------------ 106 | 107 | 108 | limiter_backup = endpoint_limiter.get_limiter_cust('pod_volume_restore') 109 | route = '/pod-volume-restore' 110 | 111 | 112 | @router.get( 113 | path=route, 114 | tags=[tag_name], 115 | summary='Get pod volume', 116 | description=route_description(tag=tag_name, 117 | route=route, 118 | limiter_calls=limiter_backup.max_request, 119 | limiter_seconds=limiter_backup.seconds), 120 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_backup.seconds, 121 | max_requests=limiter_backup.max_request))], 122 | response_model=SuccessfulRequest, 123 | responses=common_error_authenticated_response, 124 | status_code=status.HTTP_200_OK) 125 | @handle_exceptions_endpoint 126 | async def get_pvb_restore_details(restore_name: str): 127 | return await get_pod_volume_restore_details_handler(restore_name) 128 | -------------------------------------------------------------------------------- /src/api/v1/routers/vsl.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, status 2 | 3 | from constants.response import common_error_authenticated_response 4 | from schemas.request.update_vsl import UpdateVslRequestSchema 5 | 6 | from vui_common.security.helpers.rate_limiter import RateLimiter, LimiterRequests 7 | 8 | from vui_common.utils.swagger import route_description 9 | from vui_common.utils.exceptions import handle_exceptions_endpoint 10 | 11 | from vui_common.schemas.response.successful_request import SuccessfulRequest 12 | from schemas.request.delete_resource import DeleteResourceRequestSchema 13 | 14 | from controllers.vsl import (get_vsl_handler, 15 | create_vsl_handler, 16 | delete_vsl_handler, update_vsl_handler) 17 | 18 | from schemas.request.create_vsl import CreateVslRequestSchema 19 | 20 | router = APIRouter() 21 | rate_limiter = RateLimiter() 22 | 23 | 24 | tag_name = 'Volume Snapshot Locations' 25 | endpoint_limiter = LimiterRequests(tags=tag_name, 26 | default_key='L1') 27 | 28 | # ------------------------------------------------------------------------------------------------ 29 | # GET VOLUME SNAPSHOT LOCATIONS LIST 30 | # ------------------------------------------------------------------------------------------------ 31 | 32 | 33 | limiter_vsl = endpoint_limiter.get_limiter_cust('vsl') 34 | route = '/vsl' 35 | 36 | 37 | @router.get( 38 | path=route, 39 | tags=[tag_name], 40 | summary='Get locations for the snapshot', 41 | description=route_description(tag=tag_name, 42 | route=route, 43 | limiter_calls=limiter_vsl.max_request, 44 | limiter_seconds=limiter_vsl.seconds), 45 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_vsl.seconds, 46 | max_requests=limiter_vsl.max_request))], 47 | response_model=SuccessfulRequest, 48 | responses=common_error_authenticated_response, 49 | status_code=status.HTTP_200_OK) 50 | @handle_exceptions_endpoint 51 | async def get_vsls(): 52 | return await get_vsl_handler() 53 | 54 | 55 | # ------------------------------------------------------------------------------------------------ 56 | # CREATE A VOLUME SNAPSHOT LOCATION 57 | # ------------------------------------------------------------------------------------------------ 58 | 59 | 60 | limiter_create = endpoint_limiter.get_limiter_cust('vsl') 61 | route = '/vsl' 62 | 63 | 64 | @router.post( 65 | path=route, 66 | tags=[tag_name], 67 | summary='Create a backup storage location', 68 | description=route_description(tag=tag_name, 69 | route=route, 70 | limiter_calls=limiter_create.max_request, 71 | limiter_seconds=limiter_create.seconds), 72 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_create.seconds, 73 | max_requests=limiter_create.max_request))], 74 | response_model=SuccessfulRequest, 75 | responses=common_error_authenticated_response, 76 | status_code=status.HTTP_201_CREATED) 77 | @handle_exceptions_endpoint 78 | async def create_vsl(create_bsl: CreateVslRequestSchema): 79 | return await create_vsl_handler(create_bsl=create_bsl) 80 | 81 | 82 | # ------------------------------------------------------------------------------------------------ 83 | # DELETE VOLUME SNAPSHOT LOCATION 84 | # ------------------------------------------------------------------------------------------------ 85 | 86 | 87 | limiter_del = endpoint_limiter.get_limiter_cust('vsl') 88 | route = '/vsl' 89 | 90 | 91 | @router.delete( 92 | path=route, 93 | tags=[tag_name], 94 | summary='Delete storage classes mapping in config map', 95 | description=route_description(tag=tag_name, 96 | route=route, 97 | limiter_calls=limiter_del.max_request, 98 | limiter_seconds=limiter_del.seconds), 99 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_del.seconds, 100 | max_requests=limiter_del.max_request))], 101 | response_model=SuccessfulRequest, 102 | responses=common_error_authenticated_response, 103 | status_code=status.HTTP_200_OK) 104 | @handle_exceptions_endpoint 105 | async def delete_vsl(vsl: DeleteResourceRequestSchema): 106 | return await delete_vsl_handler(bsl_delete=vsl.name) 107 | 108 | # ------------------------------------------------------------------------------------------------ 109 | # UPDATE A BACKUP STORAGE LOCATION 110 | # ------------------------------------------------------------------------------------------------ 111 | 112 | 113 | limiter_update = endpoint_limiter.get_limiter_cust('vsl') 114 | route = '/vsl' 115 | 116 | 117 | @router.put( 118 | path=route, 119 | tags=[tag_name], 120 | summary='Update a bsl', 121 | description=route_description(tag=tag_name, 122 | route=route, 123 | limiter_calls=limiter_update.max_request, 124 | limiter_seconds=limiter_update.seconds), 125 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_update.seconds, 126 | max_requests=limiter_update.max_request))], 127 | response_model=SuccessfulRequest, 128 | responses=common_error_authenticated_response, 129 | status_code=status.HTTP_200_OK) 130 | @handle_exceptions_endpoint 131 | async def update_bsl(vsl: UpdateVslRequestSchema): 132 | return await update_vsl_handler(vsl=vsl) 133 | -------------------------------------------------------------------------------- /src/service/sc_mapping.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from fastapi import HTTPException 4 | from kubernetes import client 5 | from kubernetes.client.exceptions import ApiException 6 | 7 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 8 | from vui_common.logger.logger_proxy import logger 9 | 10 | core_v1 = client.CoreV1Api() 11 | custom_object = client.CustomObjectsApi() 12 | storage_v1 = client.StorageV1Api() 13 | 14 | 15 | @trace_k8s_async_method(description="Set storage class map") 16 | async def __set_storages_classes_map(namespace=os.getenv('K8S_VELERO_NAMESPACE', 'velero'), 17 | config_map_name='change-storage-classes-config', 18 | data_list=None): 19 | if data_list is None: 20 | data_list = {} 21 | else: 22 | tmp = {} 23 | for item in data_list: 24 | tmp[item['oldStorageClass']] = item['newStorageClass'] 25 | data_list = tmp 26 | try: 27 | 28 | # ConfigMap metadata 29 | config_map_metadata = client.V1ObjectMeta( 30 | name=config_map_name, 31 | namespace=namespace, 32 | labels={ 33 | "velero.io/plugin-config": "", 34 | "velero.io/change-storage-class": "RestoreItemAction" 35 | } 36 | ) 37 | 38 | # Check if the ConfigMap already exists 39 | try: 40 | existing_config_map = core_v1.read_namespaced_config_map(name=config_map_name, 41 | namespace=namespace) 42 | 43 | # If it exists, update the ConfigMap 44 | existing_config_map.data = data_list 45 | core_v1.replace_namespaced_config_map(name=config_map_name, namespace=namespace, 46 | body=existing_config_map) 47 | logger.info( 48 | "ConfigMap 'change-storage-class-config' in namespace 'velero' updated successfully.") 49 | except client.rest.ApiException as e: 50 | # If it doesn't exist, create the ConfigMap 51 | if e.status == 404: 52 | config_map_body = client.V1ConfigMap( 53 | metadata=config_map_metadata, 54 | data=data_list 55 | ) 56 | core_v1.create_namespaced_config_map(namespace=namespace, body=config_map_body) 57 | logger.info( 58 | "ConfigMap 'change-storage-class-config' in namespace 'velero' created successfully.") 59 | else: 60 | raise e 61 | except Exception as e: 62 | logger.error(f"Error writing ConfigMap 'change-storage-class-config' in namespace 'velero': {e}") 63 | 64 | 65 | @trace_k8s_async_method(description="Get storage class config map") 66 | async def get_storages_classes_map_service(config_map_name='change-storage-classes-config', 67 | namespace=os.getenv('K8S_VELERO_NAMESPACE', 'velero')): 68 | # Create an instance of the Kubernetes core API 69 | 70 | # Get the ConfigMap 71 | try: 72 | config_map = core_v1.read_namespaced_config_map(name=config_map_name, 73 | namespace=namespace) # Extract data from the ConfigMap 74 | data = config_map.data or {} 75 | except ApiException as e: 76 | if e.status == 404: 77 | # err_msg = f"ConfigMap '{config_map_name}' not found in namespace '{namespace}'" 78 | logger.warning(f"{e.status} Error reading ConfigMap '{config_map_name}' in namespace '{namespace}'") 79 | return [] 80 | else: 81 | # err_msg = f"Error reading ConfigMap '{config_map_name}' in namespace '{namespace}': {e}" 82 | logger.warning(f"{e.status} Error reading ConfigMap '{config_map_name}' in namespace '{namespace}'") 83 | return [] 84 | 85 | if len(data.items()) > 0: 86 | data_list = [{"oldStorageClass": key, "newStorageClass": value} for key, value in data.items()] 87 | # add_id_to_list(data_list) 88 | else: 89 | data_list = [] 90 | 91 | return data_list 92 | 93 | 94 | @trace_k8s_async_method(description="Update storage classes map") 95 | async def update_storages_classes_mapping_service(data_list=None): 96 | if data_list is not None: 97 | payload = await get_storages_classes_map_service() 98 | config_map = payload 99 | exists = False 100 | for item in config_map: 101 | item.pop('id', None) 102 | if item['oldStorageClass'] == data_list['oldStorageClass']: 103 | item['newStorageClass'] = data_list['newStorageClass'] 104 | exists = True 105 | if not exists: 106 | config_map.append({'oldStorageClass': data_list['oldStorageClass'], 107 | 'newStorageClass': data_list['newStorageClass']}) 108 | await __set_storages_classes_map(data_list=config_map) 109 | 110 | return True 111 | 112 | raise HTTPException(status_code=400, detail=f'Update storage classes error') 113 | 114 | 115 | @trace_k8s_async_method(description="Delete storage classes map") 116 | async def delete_storages_classes_mapping_service(data_list=None): 117 | if data_list is not None: 118 | payload = await get_storages_classes_map_service() 119 | config_map = payload 120 | for item in config_map: 121 | item.pop('id', None) 122 | if item['oldStorageClass'] == data_list['oldStorageClass'] and item['newStorageClass'] == data_list['newStorageClass']: 123 | config_map.remove(item) 124 | 125 | await __set_storages_classes_map(data_list=config_map) 126 | return True 127 | 128 | raise HTTPException(status_code=400, detail=f'Delete storage classes error') 129 | -------------------------------------------------------------------------------- /src/api/v1/routers/sc_mapping.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, status 2 | 3 | from constants.response import common_error_authenticated_response 4 | 5 | from vui_common.security.helpers.rate_limiter import RateLimiter, LimiterRequests 6 | 7 | from vui_common.utils.swagger import route_description 8 | from vui_common.utils.exceptions import handle_exceptions_endpoint 9 | 10 | from vui_common.schemas.response.successful_request import SuccessfulRequest 11 | from schemas.request.storage_class_map import StorageClassMapRequestSchema 12 | 13 | from controllers.sc_mapping import (delete_storages_classes_mapping_handler, 14 | update_storages_classes_mapping_handler, 15 | get_storages_classes_map_handler) 16 | 17 | router = APIRouter() 18 | 19 | 20 | tag_name = 'Storage Class Mapping' 21 | 22 | endpoint_limiter = LimiterRequests(tags=tag_name, 23 | default_key='L1') 24 | 25 | # ------------------------------------------------------------------------------------------------ 26 | # GET STORAGE CLASSES MAPPING 27 | # ------------------------------------------------------------------------------------------------ 28 | 29 | limiter_sc_mapping = endpoint_limiter.get_limiter_cust('sc_mapping') 30 | route = '/sc-mapping' 31 | 32 | 33 | @router.get( 34 | path=route, 35 | tags=[tag_name], 36 | summary='Get change storage classes config map', 37 | description=route_description(tag=tag_name, 38 | route=route, 39 | limiter_calls=limiter_sc_mapping.max_request, 40 | limiter_seconds=limiter_sc_mapping.seconds), 41 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_sc_mapping.seconds, 42 | max_requests=limiter_sc_mapping.max_request))], 43 | response_model=SuccessfulRequest, 44 | responses=common_error_authenticated_response, 45 | status_code=status.HTTP_200_OK) 46 | @handle_exceptions_endpoint 47 | async def get_storages_classes_map(): 48 | return await get_storages_classes_map_handler() 49 | 50 | 51 | # ------------------------------------------------------------------------------------------------ 52 | # CREATE STORAGE CLASSES MAPPING 53 | # ------------------------------------------------------------------------------------------------ 54 | 55 | limiter_sc_create = endpoint_limiter.get_limiter_cust('sc_mapping') 56 | route = '/sc-mapping' 57 | 58 | 59 | @router.post( 60 | path=route, 61 | tags=[tag_name], 62 | summary='Create storage classes config map', 63 | description=route_description(tag=tag_name, 64 | route=route, 65 | limiter_calls=limiter_sc_create.max_request, 66 | limiter_seconds=limiter_sc_create.seconds), 67 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_sc_create.seconds, 68 | max_requests=limiter_sc_create.max_request))], 69 | response_model=SuccessfulRequest, 70 | responses=common_error_authenticated_response, 71 | status_code=status.HTTP_201_CREATED) 72 | @handle_exceptions_endpoint 73 | async def create_storages_classes_map(maps: StorageClassMapRequestSchema): 74 | return await update_storages_classes_mapping_handler(maps=maps) 75 | 76 | 77 | # ------------------------------------------------------------------------------------------------ 78 | # UPDATE STORAGE CLASSES MAPPING 79 | # ------------------------------------------------------------------------------------------------ 80 | 81 | limiter_sc_update = endpoint_limiter.get_limiter_cust('sc_mapping') 82 | route = '/sc-mapping' 83 | 84 | 85 | @router.patch( 86 | path=route, 87 | tags=[tag_name], 88 | summary='Update storage classes config map', 89 | description=route_description(tag=tag_name, 90 | route=route, 91 | limiter_calls=limiter_sc_update.max_request, 92 | limiter_seconds=limiter_sc_update.seconds), 93 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_sc_update.seconds, 94 | max_requests=limiter_sc_update.max_request))], 95 | response_model=SuccessfulRequest, 96 | responses=common_error_authenticated_response, 97 | status_code=status.HTTP_200_OK) 98 | @handle_exceptions_endpoint 99 | async def update_storages_classes_map(maps: StorageClassMapRequestSchema): 100 | return await update_storages_classes_mapping_handler(maps=maps) 101 | 102 | 103 | # ------------------------------------------------------------------------------------------------ 104 | # DELETE STORAGE CLASSES MAPPING 105 | # ------------------------------------------------------------------------------------------------ 106 | 107 | 108 | limiter_sc_delete = endpoint_limiter.get_limiter_cust('sc_mapping') 109 | route = '/sc-mapping' 110 | 111 | 112 | @router.delete( 113 | path=route, 114 | tags=[tag_name], 115 | summary='Delete storage classes mapping in config map', 116 | description=route_description(tag=tag_name, 117 | route=route, 118 | limiter_calls=limiter_sc_delete.max_request, 119 | limiter_seconds=limiter_sc_delete.seconds), 120 | dependencies=[Depends(RateLimiter(interval_seconds=limiter_sc_delete.seconds, 121 | max_requests=limiter_sc_delete.max_request))], 122 | response_model=SuccessfulRequest, 123 | responses=common_error_authenticated_response, 124 | status_code=status.HTTP_200_OK) 125 | @handle_exceptions_endpoint 126 | async def delete_storages_classes_map(items: StorageClassMapRequestSchema): 127 | return await delete_storages_classes_mapping_handler(data_list=items.storageClassMapping) 128 | -------------------------------------------------------------------------------- /src/service/k8s_secret.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from fastapi import HTTPException 4 | from kubernetes import client 5 | from kubernetes.client import ApiException 6 | 7 | from vui_common.configs.config_proxy import config_app 8 | from vui_common.utils.k8s_tracer import trace_k8s_async_method 9 | 10 | 11 | @trace_k8s_async_method(description="Get velero secret list names") 12 | async def get_velero_secret_service(): 13 | try: 14 | secrets = client.CoreV1Api().list_namespaced_secret(config_app.k8s.velero_namespace) 15 | return [secret.metadata.name for secret in secrets.items] 16 | except Exception as e: 17 | print(f"Can't get secret: {e}") 18 | raise HTTPException(status_code=400, detail=f"No secret retrieved") 19 | 20 | 21 | @trace_k8s_async_method(description="Get secret's keys") 22 | async def get_secret_keys_service(namespace: str, secret_name: str): 23 | try: 24 | secret = client.CoreV1Api().read_namespaced_secret(name=secret_name, 25 | namespace=namespace) 26 | if secret.data: 27 | return list(secret.data.keys()) 28 | else: 29 | return [] 30 | except client.exceptions.ApiException as e: 31 | print(f"Error API Kubernetes: {e}") 32 | raise HTTPException(status_code=400, detail=f"Error API Kubernetes: {e}") 33 | except Exception as e: 34 | print(f"Error while obtaining Secret keys: {e}") 35 | raise HTTPException(status_code=400, detail=f"Error while obtaining Secret keys: {e}") 36 | 37 | 38 | @trace_k8s_async_method(description="get secret content") 39 | async def get_secret_service(namespace: str, secret_name: str): 40 | try: 41 | secret = client.CoreV1Api().read_namespaced_secret(name=secret_name, 42 | namespace=namespace) 43 | if secret.data: 44 | decoded_data = {key: base64.b64decode(value).decode('utf-8') for key, value in secret.data.items()} 45 | return decoded_data 46 | else: 47 | return [] 48 | except client.exceptions.ApiException as e: 49 | print(f"Error API Kubernetes: {e}") 50 | # raise HTTPException(status_code=400, detail=f"Error API Kubernetes: {e}") 51 | return None 52 | 53 | except Exception as e: 54 | print(f"Error while obtaining Secret keys: {e}") 55 | # raise HTTPException(status_code=400, detail=f"Error while obtaining Secret keys: {e}") 56 | return None 57 | 58 | 59 | @trace_k8s_async_method(description="add or update key:value in secret") 60 | async def add_or_update_key_in_secret_service(namespace, secret_name, key, value): 61 | """ 62 | Adds or updates a key in a Secret Kubernetes. 63 | 64 | Args: 65 | namespace (str): The namespace of the Secret. 66 | secret_name (str): The name of the Secret. 67 | key (str): The key to be added or updated. 68 | value (str): The value of the key (not base64 encoded). 69 | 70 | Returns: 71 | dict | None: The updated Secret, or None in case of an error 72 | """ 73 | # Upload Kubernetes configuration 74 | # config.load_kube_config() 75 | 76 | v1 = client.CoreV1Api() 77 | 78 | try: 79 | secret = v1.read_namespaced_secret(name=secret_name, namespace=namespace) 80 | 81 | if secret.data is None: 82 | secret.data = {} 83 | 84 | # Encode value in base64 85 | secret.data[key] = base64.b64encode(value.encode()).decode() 86 | 87 | updated_secret = v1.replace_namespaced_secret(name=secret_name, namespace=namespace, body=secret) 88 | print(f"Key '{key}' added/updated in Secret '{secret_name}'.") 89 | return updated_secret 90 | 91 | except ApiException as e: 92 | if e.status == 404: 93 | print(f"Secret '{secret_name}' not found in namespace '{namespace}'. Create it before adding key") 94 | # Create the Secret with the key and the value 95 | secret_data = {key: base64.b64encode(value.encode()).decode()} 96 | new_secret = client.V1Secret( 97 | metadata=client.V1ObjectMeta(name=secret_name), 98 | data=secret_data, 99 | type="Opaque" 100 | ) 101 | 102 | created_secret = v1.create_namespaced_secret(namespace=namespace, body=new_secret) 103 | print(f"Secret '{secret_name}' created with key '{key}'.") 104 | return created_secret 105 | else: 106 | print(f"Error when updating Secret: {e}") 107 | return None 108 | 109 | 110 | @trace_k8s_async_method(description="remove key from secret") 111 | def remove_key_from_secret_service(namespace, secret_name, key): 112 | """ 113 | Removes a key from a Secret Kubernetes. 114 | 115 | Args: 116 | namespace (str): The namespace of the Secret. 117 | secret_name (str): The name of the Secret. 118 | key (str): The key to be removed. 119 | 120 | Returns: 121 | dict | None: The updated Secret or None if the Secret has been deleted or does not exist. 122 | """ 123 | # Upload Kubernetes configuration 124 | # config.load_kube_config() 125 | 126 | v1 = client.CoreV1Api() 127 | 128 | try: 129 | secret = v1.read_namespaced_secret(name=secret_name, namespace=namespace) 130 | 131 | if secret.data is None or key not in secret.data: 132 | print(f"The key '{key}' does not exist in Secret '{secret_name}'.") 133 | return secret 134 | 135 | # Removes the key 136 | del secret.data[key] 137 | print(f"Key '{key}' removed from Secret '{secret_name}'.") 138 | 139 | # If Secret is empty, it deletes it 140 | if not secret.data: 141 | print(f"The Secret '{secret_name}' is now empty. Deleting it...") 142 | v1.delete_namespaced_secret(name=secret_name, namespace=namespace) 143 | return None 144 | 145 | updated_secret = v1.replace_namespaced_secret(name=secret_name, namespace=namespace, body=secret) 146 | return updated_secret 147 | 148 | except ApiException as e: 149 | if e.status == 404: 150 | print(f"Secret '{secret_name}' not found in namespace '{namespace}'.") 151 | else: 152 | print(f"Error while accessing Secret: {e}") 153 | return None 154 | --------------------------------------------------------------------------------