├── .gitignore ├── LICENSE ├── README.md ├── docs ├── README.md ├── admin.md ├── develop.md ├── example │ ├── images │ │ └── readme.md │ ├── job-template │ │ ├── file-splitter │ │ │ └── readme.md │ │ ├── ray │ │ │ └── readme.md │ │ ├── readme.md │ │ └── standalone-script-runner │ │ │ └── readme.md │ └── readme.md └── module.md ├── install ├── README.md ├── docker │ ├── Dockerfile │ ├── Dockerfile-base │ ├── README.md │ ├── config.py │ ├── docker-add-file │ │ ├── __init__.py │ │ └── rest.py │ ├── docker-compose.yml │ ├── entrypoint.sh │ ├── kubeconfig │ │ └── dev-kubeconfig │ ├── requirements-dev.txt │ └── requirements.txt ├── kubernetes │ ├── create_ns_secret.sh │ ├── cube │ │ ├── base │ │ │ ├── deploy-schedule.yaml │ │ │ ├── deploy.yaml │ │ │ ├── ingress.yaml │ │ │ ├── init.yaml │ │ │ ├── kustomization.yml │ │ │ ├── namespace.yaml │ │ │ ├── sa-rbac.yaml │ │ │ └── service.yaml │ │ ├── overlays │ │ │ ├── config │ │ │ │ ├── config.py │ │ │ │ └── entrypoint.sh │ │ │ └── kustomization.yml │ │ └── third-party │ │ │ ├── argo.yaml │ │ │ ├── dashboard.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── mysql.yaml │ │ │ └── redis.yaml │ ├── pv-pvc-pipeline.yaml │ ├── readme.md │ └── start.sh └── upgrade │ ├── Dockerfile │ ├── server.py │ ├── templates │ └── index.html │ └── upgrade.yaml └── myapp ├── .gitignore ├── __init__.py ├── assets ├── images │ ├── 404.jpg │ ├── ad │ │ ├── video-cover1-thumb.png │ │ ├── video-cover1.png │ │ ├── video-cover2-thumb.png │ │ └── video-cover2.png │ ├── apache_feather.png │ ├── babies.png │ ├── favicon.ico │ ├── favicon.png │ ├── home │ │ ├── add.png │ │ ├── automl.png │ │ ├── cart.png │ │ ├── chatbot.jpg │ │ ├── dag.jpg │ │ ├── face.jpg │ │ ├── ide.png │ │ ├── nni.png │ │ ├── openpose.jpg │ │ ├── private.png │ │ ├── search.png │ │ ├── service.png │ │ ├── theia.jpg │ │ └── vscode.png │ ├── loading.gif │ ├── logo-2.png │ ├── logo.svg │ ├── myapp-logo.png │ ├── myapp-logo@2x.png │ ├── myapp-logo@2x2.png │ ├── myapp.png │ ├── noimg.png │ └── s.png ├── js │ ├── myutils.js │ └── mywater.js └── spec │ └── .eslintrc ├── bin ├── __init__.py └── myapp ├── check_tables.py ├── cli.py ├── config.py ├── create_db.py ├── exceptions.py ├── forms.py ├── jinja_context.py ├── migrations ├── README ├── alembic.ini ├── env.py ├── script.py.mako └── versions │ └── c2a4c6139c11_.py ├── models ├── __init__.py ├── base.py ├── helpers.py ├── log.py ├── model_docker.py ├── model_job.py ├── model_notebook.py ├── model_team.py └── user_attributes.py ├── project.py ├── run.py ├── security.py ├── static ├── appbuilder │ ├── css │ │ ├── ab.css │ │ ├── ab_light.css │ │ ├── bootstrap.min.css │ │ ├── flags │ │ │ └── flags16.css │ │ ├── font-awesome.min.css │ │ ├── myapp.css │ │ └── themes │ │ │ ├── amelia.css │ │ │ ├── cerulean.css │ │ │ ├── cosmo.css │ │ │ ├── cyborg.css │ │ │ ├── darkly.css │ │ │ ├── flatly.css │ │ │ ├── journal.css │ │ │ ├── lumen.css │ │ │ ├── paper.css │ │ │ ├── readable.css │ │ │ ├── sandstone.css │ │ │ ├── simplex.css │ │ │ ├── slate.css │ │ │ ├── solar.css │ │ │ ├── spacelab.css │ │ │ ├── superhero.css │ │ │ ├── united.css │ │ │ └── yeti.css │ ├── datepicker │ │ ├── bootstrap-datepicker.css │ │ └── bootstrap-datepicker.js │ ├── fonts │ │ ├── FontAwesome.otf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ ├── fontawesome-webfont.woff2 │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ └── glyphicons-halflings-regular.woff │ ├── img │ │ ├── aol.png │ │ ├── fab.png │ │ ├── flags │ │ │ └── flags16.png │ │ ├── flickr.png │ │ ├── glyphicons-halflings-white.png │ │ ├── glyphicons-halflings.png │ │ ├── google.png │ │ ├── myopenid.png │ │ └── yahoo.png │ ├── js │ │ ├── ab.js │ │ ├── ab_actions.js │ │ ├── ab_filters.js │ │ ├── ab_keep_tab.js │ │ ├── bootstrap.min.js │ │ ├── html5shiv.js │ │ ├── jquery-latest.js │ │ ├── marked.min.js │ │ ├── myapp_into.js │ │ └── respond.min.js │ ├── select2 │ │ ├── search.png │ │ ├── select2-spinner.gif │ │ ├── select2.css │ │ ├── select2.js │ │ ├── select2.png │ │ └── select2x2.png │ └── vison │ │ ├── asset-manifest.json │ │ ├── editor.worker.js │ │ ├── favicon.ico │ │ ├── index.html │ │ ├── json.worker.js │ │ ├── manifest.json │ │ ├── robots.txt │ │ └── static │ │ ├── css │ │ └── 257.dc955ca1.chunk.css │ │ ├── js │ │ ├── 127.bc34d4d5.chunk.js │ │ ├── 257.b1982694.chunk.js │ │ ├── 257.b1982694.chunk.js.LICENSE.txt │ │ ├── 31.c3bb78a9.chunk.js │ │ ├── 392.6cc16ad5.chunk.js │ │ ├── 46.d003bec7.chunk.js │ │ ├── 46.d003bec7.chunk.js.LICENSE.txt │ │ ├── 805.53e76136.chunk.js │ │ ├── main.ff225858.js │ │ └── main.ff225858.js.LICENSE.txt │ │ └── media │ │ └── codicon.00aa018e3ae78d50160b.ttf ├── assets └── mnt ├── stats_logger.py ├── tasks ├── __init__.py ├── async_task.py ├── celery_app.py └── schedules.py ├── templates ├── 404.html ├── add.html ├── appbuilder │ ├── baselayout.html │ ├── general │ │ ├── model │ │ │ ├── edit.html │ │ │ ├── list.html │ │ │ └── show.html │ │ └── widgets │ │ │ ├── base_list.html │ │ │ ├── list.html │ │ │ ├── search.html │ │ │ └── show.html │ ├── init.html │ ├── navbar.html │ ├── navbar_menu.html │ └── navbar_right.html ├── edit.html ├── external_link.html ├── hello.html ├── home.html ├── index.html ├── link.html ├── myapp │ ├── base.html │ ├── basic.html │ ├── csrf_token.json │ ├── fab_overrides │ │ ├── list.html │ │ ├── list_role.html │ │ └── list_with_checkboxes.html │ ├── feature.html │ ├── flash_wrapper.html │ ├── paper-theme.html │ ├── partials │ │ └── _script_tag.html │ ├── theme.html │ ├── traceback.html │ └── welcome.html └── mybase.html ├── tools ├── __init__.py └── watch_workflow.py ├── translations ├── babel.cfg ├── en │ └── LC_MESSAGES │ │ ├── messages.json │ │ └── messages.po ├── messages.pot ├── requirements.txt ├── utils.py └── zh │ └── LC_MESSAGES │ ├── messages.json │ └── messages.po ├── utils ├── __init__.py ├── cache.py ├── celery.py ├── core.py ├── dates.py ├── decorators.py ├── log.py └── py │ ├── __init__.py │ ├── py_k8s.py │ └── py_prometheus.py ├── views ├── __init__.py ├── base.py ├── baseApi.py ├── home.py ├── log │ ├── __init__.py │ ├── api.py │ └── views.py ├── route.py ├── view_docker.py ├── view_job_template.py ├── view_link.py ├── view_notebook.py ├── view_pipeline.py ├── view_runhistory.py ├── view_task.py ├── view_team.py └── view_workflow.py └── vision ├── .editorconfig ├── .eslintrc ├── .gitignore ├── .prettierignore ├── .prettierrc ├── .yarnrc ├── base-tsconfig.json ├── config-overrides.js ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── index.html ├── manifest.json └── robots.txt ├── src ├── api │ ├── ajax.ts │ └── index.ts ├── components │ ├── EditorAce │ │ ├── components │ │ │ ├── BaseTypes │ │ │ │ └── ObjectType.tsx │ │ │ └── VisualizedData │ │ │ │ ├── index.tsx │ │ │ │ └── style.ts │ │ ├── index.tsx │ │ └── style.ts │ ├── FlowEditor │ │ ├── components │ │ │ ├── EditorBody │ │ │ │ ├── components │ │ │ │ │ ├── DataSet │ │ │ │ │ │ ├── index.tsx │ │ │ │ │ │ └── style.ts │ │ │ │ │ ├── Model │ │ │ │ │ │ ├── index.tsx │ │ │ │ │ │ └── style.ts │ │ │ │ │ ├── NodeType.ts │ │ │ │ │ └── Setting │ │ │ │ │ │ ├── index.tsx │ │ │ │ │ │ └── style.ts │ │ │ │ ├── index.tsx │ │ │ │ └── style.ts │ │ │ ├── EditorHead │ │ │ │ ├── index.tsx │ │ │ │ └── style.ts │ │ │ ├── EditorTool │ │ │ │ ├── index.tsx │ │ │ │ └── style.ts │ │ │ └── ErrorTips │ │ │ │ └── index.tsx │ │ └── index.tsx │ ├── Home │ │ ├── Pipeline │ │ │ ├── index.tsx │ │ │ └── style.ts │ │ └── Section │ │ │ ├── index.tsx │ │ │ └── style.ts │ └── ModuleTree │ │ ├── components │ │ ├── ModuleDetail │ │ │ ├── index.tsx │ │ │ └── style.ts │ │ ├── ModuleItem │ │ │ ├── index.tsx │ │ │ └── style.ts │ │ └── SearchItem │ │ │ ├── index.tsx │ │ │ └── style.ts │ │ ├── index.tsx │ │ └── style.ts ├── index.tsx ├── models │ ├── app │ │ └── index.ts │ ├── editor │ │ └── index.ts │ ├── element │ │ └── index.ts │ ├── hooks.ts │ ├── pipeline │ │ └── index.ts │ ├── setting │ │ └── index.ts │ ├── store.ts │ ├── task │ │ └── index.ts │ └── template │ │ └── index.ts ├── pages │ ├── Home.tsx │ └── Index.tsx ├── public │ ├── favicon.svg │ └── logo.svg ├── react-app-env.d.ts ├── routes │ ├── config.ts │ └── index.tsx ├── setupProxy.js ├── static │ └── home.ts ├── types │ ├── pipeline │ │ └── index.d.ts │ └── task │ │ └── index.d.ts └── utils │ ├── getParamter.ts │ ├── index.ts │ └── storage.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | __pycache__ 4 | install/docker/file 5 | install/docker/docker-compose.yml 6 | myapp/static/file 7 | myapp/test 8 | # /myapp/vision/.env 9 | # /myapp/static/appbuilder/vison/* 10 | /myapp/static/appbuilder/mnt 11 | /myapp/static/appbuilder/assets/ 12 | /images/tfserving/tlinux-Dockerfile 13 | /images/serving/onnxruntime/onnxruntime-1.9.1/ 14 | /images/serving/onnxruntime/onnxruntime-1.10.0/ 15 | /images/service-pipeline/kafka-github/ 16 | /myapp/visionPlus/node_modules 17 | /myapp/frontend/node_modules 18 | /myapp/music/node_modules 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 转至cube-studio项目 https://github.com/tencentmusic/cube-studio 2 | 3 | # argo dashboard 4 | 5 | argo dashboard是由TME研发的云原生argo pipeline编排。基于argo的云原生调度,包含项目管理,在线notebook,在线镜像构建,拖拉拽编排pipeline,定时调度,实例管理。 6 | 7 | [部署使用视频](https://www.bilibili.com/video/BV18g411d75r/) 8 | 9 | [模板市场地址](https://github.com/tencentmusic/cube-studio/tree/master/job-template) 10 | 11 | 平台完成部署之后如下: 12 | 13 | image 14 | 15 | # 开源共建 16 | 17 | 有意向进行开源共建的同学请微信添加767065521并备注"开源共建"进入微信群. 18 | 19 | # 功能简述 20 | 21 | ### 多集群管控 22 | 23 | 平台支持多集群调度,可以由一个web平台管控多个训练或推理集群。在不同项目组下配置当前项目组使用的集群名,然后在用户训练或部署推理时,指定对应项目组即可。 24 | 25 | ![image](https://user-images.githubusercontent.com/20157705/167534695-d63b8239-e85e-42c4-bc7b-5999b9eff882.png) 26 | 27 | 28 | ### 分布式存储 29 | 30 | 平台会自动为用户挂载用户个人目录,同一个用户在平台任何地方启动的容器目录下/mnt/$username均为用户个人子目录。可以将pvc/hostpath/memory/configmap等挂载成容器目录。同时可以在项目组中配置项目组的默认挂载,进而实现一个项目组共享同一个目录等功能。 31 | 32 | ![image](https://user-images.githubusercontent.com/20157705/167534724-733ad796-745e-47e1-9224-9e749f918cf2.png) 33 | 34 | 35 | ### 在线开发 36 | 37 | 支持在线jupyterlab/theia(vscode)等功能,多用户,多实例,支持cpu/gpu版本。另外支持在线构建docker镜像,免除算法同学docker学习成本 38 | 39 | ![image](https://user-images.githubusercontent.com/20157705/167534731-8d19cab9-1420-46cf-8a1d-a4c68823c63d.png) 40 | 41 | 42 | ### 拖拉拽pipeline编排 43 | 44 | 支持单任务调试、分布式任务日志聚合查看,pipeline调试跟踪,任务运行资源监控,以及定时调度功能(包含补录,忽略,重试,依赖,并发限制,过期淘汰等功能) 45 | 46 | ![image](https://user-images.githubusercontent.com/20157705/167534748-9adf82ae-fd08-46f1-9ba6-a60b55bb8d3b.png) 47 | 48 | 49 | ### 功能模板化 50 | 51 | 为了避免重复开发,对pipeline中的task功能进行模板化开发。平台开发者或用户可自行开发模板镜像,将镜像注册到平台,这样其他用户就可以复用这些功能。平台自带模板在job-template目录下 52 | 53 | ![image](https://user-images.githubusercontent.com/20157705/167534770-505ffce8-8172-49be-9506-b265cd6ed465.png) 54 | 55 | 56 | # 平台部署 57 | 58 | 参考install/README.md 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # 文档说明 2 | module.md 为平台部署组件功能说明 3 | develop.md 为项目代码说明 4 | admin.md 为平台运营操作说明 5 | -------------------------------------------------------------------------------- /docs/admin.md: -------------------------------------------------------------------------------- 1 | 管理员管理注意事项: 2 | 3 | # 权限管理 4 | 5 | - oa登录开发。登录方式可通过project.py文件定义的方式自行修改,或自行添加管理员专用接口。 6 | - rbac的权限管理方式。所有菜单或者模块的增删改查以及所有的后端接口都是有权限控制的,可以在Security中自由控制 7 | - 用户登录后会自动创建以及绑定用户名同名的角色。同时绑定gamma角色,加入public项目组 8 | 9 | # 项目组->模板分类 10 | 11 | expand字段 12 | ```bash 13 | { 14 | "index":2 # index 控制该分组下面的job模板在pipeline编排界面左侧的显示顺序 15 | } 16 | ``` 17 | 18 | # 项目组->项目分组 19 | ### 跨k8s集群调度的支持: 20 | 21 | 需要在config.py文件中配置CLUSTERS,并通过ENVIRONMENT环境变量指定控制台所在的集群。其他集群为训练集群。 22 | 23 | 项目组/项目分组中,可通过项目组的expand字段控制项目组的调度集群 24 | 25 | { 26 | "cluster": "dev" 27 | } 28 | 29 | ### 资源划分项目组 30 | 机器通过label进行管理,所有的调度机器由平台控制,不由用户直接控制。 31 | 32 | - 对于cpu的train/notebook/service会选择cpu=true的机器 33 | - 对于gpu的train/notebook/service会选择gpu=true的机器 34 | 35 | - 训练任务会选择train=true的机器 36 | - notebook会选择notebook=true的机器 37 | - 服务化会选择service=true的机器 38 | - 不同项目的任务会选择对应org=xx的机器。默认为org=public 39 | 40 | 管理标注哪些机器属于哪些项目后,可通过项目组的expand字段控制项目组的调度机器 41 | 42 | { 43 | "node_selector": "org=public" 44 | } 45 | 管理该项目自动挂载什么pvc,可通过项目组的expand字段控制项目组的调度机器 46 | { 47 | "volume_mount": "kubeflow-user-workspace(pvc):/mnt" 48 | } 49 | 50 | 51 | # 任务->仓库 52 | 53 | 仓库中的k8s hubsecret 以及账号密码,在保存时会更新到配置的所有集群中的各个命名空间下。并且发生用户拉起pod的情况也会附带上用户配置的k8s hubsecret以及config.py中配置的公共k8s hubsecret。 54 | 55 | 建议仓库拉取秘钥均在config.py中全局配置 56 | 57 | # 任务->任务模板 58 | 59 | ### 版本: 60 | 61 | 模板的release版本才能被用户看到 62 | 63 | ### 工作目录/启动命令: 64 | 65 | 如果配置,则会覆盖镜像的启动目录和启动命令 66 | 67 | ### 挂载目录: 68 | 69 | 会在task创建时添加模板的挂载项,一般用户需要固定挂载的模板会配置该参数(比如docker in docker 需要 docker.socket)。目前支持hostpath/pvc/memory/configmap几种挂载的配置方式 70 | 71 | ```bash 72 | kubeflow-user-workspace(pvc):/mnt pvc的挂载方式,会自动在pvc下挂载个人子目录 73 | /data/k8s/kubeflow/pipeline/workspace/xxxx(hostpath):/mnt 挂载主机目录的方式 74 | 4G(memory):/dev/shm 内存挂载为磁盘的书写方式 75 | kubernetes-config(configmap):/root/.kube 挂载configmap成文件夹 76 | ``` 77 | 78 | ### 环境变量: 79 | 80 | 会附加给每个使用该模板的task 81 | 82 | 模板中特殊的环境变量 83 | ```bash 84 | NO_RESOURCE_CHECK=true 使用该模板的task不会进行资源配置的自动校验 85 | TASK_RESOURCE_CPU=4 使用该模板的task 忽略用户的资源配置,cpu固定配置资源为4核 86 | TASK_RESOURCE_MEMORY=4G 使用该模板的task 忽略用户的资源配置,mem固定配置资源为4G 87 | TASK_RESOURCE_GPU=0 使用该模板的task 忽略用户的资源配置,gpu固定配置资源为0卡 88 | ``` 89 | ### k8s账号 90 | 91 | 为模板配置k8s账号,主要为那些需要操控k8s集群的模板而设定的。比如临时分布式训练集群的launcher端 92 | 93 | ### 扩展 94 | 95 | ```bash 96 | { 97 | "index": 7, index控制在pipeline编排界面同一个模板分组中每个模板的显示顺序 98 | "help_url": "http://xx.xx.xx.xx/xx" help_url 为pipeline编排界面每个模板的帮助文档的地址显示 99 | } 100 | ``` 101 | 102 | 103 | # 任务->任务流 104 | 105 | - 先调试单独的task,再调试pipeline的原则。 106 | - task包含debug/run/log/clear等功能,通过run-id串联对应的pod和crd 107 | - pipeline包含run/log/运行实例/定时调度记录/pod等功能。使用run-id串联所有的task 108 | - 手动运行pipeline会先清空之前手动运行的pipeline实例(workflow) 109 | - myapp/task/scheduler.py中会定时清理运行结束的pod,避免pod堆积过多 110 | - 在pieline正常运行10次以后,定时任务会取其中真实使用的资源最大值,再预留一定的空间,动态的修改用户配置的资源项,防止资源配置不合理而浪费算力 111 | 112 | 113 | # 任务->demo任务流 114 | 115 | 首页会显示所有的demo pipeline。即 pipeline 扩展字段expand(dict)中包含 116 | ``` 117 | { 118 | "demo": "true", 119 | "img": "https://xx.xx.xx.xx/xx.png" 120 | } 121 | ``` 122 | 123 | # 任务->定时调度记录 124 | 125 | 手动运行和定时运行同一个pipeline相互之间不干扰。 126 | 127 | 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /docs/develop.md: -------------------------------------------------------------------------------- 1 | # 开发框架 2 | 平台控制端为fab框架,可以参考https://github.com/tencentmusic/fab 3 | 4 | # 登录方式/首页内容/消息推送 5 | myapp/project.py中包含web首页的配置方式、登录的方式、推送消息的方式 6 | 7 | # 定时任务的开发 8 | 启动配置:config.py中CeleryConfig 9 | 代码开发:myapp/tasks/schedules.py 10 | 11 | # 监听crd变化 12 | 代码开发:myapp/tools/watch_xx.py 13 | 14 | # 数据库的更新迭代 15 | myapp/migrations/versions 16 | 17 | # 数据库结构和视图(增删改查界面) 18 | myapp/models 19 | myapp/views 20 | 21 | # pipline编排界面前端 22 | myapp/vision 23 | 24 | # 权限管理的基础逻辑 25 | myapp/security.py -------------------------------------------------------------------------------- /docs/example/job-template/file-splitter/readme.md: -------------------------------------------------------------------------------- 1 | # 模板说明 2 | 文件分割 3 | 4 | # 模板镜像 5 | 6 | `ai.tencentmusic.com/tme-public/ray:gpu-20210601` 7 | 8 | # 模板注册 9 | 参考上级目录的readme.md,注册时填写以下配置。 10 | 11 | 1、启动参数: 12 | ``` 13 | { 14 | "source_file":, 15 | "source_type": , 16 | "csv_delimiter": , 17 | "split_num": , 18 | "tar_path": , 19 | "name_prefix": , 20 | "header": , 21 | "delete_source": 22 | } 23 | ``` 24 | source_file: 必填,源文件,支持通配符 25 | source_type:非必填,源文件类型,目前暂时只支持"csv",默认"csv"。 26 | csv_delimiter:必填,csv文件的列分隔符 27 | split_num:必填,要分割成的文件个数 28 | tar_path:非必填,分割之后的文件存放路径,如果不填,则默认与source_file在同一个目录下 29 | name_prefix:非必填,分割之后文件名前缀,如果设置了,则结果文件名为-part-.,如果没有设置,则结果文件名为-part-.,其中是文件编号,从0到split_num,是源文件的扩展名,是源文件的除去路径和扩展名之后的部分。 30 | header:非必填,csv文件是否包含文件头,如果包含文件,分割后的文件也会都包含文件头。默认为true 31 | delete_source:非必填,分割完之后,是否删掉源文件。默认为true 32 | 33 | # 使用方法 34 | 略 35 | -------------------------------------------------------------------------------- /docs/example/job-template/ray/readme.md: -------------------------------------------------------------------------------- 1 | # 模板说明 2 | 模板镜像内置了python分布式任务ray框架。基于ray框架编写代码,可以实现在集群内启动多个pod分布式执行任务。 3 | 4 | # 模板镜像 5 | 6 | `ai.tencentmusic.com/tme-public/ray:gpu-20210601` 7 | 8 | # 模板注册 9 | 参考上级目录的readme.md,注册时填写以下配置。 10 | 11 | 1、挂载目录:`2G(memory):/dev/shm` 12 | 13 | 2、启动参数: 14 | ``` 15 | { 16 | "shell": { 17 | "-n": { 18 | "type": "int", 19 | "item_type": "", 20 | "label": "分布式任务worker的数量", 21 | "require": 1, 22 | "choice": [], 23 | "range": "$min,$max", 24 | "default": "3", 25 | "placeholder": "", 26 | "describe": "分布式任务worker的数量", 27 | "editable": 1, 28 | "condition": "", 29 | "sub_args": {} 30 | }, 31 | "-i": { 32 | "type": "str", 33 | "item_type": "str", 34 | "label": "每个worker的初始化脚本文件地址,用来安装环境", 35 | "require": 1, 36 | "choice": [], 37 | "range": "", 38 | "default": "", 39 | "placeholder": "每个worker的初始化脚本文件地址,用来安装环境", 40 | "describe": "每个worker的初始化脚本文件地址,用来安装环境", 41 | "editable": 1, 42 | "condition": "", 43 | "sub_args": {} 44 | }, 45 | "-f": { 46 | "type": "str", 47 | "item_type": "str", 48 | "label": "python启动命令,例如 python3 /mnt/xx/xx.py", 49 | "require": 1, 50 | "choice": [], 51 | "range": "", 52 | "default": "", 53 | "placeholder": "", 54 | "describe": "python启动命令,例如 python3 /mnt/xx/xx.py", 55 | "editable": 1, 56 | "condition": "", 57 | "sub_args": {} 58 | } 59 | } 60 | } 61 | ``` 62 | 3、k8s账号: `kubeflow-pipeline` 63 | 64 | # 使用方法 65 | 66 | 原有代码 67 | ``` 68 | # import ray 69 | 70 | def fun1(index): 71 | # 这里是耗时的任务 72 | return 'back_data' 73 | 74 | def main(): 75 | for index in [...]: 76 | fun1(index) # 直接执行任务 77 | 78 | if __name__=="__main__": 79 | main() 80 | ``` 81 | 82 | 启用ray框架的代码 83 | 84 | ``` 85 | import ray 86 | 87 | @ray.remote 88 | def fun1(index): 89 | # 这里是耗时的任务 90 | return 'back_data' 91 | 92 | def main(): 93 | tasks=[] 94 | for index in [...]: 95 | tasks.append(fun1.remote(index)) # 建立远程函数 96 | result = ray.get(tasks) # 获取任务结果 97 | 98 | if __name__=="__main__": 99 | 100 | head_service_ip = os.getenv('RAY_HOST','') 101 | if head_service_ip: 102 | # 集群模式 103 | head_host = head_service_ip+".pipeline"+":10001" 104 | ray.util.connect(head_host) 105 | else: 106 | # 本地模式 107 | ray.init() 108 | 109 | main() 110 | ``` 111 | -------------------------------------------------------------------------------- /docs/example/job-template/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # 1.模板注册流程 3 | 4 | 1、编写代码,打包镜像,推送远程仓库。 5 | 6 | 2、在平台页面上填写信息,注册模板。 7 | 8 | # 2.job模板规范 9 | 10 | ### 2.1.模板目录结构 11 | ``` 12 | 13 | - job-template # job模板合计 14 | - $job_template_name # 自定义模板 15 | - src # 项目代码 16 | - build.sh # job镜像构建过程 17 | - Dockerfile # 构建所需的Dockerfile 18 | - readme.md # 使用方法,规定格式定义 19 | ``` 20 | 可参照ray目录下的模板格式 21 | 22 | ### 2.2.关于构建: 23 | 24 | 1、 统一的构建脚本 sh job/$job_template_name/build.sh 25 | 26 | 2、 Dcokerfile文件定义镜像构建过程,构建路径为当前路径 27 | 28 | ### 2.3.关于代码: 29 | 30 | 不关注代码的实现,只要最终形成docker即可,镜像的输入参数统一为字符串 31 | 32 | ### 2.4.关于镜像 33 | 建议镜像的tag使用日期 34 | 35 | `ai.tencentmusic.com/tme-public/$image_name:$image_tag` 36 | 37 | # 3.注册模板 38 | ### 3.1.模板注册入口 39 | 在平台页面上,任务->任务模板->添加按钮 40 | 41 | 42 | ### 3.2. 其他注册参数 43 | 参照页面上的说明 44 | 45 | ### 公共魔法变量 46 | 47 | 为了便于使用,在配置中支持几个公共的魔法变量,类似占位符,在实际运行中会被展开成实际值,魔法变量的格式为${PARAM_NAME}$,目前支持的有如下几个: 48 | 49 | __${PACK_PATH}$__:包目录,即用户自己代码数据等所在目录,例如/mnt/lionpeng/ai_radio_v2。这个目录是分布式存储挂载到集群worker docker中的目录,该目录会挂载到pipeline中每一个job对应worker docker中。 50 | 51 | __${DATA_PATH}$__: 数据目录,表示pipeline一次运行的目录,这里面会存放本次运行中各job产生的数据,包括用户自己代码所产生的数据都放在这里。每次运行目录是不一样的,便于每次运行之间隔离,另外也是便于同一次运行中上下游job进行数据交互。例如/mnt/lionpeng/ai_radio_v2_runs/20201021-141656.624784。同样该目录也会挂载到pipeline中每一个job对应worker docker中。 52 | 53 | __${DATE[(-|+numd|w|h|m|s][:format]}$__: 日期变量,例如${DATE}$表示任务运行时的时间。该变量还支持偏移,偏移单位支持d(天),w(星期),h(小时),m(分钟),s(秒),y(年),M(月)。例如${DATE-1d}$,表示运行日的前一天,例如今天是20201021,则${DATE-1d}$展开后就是20201020,而${DATE+2d}$则表示运行日的后两天,即20201023。另外支持指定日期的格式化格式,默认格式是%Y%m%d,格式化符号与python datetime格式化符号一致,可参考说明。例如当前时间是2020年10月21日早上10点5分35秒,${DATE-1d:%Y-%m-%d %H:%M:%S}$的展开结果就是"2020-10-20 10:05:35" 54 | 55 | __${ONLINE_MODEL}$__:线上模型,用于在评估任务方便用户拉取线上模型进行指标对比,关于评估任务见后面详述。 56 | -------------------------------------------------------------------------------- /docs/example/job-template/standalone-script-runner/readme.md: -------------------------------------------------------------------------------- 1 | # 模板说明 2 | 3 | 运行单机脚本,支持shell,python脚本 4 | 5 | # 模板镜像 6 | 7 | `ai.tencentmusic.com/tme-public/python_data_transform:20201010` 8 | 9 | # 模板注册 10 | 参考上级目录的readme.md,注册时填写以下配置。 11 | 12 | 1、启动参数: 13 | ``` 14 | { 15 | "script_type": "", 16 | "script_name":"", 17 | "params": [ 18 | , 19 | ... 20 | ], 21 | "export_files": [ 22 | { 23 | "tar_file": "", 24 | "label": "" 25 | }, 26 | ... 27 | ] 28 | } 29 | ``` 30 | 31 | script_type: 必填。脚本类型,目前支持python和shell两种。 32 | script_name:必填。用户脚本文件名,默认在包目录中,支持相对路径。 33 | params:非必填。传递给脚本的参数数组,参数支持使用魔法变量。 34 | export_files:非必填。指定本脚本的输出文件数组,数组每个元素指定一个输出文件。其下字段有: 35 | tar_file:输出文件名字,默认在数据目录中,支持相对路径。 36 | label:用户自定义标签。可不填。 37 | 38 | # 使用方法 39 | 略 40 | -------------------------------------------------------------------------------- /docs/module.md: -------------------------------------------------------------------------------- 1 | # 各pod功能 2 | 平台控制端为fab框架,可以参考https://github.com/tencentmusic/fab 3 | 4 | - deploy.yaml为前后端服务的容器 5 | - deploy-schedule.yaml为产生平台celery任务的pod(非pipeline任务) 6 | - deploy-worker.yaml为执行平台celery任务的pod(非pipeline任务) 7 | - deploy-watch.yaml为监听容器,会监听平台中的crd的状态变化,进而更新数据库和推送消息 8 | 9 | -------------------------------------------------------------------------------- /install/README.md: -------------------------------------------------------------------------------- 1 | 2 | # 平台部署流程 3 | 4 | 基础环境依赖 5 | - docker >= 19.03 6 | - kubernetes = 1.18 7 | - kubectl >=1.18 8 | - 分布式存储,挂载到每台机器的 /data/k8s/,不需要可忽略 9 | - 单机 磁盘>=500G 单机磁盘容量要求不大,仅做镜像容器的的存储 10 | - 控制端机器 cpu>=16 mem>=32G 11 | - 任务端机器,根据需要自行配置 12 | 13 | 请参考install/kubenetes/README.md 部署依赖组件。 14 | 15 | 平台完成部署之后如下: 16 | 17 | image 18 | 19 | 20 | 21 | # 本地开发 22 | 23 | 管理平台web端可连接多个k8s集群用来在k8s集群上调度发起任务实例。同时管理多个k8s集群时,或本地调试时,可以将k8s集群的config文件存储在kubeconfig目录中,按照$ENVIRONMENT-kubeconfig 的规范命名 24 | 25 | ./docker 目录包含本地开发方案,涉及镜像构建,docker-compose启动,前端构建,后端编码等过程 26 | 27 | 参考install/docker/README.md 28 | 29 | 30 | -------------------------------------------------------------------------------- /install/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # docker build -t ai.tencentmusic.com/tme-public/kubeflow-dashboard:argo-2022.05.01 -f install/docker/Dockerfile . 2 | 3 | FROM ai.tencentmusic.com/tme-public/kubeflow-dashboard:base 4 | 5 | RUN apt-get update && apt-get install -y mysql-client zip 6 | 7 | RUN curl -sL https://deb.nodesource.com/setup_14.x | bash 8 | RUN apt-get -y install nodejs && npm install yarn -g 9 | 10 | RUN rm /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python && rm /usr/bin/pip && ln -s /usr/bin/pip3 /usr/bin/pip 11 | RUN pip install pip --upgrade && pip install docstring_parser 12 | RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/bin/ 13 | 14 | RUN pip install kubernetes==18.20.0 click==6.7 pyyaml 15 | COPY install/docker/docker-add-file/rest.py /usr/local/lib/python3.6/dist-packages/kubernetes/client/rest.py 16 | COPY install/docker/docker-add-file/__init__.py /usr/local/lib/python3.6/dist-packages/marshmallow_enum/__init__.py 17 | 18 | COPY myapp /home/myapp/myapp 19 | ENV PATH=/home/myapp/myapp/bin:$PATH 20 | ENV PYTHONPATH=/home/myapp:$PYTHONPATH 21 | 22 | 23 | USER root 24 | COPY install/docker/entrypoint.sh /entrypoint.sh 25 | 26 | # ENTRYPOINT ["/entrypoint.sh"] 27 | # HEALTHCHECK CMD ["curl", "-f", "http://localhost:80/health"] 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /install/docker/Dockerfile-base: -------------------------------------------------------------------------------- 1 | FROM python:3.6-jessie 2 | 3 | # Configure environment 4 | ENV LANG=C.UTF-8 \ 5 | LC_ALL=C.UTF-8 6 | 7 | RUN apt-get update -y && \ 8 | apt-get install -y apt-transport-https apt-utils build-essential libssl-dev \ 9 | libglib2.0-0 libnss3 libgconf-2-4 libfontconfig1 \ 10 | vim less postgresql-client redis-tools \ 11 | libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev && \ 12 | curl -sL https://deb.nodesource.com/setup_10.x | bash - 13 | 14 | 15 | RUN apt-get install -y nodejs && \ 16 | apt-get install -y --force-yes --no-install-recommends ttf-wqy-microhei ttf-wqy-zenhei xfonts-wqy 17 | 18 | RUN pip install clickhouse-sqlalchemy pysnooper pymysql gevent mysqlclient flower infi.clickhouse_orm opencv-python numpy sqlalchemy_utils 19 | 20 | WORKDIR /home/myapp 21 | 22 | COPY install/docker/requirements.txt . 23 | COPY install/docker/requirements-dev.txt . 24 | 25 | RUN pip install --upgrade setuptools pip && \ 26 | pip install -r requirements.txt -r requirements-dev.txt && \ 27 | rm -rf /root/.cache/pip && \ 28 | CHROMEDRIVER_VERSION=`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE` && \ 29 | mkdir -p /opt/chromedriver-$CHROMEDRIVER_VERSION && \ 30 | curl -sS -o /tmp/chromedriver_linux64.zip http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip && \ 31 | unzip -qq /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION && \ 32 | rm /tmp/chromedriver_linux64.zip && \ 33 | chmod +x /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver && \ 34 | ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver /usr/local/bin/chromedriver && \ 35 | wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ 36 | dpkg -i google-chrome-stable_current_amd64.deb; apt-get -fy install && \ 37 | rm google-chrome-stable_current_amd64.deb 38 | 39 | USER root 40 | 41 | EXPOSE 80 42 | 43 | -------------------------------------------------------------------------------- /install/docker/README.md: -------------------------------------------------------------------------------- 1 | 2 | # 本地调试 3 | 4 | ## deploy mysql 5 | 6 | ``` 7 | linux 8 | docker run --network host --restart always --name mysql -e MYSQL_ROOT_PASSWORD=admin -d mysql:5.7 9 | mac 10 | docker run -p 3306:3306 --restart always --name mysql -e MYSQL_ROOT_PASSWORD=admin -d mysql:5.7 11 | 12 | ``` 13 | 进入数据库创建一个db 14 | ``` 15 | CREATE DATABASE IF NOT EXISTS kubeflow DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; 16 | ``` 17 | 镜像构建 18 | 19 | 20 | ``` 21 | 构建基础镜像(包含基础环境) 22 | docker build -t ai.tencentmusic.com/tme-public/kubeflow-dashboard:base -f install/docker/Dockerfile-base . 23 | 24 | 使用基础镜像构建生产镜像 25 | docker build -t ai.tencentmusic.com/tme-public/kubeflow-dashboard:argo-2022.05.01 -f install/docker/Dockerfile . 26 | ``` 27 | 28 | 镜像拉取(如果你不参与开发可以直接使用线上镜像) 29 | ``` 30 | docker pull ai.tencentmusic.com/tme-public/kubeflow-dashboard:argo-2022.05.01 31 | ``` 32 | 33 | ## deploy myapp (docker-compose) 34 | 35 | 本地开发使用 36 | 37 | docker-compose.yaml文件在install/docker目录下,这里提供了mac和linux版本的docker-compose.yaml。 38 | 可自行修改 39 | image:刚才构建的镜像 40 | LOGIN_URL地址:登录重定向地址 41 | MYSQL_SERVICE:mysql的地址 42 | 43 | 44 | 1) init database 45 | ``` 46 | STAGE: 'init' 47 | docker-compose -f docker-compose.yml up 48 | ``` 49 | 2) build fore 50 | ``` 51 | STAGE: 'build' 52 | docker-compose -f docker-compose.yml up 53 | ``` 54 | 3) debug backend 55 | ``` 56 | STAGE: 'dev' 57 | docker-compose -f docker-compose.yml up 58 | ``` 59 | 4) Production 60 | ``` 61 | STAGE: 'prod' 62 | docker-compose -f docker-compose.yml up 63 | ``` 64 | 65 | 部署以后,登录首页 会自动创建用户,绑定角色(Gamma和username同名角色)。 66 | 67 | 可根据自己的需求为角色授权。 68 | 69 | 70 | ## 可视化页面 71 | 72 | 页面资源镜像制作: 73 | ```sh 74 | cd myapp/vision && docker build --no-cache ./ -t your_images_name:your_label --network host 75 | ``` 76 | 77 | 项目资源打包: 78 | ``` 79 | 开发环境要求: 80 | node: 14.15.0+ 81 | npm: 6.14.8+ 82 | 83 | 包管理(建议使用yarn): 84 | yarn: npm install yarn -g 85 | ``` 86 | ```sh 87 | # 初始化安装可能会遇到依赖包的版本选择,直接回车默认即可 88 | cd myapp/vision && yarn && yarn build 89 | ``` 90 | 91 | 输出路径:`/myapp/static/appbuilder` 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /install/docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | redis: 4 | image: bitnami/redis:4.0.14 5 | restart: unless-stopped 6 | environment: 7 | REDIS_PASSWORD: admin 8 | ports: 9 | - "6379:6379" 10 | 11 | myapp: 12 | image: ai.tencentmusic.com/tme-public/kubeflow-dashboard:argo-2022.05.01 13 | restart: unless-stopped 14 | command: ['bash','-c','/entrypoint.sh'] # 生产模式 15 | environment: 16 | STAGE: 'dev' 17 | REDIS_HOST: 'host.docker.internal' 18 | REDIS_PORT: '6379' 19 | REDIS_PASSWORD: admin 20 | MYSQL_SERVICE: 'mysql+pymysql://root:admin@host.docker.internal:3306/pipeline?charset=utf8' 21 | ENVIRONMENT: DEV 22 | 23 | ports: 24 | - "80:80" 25 | depends_on: 26 | - redis 27 | volumes: 28 | - ../../myapp/:/home/myapp/myapp/ 29 | - ./file:/pvc 30 | - ./entrypoint.sh:/entrypoint.sh 31 | - ./config.py:/home/myapp/myapp/config.py 32 | - ./kubeconfig:/home/myapp/kubeconfig 33 | 34 | 35 | 36 | # beat: 37 | # image: ai.tencentmusic.com/tme-public/kubeflow-dashboard:argo-2022.05.01 38 | # restart: "no" 39 | # command: ["bash","-c","celery beat --app=myapp.tasks.celery_app:celery_app --loglevel=info"] 40 | # shm_size: '100gb' 41 | # environment: 42 | # REDIS_HOST: 'host.docker.internal' 43 | # REDIS_PORT: '6379' 44 | # REDIS_PASSWORD: admin 45 | # MYSQL_SERVICE: 'mysql+pymysql://root:admin@host.docker.internal:3306/kubeflow?charset=utf8' 46 | # ENVIRONMENT: DEV 47 | # depends_on: 48 | # - redis 49 | # volumes: 50 | # - ../../myapp/:/home/myapp/myapp/ 51 | # - ./file:/pvc 52 | # - ./entrypoint.sh:/entrypoint.sh 53 | # - ./config.py:/home/myapp/myapp/config.py 54 | # - ./kubeconfig:/home/myapp/kubeconfig 55 | ## 56 | # 57 | ## 58 | # 59 | # worker: 60 | # image: ai.tencentmusic.com/tme-public/kubeflow-dashboard:argo-2022.05.01 61 | # restart: unless-stopped 62 | # command: ["bash","-c","celery worker --app=myapp.tasks.celery_app:celery_app --loglevel=info --pool=prefork -Ofair -c 40"] 63 | # environment: 64 | # REDIS_HOST: 'host.docker.internal' 65 | # REDIS_PORT: '6379' 66 | # REDIS_PASSWORD: admin 67 | # MYSQL_SERVICE: 'mysql+pymysql://root:admin@host.docker.internal:3306/kubeflow?charset=utf8' 68 | # ENVIRONMENT: DEV 69 | # depends_on: 70 | # - redis 71 | # volumes: 72 | # - ../../myapp/:/home/myapp/myapp/ 73 | # - ./file:/pvc 74 | # - ./entrypoint.sh:/entrypoint.sh 75 | # - ./config.py:/home/myapp/myapp/config.py 76 | # - ./kubeconfig:/home/myapp/kubeconfig 77 | # 78 | 79 | 80 | -------------------------------------------------------------------------------- /install/docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | rm -rf /home/myapp/myapp/static/assets 6 | ln -s /home/myapp/myapp/assets /home/myapp/myapp/static/ 7 | rm -rf /home/myapp/myapp/static/appbuilder/mnt 8 | ln -s /data/k8s/kubeflow/global/static /home/myapp/myapp/static/appbuilder/mnt 9 | 10 | 11 | if [ "$STAGE" = "init" ]; then 12 | export FLASK_APP=myapp:app 13 | python myapp/create_db.py 14 | myapp fab create-admin --username admin --firstname admin --lastname admin --email admin@tencent.com --password admin 15 | # myapp db init # 生成migrations文件夹 16 | # myapp db migrate # 生成对应版本数据库表的升级文件到versions文件夹下,需要你的数据库是已经upgrade的 17 | myapp db upgrade # 数据库表同步更新到mysql 18 | # 会创建默认的角色和权限。会创建自定义的menu权限,也才能显示自定义menu。 19 | myapp init 20 | 21 | elif [ "$STAGE" = "build" ]; then 22 | # cd /home/myapp/myapp/vision && yarn && yarn build 23 | cd /home/myapp/myapp/vision && npm install && yarn build 24 | 25 | elif [ "$STAGE" = "dev" ]; then 26 | export FLASK_APP=myapp:app 27 | # FLASK_ENV=development flask run -p 80 --with-threads --host=0.0.0.0 28 | python myapp/run.py 29 | 30 | elif [ "$STAGE" = "prod" ]; then 31 | export FLASK_APP=myapp:app 32 | python myapp/check_tables.py 33 | gunicorn --bind 0.0.0.0:80 --workers 20 --timeout 300 --limit-request-line 0 --limit-request-field_size 0 --log-level=info myapp:app 34 | else 35 | myapp --help 36 | fi 37 | 38 | 39 | -------------------------------------------------------------------------------- /install/docker/kubeconfig/dev-kubeconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/install/docker/kubeconfig/dev-kubeconfig -------------------------------------------------------------------------------- /install/docker/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | black==19.3b0 2 | coverage==4.5.3 3 | flake8-import-order==0.18.1 4 | flake8-mypy==17.8.0 5 | flake8==3.7.7 6 | flask-cors==3.0.7 7 | ipdb==0.12 8 | mypy==0.670 9 | nose==1.3.7 10 | pip-tools==3.7.0 11 | pre-commit==1.17.0 12 | psycopg2-binary==2.7.5 13 | pycodestyle==2.5.0 14 | pydruid==0.5.6 15 | pyhive==0.6.1 16 | pylint==1.9.2 17 | redis==3.2.1 18 | requests==2.22.0 19 | statsd==3.3.0 20 | tox==3.11.1 21 | bs4 -------------------------------------------------------------------------------- /install/docker/requirements.txt: -------------------------------------------------------------------------------- 1 | alembic==1.0.11 # via flask-migrate 2 | amqp==2.5.0 # via kombu 3 | apispec[yaml]==1.3.3 # via flask-appbuilder 4 | asn1crypto==0.24.0 # via cryptography 5 | attrs==19.1.0 # via jsonschema 6 | babel==2.7.0 # via flask-babel 7 | billiard==3.6.0.0 # via celery 8 | bleach==3.1.0 9 | celery==4.3.0 10 | certifi==2019.6.16 # via requests 11 | cffi==1.12.3 # via cryptography 12 | chardet==3.0.4 # via requests 13 | click==6.7 14 | colorama==0.4.1 15 | contextlib2==0.5.5 16 | croniter==0.3.30 17 | cryptography==2.7 18 | decorator==4.4.0 # via retry 19 | defusedxml==0.6.0 # via python3-openid 20 | flask-appbuilder==2.2.2 # 2.2.2 2.1.7 21 | flask-babel==0.12.2 # via flask-appbuilder 22 | flask-caching==1.7.2 23 | flask-compress==1.4.0 24 | flask-jwt-extended==3.20.0 # via flask-appbuilder 25 | flask-login==0.4.1 # via flask-appbuilder 26 | flask-migrate==2.5.2 27 | flask-openid==1.2.5 # via flask-appbuilder 28 | flask-sqlalchemy==2.4.0 # via flask-appbuilder, flask-migrate 29 | flask-talisman==0.7.0 30 | flask-wtf==0.14.2 31 | flask==1.1.1 32 | future==0.17.1 # via parsedatetime 33 | geographiclib==1.49 # via geopy 34 | geopy==1.20.0 35 | gunicorn==19.8.1 36 | humanize==0.5.1 37 | idna==2.8 38 | isodate==0.6.0 39 | itsdangerous==1.1.0 # via flask 40 | jinja2==2.10.1 # via flask, flask-babel 41 | jsonschema==3.0.1 # via flask-appbuilder 42 | kombu==4.6.3 # via celery 43 | mako==1.0.14 # via alembic 44 | markdown==3.1.1 45 | markupsafe==1.1.1 # via jinja2, mako 46 | marshmallow-enum==1.4.1 # via flask-appbuilder 47 | marshmallow-sqlalchemy==0.17.0 # via flask-appbuilder 48 | marshmallow==2.19.5 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy 49 | numpy==1.17.0 # via pandas 50 | pandas==0.24.2 51 | parsedatetime==2.4 52 | pathlib2==2.3.4 53 | polyline==1.4.0 54 | prison==0.1.2 # via flask-appbuilder 55 | py==1.8.0 # via retry 56 | pycparser==2.19 # via cffi 57 | pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended 58 | pyrsistent==0.15.4 # via jsonschema 59 | python-dateutil==2.8.0 60 | python-dotenv==0.10.3 61 | python-editor==1.0.4 # via alembic 62 | python-geohash==0.8.5 63 | python3-openid==3.1.0 # via flask-openid 64 | pytz==2019.2 # via babel, celery, pandas 65 | pyyaml==5.1.2 66 | retry==0.9.2 67 | selenium==3.141.0 68 | simplejson==3.16.0 69 | six==1.12.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, pathlib2, polyline, prison, pydruid, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json 70 | sqlalchemy-utils==0.34.1 71 | sqlalchemy==1.3.6 72 | sqlparse==0.2.4 73 | urllib3==1.25.3 # via requests, selenium 74 | vine==1.3.0 # via amqp, celery 75 | webencodings==0.5.1 # via bleach 76 | werkzeug==0.15.5 # via flask, flask-jwt-extended 77 | wtforms-json==0.3.3 78 | wtforms==2.2.1 # via flask-wtf, wtforms-json 79 | 80 | # The following packages are considered to be unsafe in a requirements file: 81 | # setuptools==41.0.1 # via jsonschema, markdown 82 | -------------------------------------------------------------------------------- /install/kubernetes/create_ns_secret.sh: -------------------------------------------------------------------------------- 1 | 2 | for namespace in 'infra' 'pipeline' 'jupyter' 3 | do 4 | kubectl create ns $namespace 5 | kubectl delete secret docker-registry hubsecret -n $namespace 6 | kubectl create secret docker-registry hubsecret --docker-username=xxxx --docker-password=xxxx -n $namespace 7 | done 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /install/kubernetes/cube/base/deploy.yaml: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: kubeflow-dashboard 6 | namespace: infra 7 | labels: 8 | app: kubeflow-dashboard 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: kubeflow-dashboard 14 | template: 15 | metadata: 16 | name: kubeflow-dashboard 17 | labels: 18 | app: kubeflow-dashboard 19 | spec: 20 | volumes: 21 | - name: tz-config 22 | hostPath: 23 | path: /usr/share/zoneinfo/Asia/Shanghai 24 | - name: kubeflow-dashboard-config 25 | configMap: 26 | name: kubeflow-dashboard-config 27 | items: 28 | - key: entrypoint.sh 29 | path: entrypoint.sh 30 | - key: config.py 31 | path: config.py 32 | - name: kubernetes-config 33 | configMap: 34 | name: kubernetes-config 35 | 36 | serviceAccount: kubeflow-dashboard 37 | containers: 38 | - name: kubeflow-dashboard 39 | image: ai.tencentmusic.com/tme-public/kubeflow-dashboard 40 | imagePullPolicy: Always # IfNotPresent 41 | command: ["bash","/entrypoint.sh"] 42 | env: 43 | - name: STAGE 44 | value: $(STAGE) 45 | - name: REDIS_HOST 46 | valueFrom: 47 | configMapKeyRef: 48 | name: deploy-config 49 | key: REDIS_HOST 50 | - name: REDIS_PORT 51 | valueFrom: 52 | configMapKeyRef: 53 | name: deploy-config 54 | key: REDIS_PORT 55 | - name: REDIS_PASSWORD 56 | valueFrom: 57 | configMapKeyRef: 58 | name: deploy-config 59 | key: REDIS_PASSWORD 60 | - name: MYSQL_SERVICE 61 | valueFrom: 62 | configMapKeyRef: 63 | name: deploy-config 64 | key: MYSQL_SERVICE 65 | - name: ENVIRONMENT 66 | valueFrom: 67 | configMapKeyRef: 68 | name: deploy-config 69 | key: ENVIRONMENT 70 | 71 | volumeMounts: 72 | - name: kubeflow-dashboard-config 73 | mountPath: /entrypoint.sh 74 | subPath: entrypoint.sh 75 | - name: kubeflow-dashboard-config 76 | mountPath: /home/myapp/myapp/config.py 77 | subPath: config.py 78 | - name: tz-config 79 | mountPath: /etc/localtime 80 | ports: 81 | - name: http 82 | containerPort: 80 83 | protocol: TCP 84 | resources: 85 | limits: 86 | cpu: 10 87 | memory: 10Gi 88 | requests: 89 | cpu: 10m 90 | memory: 100Mi 91 | livenessProbe: 92 | failureThreshold: 2 93 | httpGet: 94 | path: /health 95 | port: http 96 | initialDelaySeconds: 100 97 | periodSeconds: 60 98 | timeoutSeconds: 10 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /install/kubernetes/cube/base/init.yaml: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: kubeflow-dashboard-init 6 | namespace: infra 7 | labels: 8 | app: kubeflow-dashboard-init 9 | spec: 10 | parallelism: 1 11 | completions: 1 12 | backoffLimit: 6 13 | template: 14 | spec: 15 | restartPolicy: Never 16 | volumes: 17 | - name: tz-config 18 | hostPath: 19 | path: /usr/share/zoneinfo/Asia/Shanghai 20 | - name: kubeflow-dashboard-config 21 | configMap: 22 | name: kubeflow-dashboard-config 23 | items: 24 | - key: entrypoint.sh 25 | path: entrypoint.sh 26 | - key: config.py 27 | path: config.py 28 | 29 | serviceAccount: kubeflow-dashboard 30 | containers: 31 | - name: kubeflow-dashboard 32 | image: ai.tencentmusic.com/tme-public/kubeflow-dashboard 33 | imagePullPolicy: Always # IfNotPresent 34 | command: ["bash","/entrypoint.sh"] 35 | env: 36 | - name: STAGE 37 | value: init 38 | - name: MYSQL_SERVICE 39 | valueFrom: 40 | configMapKeyRef: 41 | name: deploy-config 42 | key: MYSQL_SERVICE 43 | volumeMounts: 44 | - name: kubeflow-dashboard-config 45 | mountPath: /entrypoint.sh 46 | subPath: entrypoint.sh 47 | - name: kubeflow-dashboard-config 48 | mountPath: /home/myapp/myapp/config.py 49 | subPath: config.py 50 | - name: tz-config 51 | mountPath: /etc/localtime 52 | 53 | 54 | -------------------------------------------------------------------------------- /install/kubernetes/cube/base/kustomization.yml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - namespace.yaml 5 | - service.yaml 6 | - sa-rbac.yaml 7 | - deploy.yaml 8 | - deploy-schedule.yaml 9 | - init.yaml 10 | - ingress.yaml -------------------------------------------------------------------------------- /install/kubernetes/cube/base/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: pipeline 5 | --- 6 | apiVersion: v1 7 | kind: Namespace 8 | metadata: 9 | name: infra 10 | 11 | --- 12 | apiVersion: v1 13 | kind: Namespace 14 | metadata: 15 | name: jupyter 16 | -------------------------------------------------------------------------------- /install/kubernetes/cube/base/sa-rbac.yaml: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: kubeflow-dashboard 6 | namespace: infra 7 | --- 8 | kind: ClusterRole 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | metadata: 11 | name: kubeflow-clusterrole 12 | rules: 13 | - apiGroups: ["*"] 14 | resources: ["pods","pods/log","services","endpoints","configmaps","nodes","deployments","mpijobs","tfjobs","pytorchjobs","frameworks"] 15 | verbs: ["create", "delete", "deletecollection", "patch", "update", "get", "list", "watch"] 16 | - apiGroups: ["*"] 17 | resources: ["*"] 18 | verbs: ["create", "delete", "deletecollection", "patch", "update", "get", "list", "watch"] 19 | --- 20 | kind: ClusterRoleBinding 21 | apiVersion: rbac.authorization.k8s.io/v1 22 | metadata: 23 | name: kubeflow-dashboard 24 | subjects: 25 | - kind: ServiceAccount 26 | name: kubeflow-dashboard 27 | namespace: infra 28 | roleRef: 29 | kind: ClusterRole 30 | name: kubeflow-clusterrole 31 | apiGroup: rbac.authorization.k8s.io 32 | --- 33 | apiVersion: v1 34 | kind: ServiceAccount 35 | metadata: 36 | name: kubeflow-pipeline 37 | namespace: pipeline 38 | 39 | --- 40 | kind: ClusterRoleBinding 41 | apiVersion: rbac.authorization.k8s.io/v1 42 | metadata: 43 | name: kubeflow-pipeline 44 | subjects: 45 | - kind: ServiceAccount 46 | name: kubeflow-pipeline 47 | namespace: pipeline 48 | roleRef: 49 | kind: ClusterRole 50 | name: kubeflow-clusterrole 51 | apiGroup: rbac.authorization.k8s.io 52 | 53 | --- 54 | 55 | apiVersion: rbac.authorization.k8s.io/v1 56 | kind: ClusterRole 57 | metadata: 58 | name: kubeflow-admin 59 | rules: 60 | - apiGroups: ["*"] 61 | resources: ["*"] 62 | verbs: ["*"] 63 | 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /install/kubernetes/cube/base/service.yaml: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: kubeflow-dashboard 6 | namespace: infra 7 | labels: 8 | app: kubeflow-dashboard 9 | spec: 10 | ports: 11 | - name: http 12 | port: 80 13 | targetPort: 80 14 | protocol: TCP 15 | selector: 16 | app: kubeflow-dashboard 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /install/kubernetes/cube/overlays/config/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | rm -rf /home/myapp/myapp/static/assets 6 | ln -s /home/myapp/myapp/assets /home/myapp/myapp/static/ 7 | rm -rf /home/myapp/myapp/static/appbuilder/mnt 8 | ln -s /data/k8s/kubeflow/global/static /home/myapp/myapp/static/appbuilder/mnt 9 | 10 | 11 | if [ "$STAGE" = "init" ]; then 12 | export FLASK_APP=myapp:app 13 | python myapp/create_db.py 14 | myapp fab create-admin --username admin --firstname admin --lastname admin --email admin@tencent.com --password admin 15 | # myapp db init # 生成migrations文件夹 16 | # myapp db migrate # 生成对应版本数据库表的升级文件到versions文件夹下,需要你的数据库是已经upgrade的 17 | myapp db upgrade # 数据库表同步更新到mysql 18 | # 会创建默认的角色和权限。会创建自定义的menu权限,也才能显示自定义menu。 19 | myapp init 20 | 21 | elif [ "$STAGE" = "build" ]; then 22 | cd /home/myapp/myapp/vision && yarn && yarn build 23 | 24 | elif [ "$STAGE" = "dev" ]; then 25 | export FLASK_APP=myapp:app 26 | FLASK_ENV=development flask run -p 80 --with-threads --host=0.0.0.0 27 | 28 | elif [ "$STAGE" = "prod" ]; then 29 | export FLASK_APP=myapp:app 30 | python myapp/check_tables.py 31 | gunicorn --bind 0.0.0.0:80 --workers 20 --timeout 300 --limit-request-line 0 --limit-request-field_size 0 --log-level=info myapp:app 32 | else 33 | myapp --help 34 | fi 35 | 36 | 37 | -------------------------------------------------------------------------------- /install/kubernetes/cube/overlays/kustomization.yml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | namespace: infra 4 | 5 | configMapGenerator: 6 | - name: kubeflow-dashboard-config 7 | files: 8 | - config/config.py 9 | - config/entrypoint.sh 10 | 11 | # kubectl delete configmap kubeflow-dashboard-config -n infra 12 | # kubectl create configmap kubeflow-dashboard-config --from-file=config -n infra 13 | 14 | - name: deploy-config 15 | literals: 16 | - STAGE=prod 17 | - REDIS_HOST=redis-master.infra 18 | - REDIS_PORT=6379 19 | - REDIS_PASSWORD=admin 20 | - MYSQL_SERVICE=mysql+pymysql://root:admin@mysql-service.infra:3306/pipeline?charset=utf8 21 | - ENVIRONMENT=DEV 22 | 23 | 24 | vars: 25 | - name: STAGE 26 | objref: 27 | kind: ConfigMap 28 | name: deploy-config 29 | apiVersion: v1 30 | fieldref: 31 | fieldpath: data.STAGE 32 | 33 | bases: 34 | - ../base 35 | - ../third-party 36 | 37 | images: 38 | - name: ai.tencentmusic.com/tme-public/kubeflow-dashboard 39 | newName: ai.tencentmusic.com/tme-public/kubeflow-dashboard 40 | newTag: argo-2022.05.01 41 | 42 | 43 | # kustomize build . | kubectl apply -f - 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /install/kubernetes/cube/third-party/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - argo.yaml 6 | - dashboard.yaml 7 | - mysql.yaml 8 | - redis.yaml 9 | -------------------------------------------------------------------------------- /install/kubernetes/pv-pvc-pipeline.yaml: -------------------------------------------------------------------------------- 1 | 2 | # 模型训练 3 | --- 4 | apiVersion: v1 5 | kind: PersistentVolume 6 | metadata: 7 | name: pipeline-kubeflow-user-workspace 8 | labels: 9 | pipeline-pvname: pipeline-kubeflow-user-workspace 10 | spec: 11 | # storageClassName: pipeline 12 | capacity: 13 | storage: 500Gi 14 | accessModes: 15 | - ReadWriteMany 16 | hostPath: 17 | path: /data/k8s/kubeflow/pipeline/workspace 18 | persistentVolumeReclaimPolicy: Retain 19 | --- 20 | kind: PersistentVolumeClaim 21 | apiVersion: v1 22 | metadata: 23 | name: kubeflow-user-workspace 24 | namespace: pipeline 25 | spec: 26 | accessModes: 27 | - ReadWriteMany 28 | resources: 29 | requests: 30 | storage: 500Gi 31 | selector: 32 | matchLabels: 33 | pipeline-pvname: pipeline-kubeflow-user-workspace 34 | 35 | --- 36 | 37 | # 模型训练 38 | --- 39 | apiVersion: v1 40 | kind: PersistentVolume 41 | metadata: 42 | name: jupyter-kubeflow-user-workspace 43 | labels: 44 | jupyter-pvname: jupyter-kubeflow-user-workspace 45 | spec: 46 | # storageClassName: pipeline 47 | capacity: 48 | storage: 500Gi 49 | accessModes: 50 | - ReadWriteMany 51 | hostPath: 52 | path: /data/k8s/kubeflow/pipeline/workspace 53 | persistentVolumeReclaimPolicy: Retain 54 | --- 55 | kind: PersistentVolumeClaim 56 | apiVersion: v1 57 | metadata: 58 | name: kubeflow-user-workspace 59 | namespace: jupyter 60 | spec: 61 | accessModes: 62 | - ReadWriteMany 63 | resources: 64 | requests: 65 | storage: 500Gi 66 | selector: 67 | matchLabels: 68 | jupyter-pvname: jupyter-kubeflow-user-workspace 69 | -------------------------------------------------------------------------------- /install/kubernetes/readme.md: -------------------------------------------------------------------------------- 1 | # 机器/k8s环境/分布式存储环境准备 2 | 机器环境准备:准备docker,准备rancher,部署k8s。如果已经有可以忽略,没有可以参考[rancher/readme.md](https://github.com/tencentmusic/cube-studio/tree/master/install/kubernetes/rancher) 3 | 4 | 5 | # 分布式存储 6 | 7 | 如果不使用分布式存储可忽略。 8 | 9 | 需要每台机器都有对应的目录/data/k8s为分布式存储目录,如果使用云存储,自定义修改pv的yaml文件 10 | ```bash 11 | mkdir -p /data/k8s/kubeflow/pipeline/workspace 12 | ``` 13 | 平台pvc会使用这些分布式存储目录下的subpath,所以如果你是rancher部署k8s集群,需要在kubelet容器中挂载主机的/data/k8s/目录到kubelet容器的/data/k8s/目录。 14 | rancher修改kubelet容器挂载目录(选中集群-升级-编辑yaml) 15 | ``` 16 | kubelet: 17 | extra_binds: 18 | - '/data/k8s:/data/k8s' 19 | ``` 20 | 21 | # 创建仓库秘钥 22 | ```bash 23 | 修改里面的docker hub拉取账号密码 24 | sh create_ns_secret.sh 25 | ``` 26 | 27 | # 通过label进行机器管理 28 | 开发训练服务机器管理: 29 | - 对于cpu的任务会选择cpu=true的机器 30 | - 对于gpu的任务会选择gpu=true的机器 31 | 32 | - pipeline任务会选择train=true的机器 33 | - notebook会选择notebook=true的机器 34 | - 不同项目的任务会选择对应org=xx的机器。默认为org=public 35 | - 可以通过gpu-type=xx表示gpu的型号 36 | 37 | 38 | # 部署平台 39 | ```bash 40 | kubectl apply -k cube/overlays 41 | ``` 42 | 43 | 组件说明 44 | - cube/base/deploy.yaml为myapp的前后端代码 45 | - cube/base/deploy-schedule.yaml 为任务产生器 46 | - cube/base/deploy-worker.yaml 为任务执行器 47 | - cube/base/deploy-watch.yaml 任务监听器 48 | 49 | 配置文件说明 50 | - cube/overlays/config/entrypoint.sh 镜像启动脚本 51 | - cube/overlays/config/config.py 配置文件,需要将其中的配置项替换为自己的 52 | 53 | 54 | ## 部署pv-pvc.yaml 55 | 56 | ```bash 57 | kubectl create -f pv-pvc-pipeline.yaml 58 | ``` 59 | 60 | # 版本升级 61 | 数据库升级,数据库记录要批量添加默认值到原有记录上,不然容易添加失败 62 | 63 | 64 | -------------------------------------------------------------------------------- /install/kubernetes/start.sh: -------------------------------------------------------------------------------- 1 | mkdir -p /data/k8s/kubeflow/pipeline/workspace 2 | 3 | node=`kubectl get node |grep worker | awk '{print $1}' | head -n 1` 4 | kubectl label node $node train=true cpu=true org=public notebook=true --overwrite 5 | # 拉取镜像 6 | 7 | curl -LO https://dl.k8s.io/release/v1.18.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/bin/ 8 | wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv4.5.1/kustomize_v4.5.1_linux_amd64.tar.gz && tar -zxvf kustomize_v4.5.1_linux_amd64.tar.gz && chmod +x kustomize && mv kustomize /usr/bin/ 9 | 10 | # 创建命名空间 11 | sh create_ns_secret.sh 12 | # 部署dashboard 13 | kubectl apply -k cube/overlays 14 | kubectl apply -f pv-pvc-pipeline.yaml 15 | 16 | # 暴 17 | echo "访问ingress入口" 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /install/upgrade/Dockerfile: -------------------------------------------------------------------------------- 1 | # docker build -t ai.tencentmusic.com/tme-public/fab-upgrade:2020.09.01.0 -f upgrade/Dockerfile . 2 | 3 | FROM python:3.6 4 | 5 | RUN pip install flask 6 | 7 | COPY upgrade /upgrade 8 | 9 | WORKDIR /upgrade 10 | 11 | CMD ['python','server.py'] 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /install/upgrade/server.py: -------------------------------------------------------------------------------- 1 | from flask import render_template 2 | from flask import Flask,request 3 | import logging 4 | app = Flask(__name__) 5 | 6 | 7 | 8 | 9 | @app.route('/') 10 | def index(): 11 | logging.info('>>>>>>>>>>>>>') 12 | ip = request.remote_addr 13 | print(ip) 14 | logging.info(ip) 15 | logging.info('<<<<<<<<<<<<<') 16 | return render_template("./index.html") 17 | 18 | if __name__ == '__main__': 19 | app.run(host='0.0.0.0', debug=True, port='80') -------------------------------------------------------------------------------- /install/upgrade/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 服务维护中 9 | 10 | 33 | 34 | 35 | 36 |
37 |
38 |
39 | 40 | 系统维护中 42 | 44 | 45 | 47 | 49 | 50 | 51 |
52 |
53 | 系统维护中 54 |
具体恢复时间请咨询 系统管理员
55 |
56 |
57 |
58 | 59 | 60 | -------------------------------------------------------------------------------- /install/upgrade/upgrade.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: fab-upgrade 7 | namespace: infra 8 | labels: 9 | app: fab-upgrade 10 | spec: 11 | ports: 12 | - name: http 13 | port: 80 14 | targetPort: 80 15 | protocol: TCP 16 | selector: 17 | app: fab-upgrade 18 | 19 | 20 | --- 21 | 22 | apiVersion: apps/v1 23 | kind: Deployment 24 | metadata: 25 | name: fab-upgrade 26 | namespace: infra 27 | labels: 28 | app: fab-upgrade 29 | spec: 30 | replicas: 1 31 | selector: 32 | matchLabels: 33 | app: fab-upgrade 34 | template: 35 | metadata: 36 | name: fab-upgrade 37 | labels: 38 | app: fab-upgrade 39 | spec: 40 | containers: 41 | - name: fab-upgrade 42 | image: ai.tencentmusic.com/tme-public/fab-upgrade:2020.09.01.0 43 | imagePullPolicy: Always # IfNotPresent 44 | command: ["python","server.py"] # 如果修改了entrypoint要使用命令执行 45 | # command: ["sleep","300000"] 46 | ports: 47 | - name: http 48 | containerPort: 80 49 | protocol: TCP -------------------------------------------------------------------------------- /myapp/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | app.db 3 | ./tmp 4 | ./build/* 5 | .idea 6 | env 7 | venv 8 | *.sublime* 9 | appbuilder1 10 | install/docker/kubeconfig 11 | 12 | 13 | /musicfeast/ 14 | -------------------------------------------------------------------------------- /myapp/assets/images/404.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/404.jpg -------------------------------------------------------------------------------- /myapp/assets/images/ad/video-cover1-thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/ad/video-cover1-thumb.png -------------------------------------------------------------------------------- /myapp/assets/images/ad/video-cover1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/ad/video-cover1.png -------------------------------------------------------------------------------- /myapp/assets/images/ad/video-cover2-thumb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/ad/video-cover2-thumb.png -------------------------------------------------------------------------------- /myapp/assets/images/ad/video-cover2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/ad/video-cover2.png -------------------------------------------------------------------------------- /myapp/assets/images/apache_feather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/apache_feather.png -------------------------------------------------------------------------------- /myapp/assets/images/babies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/babies.png -------------------------------------------------------------------------------- /myapp/assets/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/favicon.ico -------------------------------------------------------------------------------- /myapp/assets/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/favicon.png -------------------------------------------------------------------------------- /myapp/assets/images/home/add.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/add.png -------------------------------------------------------------------------------- /myapp/assets/images/home/automl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/automl.png -------------------------------------------------------------------------------- /myapp/assets/images/home/cart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/cart.png -------------------------------------------------------------------------------- /myapp/assets/images/home/chatbot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/chatbot.jpg -------------------------------------------------------------------------------- /myapp/assets/images/home/dag.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/dag.jpg -------------------------------------------------------------------------------- /myapp/assets/images/home/face.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/face.jpg -------------------------------------------------------------------------------- /myapp/assets/images/home/ide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/ide.png -------------------------------------------------------------------------------- /myapp/assets/images/home/nni.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/nni.png -------------------------------------------------------------------------------- /myapp/assets/images/home/openpose.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/openpose.jpg -------------------------------------------------------------------------------- /myapp/assets/images/home/private.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/private.png -------------------------------------------------------------------------------- /myapp/assets/images/home/search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/search.png -------------------------------------------------------------------------------- /myapp/assets/images/home/service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/service.png -------------------------------------------------------------------------------- /myapp/assets/images/home/theia.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/theia.jpg -------------------------------------------------------------------------------- /myapp/assets/images/home/vscode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/home/vscode.png -------------------------------------------------------------------------------- /myapp/assets/images/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/loading.gif -------------------------------------------------------------------------------- /myapp/assets/images/logo-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/logo-2.png -------------------------------------------------------------------------------- /myapp/assets/images/myapp-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/myapp-logo.png -------------------------------------------------------------------------------- /myapp/assets/images/myapp-logo@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/myapp-logo@2x.png -------------------------------------------------------------------------------- /myapp/assets/images/myapp-logo@2x2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/myapp-logo@2x2.png -------------------------------------------------------------------------------- /myapp/assets/images/myapp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/myapp.png -------------------------------------------------------------------------------- /myapp/assets/images/noimg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/noimg.png -------------------------------------------------------------------------------- /myapp/assets/images/s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/assets/images/s.png -------------------------------------------------------------------------------- /myapp/assets/js/myutils.js: -------------------------------------------------------------------------------- 1 | 2 | // 转到携带参数的新网址 3 | function to_new_url(attr,value) { 4 | var url = document.location.toString(); 5 | var arrUrl = url.split("?"); 6 | var path = arrUrl[0]; 7 | var new_url=''; 8 | if(arrUrl.length>1){ 9 | var para = arrUrl[1]; 10 | var vars = para.split("&"); 11 | 12 | var new_para=''; 13 | exist=false; 14 | for (var i=0;i0) 16 | new_para+="&"; 17 | var pair = vars[i].split("="); 18 | new_para+=pair[0]; 19 | if(pair[0] == attr){ 20 | new_para+='='+value; 21 | exist=true; 22 | } 23 | else{ 24 | new_para+='='+pair[1]; 25 | } 26 | } 27 | 28 | if(!exist){ 29 | new_para+='&'+attr+'='+value 30 | } 31 | new_url=path+"?"+new_para 32 | }else{ 33 | new_url=path+"?"+attr+'='+value 34 | } 35 | console.log(new_url); 36 | window.open(new_url,'_self'); 37 | } 38 | 39 | 40 | function set_change(attr) { 41 | var myselect=document.getElementById(attr); 42 | var new_text=myselect.options[myselect.selectedIndex].text; 43 | to_new_url(attr,new_text) 44 | } 45 | 46 | -------------------------------------------------------------------------------- /myapp/assets/spec/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": [ 3 | "jest", 4 | "no-only-tests" 5 | ], 6 | "env": { 7 | "jest/globals": true 8 | }, 9 | "extends": ["plugin:jest/recommended"], 10 | "rules": { 11 | "import/no-extraneous-dependencies": ["error", {"devDependencies": true}], 12 | "jest/consistent-test-it": "error", 13 | "no-only-tests/no-only-tests": "error" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /myapp/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/bin/__init__.py -------------------------------------------------------------------------------- /myapp/bin/myapp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import click 3 | from flask.cli import FlaskGroup 4 | from myapp.cli import create_app 5 | 6 | 7 | @click.group(cls=FlaskGroup, create_app=create_app) 8 | def cli(): 9 | """This is a management script for the Myapp application.""" 10 | pass 11 | 12 | 13 | if __name__ == '__main__': 14 | cli() 15 | -------------------------------------------------------------------------------- /myapp/check_tables.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import pysnooper 4 | # @pysnooper.snoop() 5 | def check_tables(): 6 | SQLALCHEMY_DATABASE_URI = os.getenv('MYSQL_SERVICE','') 7 | if SQLALCHEMY_DATABASE_URI: 8 | import sqlalchemy.engine.url as url 9 | uri = url.make_url(SQLALCHEMY_DATABASE_URI) 10 | """Inits the Myapp application""" 11 | import pymysql 12 | # 创建连接 13 | conn = pymysql.connect(host=uri.host,port=uri.port, user=uri.username, password=uri.password, charset='utf8') 14 | # 创建游标 15 | cursor = conn.cursor() 16 | 17 | # 创建数据库的sql(如果数据库存在就不创建,防止异常) 18 | sql = "SELECT table_name FROM information_schema.tables WHERE table_schema='pipeline'" 19 | cursor.execute(sql) 20 | results = list(cursor.fetchall()) 21 | results = [item[0] for item in results] 22 | print(results) 23 | for table_name in ['ab_permission','ab_permission_view','ab_permission_view_role','ab_register_user','ab_role','ab_user','ab_user_role','ab_view_menu','alembic_version','docker','job_template','logs','notebook','pipeline','project','project_user','run','task','user_attribute','workflow']: 24 | if table_name not in results: 25 | print('pipeline db下,table不完整,缺少%s表,可等待init任务完成,或\n1、kubectl delete -k cube/overlays\n2、drop database pipeline\n3、kubectl apply -k cube/overlays'%table_name) 26 | exit(1) 27 | 28 | check_tables() -------------------------------------------------------------------------------- /myapp/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/config.py -------------------------------------------------------------------------------- /myapp/create_db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import pysnooper 4 | @pysnooper.snoop() 5 | def init_db(): 6 | SQLALCHEMY_DATABASE_URI = os.getenv('MYSQL_SERVICE','') 7 | if SQLALCHEMY_DATABASE_URI: 8 | import sqlalchemy.engine.url as url 9 | uri = url.make_url(SQLALCHEMY_DATABASE_URI) 10 | """Inits the Myapp application""" 11 | import pymysql 12 | # 创建连接 13 | conn = pymysql.connect(host=uri.host,port=uri.port, user=uri.username, password=uri.password, charset='utf8') 14 | # 创建游标 15 | cursor = conn.cursor() 16 | 17 | # 创建数据库的sql(如果数据库存在就不创建,防止异常) 18 | sql = "CREATE DATABASE IF NOT EXISTS pipeline DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" 19 | # 执行创建数据库的sql 20 | cursor.execute(sql) 21 | conn.commit() 22 | 23 | sql = "CREATE DATABASE IF NOT EXISTS argo DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;" 24 | # 执行创建数据库的sql 25 | cursor.execute(sql) 26 | conn.commit() 27 | 28 | init_db() 29 | 30 | -------------------------------------------------------------------------------- /myapp/exceptions.py: -------------------------------------------------------------------------------- 1 | class MyappException(Exception): 2 | status = 500 3 | 4 | def __init__(self, msg): 5 | super(MyappException, self).__init__(msg) 6 | 7 | 8 | class MyappTimeoutException(MyappException): 9 | pass 10 | 11 | 12 | class MyappSecurityException(MyappException): 13 | status = 401 14 | 15 | def __init__(self, msg, link=None): 16 | super(MyappSecurityException, self).__init__(msg) 17 | self.link = link 18 | 19 | 20 | class MetricPermException(MyappException): 21 | pass 22 | 23 | 24 | class NoDataException(MyappException): 25 | status = 400 26 | 27 | 28 | class NullValueException(MyappException): 29 | status = 400 30 | 31 | 32 | class MyappTemplateException(MyappException): 33 | pass 34 | 35 | 36 | class SpatialException(MyappException): 37 | pass 38 | 39 | 40 | class DatabaseNotFound(MyappException): 41 | status = 400 42 | -------------------------------------------------------------------------------- /myapp/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /myapp/migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | 12 | # Logging configuration 13 | [loggers] 14 | keys = root,sqlalchemy,alembic 15 | 16 | [handlers] 17 | keys = console 18 | 19 | [formatters] 20 | keys = generic 21 | 22 | [logger_root] 23 | level = WARN 24 | handlers = console 25 | qualname = 26 | 27 | [logger_sqlalchemy] 28 | level = WARN 29 | handlers = 30 | qualname = sqlalchemy.engine 31 | 32 | [logger_alembic] 33 | level = INFO 34 | handlers = 35 | qualname = alembic 36 | 37 | [handler_console] 38 | class = StreamHandler 39 | args = (sys.stderr,) 40 | level = NOTSET 41 | formatter = generic 42 | 43 | [formatter_generic] 44 | format = %(levelname)-5.5s [%(name)s] %(message)s 45 | datefmt = %H:%M:%S 46 | -------------------------------------------------------------------------------- /myapp/migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | import logging 4 | from logging.config import fileConfig 5 | 6 | from sqlalchemy import engine_from_config 7 | from sqlalchemy import pool 8 | 9 | from alembic import context 10 | 11 | # this is the Alembic Config object, which provides 12 | # access to the values within the .ini file in use. 13 | config = context.config 14 | 15 | # Interpret the config file for Python logging. 16 | # This line sets up loggers basically. 17 | fileConfig(config.config_file_name) 18 | logger = logging.getLogger('alembic.env') 19 | 20 | # add your model's MetaData object here 21 | # for 'autogenerate' support 22 | # from myapp import mymodel 23 | # target_metadata = mymodel.Base.metadata 24 | from flask import current_app 25 | config.set_main_option( 26 | 'sqlalchemy.url', current_app.config.get( 27 | 'SQLALCHEMY_DATABASE_URI').replace('%', '%%')) 28 | target_metadata = current_app.extensions['migrate'].db.metadata 29 | 30 | # other values from the config, defined by the needs of env.py, 31 | # can be acquired: 32 | # my_important_option = config.get_main_option("my_important_option") 33 | # ... etc. 34 | 35 | 36 | def run_migrations_offline(): 37 | """Run migrations in 'offline' mode. 38 | 39 | This configures the context with just a URL 40 | and not an Engine, though an Engine is acceptable 41 | here as well. By skipping the Engine creation 42 | we don't even need a DBAPI to be available. 43 | 44 | Calls to context.execute() here emit the given string to the 45 | script output. 46 | 47 | """ 48 | url = config.get_main_option("sqlalchemy.url") 49 | context.configure( 50 | url=url, target_metadata=target_metadata, literal_binds=True 51 | ) 52 | 53 | with context.begin_transaction(): 54 | context.run_migrations() 55 | 56 | 57 | def run_migrations_online(): 58 | """Run migrations in 'online' mode. 59 | 60 | In this scenario we need to create an Engine 61 | and associate a connection with the context. 62 | 63 | """ 64 | 65 | # this callback is used to prevent an auto-migration from being generated 66 | # when there are no changes to the schema 67 | # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html 68 | def process_revision_directives(context, revision, directives): 69 | if getattr(config.cmd_opts, 'autogenerate', False): 70 | script = directives[0] 71 | if script.upgrade_ops.is_empty(): 72 | directives[:] = [] 73 | logger.info('No changes in schema detected.') 74 | 75 | connectable = engine_from_config( 76 | config.get_section(config.config_ini_section), 77 | prefix='sqlalchemy.', 78 | poolclass=pool.NullPool, 79 | ) 80 | 81 | with connectable.connect() as connection: 82 | context.configure( 83 | connection=connection, 84 | target_metadata=target_metadata, 85 | process_revision_directives=process_revision_directives, 86 | **current_app.extensions['migrate'].configure_args 87 | ) 88 | 89 | with context.begin_transaction(): 90 | context.run_migrations() 91 | 92 | 93 | if context.is_offline_mode(): 94 | run_migrations_offline() 95 | else: 96 | run_migrations_online() 97 | -------------------------------------------------------------------------------- /myapp/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /myapp/models/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . import log 3 | from . import model_job 4 | from . import user_attributes # noqa 5 | -------------------------------------------------------------------------------- /myapp/models/log.py: -------------------------------------------------------------------------------- 1 | """A collection of ORM sqlalchemy models for Myapp""" 2 | from contextlib import closing 3 | from copy import copy, deepcopy 4 | from datetime import datetime 5 | from flask_appbuilder import Model 6 | from sqlalchemy import ( 7 | Boolean, 8 | Column, 9 | create_engine, 10 | DateTime, 11 | ForeignKey, 12 | Integer, 13 | MetaData, 14 | String, 15 | Table, 16 | Text, 17 | ) 18 | from sqlalchemy.engine import url 19 | from sqlalchemy.engine.url import make_url 20 | from sqlalchemy.orm import relationship, sessionmaker, subqueryload 21 | from myapp import ( 22 | app, 23 | appbuilder, 24 | conf, 25 | db 26 | ) 27 | from myapp.models.base import MyappModelBase 28 | from myapp import app, db, is_feature_enabled, security_manager 29 | from myapp.security import MyUser 30 | 31 | 32 | config = app.config 33 | custom_password_store = config.get("SQLALCHEMY_CUSTOM_PASSWORD_STORE") 34 | stats_logger = config.get("STATS_LOGGER") 35 | 36 | PASSWORD_MASK = "X" * 10 37 | 38 | 39 | class Log(Model,MyappModelBase): 40 | 41 | """ORM object used to log Myapp actions to the database""" 42 | 43 | __tablename__ = "logs" 44 | 45 | id = Column(Integer, primary_key=True) 46 | 47 | user_id = Column(Integer, ForeignKey("ab_user.id")) 48 | user = relationship( 49 | MyUser, foreign_keys=[user_id] 50 | ) 51 | action = Column(String(512)) 52 | method = Column(String(50)) 53 | path = Column(String(200)) 54 | status = Column(Integer) 55 | json = Column(Text) 56 | dttm = Column(DateTime, default=datetime.now) # 不要使用datetime.now()不然是程序启动的固定时间了 57 | duration_ms = Column(Integer) 58 | referrer = Column(String(1024)) 59 | 60 | 61 | label_columns={ 62 | 'user':'用户', 63 | "action": "函数", 64 | "method": "方法", 65 | "path": "网址", 66 | "status": "状态", 67 | "dttm": "时间", 68 | "duration_ms": "响应延迟", 69 | "referrer": "相关人", 70 | } 71 | 72 | 73 | -------------------------------------------------------------------------------- /myapp/models/model_docker.py: -------------------------------------------------------------------------------- 1 | from flask_appbuilder import Model 2 | from sqlalchemy import Column, Integer, String, ForeignKey,Float 3 | from sqlalchemy.orm import relationship 4 | import datetime,time,json 5 | from sqlalchemy import ( 6 | Boolean, 7 | Column, 8 | create_engine, 9 | DateTime, 10 | ForeignKey, 11 | Integer, 12 | MetaData, 13 | String, 14 | Table, 15 | Text, 16 | Enum, 17 | ) 18 | import pysnooper 19 | from myapp.models.base import MyappModelBase 20 | from myapp.models.helpers import AuditMixinNullable, ImportMixin 21 | from flask import escape, g, Markup, request 22 | from myapp import app,db 23 | from myapp.models.helpers import ImportMixin 24 | # 添加自定义model 25 | from sqlalchemy import Column, Integer, String, ForeignKey ,Date,DateTime 26 | from flask_appbuilder.models.decorators import renders 27 | from flask import Markup 28 | import datetime 29 | metadata = Model.metadata 30 | conf = app.config 31 | # from myapp.utils.py.py_k8s import K8s 32 | 33 | 34 | # 定义model 35 | class Docker(Model,AuditMixinNullable,MyappModelBase): 36 | __tablename__ = 'docker' 37 | id = Column(Integer, primary_key=True) 38 | project_id = Column(Integer, ForeignKey('project.id'), nullable=False) # 定义外键 39 | project = relationship( 40 | "Project", foreign_keys=[project_id] 41 | ) 42 | describe = Column(String(200), nullable=True) 43 | base_image = Column(String(200), nullable=True) 44 | target_image=Column(String(200), nullable=True,default='') 45 | last_image = Column(String(200), nullable=True, default='') 46 | need_gpu = Column(Boolean, nullable=True, default=False) 47 | consecutive_build = Column(Boolean, default=True) # 连续构建 48 | expand = Column(Text(65536), default='{}') 49 | 50 | def __repr__(self): 51 | return self.label 52 | 53 | # 清空激活 54 | @property 55 | def save(self): 56 | return Markup(f'保存') 57 | 58 | # 清空激活 59 | @property 60 | def debug(self): 61 | return Markup(f'调试 | 清理') 62 | 63 | @property 64 | def image_history(self): 65 | return Markup(f'基础镜像:{self.base_image}
当前镜像:{self.last_image if self.last_image else self.base_image}
目标镜像:{self.target_image}') 66 | -------------------------------------------------------------------------------- /myapp/models/user_attributes.py: -------------------------------------------------------------------------------- 1 | 2 | from flask_appbuilder import Model 3 | from sqlalchemy import Column, ForeignKey, Integer 4 | from sqlalchemy.orm import relationship 5 | 6 | from myapp import security_manager 7 | from myapp.models.helpers import AuditMixinNullable 8 | 9 | 10 | class UserAttribute(Model, AuditMixinNullable): 11 | 12 | """ 13 | Custom attributes attached to the user. 14 | 15 | Extending the user attribute is tricky due to its dependency on the 16 | authentication typew an circular dependencies in Myapp. Instead, we use 17 | a custom model for adding attributes. 18 | 19 | """ 20 | 21 | __tablename__ = "user_attribute" 22 | id = Column(Integer, primary_key=True) # pylint: disable=invalid-name 23 | user_id = Column(Integer, ForeignKey("ab_user.id")) 24 | user = relationship( 25 | security_manager.user_model, backref="extra_attributes", foreign_keys=[user_id] 26 | ) 27 | 28 | -------------------------------------------------------------------------------- /myapp/run.py: -------------------------------------------------------------------------------- 1 | from myapp import app 2 | # 如果使用 flask run 命令启动,将忽视 这里配置8080,而采用默认的5000端口 3 | app.run(host="0.0.0.0", port=80, debug=True) 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/css/myapp.css: -------------------------------------------------------------------------------- 1 | .list-container { 2 | position: relative; 3 | } 4 | .list-container .pagination-container { 5 | display: flex; 6 | flex-direction: row; 7 | flex-wrap: wrap; 8 | justify-content: center; 9 | align-items: center; 10 | padding-bottom: 20px; 11 | } 12 | 13 | .list-container .pagination-container .pagination { 14 | margin: 0 15px; 15 | } 16 | .list-container .pagination-container strong { 17 | margin-right: 5px; 18 | } 19 | 20 | .list-container .list-add-action { 21 | position: absolute; 22 | top: -30px; 23 | right: 15px; 24 | } 25 | 26 | .list-container .form-actions-container { 27 | padding: 0 0 20px 10px; 28 | display: inline; 29 | } 30 | 31 | .list-container .filter-action { 32 | margin: 10px 10px 0 10px; 33 | padding-bottom: 15px; 34 | } 35 | 36 | .list-container .filters-container table tr:first-child td { 37 | border-top: none; 38 | } 39 | .list-container .filters-container table tr:last-child td { 40 | border-bottom: 1px solid #b3b3b3; 41 | } 42 | 43 | .list-search-container .dropdown-toggle { 44 | position: absolute; 45 | top: -30px; 46 | right: 50px; 47 | border: 0; 48 | padding: 0 18px; 49 | } 50 | .list-search-container .fa-filter { 51 | position: relative; 52 | left: -8px; 53 | } 54 | 55 | .list-search-container .dropdown-menu { 56 | top: 0px; 57 | right: 0; 58 | left: auto; 59 | float: none; 60 | } 61 | 62 | .list-container .list-add-action { 63 | position: absolute; 64 | top: -30px; 65 | right: 15px; 66 | } 67 | .list-add-action .btn.btn-sm { 68 | padding: 5px 6px; 69 | font-size: 10px; 70 | line-height: 2px; 71 | border-radius: 50%; 72 | box-shadow: 2px 2px 4px -1px rgba(0, 0, 0, 1); 73 | } -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /myapp/static/appbuilder/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/aol.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/aol.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/fab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/fab.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/flags/flags16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/flags/flags16.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/flickr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/flickr.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/google.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/google.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/myopenid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/myopenid.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/img/yahoo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/img/yahoo.png -------------------------------------------------------------------------------- /myapp/static/appbuilder/js/ab_keep_tab.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | //--------------------------------------- 4 | // Function for keeping tab focus 5 | // after page reload, uses cookies 6 | //--------------------------------------- 7 | $(function() 8 | { 9 | $('a[data-toggle="tab"]').on('shown.bs.tab', function () { 10 | //save the latest tab; use cookies if you like 'em better: 11 | localStorage.setItem('lastTab', $(this).attr('href')); 12 | }); 13 | 14 | //go to the latest tab, if it exists: 15 | var lastTab = localStorage.getItem('lastTab'); 16 | if (lastTab) { 17 | $('a[href=' + lastTab + ']').tab('show'); 18 | } 19 | else 20 | { 21 | // Set the first tab if cookie do not exist 22 | $('a[data-toggle="tab"]:first').tab('show'); 23 | } 24 | 25 | //--------------------------------------- 26 | // Function for keeping accordion focus 27 | // after page reload, uses cookies 28 | //--------------------------------------- 29 | 30 | $('.panel-collapse').on('shown.bs.collapse', function () { 31 | //save the latest accordion; use cookies if you like 'em better: 32 | localStorage.setItem('lastAccordion', $(this).attr('id')); 33 | }); 34 | 35 | 36 | $('.panel-collapse').on('hidden.bs.collapse', function () { 37 | //remove the latest accordion; use cookies if you like 'em better: 38 | localStorage.removeItem('lastAccordion'); 39 | }); 40 | 41 | 42 | //go to the latest accordion, if it exists: 43 | var lastAccordion = localStorage.getItem('lastAccordion'); 44 | if (lastAccordion) { 45 | if (!($('#' + lastAccordion).hasClass('collapse in'))) 46 | { 47 | $('#' + lastAccordion).collapse('show'); 48 | } 49 | } 50 | 51 | }); 52 | 53 | 54 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/js/html5shiv.js: -------------------------------------------------------------------------------- 1 | /* 2 | HTML5 Shiv v3.6.2pre | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed 3 | */ 4 | (function(l,f){function m(){var a=e.elements;return"string"==typeof a?a.split(" "):a}function i(a){var b=n[a[o]];b||(b={},h++,a[o]=h,n[h]=b);return b}function p(a,b,c){b||(b=f);if(g)return b.createElement(a);c||(c=i(b));b=c.cache[a]?c.cache[a].cloneNode():r.test(a)?(c.cache[a]=c.createElem(a)).cloneNode():c.createElem(a);return b.canHaveChildren&&!s.test(a)?c.frag.appendChild(b):b}function t(a,b){if(!b.cache)b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag(); 5 | a.createElement=function(c){return!e.shivMethods?b.createElem(c):p(c,a,b)};a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+m().join().replace(/\w+/g,function(a){b.createElem(a);b.frag.createElement(a);return'c("'+a+'")'})+");return n}")(e,b.frag)}function q(a){a||(a=f);var b=i(a);if(e.shivCSS&&!j&&!b.hasCSS){var c,d=a;c=d.createElement("p");d=d.getElementsByTagName("head")[0]||d.documentElement;c.innerHTML="x"; 6 | c=d.insertBefore(c.lastChild,d.firstChild);b.hasCSS=!!c}g||t(a,b);return a}var k=l.html5||{},s=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,r=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,j,o="_html5shiv",h=0,n={},g;(function(){try{var a=f.createElement("a");a.innerHTML="";j="hidden"in a;var b;if(!(b=1==a.childNodes.length)){f.createElement("a");var c=f.createDocumentFragment();b="undefined"==typeof c.cloneNode|| 7 | "undefined"==typeof c.createDocumentFragment||"undefined"==typeof c.createElement}g=b}catch(d){g=j=!0}})();var e={elements:k.elements||"abbr article aside audio bdi canvas data datalist details figcaption figure footer header hgroup mark meter nav output progress section summary time video",version:"3.6.2pre",shivCSS:!1!==k.shivCSS,supportsUnknownElements:g,shivMethods:!1!==k.shivMethods,type:"default",shivDocument:q,createElement:p,createDocumentFragment:function(a,b){a||(a=f);if(g)return a.createDocumentFragment(); 8 | for(var b=b||i(a),c=b.frag.cloneNode(),d=0,e=m(),h=e.length;d机器学习平台
-------------------------------------------------------------------------------- /myapp/static/appbuilder/vison/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "machine_learnring", 3 | "name": "tme_machine_learnring_platform", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/vison/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/vison/static/js/257.b1982694.chunk.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /* 2 | object-assign 3 | (c) Sindre Sorhus 4 | @license MIT 5 | */ 6 | 7 | /*! 8 | Copyright (c) 2018 Jed Watson. 9 | Licensed under the MIT License (MIT), see 10 | http://jedwatson.github.io/classnames 11 | */ 12 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/vison/static/js/46.d003bec7.chunk.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /*! 2 | * Draggabilly v2.3.0 3 | * Make that shiz draggable 4 | * https://draggabilly.desandro.com 5 | * MIT license 6 | */ 7 | 8 | /*! 9 | * Unidragger v2.3.1 10 | * Draggable base class 11 | * MIT license 12 | */ 13 | 14 | /*! 15 | * Unipointer v2.3.0 16 | * base class for doing one thing with pointer event 17 | * MIT license 18 | */ 19 | 20 | /*! 21 | * getSize v2.0.3 22 | * measure size of elements 23 | * MIT license 24 | */ 25 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/vison/static/js/main.ff225858.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /* 2 | object-assign 3 | (c) Sindre Sorhus 4 | @license MIT 5 | */ 6 | 7 | /*! 8 | * cookie 9 | * Copyright(c) 2012-2014 Roman Shtylman 10 | * Copyright(c) 2015 Douglas Christopher Wilson 11 | * MIT Licensed 12 | */ 13 | 14 | /*! ***************************************************************************** 15 | Copyright (c) Microsoft Corporation. 16 | 17 | Permission to use, copy, modify, and/or distribute this software for any 18 | purpose with or without fee is hereby granted. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 21 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 22 | AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 23 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 24 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 25 | OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 26 | PERFORMANCE OF THIS SOFTWARE. 27 | ***************************************************************************** */ 28 | 29 | /** @license React v0.20.2 30 | * scheduler.production.min.js 31 | * 32 | * Copyright (c) Facebook, Inc. and its affiliates. 33 | * 34 | * This source code is licensed under the MIT license found in the 35 | * LICENSE file in the root directory of this source tree. 36 | */ 37 | 38 | /** @license React v16.13.1 39 | * react-is.production.min.js 40 | * 41 | * Copyright (c) Facebook, Inc. and its affiliates. 42 | * 43 | * This source code is licensed under the MIT license found in the 44 | * LICENSE file in the root directory of this source tree. 45 | */ 46 | 47 | /** @license React v17.0.2 48 | * react-dom.production.min.js 49 | * 50 | * Copyright (c) Facebook, Inc. and its affiliates. 51 | * 52 | * This source code is licensed under the MIT license found in the 53 | * LICENSE file in the root directory of this source tree. 54 | */ 55 | 56 | /** @license React v17.0.2 57 | * react-jsx-runtime.production.min.js 58 | * 59 | * Copyright (c) Facebook, Inc. and its affiliates. 60 | * 61 | * This source code is licensed under the MIT license found in the 62 | * LICENSE file in the root directory of this source tree. 63 | */ 64 | 65 | /** @license React v17.0.2 66 | * react.production.min.js 67 | * 68 | * Copyright (c) Facebook, Inc. and its affiliates. 69 | * 70 | * This source code is licensed under the MIT license found in the 71 | * LICENSE file in the root directory of this source tree. 72 | */ 73 | -------------------------------------------------------------------------------- /myapp/static/appbuilder/vison/static/media/codicon.00aa018e3ae78d50160b.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/static/appbuilder/vison/static/media/codicon.00aa018e3ae78d50160b.ttf -------------------------------------------------------------------------------- /myapp/static/assets: -------------------------------------------------------------------------------- 1 | /home/myapp/myapp/assets -------------------------------------------------------------------------------- /myapp/static/mnt: -------------------------------------------------------------------------------- 1 | /mnt/ceph_fuse/k8s/kubeflow/pipline-test/workspace -------------------------------------------------------------------------------- /myapp/stats_logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from colorama import Fore, Style 4 | 5 | class BaseStatsLogger(object): 6 | """Base class for logging realtime events""" 7 | 8 | def __init__(self, prefix="myapp"): 9 | self.prefix = prefix 10 | 11 | def key(self, key): 12 | if self.prefix: 13 | return self.prefix + key 14 | return key 15 | 16 | def incr(self, key): 17 | """Increment a counter""" 18 | raise NotImplementedError() 19 | 20 | def decr(self, key): 21 | """Decrement a counter""" 22 | raise NotImplementedError() 23 | 24 | def timing(self, key, value): 25 | raise NotImplementedError() 26 | 27 | def gauge(self, key): 28 | """Setup a gauge""" 29 | raise NotImplementedError() 30 | 31 | 32 | class DummyStatsLogger(BaseStatsLogger): 33 | def incr(self, key): 34 | logging.debug(Fore.CYAN + "[stats_logger] (incr) " + key + Style.RESET_ALL) 35 | 36 | def decr(self, key): 37 | logging.debug((Fore.CYAN + "[stats_logger] (decr) " + key + Style.RESET_ALL)) 38 | 39 | def timing(self, key, value): 40 | logging.debug( 41 | (Fore.CYAN + f"[stats_logger] (timing) {key} | {value} " + Style.RESET_ALL) 42 | ) 43 | 44 | def gauge(self, key, value): 45 | logging.debug( 46 | ( 47 | Fore.CYAN 48 | + "[stats_logger] (gauge) " 49 | + f"{key} | {value}" 50 | + Style.RESET_ALL 51 | ) 52 | ) 53 | 54 | 55 | try: 56 | from statsd import StatsClient 57 | 58 | class StatsdStatsLogger(BaseStatsLogger): 59 | def __init__( 60 | self, host="localhost", port=8125, prefix="myapp", statsd_client=None 61 | ): 62 | """ 63 | Initializes from either params or a supplied, pre-constructed statsd client. 64 | 65 | If statsd_client argument is given, all other arguments are ignored and the 66 | supplied client will be used to emit metrics. 67 | """ 68 | if statsd_client: 69 | self.client = statsd_client 70 | else: 71 | self.client = StatsClient(host=host, port=port, prefix=prefix) 72 | 73 | def incr(self, key): 74 | self.client.incr(key) 75 | 76 | def decr(self, key): 77 | self.client.decr(key) 78 | 79 | def timing(self, key, value): 80 | self.client.timing(key, value) 81 | 82 | def gauge(self, key): 83 | # pylint: disable=no-value-for-parameter 84 | self.client.gauge(key) 85 | 86 | 87 | except Exception: 88 | pass 89 | -------------------------------------------------------------------------------- /myapp/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from . import schedules 2 | from . import async_task -------------------------------------------------------------------------------- /myapp/tasks/async_task.py: -------------------------------------------------------------------------------- 1 | 2 | """Utility functions used across Myapp""" 3 | import sys,os 4 | import numpy as np 5 | from bs4 import BeautifulSoup 6 | import requests,base64,hashlib 7 | from collections import namedtuple 8 | import datetime 9 | from email.utils import make_msgid, parseaddr 10 | import logging 11 | import time,json 12 | from urllib.error import URLError 13 | import urllib.request 14 | import pysnooper 15 | import re 16 | import croniter 17 | from dateutil.tz import tzlocal 18 | import shutil 19 | import os,sys,io,json,datetime,time 20 | import subprocess 21 | from datetime import datetime, timedelta 22 | import os 23 | import sys 24 | import time 25 | import datetime 26 | from myapp.utils.py.py_k8s import K8s 27 | from myapp.utils.celery import session_scope 28 | from myapp.project import push_message,push_admin 29 | from myapp.tasks.celery_app import celery_app 30 | # Myapp framework imports 31 | from myapp import app, db, security_manager 32 | from myapp.models.model_job import ( 33 | Pipeline, 34 | RunHistory, 35 | Workflow, 36 | Task 37 | ) 38 | from myapp.models.model_notebook import Notebook 39 | from myapp.security import ( 40 | MyUser 41 | ) 42 | from sqlalchemy.exc import InvalidRequestError,OperationalError 43 | from sqlalchemy import or_ 44 | from myapp.models.model_docker import Docker 45 | conf = app.config 46 | 47 | 48 | 49 | @celery_app.task(name="task.check_docker_commit", bind=True) # , soft_time_limit=15 50 | def check_docker_commit(task,docker_id): # 在页面中测试时会自定接收者和id 51 | with session_scope(nullpool=True) as dbsession: 52 | try: 53 | docker = dbsession.query(Docker).filter_by(id=int(docker_id)).first() 54 | pod_name = "docker-commit-%s-%s" % (docker.created_by.username, str(docker.id)) 55 | namespace = conf.get('NOTEBOOK_NAMESPACE') 56 | k8s_client = K8s(conf.get('CLUSTERS').get(conf.get('ENVIRONMENT')).get('KUBECONFIG')) 57 | begin_time=datetime.datetime.now() 58 | now_time=datetime.datetime.now() 59 | while((now_time-begin_time).seconds<1800): # 也就是最多commit push 30分钟 60 | time.sleep(12000) 61 | commit_pods = k8s_client.get_pods(namespace=namespace,pod_name=pod_name) 62 | if commit_pods: 63 | commit_pod=commit_pods[0] 64 | if commit_pod['status']=='Succeeded': 65 | docker.last_image=docker.target_image 66 | dbsession.commit() 67 | break 68 | # 其他异常状态直接报警 69 | if commit_pod['status']!='Running': 70 | push_message(conf.get('ADMIN_USER').split(','),'commit pod %s not running'%commit_pod['name']) 71 | break 72 | else: 73 | break 74 | 75 | except Exception as e: 76 | print(e) 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /myapp/tasks/celery_app.py: -------------------------------------------------------------------------------- 1 | 2 | """Utility functions used across Myapp""" 3 | 4 | # Myapp framework imports 5 | from myapp import app 6 | from myapp.utils.core import get_celery_app 7 | 8 | # 全局配置,全部celery app。所有任务都挂在这个app下面 9 | conf = app.config 10 | # print(conf) 11 | # 获取CELERY_CONFIG中定义的任务配置,这个是所有任务的统一配置项,所有任务都应用这个配置项。 12 | # 但这并不是真正的定义任务。因为任务是task修饰定义的。并且是有逻辑的 13 | celery_app = get_celery_app(conf) 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /myapp/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | 3 | {% block content %} 4 | 5 |
6 | 404 7 |
8 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/add.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/general/model/add.html" %} 2 | 3 | {% block add_form %} 4 | This Text is before the add form widget 5 | {{ super() }} 6 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/appbuilder/baselayout.html: -------------------------------------------------------------------------------- 1 | {% extends 'appbuilder/init.html' %} 2 | {% import 'appbuilder/baselib.html' as baselib %} 3 | 4 | {% block body %} 5 | {% include 'appbuilder/general/confirm.html' %} 6 | {% include 'appbuilder/general/alert.html' %} 7 | {% include 'myapp/feature.html' %} 8 | 9 | {% block navbar %} 10 |
11 | {% include 'appbuilder/navbar.html' %} 12 |
13 | {% endblock %} 14 | 15 | {% block uncontained %}{% endblock %} 16 | 17 |
18 |
19 | {% block messages %} 20 | {% include 'myapp/flash_wrapper.html' %} 21 | {% endblock %} 22 | {% block content %} 23 | {% endblock %} 24 |
25 |
26 |
27 | {% block content_fluid %} 28 | {% endblock %} 29 |
30 | 31 | {% block footer %} 32 |
33 | 38 |
39 | {% endblock %} 40 | 41 | 42 | 43 | 46 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/appbuilder/general/model/edit.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | {% import 'appbuilder/general/lib.html' as lib %} 3 | 4 | {% block content %} 5 | {{ lib.panel_begin(title, "edit") }} 6 | 7 | {% if related_views is defined %} 8 | 14 | 15 |
16 | {% for view in related_views %} 17 |
18 | {{ widgets.get('related_views')[loop.index - 1]()|safe }} 19 |
20 | {% endfor %} 21 | {% endif %} 22 | 23 | {% block edit_form %} 24 |
25 | {{ widgets.get('edit')(form_action=form_action)|safe }} 26 |
27 | {% endblock %} 28 | 29 | {% if related_views is defined %}
{% endif %} 30 | 31 | {{ lib.panel_end() }} 32 | {% endblock %} 33 | 34 | {% block add_tail_js %} 35 | 36 | {% endblock %} 37 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/general/model/list.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | {% import 'appbuilder/general/lib.html' as lib %} 3 | 4 | {% block content %} 5 | {{ lib.panel_begin(title) }} 6 | 7 |
8 | {% block list_search scoped %} 9 | {{ widgets.get('search')()|safe }} 10 | {% endblock %} 11 | 12 | {% block list_list scoped %} 13 | {{ widgets.get('list')()|safe }} 14 | {% endblock %} 15 |
16 | 17 | {{ lib.panel_end() }} 18 | {% endblock %} 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/general/model/show.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | {% import 'appbuilder/general/lib.html' as lib %} 3 | 4 | {% block content %} 5 | {{ lib.panel_begin(title) }} 6 | 7 | {% if related_views is defined %} 8 | 16 | 17 |
18 | {% for view in related_views %} 19 |
20 | {{ widgets.get('related_views')[loop.index - 1](pk = pk)|safe }} 21 |
22 | {% endfor %} 23 | {% endif %} 24 | 25 | {% block show_form %} 26 |
27 | {{ widgets.get('show')()|safe }} 28 |
29 | {% endblock show_form %} 30 | 31 | {% if related_views is defined %}
{% endif %} 32 | {{ lib.panel_end() }} 33 | 34 | {% endblock content %} 35 | 36 | {% block add_tail_js %} 37 | 38 | {% endblock %} 39 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/general/widgets/base_list.html: -------------------------------------------------------------------------------- 1 | 2 | {% import 'appbuilder/general/lib.html' as lib %} 3 | 4 | {% set can_add = "can_add" | is_item_visible(modelview_name) %} 5 | {% set can_show = "can_show" | is_item_visible(modelview_name) %} 6 | {% set can_edit = "can_edit" | is_item_visible(modelview_name) %} 7 | {% set can_delete = "can_delete" | is_item_visible(modelview_name) %} 8 | {% set actions = actions | get_actions_on_list(modelview_name) %} 9 | 10 | {% if can_add %} 11 | 12 | {% set path = url_for(modelview_name + '.add') %} 13 | {% set path = path | set_link_filters(filters) %} 14 |  {{ lib.lnk_add(path) }} 15 | 16 | {% endif %} 17 | 18 | {% if count > 0 %} 19 | 20 | {% block begin_content scoped %} 21 | {% endblock %} 22 | 23 | {% block begin_loop_header scoped %} 24 | {% endblock %} 25 | 26 | {% block begin_loop_values %} 27 | {% endblock %} 28 | 29 | {% block end_content scoped %} 30 | {% endblock %} 31 | 32 |
33 | {{ lib.render_actions(actions, modelview_name) }} 34 |
35 | {{ lib.action_form(actions,modelview_name) }} 36 | 37 |
38 | {{ _('Record Count') }}: {{ count }} 39 | {{ lib.render_pagination(page, page_size, count, modelview_name) }} 40 | {{ lib.render_set_page_size(page, page_size, count, modelview_name) }} 41 |
42 | 45 | 46 | {% else %} 47 | {{_("No records found")}} 48 | {% endif %} -------------------------------------------------------------------------------- /myapp/templates/appbuilder/general/widgets/search.html: -------------------------------------------------------------------------------- 1 | {% import 'appbuilder/general/lib.html' as lib %} 2 | 3 |
4 | 35 |
36 | 37 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/general/widgets/show.html: -------------------------------------------------------------------------------- 1 | {% import 'appbuilder/general/lib.html' as lib %} 2 | {% include 'appbuilder/general/confirm.html' %} 3 | {% include 'appbuilder/general/alert.html' %} 4 | 5 | {% if fieldsets %} 6 | 7 | {% for fieldset_item in fieldsets %} 8 | {% if fieldset_item[1].get('expanded') == None %} 9 | {% set expanded = True %} 10 | {% else %} 11 | {% set expanded = fieldset_item[1].get('expanded') %} 12 | {% endif %} 13 | {% call lib.accordion_tag(loop.index,fieldset_item[0], expanded) %} 14 |
15 | 16 | {% for item in fieldset_item[1].get('fields') %} 17 | 18 | 19 | 20 | 21 | {% endfor %} 22 |
{{label_columns.get(item)}}{{value_columns[include_columns.index(item)]}}
23 | {% endcall %} 24 | {% endfor %} 25 | 26 | {% else %} 27 |
28 | 29 | 30 | {% for item in include_columns %} 31 | 32 | 33 | 38 | 39 | {% endfor %} 40 |
{{label_columns.get(item)}} 34 | {% set formatter = formatters_columns.get(item) %} 35 | {% set v = value_columns[loop.index-1]%} 36 | {{formatter(v) if formatter else v}} 37 |
41 | {% endif %} 42 | 43 | 44 |
45 | {{ lib.render_action_links(actions, pk, modelview_name) }} 46 | {{ lib.lnk_back() }} 47 |
48 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/init.html: -------------------------------------------------------------------------------- 1 | {% import 'appbuilder/baselib.html' as baselib with context %} 2 | 3 | 4 | {% if appbuilder %} 5 | {% set app_name = appbuilder.app_name %} 6 | {% endif %} 7 | 8 | 9 | 10 | 11 | {% block page_title %}{{app_name}}{% endblock %} 12 | 13 | {% block head_meta %} 14 | 15 | 16 | 17 | {% endblock %} 18 | {% block head_css %} 19 | 20 | 21 | 22 | {% if appbuilder.app_theme %} 23 | 24 | {% endif %} 25 | 26 | 27 | 31 | 32 | 33 | 34 | 35 | 36 | {% endblock %} 37 | {% block head_js %} 38 | 39 | 40 | 41 | {% endblock %} 42 | 43 | 44 | 45 | 46 | 47 | {% block body %} 48 | {% endblock %} 49 | 50 | 51 | {% block tail_js %} 52 | 53 | 54 | 55 | 56 | {% endblock %} 57 | 58 | {% block add_tail_js %} 59 | {% endblock %} 60 | 61 | {% block tail %} 62 | {% endblock %} 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/navbar.html: -------------------------------------------------------------------------------- 1 | {% set menu = appbuilder.menu %} 2 | {% set security_manager = appbuilder.sm %} 3 | {% set languages = appbuilder.languages %} 4 | {% set WARNING_MSG = appbuilder.app.config.get('WARNING_MSG') %} 5 | {% set app_icon_height = appbuilder.app.config.get('APP_ICON_HEIGHT', '100%') %} 6 | {% set logo_target_path = appbuilder.app.config.get('LOGO_TARGET_PATH') or '/profile/{}/'.format(current_user.username) 7 | %} 8 | 9 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/navbar_menu.html: -------------------------------------------------------------------------------- 1 | {% macro menu_item(item) %} 2 | 3 | {% if item.icon %} 4 |   5 | {% endif %} 6 | {{_(item.label)}} 7 | {% endmacro %} 8 | 9 | 10 | {% for item1 in menu.get_list() %} 11 | {% if item1 and security_manager.can_access("menu_access",item1.name) %} 12 | {% if item1.childs %} 13 | 32 | {% else %} 33 |
  • 34 | {{ menu_item(item1) }} 35 |
  • 36 | {% endif %} 37 | {% endif %} 38 | {% endfor %} 39 | -------------------------------------------------------------------------------- /myapp/templates/appbuilder/navbar_right.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% set bug_report_url = appbuilder.app.config.get('BUG_REPORT_URL') %} 4 | {% set documentation_url = appbuilder.app.config.get('DOCUMENTATION_URL') %} 5 | {% set locale = session['locale'] %} 6 | {% if not locale %} 7 | {% set locale = 'en' %} 8 | {% endif %} 9 | 10 | 11 | {% if documentation_url %} 12 |
  • 13 | 20 |  doc 21 | 22 |
  • 23 | {% endif %} 24 | {% if bug_report_url %} 25 |
  • 26 | 33 |   34 | 35 |
  • 36 | {% endif %} 37 | 38 | 39 | {% if not current_user.is_anonymous %} 40 | 57 | {% else %} 58 |
  • 59 | {{_("Login")}}
  • 60 | {% endif %} 61 | -------------------------------------------------------------------------------- /myapp/templates/edit.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/general/model/edit.html" %} 2 | 3 | {% block add_form %} 4 | This Text is before the add form widget 5 | {{ super() }} 6 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/external_link.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | 3 | {% block content %} 4 | 11 | 14 | {% if data['loading'] %} 15 |
    Hello
    16 | {% endif %} 17 | 60 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/hello.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | {% block content %} 3 |

    {{ msg }}

    4 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/home.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | 3 | {% block add_tail_js %} 4 | 5 | {% endblock %} 6 | 7 | {% block content %} 8 |
    9 | 10 |
    11 | 51 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/base.html" %} 2 | {% block content %} 3 |
    4 |
    5 |

    {{_("My App on F.A.B.")}}

    6 |

    {{_("My first app using F.A.B, bla, bla, bla")}}

    7 |
    8 |
    9 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/myapp/base.html: -------------------------------------------------------------------------------- 1 | {% extends "appbuilder/baselayout.html" %} 2 | 3 | {% block head_css %} 4 | {{super()}} 5 | 6 | 7 | {% for entry in get_unloaded_chunks(css_manifest('theme'), loaded_chunks) %} 8 | 9 | {% endfor %} 10 | {% endblock %} 11 | 12 | {% block head_js %} 13 | {{super()}} 14 | {% with filename="theme" %} 15 | {% include "myapp/partials/_script_tag.html" %} 16 | {% endwith %} 17 | {% endblock %} 18 | 19 | {% block tail_js %} 20 | {{super()}} 21 | {% with filename="preamble" %} 22 | {% include "myapp/partials/_script_tag.html" %} 23 | {% endwith %} 24 | {% endblock %} -------------------------------------------------------------------------------- /myapp/templates/myapp/basic.html: -------------------------------------------------------------------------------- 1 | {% import 'appbuilder/general/lib.html' as lib %} 2 | 3 | 4 | 5 | 6 | {% block title %} 7 | {% if title %} 8 | {{ title }} 9 | {% elif appbuilder and appbuilder.app_name %} 10 | {{ appbuilder.app_name }} 11 | {% endif %} 12 | {% endblock %} 13 | 14 | {% block head_meta %}{% endblock %} 15 | {% block head_css %} 16 | 17 | 18 | 19 | 20 | {% for entry in get_unloaded_chunks(css_manifest('theme'), loaded_chunks) %} 21 | 22 | {% endfor %} 23 | 24 | {% if entry %} 25 | {% set entry_files = css_manifest(entry) %} 26 | {% for entry in get_unloaded_chunks(entry_files, loaded_chunks) %} 27 | 28 | {% endfor %} 29 | {% endif %} 30 | 31 | {% endblock %} 32 | 33 | {% with filename="theme" %} 34 | {% include "myapp/partials/_script_tag.html" %} 35 | {% endwith %} 36 | 37 | 38 | 39 | 40 | 41 | {# 顶层导航栏 #} 42 | {% block navbar %} 43 | {% if not standalone_mode %} 44 |
    45 | {% include 'appbuilder/navbar.html' %} 46 |
    47 | {% endif %} 48 | {% endblock %} 49 | {# 当前页面的body #} 50 | {% block body %} 51 |
    52 | 54 |
    55 | {% endblock %} 56 | 57 | 58 | 75 | {% block tail_js %} 76 | {% if entry %} 77 | {% with filename=entry %} 78 | {% include "myapp/partials/_script_tag.html" %} 79 | {% endwith %} 80 | {% endif %} 81 | {% endblock %} 82 | 83 | 84 | 85 | 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /myapp/templates/myapp/csrf_token.json: -------------------------------------------------------------------------------- 1 | { 2 | "csrf_token": "{{ csrf_token() if csrf_token else '' }}" 3 | } 4 | -------------------------------------------------------------------------------- /myapp/templates/myapp/fab_overrides/list.html: -------------------------------------------------------------------------------- 1 | 2 | {% extends 'appbuilder/general/widgets/list.html' %} 3 | 4 | {% block begin_content scoped %} 5 |
    6 | 7 | {% endblock %} 8 | 9 | -------------------------------------------------------------------------------- /myapp/templates/myapp/fab_overrides/list_role.html: -------------------------------------------------------------------------------- 1 | 2 | {% extends 'appbuilder/general/widgets/roles/list.html' %} 3 | 4 | {% block begin_content scoped %} 5 |
    6 |
    7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /myapp/templates/myapp/feature.html: -------------------------------------------------------------------------------- 1 |
    2 | 知道了 3 |
    4 |
    5 | 27 |
    -------------------------------------------------------------------------------- /myapp/templates/myapp/flash_wrapper.html: -------------------------------------------------------------------------------- 1 | 2 |
    3 | {% include 'appbuilder/flash.html' %} 4 |
    5 | -------------------------------------------------------------------------------- /myapp/templates/myapp/partials/_script_tag.html: -------------------------------------------------------------------------------- 1 | 2 | {% block partial_js %} 3 | {% for entry in get_unloaded_chunks(js_manifest(filename), loaded_chunks) %} 4 | 5 | {% endfor %} 6 | {% endblock %} 7 | -------------------------------------------------------------------------------- /myapp/templates/myapp/traceback.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

    Sorry, something went wrong

    5 |

    500 - Internal Server Error

    6 |
    7 |

    Stacktrace

    8 |
    9 | 10 |
    11 |         {{ error_msg }}
    12 |       
    13 |
    14 | 15 | 16 | -------------------------------------------------------------------------------- /myapp/templates/myapp/welcome.html: -------------------------------------------------------------------------------- 1 | 2 | {% extends "myapp/basic.html" %} 3 | 4 | {% block navbar %} 5 | {% endblock %} 6 | 7 | {% block tail_js %} 8 | {% with filename="welcome" %} 9 | {% include "myapp/partials/_script_tag.html" %} 10 | {% endwith %} 11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /myapp/templates/mybase.html: -------------------------------------------------------------------------------- 1 | {% extends 'appbuilder/baselayout.html' %} 2 | {#引入自定义css#} 3 | {% block head_css %} 4 | {{ super() }} 5 | 6 | {% endblock %} 7 | 8 | {# 在模板首部导入js文件,使用 head_js #} 9 | {% block head_js %} 10 | {{ super() }} 11 | 12 | {% endblock %} 13 | 14 | 15 | {# 在模板末尾导入js文件,使用tail_js#} 16 | {% block tail_js %} 17 | {{ super() }} 18 | 19 | {% endblock %} -------------------------------------------------------------------------------- /myapp/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/tools/__init__.py -------------------------------------------------------------------------------- /myapp/translations/babel.cfg: -------------------------------------------------------------------------------- 1 | [python: **.py] 2 | [jinja2: **/templates/**.html] 3 | encoding = utf-8 4 | -------------------------------------------------------------------------------- /myapp/translations/en/LC_MESSAGES/messages.json: -------------------------------------------------------------------------------- 1 | {"domain":"myapp","locale_data":{"myapp":{"":{"domain":"myapp","plural_forms":"nplurals=2; plural=(n != 1)","lang":"en"}}}} -------------------------------------------------------------------------------- /myapp/translations/en/LC_MESSAGES/messages.po: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/translations/en/LC_MESSAGES/messages.po -------------------------------------------------------------------------------- /myapp/translations/messages.pot: -------------------------------------------------------------------------------- 1 | # Translations template for PROJECT. 2 | # Copyright (C) 2015 ORGANIZATION 3 | # This file is distributed under the same license as the PROJECT project. 4 | # FIRST AUTHOR , 2015. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PROJECT VERSION\n" 10 | "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" 11 | "POT-Creation-Date: 2015-10-26 13:23+0000\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=utf-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Generated-By: Babel 2.0\n" 19 | 20 | #: app/templates/404.html:4 21 | msgid "Page not found" 22 | msgstr "" 23 | 24 | -------------------------------------------------------------------------------- /myapp/translations/requirements.txt: -------------------------------------------------------------------------------- 1 | 2 | Babel==2.5.3 3 | -------------------------------------------------------------------------------- /myapp/translations/utils.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import os 4 | 5 | # Global caching for JSON language packs 6 | ALL_LANGUAGE_PACKS = {"en": {}} 7 | 8 | DIR = os.path.dirname(os.path.abspath(__file__)) 9 | 10 | 11 | def get_language_pack(locale): 12 | pack = ALL_LANGUAGE_PACKS.get(locale) 13 | if not pack: 14 | filename = DIR + "/{}/LC_MESSAGES/messages.json".format(locale) 15 | try: 16 | with open(filename) as f: 17 | pack = json.load(f) 18 | ALL_LANGUAGE_PACKS[locale] = pack 19 | except Exception: 20 | # Assuming english, client side falls back on english 21 | pass 22 | return pack 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /myapp/translations/zh/LC_MESSAGES/messages.json: -------------------------------------------------------------------------------- 1 | { 2 | "domain": "myapp", 3 | "locale_data": { 4 | "myapp": { 5 | "": { 6 | "domain": "myapp", 7 | "plural_forms": "nplurals=1; plural=0;", 8 | "lang": "zh" 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /myapp/translations/zh/LC_MESSAGES/messages.po: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/translations/zh/LC_MESSAGES/messages.po -------------------------------------------------------------------------------- /myapp/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/utils/__init__.py -------------------------------------------------------------------------------- /myapp/utils/cache.py: -------------------------------------------------------------------------------- 1 | from flask import request 2 | 3 | def view_cache_key(*unused_args, **unused_kwargs) -> str: 4 | args_hash = hash(frozenset(request.args.items())) 5 | return "view/{}/{}".format(request.path, args_hash) 6 | 7 | 8 | def memoized_func(key=view_cache_key, attribute_in_key=None): 9 | """Use this decorator to cache functions that have predefined first arg. 10 | 11 | enable_cache is treated as True by default, 12 | except enable_cache = False is passed to the decorated function. 13 | 14 | force means whether to force refresh the cache and is treated as False by default, 15 | except force = True is passed to the decorated function. 16 | 17 | timeout of cache is set to 600 seconds by default, 18 | except cache_timeout = {timeout in seconds} is passed to the decorated function. 19 | 20 | memoized_func uses simple_cache and stored the data in memory. 21 | Key is a callable function that takes function arguments and 22 | returns the caching key. 23 | """ 24 | def wrap(f): 25 | # noop 26 | def wrapped_f(self, *args, **kwargs): 27 | return f(self, *args, **kwargs) 28 | 29 | return wrapped_f 30 | 31 | return wrap 32 | -------------------------------------------------------------------------------- /myapp/utils/celery.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | from typing import Iterator 4 | 5 | from contextlib2 import contextmanager 6 | from sqlalchemy import create_engine 7 | from sqlalchemy.exc import SQLAlchemyError 8 | from sqlalchemy.orm import Session, sessionmaker 9 | from sqlalchemy.pool import NullPool 10 | 11 | from myapp import app, db 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | # Null pool is used for the celery workers due process forking side effects. 16 | @contextmanager 17 | def session_scope(nullpool: bool) -> Iterator[Session]: 18 | """Provide a transactional scope around a series of operations.""" 19 | database_uri = app.config["SQLALCHEMY_DATABASE_URI"] 20 | if nullpool: 21 | engine = create_engine(database_uri, poolclass=NullPool) 22 | session_class = sessionmaker() 23 | session_class.configure(bind=engine) 24 | session = session_class() 25 | else: 26 | session = db.session() 27 | session.commit() # HACK 28 | 29 | try: 30 | yield session 31 | session.commit() 32 | except SQLAlchemyError as ex: 33 | session.rollback() 34 | logger.exception(ex) 35 | raise 36 | finally: 37 | session.close() 38 | -------------------------------------------------------------------------------- /myapp/utils/dates.py: -------------------------------------------------------------------------------- 1 | 2 | from datetime import datetime 3 | 4 | import pytz 5 | 6 | EPOCH = datetime(1970, 1, 1) 7 | 8 | 9 | def datetime_to_epoch(dttm): 10 | if dttm.tzinfo: 11 | dttm = dttm.replace(tzinfo=pytz.utc) 12 | epoch_with_tz = pytz.utc.localize(EPOCH) 13 | return (dttm - epoch_with_tz).total_seconds() * 1000 14 | return (dttm - EPOCH).total_seconds() * 1000 15 | 16 | 17 | def now_as_float(): 18 | return datetime_to_epoch(datetime.utcnow()) 19 | -------------------------------------------------------------------------------- /myapp/utils/decorators.py: -------------------------------------------------------------------------------- 1 | 2 | from datetime import datetime, timedelta 3 | from functools import wraps 4 | import logging 5 | 6 | from contextlib2 import contextmanager 7 | from flask import request 8 | 9 | from myapp import app 10 | from myapp.utils.dates import now_as_float 11 | 12 | 13 | 14 | 15 | @contextmanager 16 | def stats_timing(stats_key, stats_logger): 17 | """Provide a transactional scope around a series of operations.""" 18 | start_ts = now_as_float() 19 | try: 20 | yield start_ts 21 | except Exception as e: 22 | raise e 23 | finally: 24 | stats_logger.timing(stats_key, now_as_float() - start_ts) 25 | 26 | 27 | def etag(check_perms=bool): 28 | 29 | def decorator(f): 30 | @wraps(f) 31 | def wrapper(*args, **kwargs): 32 | # check if the user can access the resource 33 | check_perms(*args, **kwargs) 34 | 35 | if request.method == "POST": 36 | return f(*args, **kwargs) 37 | 38 | response = None 39 | 40 | if response is None: 41 | response = f(*args, **kwargs) 42 | 43 | return response.make_conditional(request) 44 | 45 | return wrapper 46 | 47 | return decorator 48 | -------------------------------------------------------------------------------- /myapp/utils/log.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | import datetime 3 | import functools 4 | import json 5 | 6 | from flask import current_app, g, request 7 | 8 | import pysnooper 9 | 10 | # @pysnooper.snoop(depth=2) 11 | class AbstractEventLogger(ABC): 12 | @abstractmethod 13 | def log(self, user_id, action,duration_ms, *args, **kwargs): 14 | pass 15 | 16 | def log_this(self, f): 17 | @functools.wraps(f) 18 | def wrapper(*args, **kwargs): 19 | user_id = None 20 | if g.user: 21 | user_id = g.user.get_id() 22 | d = request.form.to_dict() or {} 23 | 24 | # request parameters can overwrite post body 25 | request_params = request.args.to_dict() 26 | d.update(request_params) 27 | d.update(kwargs) 28 | 29 | d.update(request.json or {}) 30 | 31 | self.stats_logger.incr(f.__name__) 32 | start_dttm = datetime.datetime.now() 33 | value = f(*args, **kwargs) 34 | duration_ms = (datetime.datetime.now() - start_dttm).total_seconds() * 1000 35 | 36 | if user_id: 37 | self.log( 38 | user_id, 39 | f.__name__, 40 | duration_ms=duration_ms, 41 | **d 42 | ) 43 | return value 44 | 45 | return wrapper 46 | 47 | @property 48 | def stats_logger(self): 49 | return current_app.config.get("STATS_LOGGER") 50 | 51 | # @pysnooper.snoop(depth=2) 52 | class DBEventLogger(AbstractEventLogger): 53 | 54 | def log(self, user_id, action,duration_ms, *args, **kwargs): 55 | from myapp.models.log import Log 56 | referrer = request.referrer[:1000] if request.referrer else None 57 | 58 | log = Log( 59 | action=action, 60 | json=json.dumps(kwargs,indent=4,ensure_ascii=False), 61 | duration_ms=duration_ms, 62 | referrer=referrer, 63 | user_id=user_id, 64 | method=request.method, 65 | path=request.path, 66 | dttm=datetime.datetime.now() 67 | ) 68 | 69 | sesh = current_app.appbuilder.get_session 70 | sesh.add(log) 71 | sesh.commit() 72 | -------------------------------------------------------------------------------- /myapp/utils/py/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/utils/py/__init__.py -------------------------------------------------------------------------------- /myapp/views/__init__.py: -------------------------------------------------------------------------------- 1 | from . import base 2 | from . import home 3 | from . import route 4 | from . import view_team 5 | 6 | from . import view_notebook 7 | from . import view_docker 8 | 9 | from . import view_job_template 10 | from . import view_task 11 | from . import view_pipeline 12 | from . import view_runhistory 13 | from . import view_workflow 14 | from . import view_link 15 | from .log import views 16 | from .log import api as log_api 17 | 18 | 19 | -------------------------------------------------------------------------------- /myapp/views/log/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from flask_babel import lazy_gettext as _ 3 | 4 | 5 | class LogMixin: 6 | label_title='日志' 7 | 8 | list_columns = ("user", "action", "dttm") 9 | edit_columns = ("user", "action", "dttm", "json") 10 | base_order = ("dttm", "desc") 11 | # label_columns = { 12 | # "user": _("User"), 13 | # "action": _("Action"), 14 | # "dttm": _("dttm"), 15 | # "json": _("JSON"), 16 | # } 17 | -------------------------------------------------------------------------------- /myapp/views/log/api.py: -------------------------------------------------------------------------------- 1 | 2 | from flask_appbuilder import ModelRestApi 3 | from flask_appbuilder.models.sqla.interface import SQLAInterface 4 | 5 | from myapp import app, appbuilder 6 | from myapp.models.log import Log 7 | from . import LogMixin 8 | 9 | 10 | class LogRestApi(LogMixin, ModelRestApi): 11 | datamodel = SQLAInterface(Log) 12 | 13 | class_permission_name = "LogModelView" 14 | method_permission_name = { 15 | "get_list": "list", 16 | "get": "show", 17 | "post": "add", 18 | "put": "edit", 19 | "delete": "delete", 20 | "info": "list", 21 | } 22 | resource_name = "log" 23 | allow_browser_login = True 24 | list_columns = ("user.username", "action", "dttm") 25 | 26 | 27 | if ( 28 | not app.config.get("FAB_ADD_SECURITY_VIEWS") is False 29 | or app.config.get("MYAPP_LOG_VIEW") is False 30 | ): 31 | appbuilder.add_api(LogRestApi) 32 | -------------------------------------------------------------------------------- /myapp/views/log/views.py: -------------------------------------------------------------------------------- 1 | from flask_appbuilder.models.sqla.interface import SQLAInterface 2 | from flask_babel import gettext as __ 3 | 4 | from myapp import app, appbuilder 5 | from myapp.models.log import Log 6 | from myapp.views.base import MyappModelView 7 | from . import LogMixin 8 | 9 | 10 | class LogModelView(LogMixin, MyappModelView): 11 | datamodel = SQLAInterface(Log) 12 | list_columns = ['user','method','path','duration_ms','dttm'] 13 | 14 | 15 | if ( 16 | not app.config.get("FAB_ADD_SECURITY_VIEWS") is False 17 | or app.config.get("MYAPP_LOG_VIEW") is False 18 | ): 19 | appbuilder.add_view( 20 | LogModelView, 21 | "Action Log", 22 | label=__("Action Log"), 23 | category="Security", 24 | category_label=__("Security"), 25 | icon="fa-list-ol", 26 | ) 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /myapp/views/route.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from myapp import ( 4 | app, 5 | appbuilder, 6 | conf, 7 | db, 8 | event_logger, 9 | get_feature_flags, 10 | is_feature_enabled, 11 | results_backend, 12 | security_manager, 13 | ) 14 | 15 | 16 | config = app.config 17 | 18 | 19 | @app.route("/health") 20 | def health(): 21 | return "OK" 22 | 23 | 24 | @app.route("/healthcheck") 25 | def healthcheck(): 26 | return "OK" 27 | 28 | 29 | @app.route("/ping") 30 | def ping(): 31 | return "OK" 32 | 33 | 34 | -------------------------------------------------------------------------------- /myapp/views/view_link.py: -------------------------------------------------------------------------------- 1 | 2 | from flask_babel import gettext as __ 3 | from flask_babel import lazy_gettext as _ 4 | # 将model添加成视图,并控制在前端的显示 5 | from myapp import app, appbuilder,db,event_logger 6 | 7 | from flask import ( 8 | current_app, 9 | abort, 10 | flash, 11 | g, 12 | Markup, 13 | make_response, 14 | redirect, 15 | render_template, 16 | request, 17 | send_from_directory, 18 | Response, 19 | url_for, 20 | ) 21 | from types import FunctionType 22 | from flask_appbuilder import CompactCRUDMixin, expose 23 | import pysnooper,datetime,time,json 24 | conf = app.config 25 | from myapp.views.base import BaseMyappView 26 | 27 | 28 | all_links = conf.get('ALL_LINKS',{}) 29 | for link in all_links: 30 | appbuilder.add_link( 31 | link['label'], 32 | label=_(link['label']), 33 | href=link['url'], 34 | category_icon="fa-link", 35 | icon="fa-link", 36 | category="link", 37 | category_label=__("链接"), 38 | ) 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /myapp/views/view_runhistory.py: -------------------------------------------------------------------------------- 1 | from flask import render_template,redirect 2 | from flask_appbuilder.models.sqla.interface import SQLAInterface 3 | 4 | # 将model添加成视图,并控制在前端的显示 5 | from myapp.models.model_job import Job_Template,Task,Pipeline,Workflow,RunHistory 6 | 7 | from myapp import app, appbuilder,db,event_logger 8 | from wtforms import BooleanField, IntegerField,StringField, SelectField,FloatField,DateField,DateTimeField,SelectMultipleField,FormField,FieldList 9 | from flask_appbuilder.fieldwidgets import BS3TextFieldWidget,BS3PasswordFieldWidget,DatePickerWidget,DateTimePickerWidget,Select2ManyWidget,Select2Widget,BS3TextAreaFieldWidget 10 | from flask_babel import gettext as __ 11 | from flask_babel import lazy_gettext as _ 12 | from sqlalchemy import and_, or_, select 13 | 14 | from .baseApi import ( 15 | MyappModelRestApi 16 | ) 17 | 18 | from myapp import security_manager 19 | from werkzeug.datastructures import FileStorage 20 | from .base import ( 21 | api, 22 | BaseMyappView, 23 | check_ownership, 24 | data_payload_response, 25 | DeleteMixin, 26 | generate_download_headers, 27 | get_error_msg, 28 | get_user_roles, 29 | handle_api_exception, 30 | json_error_response, 31 | json_success, 32 | MyappFilter, 33 | MyappModelView, 34 | ) 35 | from flask_appbuilder import CompactCRUDMixin, expose 36 | import pysnooper,datetime,time,json 37 | conf = app.config 38 | logging = app.logger 39 | 40 | 41 | class RunHistory_Filter(MyappFilter): 42 | # @pysnooper.snoop() 43 | def apply(self, query, func): 44 | user_roles = [role.name.lower() for role in list(self.get_user_roles())] 45 | if "admin" in user_roles: 46 | return query 47 | 48 | pipeline_ids = security_manager.get_create_pipeline_ids(db.session) 49 | return query.filter( 50 | or_( 51 | self.model.pipeline_id.in_(pipeline_ids), 52 | # self.model.project.name.in_(['public']) 53 | ) 54 | ) 55 | 56 | 57 | 58 | class RunHistory_ModelView_Base(): 59 | label_title='定时调度历史' 60 | datamodel = SQLAInterface(RunHistory) 61 | base_order = ('id', 'desc') 62 | order_columns = ['id'] 63 | 64 | list_columns = ['pipeline_url','creator','created_on','execution_date','status_url','log','history'] 65 | edit_columns = ['status'] 66 | base_filters = [["id", RunHistory_Filter, lambda: []]] # 设置权限过滤器 67 | add_form_extra_fields = { 68 | "status": SelectField( 69 | _(datamodel.obj.lab('status')), 70 | description="状态comed为已识别未提交,created为已提交", 71 | widget=Select2Widget(), 72 | choices=[['comed', 'comed'], ['created', 'created']] 73 | ), 74 | } 75 | edit_form_extra_fields = add_form_extra_fields 76 | 77 | 78 | class RunHistory_ModelView(RunHistory_ModelView_Base,MyappModelView,DeleteMixin): 79 | datamodel = SQLAInterface(RunHistory) 80 | 81 | appbuilder.add_view(RunHistory_ModelView,"定时调度记录",icon = 'fa-clock-o',category = '任务') 82 | 83 | 84 | 85 | 86 | # 添加api 87 | class RunHistory_ModelView_Api(RunHistory_ModelView_Base,MyappModelRestApi): 88 | datamodel = SQLAInterface(RunHistory) 89 | route_base = '/runhistory_modelview/api' 90 | 91 | appbuilder.add_api(RunHistory_ModelView_Api) 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /myapp/vision/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [Makefile] 16 | indent_style = tab 17 | -------------------------------------------------------------------------------- /myapp/vision/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "env": { 4 | "browser": true, 5 | "commonjs": true, 6 | "node": true, 7 | "es6": true 8 | }, 9 | "extends": [ 10 | "eslint:recommended", 11 | "plugin:react/recommended", 12 | "plugin:@typescript-eslint/recommended" 13 | ], 14 | "globals": { 15 | "describe": true, 16 | "beforeEach": true, 17 | "it": true, 18 | "expect": true 19 | }, 20 | "plugins": [ 21 | "react", 22 | "@typescript-eslint" 23 | ], 24 | "parser": "@typescript-eslint/parser", 25 | "parserOptions": { 26 | "ecmaFeatures": { 27 | "jsx": true 28 | }, 29 | "ecmaVersion": 2018, 30 | "sourceType": "module", 31 | "allowImportExportEverywhere": true 32 | }, 33 | "rules": { 34 | "no-alert": "error", 35 | "no-eval": "error", 36 | "spaced-comment": [ 37 | "error", 38 | "always" 39 | ], 40 | // "linebreak-style": ["error", "windows"], 41 | "linebreak-style": 0, 42 | // "linebreak-style": ["error", "unix"], 43 | "react/prop-types": 0, 44 | "no-console": 0, 45 | "no-useless-escape": 0, 46 | "@typescript-eslint/no-explicit-any": 0, 47 | "@typescript-eslint/camelcase": 0, 48 | "@typescript-eslint/explicit-function-return-type": 0 49 | } 50 | } -------------------------------------------------------------------------------- /myapp/vision/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | /dist 14 | 15 | # misc 16 | .DS_Store 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | /yarn.lock 22 | -------------------------------------------------------------------------------- /myapp/vision/.prettierignore: -------------------------------------------------------------------------------- 1 | **/*.md 2 | **/*.svg 3 | **/*.html 4 | package.json 5 | dist/ 6 | -------------------------------------------------------------------------------- /myapp/vision/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all", 4 | "printWidth": 120, 5 | "semi": true, 6 | "overrides": [ 7 | { 8 | "files": ".prettierrc", 9 | "options": { 10 | "parser": "json" 11 | } 12 | } 13 | ], 14 | "bracketSpacing": true, 15 | "arrowParens": "avoid", 16 | "endOfLine": "lf" 17 | } 18 | -------------------------------------------------------------------------------- /myapp/vision/.yarnrc: -------------------------------------------------------------------------------- 1 | registry "https://mirrors.tencent.com/npm/" -------------------------------------------------------------------------------- /myapp/vision/base-tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx", 22 | "baseUrl": ".", 23 | "paths": { 24 | "@src/*": [ 25 | "./src/*" 26 | ] 27 | } 28 | }, 29 | "include": [ 30 | "src" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /myapp/vision/config-overrides.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin'); 3 | const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; 4 | const path = require('path'); 5 | const paths = require('react-scripts/config/paths'); 6 | const { override, fixBabelImports, addLessLoader } = require('customize-cra'); 7 | // 合并项目,修改打包输出的路径 8 | paths.appBuild = path.join(path.dirname(paths.appBuild), '../static/appbuilder/vison'); 9 | // paths.appBuild = path.join(path.dirname(paths.appBuild), './build'); 10 | 11 | function webpack(config){ 12 | config.devtool = false; 13 | // alias 14 | config.resolve.alias = { 15 | ...config.resolve.alias, 16 | '@src': path.resolve(__dirname, 'src'), 17 | }; 18 | 19 | // plugin 20 | config.plugins.push( 21 | new MonacoWebpackPlugin({ 22 | languages: ['json'], 23 | }), 24 | ); 25 | 26 | // if (process.env.NODE_ENV !== 'production') { 27 | // config.plugins.push( 28 | // new BundleAnalyzerPlugin() 29 | // ) 30 | // } 31 | 32 | return config; 33 | } 34 | module.exports = { 35 | webpack: config => { 36 | config.devtool = false; 37 | // alias 38 | config.resolve.alias = { 39 | ...config.resolve.alias, 40 | '@src': path.resolve(__dirname, 'src'), 41 | }; 42 | 43 | // plugin 44 | config.plugins.push( 45 | new MonacoWebpackPlugin({ 46 | languages: ['json'], 47 | }), 48 | ); 49 | 50 | // if (process.env.NODE_ENV !== 'production') { 51 | // config.plugins.push( 52 | // new BundleAnalyzerPlugin() 53 | // ) 54 | // } 55 | 56 | return config; 57 | }, 58 | }; 59 | 60 | // module.exports = override( 61 | // fixBabelImports('import', { 62 | // libraryName: 'antd', 63 | // libraryDirectory: 'es', 64 | // style: true, 65 | // }), 66 | // addLessLoader({ 67 | // javascriptEnabled: true, 68 | // modifyVars: { '@primary-color': '#1DA57A' }, 69 | // }), 70 | // ); 71 | -------------------------------------------------------------------------------- /myapp/vision/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vite-ml-platform", 3 | "version": "0.1.0", 4 | "license": "UNLICENSED", 5 | "private": true, 6 | "dependencies": { 7 | "@ant-design/icons": "^4.7.0", 8 | "@antv/g6": "^4.6.4", 9 | "@fluentui/react": "^8.5.1", 10 | "@reduxjs/toolkit": "^1.5.1", 11 | "@typescript-eslint/parser": "^5.10.0", 12 | "antd": "4.17.4", 13 | "array-move": "^4.0.0", 14 | "axios": "^0.21.4", 15 | "babel-plugin-import": "^1.13.3", 16 | "cookie": "^0.4.1", 17 | "customize-cra": "^1.0.0", 18 | "eslint-plugin-react": "^7.28.0", 19 | "less": "^4.1.2", 20 | "less-loader": "^10.2.0", 21 | "moment": "^2.29.3", 22 | "react": "^17.0.0", 23 | "react-copy-to-clipboard": "^5.0.4", 24 | "react-dom": "^17.0.0", 25 | "react-flow-renderer": "^9.4.0", 26 | "react-json-view": "^1.21.3", 27 | "react-monaco-editor": "^0.44.0", 28 | "react-redux": "^7.2.4", 29 | "react-router-dom": "^5.2.0", 30 | "react-sortable-hoc": "^2.0.0", 31 | "xgplayer": "^2.28.0" 32 | }, 33 | "scripts": { 34 | "dev": "react-app-rewired start", 35 | "build": "react-app-rewired build", 36 | "test": "react-app-rewired test", 37 | "prettier": "prettier --write '**/*.{tsx,ts,less}'" 38 | }, 39 | "browserslist": { 40 | "production": [ 41 | ">0.2%", 42 | "not dead", 43 | "not op_mini all" 44 | ], 45 | "development": [ 46 | "last 1 chrome version", 47 | "last 1 firefox version", 48 | "last 1 safari version" 49 | ] 50 | }, 51 | "devDependencies": { 52 | "@types/cookie": "^0.4.0", 53 | "@types/react": "^17.0.0", 54 | "@types/react-copy-to-clipboard": "^5.0.2", 55 | "@types/react-dom": "^17.0.0", 56 | "@types/react-redux": "^7.1.16", 57 | "@types/react-router-dom": "^5.1.7", 58 | "@typescript-eslint/eslint-plugin": "^4.29.1", 59 | "eslint": "^7.32.0", 60 | "monaco-editor-webpack-plugin": "^4.1.1", 61 | "prettier": "^2.2.1", 62 | "react-app-rewired": "^2.1.9", 63 | "react-scripts": "^5.0.0", 64 | "typescript": "^4.5.4", 65 | "web-vitals": "^1.0.1", 66 | "webpack-bundle-analyzer": "^4.4.2" 67 | }, 68 | "homepage": "./" 69 | } 70 | -------------------------------------------------------------------------------- /myapp/vision/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tencentmusic/argo-workflow/d3c1bfb7249a39a7f458254a69a177e379453c0f/myapp/vision/public/favicon.ico -------------------------------------------------------------------------------- /myapp/vision/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 机器学习平台 12 | 13 | 14 | 15 | 16 |
    17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /myapp/vision/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "machine_learnring", 3 | "name": "tme_machine_learnring_platform", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /myapp/vision/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /myapp/vision/src/components/EditorAce/components/BaseTypes/ObjectType.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | // import { Icon } from '@fluentui/react'; 3 | 4 | interface Props { 5 | data: any; 6 | } 7 | 8 | const ObjectType: React.FC = () => { 9 | // const [expanded, setExpanded] = useState(true); 10 | 11 | return ( 12 |
    13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
    21 | ); 22 | }; 23 | 24 | export default ObjectType; 25 | -------------------------------------------------------------------------------- /myapp/vision/src/components/EditorAce/components/VisualizedData/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | // import ObjectType from '../BaseTypes/ObjectType'; 3 | // import style from './style'; 4 | 5 | const VisualizedData: React.FC = () => { 6 | // const [data, setData] = useState({}); 7 | // const [text, setText] = useState('0'); 8 | 9 | return ( 10 |
    11 | {/* {text.length > 0 ? ( 12 | 13 | ) : ( 14 |
    No data
    15 | )} */} 16 |
    17 | ); 18 | }; 19 | 20 | export default VisualizedData; 21 | -------------------------------------------------------------------------------- /myapp/vision/src/components/EditorAce/components/VisualizedData/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | const noDataStyle = mergeStyles({ 3 | fontStyle: 'italic', 4 | color: '#6c757d', 5 | padding: '.5rem', 6 | }); 7 | 8 | export default { 9 | noDataStyle, 10 | }; 11 | -------------------------------------------------------------------------------- /myapp/vision/src/components/EditorAce/style.ts: -------------------------------------------------------------------------------- 1 | import { IIconProps, mergeStyleSets, mergeStyles } from '@fluentui/react'; 2 | const modalStyles = mergeStyles({ 3 | position: 'fixed', 4 | width: '100%', 5 | height: '100vh', 6 | backgroundColor: 'rgba(0, 0, 0, .5)', 7 | zIndex: 999, 8 | display: 'flex', 9 | justifyContent: 'center', 10 | alignItems: 'center', 11 | }); 12 | const cancelIcon: IIconProps = { iconName: 'Cancel' }; 13 | const contentStyles = mergeStyleSets({ 14 | container: { 15 | minWidth: '75%', 16 | minHeight: '80%', 17 | overflow: 'hidden', 18 | display: 'flex', 19 | borderRadius: 2, 20 | backgroundColor: '#fff', 21 | padding: '0 24px', 22 | }, 23 | header: { 24 | display: 'flex', 25 | alignItems: 'center', 26 | justifyContent: 'space-between', 27 | padding: '10px 0 14px 0', 28 | }, 29 | body: { 30 | flex: 1, 31 | height: 0, 32 | position: 'relative', 33 | display: 'flex', 34 | }, 35 | footer: { 36 | position: 'relative', 37 | display: 'flex', 38 | padding: '24px 0', 39 | justifyContent: 'flex-end', 40 | }, 41 | }); 42 | const resizeLine = mergeStyles({ 43 | width: 8, 44 | height: '100%', 45 | cursor: 'col-resize', 46 | background: '#eee', 47 | transition: 'background 0.2s', 48 | margin: '0 15px 0 2px', 49 | display: 'flex', 50 | alignItems: 'center', 51 | justifyContent: 'center', 52 | overflow: 'hidden', 53 | }); 54 | const resizeLineIconStyles = mergeStyles({ 55 | display: 'flex', 56 | justifyContent: 'center', 57 | alignItems: 'center', 58 | boxSizing: 'border-box', 59 | paddingRight: '19px', 60 | width: 8, 61 | height: 20, 62 | fontSize: 20, 63 | pointerEvents: 'none', 64 | }); 65 | 66 | export default { 67 | modalStyles, 68 | cancelIcon, 69 | contentStyles, 70 | resizeLine, 71 | resizeLineIconStyles, 72 | }; 73 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorBody/components/DataSet/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Icon, Layer } from '@fluentui/react'; 3 | import { Handle, Position, NodeProps } from 'react-flow-renderer'; 4 | import Model from '../Model'; 5 | import style from './style'; 6 | 7 | const DataSet: React.FC = props => { 8 | return ( 9 | <> 10 | 11 | {/*
    12 |
    13 |
    14 |
    15 |
    16 | 17 |
    {props?.data?.label}
    18 |
    19 |
    20 |
    21 |
    */} 22 |
    23 |
    24 | 25 |
    26 |
    27 |
    {props?.data?.label}
    28 | { 29 | props?.data?.info ?
    {props?.data?.info['describe']}
    : null 30 | } 31 |
    32 |
    33 | 34 | {/* 配置板 */} 35 | 36 | 37 | 38 | 39 | ); 40 | }; 41 | 42 | export default DataSet; 43 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorBody/components/DataSet/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | 3 | const baseContainerStyle = { 4 | height: 54, 5 | minWidth: 272, 6 | borderRadius: 100, 7 | borderStyle: 'solid', 8 | display: 'flex', 9 | flexDirection: 'row', 10 | backgroundColor: '#fff', 11 | fontSize: 12, 12 | cursor: 'move', 13 | boxSizing: 'border-box', 14 | transition: 'all 0.3s' 15 | }; 16 | const nodeContainer = mergeStyles({ 17 | ...baseContainerStyle, 18 | ...{ borderWidth: 1, borderColor: '#b1b1b7' }, 19 | }); 20 | const nodeOnSelect = mergeStyles({ 21 | ...baseContainerStyle, 22 | ...{ borderWidth: 1, borderColor: '#006dce', backgroundColor: '#f1f7fd' }, 23 | }); 24 | const nodeBar = mergeStyles({ 25 | width: 8, 26 | flexShrink: 0, 27 | borderRadius: '3px 0 0 3px', 28 | borderRight: '1px solid #8a8886', 29 | margin: '-8px 0 -8px -8px', 30 | }); 31 | const nodeContent = mergeStyles({ 32 | boxSizing: 'border-box', 33 | display: 'flex', 34 | flexDirection: 'column', 35 | overflow: 'hidden', 36 | }); 37 | const nodeConentTitleBar = mergeStyles({ 38 | display: 'flex', 39 | marginLeft: 7, 40 | minHeight: 26, 41 | }); 42 | const nodeIconWrapper = mergeStyles({ 43 | fontSize: 16, 44 | width: 64, 45 | display: 'flex', 46 | alignItems: 'center', 47 | justifyContent: 'center', 48 | backgroundColor: 'rgb(0, 120, 212)', 49 | borderTopLeftRadius: 100, 50 | borderBottomLeftRadius: 100, 51 | margin: '-1px 0 -1px -1px' 52 | }); 53 | const nodeIcon = mergeStyles({ 54 | userSelect: 'none', 55 | boxSizing: 'border-box', 56 | color: '#fff', 57 | fontSize: 20, 58 | marginLeft: 8 59 | }); 60 | const nodeTitle = mergeStyles({ 61 | whiteSpace: 'nowrap', 62 | textOverflow: 'ellipsis', 63 | overflow: 'hidden', 64 | paddingLeft: 8, 65 | paddingBottom: 2, 66 | marginBottom: 2, 67 | fontWeight: 600, 68 | fontSize: 14, 69 | borderBottom: '1px dashed #c1c1c1', 70 | userSelect: 'none', 71 | }); 72 | const handleStyle = mergeStyles({ 73 | width: '12px !important', 74 | height: '12px !important', 75 | bottom: '-7px !important', 76 | // top: '-7px !important', 77 | borderColor: '#b1b1b7 !important', 78 | backgroundColor: '#fff !important', 79 | transition: 'all 0.3s', 80 | '&:hover': { 81 | borderWidth: '2px !important', 82 | borderColor: 'var(--hover-color) !important', 83 | cursor: 'pointer !important', 84 | }, 85 | }); 86 | const hidden = mergeStyles({ 87 | visibility: 'hidden', 88 | }); 89 | 90 | const nodeContentWrapper = mergeStyles({ 91 | height: '100%', 92 | width: '100%', 93 | display: 'flex', 94 | justifyContent: 'center', 95 | flexDirection: 'column', 96 | }); 97 | 98 | const nodeTips = mergeStyles({ 99 | color: 'rgb(177, 177, 183)', 100 | paddingLeft: 8, 101 | }); 102 | 103 | export default { 104 | nodeTips, 105 | nodeContentWrapper, 106 | nodeContainer, 107 | nodeOnSelect, 108 | nodeBar, 109 | nodeContent, 110 | nodeConentTitleBar, 111 | nodeIconWrapper, 112 | nodeIcon, 113 | nodeTitle, 114 | handleStyle, 115 | hidden, 116 | }; 117 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorBody/components/Model/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | const modelContainer = mergeStyles({ 3 | display: 'flex', 4 | flexDirection: 'column', 5 | width: 350, 6 | position: 'absolute', 7 | top: 0, 8 | right: 0, 9 | bottom: 0, 10 | height: 'auto', 11 | backgroundColor: '#fff', 12 | borderLeft: '1px solid rgb(234, 234, 234)', 13 | boxShadow: 'rgb(0 0 0 / 18%) 0px 1.6px 3.6px 0px, rgb(0 0 0 / 22%) 0px 0.3px 0.9px 0px', 14 | zIndex: 998, 15 | }); 16 | const modelHeader = mergeStyles({ 17 | display: 'flex', 18 | width: '100%', 19 | alignItems: 'center', 20 | justifyContent: 'space-between', 21 | color: 'balck', 22 | boxSizing: 'border-box', 23 | padding: '10px 20px', 24 | }); 25 | const headerTitle = mergeStyles({ 26 | fontSize: 20, 27 | textOverflow: 'ellipsis', 28 | overflow: 'hidden', 29 | whiteSpace: 'nowrap', 30 | color: 'black', 31 | fontWeight: 600, 32 | }); 33 | const modelContent = mergeStyles({ 34 | flex: '1 1 0%', 35 | overflowY: 'auto', 36 | '&::-webkit-scrollbar': { 37 | width: 4, 38 | }, 39 | '&::-webkit-scrollbar-thumb': { 40 | minHeight: '15px', 41 | border: '6px solid transparent', 42 | backgroundClip: 'padding-box', 43 | backgroundColor: 'rgb(200, 200, 200)', 44 | }, 45 | }); 46 | const contentWrapper = mergeStyles({ 47 | padding: '0 20px 20px', 48 | overflowX: 'hidden', 49 | wordBreak: 'break-word', 50 | userSelect: 'text', 51 | borderTop: '1px solid #eaeaea', 52 | }); 53 | const splitLine = mergeStyles({ 54 | margin: '10px -20px 0 -20px', 55 | borderTop: '1px solid #eaeaea', 56 | }); 57 | const settingControl = mergeStyles({ 58 | height: 'auto', 59 | display: 'flex', 60 | flexDirection: 'column', 61 | paddingTop: 20, 62 | }); 63 | const saveButton = mergeStyles({ 64 | alignSelf: 'center', 65 | }); 66 | const debugButton = mergeStyles({ 67 | display: 'flex', 68 | flexDirection: 'row', 69 | justifyContent: 'space-evenly', 70 | paddingTop: 20, 71 | }); 72 | const templateConfig = mergeStyles({ 73 | color: 'rgb(96, 94, 92)', 74 | fontSize: 10, 75 | paddingTop: 5, 76 | paddingLeft: 2, 77 | '> a': { 78 | textDecoration: 'none', 79 | }, 80 | }); 81 | const argsDescription = mergeStyles({ 82 | color: 'rgb(96, 94, 92)', 83 | fontSize: 10, 84 | '> a': { 85 | textDecoration: 'none', 86 | }, 87 | }); 88 | const textLabelStyle = mergeStyles({ 89 | display: 'flex', 90 | alignItems: 'center', 91 | justifyContent: 'space-between', 92 | fontWeight: 600, 93 | color: 'rgb(50, 49, 48)', 94 | boxSizing: 'border-box', 95 | boxShadow: 'none', 96 | margin: '0px', 97 | padding: '5px 0px', 98 | overflowWrap: 'break-word', 99 | lineHeight: '30px', 100 | }); 101 | 102 | export default { 103 | modelContainer, 104 | modelHeader, 105 | headerTitle, 106 | modelContent, 107 | contentWrapper, 108 | splitLine, 109 | settingControl, 110 | saveButton, 111 | debugButton, 112 | templateConfig, 113 | argsDescription, 114 | textLabelStyle, 115 | }; 116 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorBody/components/NodeType.ts: -------------------------------------------------------------------------------- 1 | import dataSet from './DataSet'; 2 | 3 | const NodeType = { 4 | dataSet, 5 | }; 6 | 7 | export default NodeType; 8 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorBody/components/Setting/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | const settingContainer = mergeStyles({ 3 | display: 'flex', 4 | flexDirection: 'column', 5 | width: 350, 6 | position: 'absolute', 7 | top: 0, 8 | right: 0, 9 | bottom: 0, 10 | height: 'auto', 11 | backgroundColor: '#fff', 12 | borderLeft: '1px solid rgb(234, 234, 234)', 13 | boxShadow: 'rgb(0 0 0 / 18%) 0px 1.6px 3.6px 0px, rgb(0 0 0 / 22%) 0px 0.3px 0.9px 0px', 14 | zIndex: 999, 15 | }); 16 | const settingHeader = mergeStyles({ 17 | display: 'flex', 18 | width: '100%', 19 | alignItems: 'center', 20 | justifyContent: 'space-between', 21 | color: 'balck', 22 | boxSizing: 'border-box', 23 | padding: '10px 20px', 24 | }); 25 | const headerTitle = mergeStyles({ 26 | fontSize: 20, 27 | fontWeight: 600, 28 | textOverflow: 'ellipsis', 29 | overflow: 'hidden', 30 | whiteSpace: 'nowrap', 31 | color: 'black', 32 | }); 33 | const settingContent = mergeStyles({ 34 | flex: '1 1 0%', 35 | overflowY: 'auto', 36 | '&::-webkit-scrollbar': { 37 | width: 4, 38 | }, 39 | '&::-webkit-scrollbar-thumb': { 40 | minHeight: '15px', 41 | border: '6px solid transparent', 42 | backgroundClip: 'padding-box', 43 | backgroundColor: 'rgb(200, 200, 200)', 44 | }, 45 | }); 46 | const contentWrapper = mergeStyles({ 47 | padding: '0 20px 20px', 48 | overflowX: 'hidden', 49 | wordBreak: 'break-word', 50 | userSelect: 'text', 51 | borderTop: '1px solid #eaeaea', 52 | }); 53 | const splitLine = mergeStyles({ 54 | margin: '10px -20px 0 -20px', 55 | borderTop: '1px solid #eaeaea', 56 | }); 57 | 58 | export default { 59 | settingContainer, 60 | settingHeader, 61 | headerTitle, 62 | settingContent, 63 | contentWrapper, 64 | splitLine, 65 | }; 66 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorBody/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | 3 | const itemStyles = mergeStyles({ 4 | position: 'relative', 5 | }); 6 | const flowWrapper = mergeStyles({ 7 | width: '100%', 8 | height: '100%', 9 | }); 10 | const spinnerWrapper = mergeStyles({ 11 | position: 'absolute', 12 | top: 0, 13 | left: 0, 14 | display: 'flex', 15 | justifyContent: 'center', 16 | alignItems: 'center', 17 | width: '100%', 18 | height: '100%', 19 | overflow: 'hidden', 20 | backgroundColor: 'rgba(255, 255, 255, 0.6)', 21 | zIndex: 9999, 22 | }); 23 | 24 | export default { 25 | itemStyles, 26 | flowWrapper, 27 | spinnerWrapper, 28 | }; 29 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorHead/style.ts: -------------------------------------------------------------------------------- 1 | import { IStackStyles, IStackItemStyles, mergeStyles } from '@fluentui/react'; 2 | 3 | const headStyle: Partial = { 4 | root: { 5 | backgroundColor: '#fff', 6 | padding: '10px 16px', 7 | borderBottom: '1px solid #dadada', 8 | }, 9 | }; 10 | const headNameStyle = mergeStyles({ 11 | boxSizing: 'border-box', 12 | marginLeft: '8px', 13 | padding: '0 8px', 14 | height: '32px', 15 | lineHeight: '32px', 16 | cursor: 'text', 17 | marginBottom: '4px', 18 | fontSize: '16px', 19 | fontWeight: 600, 20 | overflow: 'hidden', 21 | textOverflow: 'ellipsis', 22 | whiteSpace: 'nowrap', 23 | color: 'black', 24 | }); 25 | const buttonItemStyle: Partial = { 26 | root: { 27 | marginRight: '16px', 28 | }, 29 | }; 30 | 31 | const hidden = mergeStyles({ 32 | visibility: 'hidden', 33 | }); 34 | 35 | export default { 36 | headStyle, 37 | headNameStyle, 38 | buttonItemStyle, 39 | hidden, 40 | }; 41 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/EditorTool/style.ts: -------------------------------------------------------------------------------- 1 | import { 2 | IStackItemStyles, 3 | ICommandBarStyles, 4 | IButtonStyles, 5 | IIconStyles, 6 | IToggleStyles, 7 | IComboBoxStyles, 8 | mergeStyles, 9 | } from '@fluentui/react'; 10 | 11 | const editorToolStyle: Partial = { 12 | root: { 13 | display: 'flex', 14 | flexDirection: 'row', 15 | backgroundColor: '#fff', 16 | padding: '2px 0px', 17 | borderBottom: '1px solid #dadada', 18 | '#authoring-page-toolbar': { 19 | flex: 1, 20 | }, 21 | }, 22 | }; 23 | 24 | const autoSavedTips = mergeStyles({ 25 | paddingRight: 20, 26 | lineHeight: '1', 27 | }); 28 | 29 | const commandBarStyle: Partial = { 30 | root: { 31 | height: '40px', 32 | padding: '0px 14px 0px 8px', 33 | borderBottom: 'none', 34 | backgroundColor: '#fff', 35 | }, 36 | }; 37 | 38 | const commonButton: IButtonStyles = { 39 | root: { 40 | width: '40px', 41 | height: '40px', 42 | borderRadius: '2px', 43 | minWidth: '40px', 44 | }, 45 | }; 46 | const commonIcon: IIconStyles = { 47 | root: { color: '#015cda', fontSize: 18 }, 48 | }; 49 | 50 | const toggleStyle: Partial = { 51 | root: { 52 | display: 'flex', 53 | alignItems: 'center', 54 | padding: '8px 0px 8px 8px', 55 | height: '100%', 56 | boxSizing: 'border-box', 57 | borderLeft: '1px solid rgb(234, 234, 234)', 58 | marginLeft: '8px', 59 | }, 60 | container: { 61 | display: 'inline-flex', 62 | }, 63 | }; 64 | 65 | const comboBoxStyle: Partial = { 66 | container: { 67 | borderLeft: '1px solid #eaeaea', 68 | marginLeft: '4px', 69 | padding: '0px 5px 0px 10px', 70 | width: 'auto', 71 | }, 72 | root: { 73 | backgroundColor: '#ffffff', 74 | padding: '0px 20px 0px 4px', 75 | selectors: { 76 | '&.is-open::after': { borderBottom: '2px solid #015cda' }, 77 | '&::after': { 78 | border: 'none', 79 | borderBottom: 0, 80 | borderRadius: 'none', 81 | }, 82 | '.ms-Button': { 83 | width: 20, 84 | }, 85 | '.ms-Icon': { 86 | fontSize: 8, 87 | }, 88 | }, 89 | }, 90 | input: { 91 | width: 35, 92 | backgroundColor: '#ffffff', 93 | }, 94 | }; 95 | 96 | export default { 97 | editorToolStyle, 98 | autoSavedTips, 99 | commandBarStyle, 100 | commonButton, 101 | commonIcon, 102 | toggleStyle, 103 | comboBoxStyle, 104 | }; 105 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/components/ErrorTips/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | import { MessageBar, MessageBarType } from '@fluentui/react'; 3 | import { useAppSelector, useAppDispatch } from '../../../../models/hooks'; 4 | import { selectErrMsg, updateErrMsg } from '../../../../models/app'; 5 | 6 | const ErrorTips: React.FC = () => { 7 | const dispatch = useAppDispatch(); 8 | const errMsg = useAppSelector(selectErrMsg); 9 | 10 | // tips 3s 后隐藏 11 | useEffect(() => { 12 | if (errMsg) { 13 | setTimeout(() => { 14 | dispatch(updateErrMsg(null)); 15 | }, 3000); 16 | } 17 | }, [errMsg]); 18 | 19 | return ( 20 | errMsg && ( 21 | { 32 | dispatch(updateErrMsg(null)); 33 | }} 34 | isMultiline={false} 35 | > 36 | {errMsg?.msg || ''} 37 | 38 | ) 39 | ); 40 | }; 41 | 42 | export default ErrorTips; 43 | -------------------------------------------------------------------------------- /myapp/vision/src/components/FlowEditor/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | import { Stack } from '@fluentui/react'; 3 | import { useAppDispatch } from '@src/models/hooks'; 4 | import { getPipeline } from '@src/models/pipeline'; 5 | import EditorBody from './components/EditorBody'; 6 | import EditorHead from './components/EditorHead'; 7 | import EditorTool from './components/EditorTool'; 8 | 9 | const { Item } = Stack; 10 | 11 | const FlowEditor: React.FC = () => { 12 | const dispatch = useAppDispatch(); 13 | 14 | useEffect(() => { 15 | // 获取 pipeline 信息 16 | dispatch(getPipeline()); 17 | }, []); 18 | 19 | return ( 20 | 33 | 34 | {/* 头部信息 */} 35 | 36 | {/* 工具栏 */} 37 | 38 | {/* 流水线交互面板 */} 39 | 40 | 41 | 42 | ); 43 | }; 44 | 45 | export default FlowEditor; 46 | -------------------------------------------------------------------------------- /myapp/vision/src/components/Home/Pipeline/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Stack } from '@fluentui/react'; 3 | import style from './style'; 4 | 5 | const Pipeline: React.FC = () => { 6 | return ; 7 | }; 8 | 9 | export default Pipeline; 10 | -------------------------------------------------------------------------------- /myapp/vision/src/components/Home/Pipeline/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/merge-styles'; 2 | 3 | const pipelineList = mergeStyles({ 4 | marginTop: '16px !important', 5 | padding: '0 10px 24px', 6 | }); 7 | 8 | export default { pipelineList }; 9 | -------------------------------------------------------------------------------- /myapp/vision/src/components/Home/Section/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/merge-styles'; 2 | 3 | const sectionStyles = mergeStyles({ 4 | marginBottom: 16, 5 | padding: '0 10px 24px 10px', 6 | borderBottom: '1px solid #dadada', 7 | selectors: { 8 | '.subtitle': { 9 | marginBottom: 20, 10 | height: 24, 11 | lineHeight: '1.1', 12 | fontSize: 20, 13 | fontWeight: 'bold', 14 | fontFamily: '"Raleway","Helvetica Neue",Helvetica,Arial,sans-serif;', 15 | }, 16 | '.expand-button': { 17 | color: '#005cd2', 18 | marginRight: 24, 19 | fontSize: 14, 20 | marginBottom: 12, 21 | }, 22 | }, 23 | }); 24 | 25 | const sampleStyles = mergeStyles({ 26 | minHeight: 186, 27 | position: 'relative', 28 | display: 'flex', 29 | flexWrap: 'wrap', 30 | }); 31 | const sampleHide = mergeStyles({ 32 | height: 186, 33 | overflow: 'hidden', 34 | }); 35 | 36 | const sampleCardStyle = mergeStyles({ 37 | width: 200, 38 | marginRight: 24, 39 | marginBottom: 24, 40 | }); 41 | 42 | const cardContainer = mergeStyles({ 43 | width: 200, 44 | height: 124, 45 | borderRadius: 2, 46 | display: 'flex', 47 | flexDirection: 'column', 48 | cursor: 'pointer', 49 | backgroundColor: '#fff', 50 | border: '1px solid #dadada', 51 | selectors: { 52 | ':hover': { 53 | boxShadow: 'rgb(0 0 0 / 18%) 0px 6.4px 14.4px 0px, rgb(0 0 0 / 22%) 0px 1.2px 3.6px 0px', 54 | }, 55 | }, 56 | }); 57 | 58 | const addIconStyles = mergeStyles({ 59 | fontSize: 38, 60 | padding: '42px 0 29px', 61 | cursor: 'pointer', 62 | textAlign: 'center', 63 | color: '#005cd2', 64 | }); 65 | 66 | const sampleImgStyles = mergeStyles({ 67 | height: '100%', 68 | backgroundRepeat: 'no-repeat', 69 | backgroundSize: 'cover', 70 | selectors: { 71 | img: { 72 | width: '100%', 73 | height: '100%', 74 | }, 75 | }, 76 | }); 77 | 78 | const cardTitleStyles = mergeStyles({ 79 | marginTop: 16, 80 | textAlign: 'center', 81 | cursor: 'pointer', 82 | whiteSpace: 'nowrap', 83 | display: 'flex', 84 | flexFlow: 'column nowrap', 85 | width: 'auto', 86 | height: 'auto', 87 | textOverflow: 'ellipsis', 88 | }); 89 | 90 | const videoPlayerStyles = mergeStyles({ 91 | backgroundColor: '#000', 92 | width: 800, 93 | height: 500, 94 | }); 95 | 96 | export default { 97 | sectionStyles, 98 | sampleStyles, 99 | sampleHide, 100 | sampleCardStyle, 101 | cardContainer, 102 | addIconStyles, 103 | sampleImgStyles, 104 | cardTitleStyles, 105 | videoPlayerStyles, 106 | }; 107 | -------------------------------------------------------------------------------- /myapp/vision/src/components/ModuleTree/components/ModuleDetail/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | 3 | const calloutContent = mergeStyles({ 4 | display: 'flex', 5 | flexFlow: 'column nowrap', 6 | width: '360px', 7 | height: 'auto', 8 | boxSizing: 'border-box', 9 | maxHeight: 'inherit', 10 | padding: '16px 0px', 11 | fontSize: '12px', 12 | background: '#fff', 13 | borderRadius: '2px', 14 | }); 15 | 16 | const moduleDetailItemStyle = mergeStyles({ 17 | fontSize: '12px', 18 | padding: '0 16px', 19 | overflowX: 'visible', 20 | overflowY: 'hidden', 21 | }); 22 | 23 | const moduleDetailTitle = mergeStyles({ 24 | padding: '0px 16px', 25 | fontWeight: 600, 26 | fontSize: '14px', 27 | lineHeight: '1', 28 | marginTop: '0px', 29 | marginBottom: '8px', 30 | }); 31 | 32 | const moduleDetailLabel = mergeStyles({ 33 | fontSize: '12px', 34 | fontWeight: 'bold', 35 | lineHeight: '16px', 36 | color: 'rgb(89, 89, 89)', 37 | padding: '0px', 38 | margin: '0px 0px 4px', 39 | }); 40 | const moduleDetailBody = mergeStyles({ 41 | lineHeight: '14px', 42 | marginBottom: '22px', 43 | p: { 44 | margin: '0', 45 | padding: '0', 46 | }, 47 | }); 48 | const moduleButton = mergeStyles({ 49 | display: 'flex', 50 | justifyContent: 'center', 51 | alignItems: 'center', 52 | }); 53 | 54 | export default { 55 | calloutContent, 56 | moduleDetailItemStyle, 57 | moduleDetailTitle, 58 | moduleDetailLabel, 59 | moduleDetailBody, 60 | moduleButton, 61 | }; 62 | -------------------------------------------------------------------------------- /myapp/vision/src/components/ModuleTree/components/ModuleItem/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect, DragEvent } from 'react'; 2 | import { Icon } from '@fluentui/react'; 3 | import { useAppDispatch } from '@src/models/hooks'; 4 | import { updateCallout, updateCurrent, updateInfo } from '@src/models/template'; 5 | import style from './style'; 6 | 7 | interface ModelProps { 8 | model: any; 9 | } 10 | 11 | const ModuleItem: React.FC = props => { 12 | const dispatch = useAppDispatch(); 13 | const [modelInfo, setModelInfo] = useState(props.model); 14 | 15 | // 鼠标滑动事件 16 | const handleMouseEvent = (e: any) => { 17 | if (e.type === 'mouseenter') { 18 | dispatch(updateCurrent(e.target)); 19 | } 20 | dispatch(updateCallout(e.type !== 'mouseenter')); 21 | dispatch(updateInfo(modelInfo)); 22 | }; 23 | 24 | // 鼠标点击事件 25 | const handleMouseDown = () => { 26 | dispatch(updateCallout(true)); 27 | }; 28 | 29 | // 拖拽开始事件 30 | const onDragStart = (event: DragEvent) => { 31 | event.dataTransfer.setData('application/reactflow', JSON.stringify(modelInfo)); 32 | event.dataTransfer.effectAllowed = 'move'; 33 | }; 34 | 35 | useEffect(() => { 36 | setModelInfo(props.model); 37 | }, [props.model]); 38 | 39 | return ( 40 |
    41 |
    42 |
    43 |
    51 |
    52 | 61 |
    {modelInfo.name}
    62 |
    63 |
    64 | 65 | {modelInfo.describe} 66 | 67 |
    68 |
    69 |
    70 |
    71 |
    72 | ); 73 | }; 74 | 75 | export default ModuleItem; 76 | -------------------------------------------------------------------------------- /myapp/vision/src/components/ModuleTree/components/ModuleItem/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | 3 | const moduleItem = mergeStyles({ 4 | width: '100%', 5 | userSelect: 'none', 6 | cursor: 'pointer', 7 | }); 8 | 9 | const moduleListModuleCard = mergeStyles({ 10 | margin: '8px 16px', 11 | border: '1px solid rgb(234, 234, 234)', 12 | background: 'rgb(248, 248, 248)', 13 | color: 'rgb(55, 55, 55)', 14 | borderRadius: '2px', 15 | padding: '8px', 16 | display: 'flex', 17 | fontSize: '12px', 18 | flexDirection: 'column', 19 | '&:hover': { 20 | backgroundColor: '#eee', 21 | }, 22 | '> div': { 23 | pointerEvents: 'none', 24 | display: 'flex', 25 | flexDirection: 'row', 26 | flexGrow: 1, 27 | alignItems: 'center', 28 | '> header': { 29 | flexGrow: 1, 30 | color: 'rgb(0, 0, 0)', 31 | fontWeight: 600, 32 | fontSize: '12px', 33 | textOverflow: 'ellipsis', 34 | overflow: 'hidden', 35 | whiteSpace: 'nowrap', 36 | lineHeight: '16px', 37 | }, 38 | '> summary': { 39 | flexGrow: 1, 40 | color: 'rgb(55, 55, 55)', 41 | fontSize: '12px', 42 | }, 43 | }, 44 | }); 45 | 46 | export default { 47 | moduleItem, 48 | moduleListModuleCard, 49 | }; 50 | -------------------------------------------------------------------------------- /myapp/vision/src/components/ModuleTree/components/SearchItem/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { Icon } from '@fluentui/react'; 3 | import style from './style'; 4 | 5 | interface ModelProps { 6 | model: any; 7 | onClick: () => void; 8 | } 9 | 10 | const SearchItem: React.FC = props => { 11 | const [modelInfo, setModelInfo] = useState(props.model); 12 | 13 | useEffect(() => { 14 | setModelInfo(props.model); 15 | }, [props.model]); 16 | 17 | return ( 18 |
    19 |
    20 |
    21 |
    22 |
    23 | 32 |
    {modelInfo.name}
    33 |
    34 |
    35 | 36 | {modelInfo.describe} 37 | 38 |
    39 |
    40 |
    41 |
    42 |
    43 | ); 44 | }; 45 | 46 | export default SearchItem; 47 | -------------------------------------------------------------------------------- /myapp/vision/src/components/ModuleTree/components/SearchItem/style.ts: -------------------------------------------------------------------------------- 1 | import { mergeStyles } from '@fluentui/react'; 2 | 3 | const moduleItem = mergeStyles({ 4 | width: '100%', 5 | userSelect: 'none', 6 | cursor: 'pointer', 7 | }); 8 | 9 | const moduleListModuleCard = mergeStyles({ 10 | margin: '4px 8px', 11 | border: '1px solid rgb(234, 234, 234)', 12 | background: 'rgb(248, 248, 248)', 13 | color: 'rgb(55, 55, 55)', 14 | borderRadius: '2px', 15 | padding: '5px 8px', 16 | display: 'flex', 17 | fontSize: '12px', 18 | flexDirection: 'column', 19 | '&:hover': { 20 | backgroundColor: '#eee', 21 | }, 22 | '> div': { 23 | pointerEvents: 'none', 24 | display: 'flex', 25 | flexDirection: 'row', 26 | flexGrow: 1, 27 | alignItems: 'center', 28 | '> header': { 29 | flexGrow: 1, 30 | color: 'rgb(0, 0, 0)', 31 | fontWeight: 600, 32 | fontSize: '12px', 33 | textOverflow: 'ellipsis', 34 | overflow: 'hidden', 35 | whiteSpace: 'nowrap', 36 | lineHeight: '16px', 37 | }, 38 | '> summary': { 39 | flexGrow: 1, 40 | color: 'rgb(55, 55, 55)', 41 | fontSize: '12px', 42 | }, 43 | }, 44 | }); 45 | 46 | export default { 47 | moduleItem, 48 | moduleListModuleCard, 49 | }; 50 | -------------------------------------------------------------------------------- /myapp/vision/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import { HashRouter as Router } from 'react-router-dom'; 4 | import { mergeStyles, initializeIcons } from '@fluentui/react'; 5 | import AppRouter from './routes'; 6 | import { store } from './models/store'; 7 | import { Provider } from 'react-redux'; 8 | 9 | // fluentui icon 资源初始化 10 | initializeIcons(); 11 | 12 | // 全局样式 13 | mergeStyles({ 14 | ':global(body,html,#app)': { 15 | margin: 0, 16 | padding: 0, 17 | height: '100vh', 18 | }, 19 | ':global(.react-flow__edge-path)': { 20 | strokeWidth: '2 !important', 21 | }, 22 | }); 23 | 24 | ReactDOM.render( 25 | 26 | 27 | 28 | 29 | 30 | 31 | , 32 | document.getElementById('app'), 33 | ); 34 | -------------------------------------------------------------------------------- /myapp/vision/src/models/app/index.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import cookie from 'cookie'; 3 | import { RootState } from '../store'; 4 | 5 | const { myapp_username, t_uid, km_uid } = cookie.parse(document.cookie); 6 | 7 | export interface AppState { 8 | errMsg: any; 9 | userName: string; 10 | } 11 | 12 | const initialState: AppState = { 13 | errMsg: null, 14 | userName: myapp_username || t_uid || km_uid, 15 | }; 16 | 17 | const AppSlice = createSlice({ 18 | name: 'element', 19 | initialState, 20 | reducers: { 21 | updateErrMsg: (state, action: PayloadAction) => { 22 | state.errMsg = action.payload; 23 | }, 24 | }, 25 | }); 26 | 27 | export const { updateErrMsg } = AppSlice.actions; 28 | 29 | export const selectErrMsg = (state: RootState): AppState['errMsg'] => state.app.errMsg; 30 | export const selectUserName = (state: RootState): AppState['userName'] => state.app.userName; 31 | 32 | export default AppSlice.reducer; 33 | -------------------------------------------------------------------------------- /myapp/vision/src/models/editor/index.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import { RootState } from '../store'; 3 | 4 | export interface EditorState { 5 | showEditor: boolean; 6 | key: string; 7 | value: string; 8 | } 9 | 10 | const initialState: EditorState = { 11 | showEditor: false, 12 | key: '', 13 | value: '', 14 | }; 15 | 16 | const EditorSlice = createSlice({ 17 | name: 'element', 18 | initialState, 19 | reducers: { 20 | updateShowEditor: (state, action: PayloadAction) => { 21 | state.showEditor = action.payload; 22 | }, 23 | updateKeyValue: (state, action: PayloadAction<{ key: string; value: string }>) => { 24 | state.key = action.payload.key; 25 | state.value = action.payload.value; 26 | }, 27 | updateValue: (state, action: PayloadAction) => { 28 | state.value = action.payload; 29 | }, 30 | }, 31 | }); 32 | 33 | export const { updateShowEditor, updateValue, updateKeyValue } = EditorSlice.actions; 34 | 35 | export const selectShowEditor = (state: RootState): EditorState['showEditor'] => state.editor.showEditor; 36 | export const selectKey = (state: RootState): EditorState['key'] => state.editor.key; 37 | export const selectValue = (state: RootState): EditorState['value'] => state.editor.value; 38 | 39 | export default EditorSlice.reducer; 40 | -------------------------------------------------------------------------------- /myapp/vision/src/models/element/index.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import { Elements } from 'react-flow-renderer'; 3 | import { RootState } from '../store'; 4 | 5 | export interface ElementState { 6 | elements: Elements; 7 | selected: Elements; 8 | } 9 | 10 | const initialState: ElementState = { 11 | elements: [], 12 | selected: [], 13 | }; 14 | 15 | const elementSlice = createSlice({ 16 | name: 'element', 17 | initialState, 18 | reducers: { 19 | updateElements: (state, action: PayloadAction) => { 20 | state.elements = action.payload; 21 | }, 22 | updateSelected: (state, action: PayloadAction) => { 23 | state.selected = action.payload; 24 | }, 25 | }, 26 | }); 27 | 28 | export const { updateElements, updateSelected } = elementSlice.actions; 29 | 30 | export const selectElements = (state: RootState): ElementState['elements'] => state.element.elements; 31 | export const selectSelected = (state: RootState): ElementState['selected'] => state.element.selected; 32 | 33 | export default elementSlice.reducer; 34 | -------------------------------------------------------------------------------- /myapp/vision/src/models/hooks.ts: -------------------------------------------------------------------------------- 1 | import { TypedUseSelectorHook, useDispatch, useSelector } from 'react-redux'; 2 | import type { RootState, AppDispatch } from './store'; 3 | 4 | export const useAppDispatch = (): AppDispatch => useDispatch(); 5 | export const useAppSelector: TypedUseSelectorHook = useSelector; 6 | -------------------------------------------------------------------------------- /myapp/vision/src/models/setting/index.ts: -------------------------------------------------------------------------------- 1 | import { createSlice } from '@reduxjs/toolkit'; 2 | import { RootState } from '../store'; 3 | 4 | export interface SettingState { 5 | show: boolean; 6 | } 7 | 8 | const initialState: SettingState = { 9 | show: false, 10 | }; 11 | 12 | const settingSlice = createSlice({ 13 | name: 'setting', 14 | initialState, 15 | reducers: { 16 | toggle: state => { 17 | state.show = !state.show; 18 | }, 19 | }, 20 | }); 21 | 22 | export const { toggle } = settingSlice.actions; 23 | 24 | export const selectShow = (state: RootState): SettingState['show'] => state.setting.show; 25 | 26 | export default settingSlice.reducer; 27 | -------------------------------------------------------------------------------- /myapp/vision/src/models/store.ts: -------------------------------------------------------------------------------- 1 | import { configureStore, ThunkAction, Action } from '@reduxjs/toolkit'; 2 | import { enableMapSet } from 'immer'; 3 | import appReducer from './app'; 4 | import editorReducer from './editor'; 5 | import elementReducer from './element'; 6 | import pipelineReducer from './pipeline'; 7 | import settingReducer from './setting'; 8 | import taskReducer from './task'; 9 | import templateReduce from './template'; 10 | 11 | enableMapSet(); 12 | 13 | export const store = configureStore({ 14 | reducer: { 15 | app: appReducer, 16 | editor: editorReducer, 17 | element: elementReducer, 18 | pipeline: pipelineReducer, 19 | setting: settingReducer, 20 | task: taskReducer, 21 | template: templateReduce, 22 | }, 23 | middleware: getDefaultMiddleware => 24 | getDefaultMiddleware({ 25 | serializableCheck: false, 26 | }), 27 | }); 28 | 29 | export type AppDispatch = typeof store.dispatch; 30 | export type RootState = ReturnType; 31 | export type AppThunk = ThunkAction>; 32 | -------------------------------------------------------------------------------- /myapp/vision/src/models/task/index.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import api from '@src/api'; 3 | import { RootState, AppThunk } from '../store'; 4 | import { selectPipelineId } from '../pipeline'; 5 | 6 | export interface TaskState { 7 | taskList: any; 8 | loading: boolean; 9 | taskId: number; 10 | } 11 | interface TaskList { 12 | id: string | number; 13 | changed: any; 14 | } 15 | 16 | const initialState: TaskState = { 17 | taskList: null, 18 | loading: false, 19 | taskId: 0, 20 | }; 21 | 22 | const taskSlice = createSlice({ 23 | name: 'task', 24 | initialState, 25 | reducers: { 26 | updateTaskList: (state, action: PayloadAction) => { 27 | if (!state.taskList) { 28 | const cur = new Map(); 29 | cur.set(action.payload.id, action.payload.changed); 30 | state.taskList = cur; 31 | } else { 32 | state.taskList = state.taskList.set(action.payload.id, action.payload.changed); 33 | } 34 | }, 35 | updateLoading: (state, action: PayloadAction) => { 36 | state.loading = action.payload; 37 | }, 38 | resetTaskList: state => { 39 | state.taskList = new Map(); 40 | }, 41 | updateTaskId: (state, action: PayloadAction) => { 42 | state.taskId = action.payload; 43 | }, 44 | }, 45 | }); 46 | 47 | export const { updateTaskList, updateLoading, resetTaskList, updateTaskId } = taskSlice.actions; 48 | 49 | export const selectTaskList = (state: RootState): TaskState['taskList'] => state.task.taskList; 50 | export const selectLoading = (state: RootState): TaskState['loading'] => state.task.loading; 51 | export const selectTaskId = (state: RootState): TaskState['taskId'] => state.task.taskId; 52 | 53 | // 依次提交保存改动的 task 配置 54 | export const saveTaskList = async (): Promise => async (dispatch, getState) => { 55 | dispatch(updateLoading(true)); 56 | const state = getState(); 57 | const taskList = selectTaskList(state); 58 | const queue: any = []; 59 | // taskList 为 Map 结构 60 | taskList && 61 | taskList.forEach((value: any, key: any) => { 62 | if (Object.keys(value).length > 0) { 63 | queue.push(api.task_modelview_edit(selectPipelineId(state), key, value)); 64 | } 65 | }); 66 | const result = await Promise.allSettled(queue); 67 | dispatch(updateLoading(false)); 68 | dispatch(resetTaskList()); // 清空已更新的 changed 69 | return result; 70 | }; 71 | 72 | export default taskSlice.reducer; 73 | -------------------------------------------------------------------------------- /myapp/vision/src/models/template/index.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import { RootState } from '../store'; 3 | 4 | export interface TemplateState { 5 | show: boolean; 6 | callout: boolean; 7 | current: any; 8 | info: CurrentInfo; 9 | } 10 | export interface CurrentInfo { 11 | createdBy: string; 12 | describe: string; 13 | id: number; 14 | imagesName: string; 15 | lastChanged: string; 16 | name: string; 17 | version: string; 18 | [key: string]: any; 19 | } 20 | 21 | const initialState: TemplateState = { 22 | show: true, 23 | callout: true, 24 | current: null, 25 | info: { 26 | createdBy: '', 27 | describe: '', 28 | id: 69, 29 | imagesName: '', 30 | lastChanged: '', 31 | name: '', 32 | version: '', 33 | }, 34 | }; 35 | 36 | const templateSlice = createSlice({ 37 | name: 'template', 38 | initialState, 39 | reducers: { 40 | toggle: state => { 41 | state.show = !state.show; 42 | }, 43 | updateCallout: (state, action: PayloadAction) => { 44 | state.callout = action.payload; 45 | }, 46 | updateCurrent: (state, action: PayloadAction) => { 47 | state.current = action.payload; 48 | }, 49 | updateInfo: (state, action: PayloadAction) => { 50 | state.info = action.payload; 51 | }, 52 | }, 53 | }); 54 | 55 | export const { toggle, updateCallout, updateCurrent, updateInfo } = templateSlice.actions; 56 | 57 | export const selectShow = (state: RootState): TemplateState['show'] => state.template.show; 58 | export const selectCallout = (state: RootState): TemplateState['callout'] => state.template.callout; 59 | export const selectCurrent = (state: RootState): TemplateState['current'] => state.template.current; 60 | export const selectInfo = (state: RootState): TemplateState['info'] => state.template.info; 61 | 62 | export default templateSlice.reducer; 63 | -------------------------------------------------------------------------------- /myapp/vision/src/pages/Index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Stack, IStackStyles } from '@fluentui/react'; 3 | import EditorAce from '@src/components/EditorAce'; 4 | import FlowEditor from '@src/components/FlowEditor'; 5 | import ModuleTree from '@src/components/ModuleTree'; 6 | 7 | // app 页面初始化样式 8 | const appContainerStyle: IStackStyles = { 9 | root: { 10 | width: '100%', 11 | height: '100%', 12 | overflow: 'hidden', 13 | }, 14 | }; 15 | 16 | const App: React.FC = () => { 17 | return ( 18 | 19 | {/* 任务模板库 */} 20 | 21 | {/* 流水线编辑 */} 22 | 23 | {/* JSON 编辑器 */} 24 | 25 | 26 | ); 27 | }; 28 | 29 | export default App; 30 | -------------------------------------------------------------------------------- /myapp/vision/src/public/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /myapp/vision/src/public/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /myapp/vision/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /myapp/vision/src/routes/config.ts: -------------------------------------------------------------------------------- 1 | import React, { lazy } from 'react'; 2 | 3 | export interface IRoute { 4 | name: string; 5 | path: string; 6 | component: React.LazyExoticComponent>; 7 | } 8 | 9 | const config: Array = [ 10 | { 11 | name: 'index', 12 | path: '/', 13 | component: lazy(() => import('../pages/Index')), 14 | }, 15 | { 16 | name: 'home', 17 | path: '/home', 18 | component: lazy(() => import('../pages/Home')), 19 | }, 20 | ]; 21 | 22 | export default config; 23 | -------------------------------------------------------------------------------- /myapp/vision/src/routes/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { Suspense } from 'react'; 2 | import { Route, Switch } from 'react-router-dom'; 3 | import config, { IRoute } from './config'; 4 | 5 | const AppRouter: React.FC = () => { 6 | return ( 7 | 8 | {config.map((route: IRoute) => ( 9 | ( 14 | 15 | 16 | 17 | )} 18 | > 19 | ))} 20 | 21 | ); 22 | }; 23 | 24 | export default AppRouter; 25 | -------------------------------------------------------------------------------- /myapp/vision/src/setupProxy.js: -------------------------------------------------------------------------------- 1 | const { createProxyMiddleware } = require('http-proxy-middleware'); 2 | 3 | // https://create-react-app.dev/docs/proxying-api-requests-in-development/ 4 | module.exports = function (app) { 5 | app.use( 6 | ['**/api/**', '/myapp', '**/list/**'], 7 | createProxyMiddleware({ 8 | target: 'http://localhost', 9 | changeOrigin: true, 10 | }) 11 | ); 12 | }; -------------------------------------------------------------------------------- /myapp/vision/src/static/home.ts: -------------------------------------------------------------------------------- 1 | export const pipelineDemo = []; 2 | 3 | export const videoDemo = [ 4 | { 5 | name: '新人制作一个pipeline', 6 | img: '/static/assets/images/ad/video-cover2-thumb.png', 7 | url: 'https://docker-76009.sz.gfp.tencent-cloud.com/kubeflow/make_pipeline.mp4', 8 | type: 'video', 9 | }, 10 | { 11 | name: '开发定制一个任务模板', 12 | img: '/static/assets/images/ad/video-cover2-thumb.png', 13 | url: 'https://docker-76009.sz.gfp.tencent-cloud.com/kubeflow/make_job_template.mp4', 14 | type: 'video', 15 | }, 16 | ]; 17 | -------------------------------------------------------------------------------- /myapp/vision/src/types/pipeline/index.d.ts: -------------------------------------------------------------------------------- 1 | // 新建流水线参数类型 2 | export interface IPipelineAdd { 3 | describe: string; 4 | name: string; 5 | node_selector: string; 6 | schedule_type: string; 7 | image_pull_policy: string; 8 | parallelism: number; 9 | project: number; 10 | } 11 | 12 | // 流水线可编辑参数类型 13 | export interface IPipelineEdit { 14 | project?: string; 15 | name?: string; 16 | describe?: string; 17 | namespace?: string; 18 | schedule_type?: string; 19 | cron_time?: string; 20 | node_selector?: string; 21 | image_pull_policy?: string; 22 | parallelism?: number; 23 | dag_json?: string; 24 | global_env?: string; 25 | expand: string; 26 | } 27 | -------------------------------------------------------------------------------- /myapp/vision/src/types/task/index.d.ts: -------------------------------------------------------------------------------- 1 | // 新增 task 的参数类型 2 | export interface ITaskAdd { 3 | job_template: number; 4 | label: string; 5 | name: string; 6 | node_selector: string; 7 | pipeline: number; 8 | resource_cpu: string; 9 | resource_memory: string; 10 | [propname: string]: any; 11 | } 12 | 13 | // task 可编辑参数类型 14 | export interface ITaskEdit { 15 | args?: string; 16 | command?: string; 17 | label?: string; 18 | name?: string; 19 | node_selector?: string; 20 | resource_cpu?: string; 21 | resource_gpu?: string; 22 | resource_memory?: string; 23 | retry?: string; 24 | timeout?: string; 25 | volume_mount?: string; 26 | working_dir?: string; 27 | } 28 | -------------------------------------------------------------------------------- /myapp/vision/src/utils/getParamter.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @func 3 | * @desc 一个带参数的函数 4 | * @param {string} name - 要查询的参数 5 | * @param {string} url - 被查询的url 6 | * @param {booleam} bool - 默认为true 7 | * @return 返回值 8 | */ 9 | function getDetailed(name: string, str: string) { 10 | const obj: any = {}; 11 | 12 | if (name !== '') { 13 | const reg = new RegExp('(^|&)' + name + '=([^&]*)(&|$)', 'i'); 14 | const r = str.match(reg); 15 | 16 | if (r !== null) { 17 | obj[name] = r[2]; 18 | } else { 19 | obj[name] = ''; 20 | } 21 | } else { 22 | // 查询所有的参数 23 | const arr = str.split('&'); 24 | 25 | arr.forEach(element => { 26 | const temp = element.split('='); 27 | 28 | obj[temp[0]] = temp[1]; 29 | }); 30 | } 31 | return obj; 32 | } 33 | const getParamter = function (name: string, url: string, bool = false): any { 34 | url = decodeURIComponent(url || window.location.href); 35 | const answer: any = {}; 36 | const indexBefore = url.indexOf('?'); // search字段的'?'位置 37 | let end = url.indexOf('#'); 38 | 39 | if (end === -1) { 40 | // 判断url中是否有hash 41 | end = url.length; 42 | } 43 | 44 | // 可以取hash字段 45 | if (end !== url.length && bool) { 46 | let strAfter = url.substring(end + 1); // hash字段 47 | const indexAfter = strAfter.indexOf('?'); // hash字段的'?'位置 48 | 49 | if (indexAfter !== -1) { 50 | strAfter = strAfter.substring(indexAfter + 1); 51 | Object.assign(answer, getDetailed(name, strAfter)); 52 | } 53 | } 54 | // 可以取search字段 55 | if (indexBefore !== -1) { 56 | const strBefore = url.substring(indexBefore + 1, end); // search字段 57 | 58 | Object.assign(answer, getDetailed(name, strBefore)); 59 | } 60 | return answer; 61 | }; 62 | 63 | export default getParamter; 64 | -------------------------------------------------------------------------------- /myapp/vision/src/utils/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @description 函数防抖 3 | * @param fn 函数 4 | * @param delay 间隔时间 5 | * @returns {Function} 6 | */ 7 | export function debounce void>(fn: F, delay: number): F { 8 | let timer: any = null; 9 | return function (this: any, ...args: any[]) { 10 | clearTimeout(timer); 11 | timer = window.setTimeout(() => fn.apply(this, args), delay); 12 | } as F; 13 | } 14 | 15 | /** 16 | * @description 校验是否为 JSON 字符串 17 | * @param str 字符串 18 | * @returns {Boolean} 19 | */ 20 | export function isJsonString(str: string): boolean { 21 | try { 22 | JSON.parse(str); 23 | return true; 24 | } catch (error) { 25 | return false; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /myapp/vision/src/utils/storage.ts: -------------------------------------------------------------------------------- 1 | const storage = { 2 | get: (name: string): any => { 3 | const result = window.localStorage.getItem(name); 4 | 5 | if (result && result.match(/^(\{|\[).*(?=[}\]])/)) { 6 | return JSON.parse(result); 7 | } 8 | return result || ''; 9 | }, 10 | set: (name: string, value: unknown): void => { 11 | window.localStorage.setItem(name, typeof value === 'string' ? value : JSON.stringify(value)); 12 | }, 13 | }; 14 | 15 | export default storage; 16 | -------------------------------------------------------------------------------- /myapp/vision/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./base-tsconfig.json", 3 | "compilerOptions": {} 4 | } 5 | --------------------------------------------------------------------------------