├── .commitlintrc ├── .dockerignore ├── .github └── workflows │ ├── app-cd.yaml │ ├── sync-dev.yml │ └── tool-ci.yaml ├── .gitignore ├── .husky ├── commit-msg └── pre-commit ├── .npmrc ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── build_doc.sh ├── codecov.yml ├── dds.py ├── ddsop.py ├── deepdataspace ├── __init__.py ├── algos │ ├── __init__.py │ ├── calculate_fnfp.py │ ├── graph.py │ └── refine_by_seed.py ├── constants.py ├── environs.py ├── globals.py ├── io │ ├── __init__.py │ └── importer.py ├── model │ ├── __init__.py │ ├── _base.py │ ├── category.py │ ├── dataset.py │ ├── image.py │ ├── label.py │ ├── label_task.py │ ├── object.py │ └── user.py ├── plugins │ ├── __init__.py │ ├── coco2017 │ │ ├── __init__.py │ │ └── importer.py │ └── tsv │ │ ├── __init__.py │ │ ├── importer.py │ │ ├── process.py │ │ └── server.py ├── process │ ├── __init__.py │ ├── calculate_fnfp.py │ └── processor.py ├── samples │ └── coco_dataset_meta.py ├── scripts │ ├── __init__.py │ ├── dataset_cmds.py │ ├── enter_shell.py │ ├── label_project_cmds.py │ ├── migrate │ │ ├── 2023053101.py │ │ ├── 2023061401.py │ │ └── __init__.py │ ├── start.py │ └── user_cmds.py ├── server │ ├── __init__.py │ ├── apps.py │ ├── manage.py │ ├── middlewares │ │ ├── __init__.py │ │ └── request_perf.py │ ├── resources │ │ ├── __init__.py │ │ ├── api_v1 │ │ │ ├── __init__.py │ │ │ ├── comparisons.py │ │ │ ├── datasets.py │ │ │ ├── image_flags.py │ │ │ ├── images.py │ │ │ ├── label_clone.py │ │ │ ├── label_tasks.py │ │ │ ├── lints.py │ │ │ ├── login.py │ │ │ └── ping.py │ │ └── files │ │ │ ├── __init__.py │ │ │ ├── dataset_flags.py │ │ │ └── local_file.py │ ├── settings.py │ ├── static │ │ ├── 124.69a3d116.async.js │ │ ├── 20.85fcbb81.async.js │ │ ├── 222.33ccd216.async.js │ │ ├── 233.3d91ac9a.async.js │ │ ├── 422.f952182a.async.js │ │ ├── 7.e4c5d564.async.js │ │ ├── 742.bbe64d32.async.js │ │ ├── 845.7d656ce9.async.js │ │ ├── 851.1072d61f.async.js │ │ ├── 9.9f27f1a0.async.js │ │ ├── favicon.png │ │ ├── index.html │ │ ├── layouts__index.cf0de806.chunk.css │ │ ├── layouts__index.f60cee94.async.js │ │ ├── p__404.0f1cb5ec.async.js │ │ ├── p__Annotator__index.8750e2fd.async.js │ │ ├── p__DatasetList__index.91c4bfd3.async.js │ │ ├── p__DatasetList__index.cbd96143.chunk.css │ │ ├── p__Dataset__index.419888be.async.js │ │ ├── p__Dataset__index.c6647b2a.chunk.css │ │ ├── p__Lab__Datasets__index.6cbc99df.async.js │ │ ├── p__Lab__Datasets__index.e9f4c3ea.chunk.css │ │ ├── p__Lab__FlagTool__index.c0569e06.chunk.css │ │ ├── p__Lab__FlagTool__index.c0cac9b9.async.js │ │ ├── p__Lab__index.28ea9971.chunk.css │ │ ├── p__Lab__index.5de6b129.async.js │ │ ├── p__Login__index.26727079.chunk.css │ │ ├── p__Login__index.572dbaae.async.js │ │ ├── p__Project__Detail__index.9b99b574.chunk.css │ │ ├── p__Project__Detail__index.f6bb0552.async.js │ │ ├── p__Project__Workspace__index.2c2baf46.async.js │ │ ├── p__Project__Workspace__index.983f52a6.chunk.css │ │ ├── p__Project__index.711b5411.chunk.css │ │ ├── p__Project__index.88bd2c95.async.js │ │ ├── static │ │ │ ├── annotate_coop.44ac433d.png │ │ │ ├── annotate_quick.b8733d56.png │ │ │ ├── card_cover_4.b81bbca8.png │ │ │ ├── flagtool.a991e5bd.png │ │ │ └── home_banner.0fcae71f.png │ │ ├── t__plugin-layout__Layout.2ad2010d.async.js │ │ ├── t__plugin-layout__Layout.8f39539f.chunk.css │ │ ├── umi.c30875f2.js │ │ ├── umi.e762716e.css │ │ └── wrappers__auth.d0b0c30b.async.js │ ├── templates │ │ └── index.html │ ├── urls.py │ ├── utils.py │ ├── views │ │ ├── __init__.py │ │ └── index.py │ └── wsgi.py ├── services │ ├── __init__.py │ ├── celery.py │ ├── config.py │ ├── dds.config.sample.yaml │ ├── dds.py │ ├── django.py │ ├── mongodb.py │ ├── redis.py │ ├── service.py │ └── sqlite.py ├── task │ ├── __init__.py │ ├── celery.py │ ├── import_dataset.py │ ├── ping.py │ ├── process_dataset.py │ └── settings.py └── utils │ ├── __init__.py │ ├── classes.py │ ├── file.py │ ├── function.py │ ├── http.py │ ├── import_utils.py │ ├── network.py │ ├── os.py │ └── string.py ├── docker-compose.yaml ├── docker ├── init-dds.py └── init-mongo.js ├── package.json ├── packages ├── app │ ├── .env │ ├── .eslintrc.js │ ├── .gitignore │ ├── .lintstagedrc │ ├── .npmrc │ ├── .prettierignore │ ├── .prettierrc │ ├── .stylelintrc.js │ ├── .umirc.ts │ ├── README.md │ ├── jest.config.ts │ ├── mock │ │ ├── dataset.ts │ │ ├── project.ts │ │ ├── task.ts │ │ └── temp.ts │ ├── package.json │ ├── src │ │ ├── app.tsx │ │ ├── assets │ │ │ ├── .gitkeep │ │ │ ├── images │ │ │ │ ├── cards │ │ │ │ │ ├── annotate_coop.png │ │ │ │ │ ├── annotate_quick.png │ │ │ │ │ ├── card_cover_0.png │ │ │ │ │ ├── card_cover_1.png │ │ │ │ │ ├── card_cover_2.png │ │ │ │ │ ├── card_cover_3.png │ │ │ │ │ ├── card_cover_4.png │ │ │ │ │ ├── card_cover_5.png │ │ │ │ │ └── flagtool.png │ │ │ │ ├── favicon.png │ │ │ │ ├── home_banner.png │ │ │ │ ├── logo_title.png │ │ │ │ ├── old_favicon.png │ │ │ │ └── old_logo_title.png │ │ │ └── svg │ │ │ │ ├── auto-awesome.svg │ │ │ │ ├── auto-fix.svg │ │ │ │ ├── classification.svg │ │ │ │ ├── dash-box.svg │ │ │ │ ├── datasetClassify.svg │ │ │ │ ├── datasetDetection.svg │ │ │ │ ├── datasetKeypoint.svg │ │ │ │ ├── datasetMatting.svg │ │ │ │ ├── datasetSegment.svg │ │ │ │ ├── delete.svg │ │ │ │ ├── download.svg │ │ │ │ ├── edit.svg │ │ │ │ ├── email-send.svg │ │ │ │ ├── flag.svg │ │ │ │ ├── hand.svg │ │ │ │ ├── keypoints.svg │ │ │ │ ├── maxSize.svg │ │ │ │ ├── minSize.svg │ │ │ │ ├── minus.svg │ │ │ │ ├── multVisible.svg │ │ │ │ ├── plus.svg │ │ │ │ ├── random.svg │ │ │ │ ├── rect.svg │ │ │ │ ├── settings.svg │ │ │ │ ├── square.svg │ │ │ │ └── visible.svg │ │ ├── components │ │ │ ├── CategoryFilter │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── DatasetItem │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── DisplayOptions │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── DropdownSelector │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── ImportImgsForm │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── LabelOptions │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── LoginModal │ │ │ │ └── index.tsx │ │ │ └── UploadImageList │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ ├── constants │ │ │ └── index.ts │ │ ├── favicon.png │ │ ├── global.less │ │ ├── global.ts │ │ ├── layouts │ │ │ ├── index.less │ │ │ ├── index.tsx │ │ │ └── menu.tsx │ │ ├── locales │ │ │ ├── en-US.ts │ │ │ └── zh-CN.ts │ │ ├── models │ │ │ ├── dataset │ │ │ │ ├── common.tsx │ │ │ │ ├── comparisons.ts │ │ │ │ ├── filters.ts │ │ │ │ ├── flag.ts │ │ │ │ └── type.ts │ │ │ ├── datasets.ts │ │ │ ├── global.ts │ │ │ └── user.ts │ │ ├── pages │ │ │ ├── 404.tsx │ │ │ ├── Annotator │ │ │ │ ├── components │ │ │ │ │ ├── FormModal │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ └── index.tsx │ │ │ │ │ └── ImageList │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ └── index.tsx │ │ │ │ ├── constants.ts │ │ │ │ ├── index.less │ │ │ │ ├── index.tsx │ │ │ │ └── model.ts │ │ │ ├── Dataset │ │ │ │ ├── components │ │ │ │ │ ├── ComparisonsBar │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── DiffLabelsTip │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ └── index.tsx │ │ │ │ │ └── Header │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ └── index.tsx │ │ │ │ ├── index.less │ │ │ │ ├── index.tsx │ │ │ │ └── model.ts │ │ │ ├── DatasetList │ │ │ │ ├── index.less │ │ │ │ ├── index.tsx │ │ │ │ └── model.ts │ │ │ ├── Lab │ │ │ │ ├── Datasets │ │ │ │ │ ├── index.less │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── model.ts │ │ │ │ ├── FlagTool │ │ │ │ │ ├── components │ │ │ │ │ │ ├── FlagToolsBar │ │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ │ └── index.tsx │ │ │ │ │ │ └── Header │ │ │ │ │ │ │ ├── index.less │ │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── index.less │ │ │ │ │ ├── index.tsx │ │ │ │ │ └── model.ts │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ ├── Login │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ └── Project │ │ │ │ ├── Detail │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ │ ├── Workspace │ │ │ │ ├── index.less │ │ │ │ └── index.tsx │ │ │ │ ├── components │ │ │ │ ├── ProgressBar │ │ │ │ │ └── index.tsx │ │ │ │ ├── ProjectEditModal │ │ │ │ │ └── index.tsx │ │ │ │ ├── ProjectExportModal │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ProjectInfoModal │ │ │ │ │ └── index.tsx │ │ │ │ ├── TableTags │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── TaskAssignModal │ │ │ │ │ └── index.tsx │ │ │ │ ├── TaskDetailModal │ │ │ │ │ └── index.tsx │ │ │ │ └── TaskProgress │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── constants.ts │ │ │ │ ├── index.less │ │ │ │ ├── index.tsx │ │ │ │ └── models │ │ │ │ ├── auth.ts │ │ │ │ ├── detail.ts │ │ │ │ ├── list.ts │ │ │ │ └── workspace.ts │ │ ├── routes.ts │ │ ├── services │ │ │ ├── common.ts │ │ │ ├── dataset.ts │ │ │ ├── errorCode.ts │ │ │ ├── project.ts │ │ │ └── user.ts │ │ ├── types │ │ │ ├── annotator.ts │ │ │ ├── api.ts │ │ │ ├── coco.ts │ │ │ ├── dataset.ts │ │ │ ├── index.ts │ │ │ └── project.ts │ │ ├── typings.d.ts │ │ ├── utils │ │ │ ├── adapter.ts │ │ │ └── datasets.ts │ │ └── wrappers │ │ │ └── auth.tsx │ ├── tests │ │ ├── components │ │ │ ├── DropdownSelector.test.tsx │ │ │ └── __snapshots__ │ │ │ │ └── DropdownSelector.test.tsx.snap │ │ ├── file-transform.js │ │ ├── hooks │ │ │ └── useWindowResize.test.ts │ │ ├── jest-setup.ts │ │ ├── layouts │ │ │ ├── __snapshots__ │ │ │ │ ├── layout.test.tsx.snap │ │ │ │ └── menu.test.tsx.snap │ │ │ ├── layout.test.tsx │ │ │ └── menu.test.tsx │ │ ├── models │ │ │ ├── dataset.test.tsx │ │ │ ├── datasets.test.tsx │ │ │ ├── global.test.tsx │ │ │ └── user.test.tsx │ │ ├── pages │ │ │ ├── 404.test.tsx │ │ │ └── DatasetList.test.tsx │ │ ├── test-utils.tsx │ │ └── utils │ │ │ ├── digit.test.ts │ │ │ ├── file.test.ts │ │ │ └── url.test.ts │ └── tsconfig.json ├── components │ ├── .dumirc.ts │ ├── .fatherrc.ts │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── docs │ │ ├── guide.md │ │ └── index.md │ ├── package.json │ ├── public │ │ └── favicon.png │ ├── src │ │ ├── Annotator │ │ │ ├── assets │ │ │ │ ├── add-prompt.svg │ │ │ │ ├── attribute.svg │ │ │ │ ├── brush-add.svg │ │ │ │ ├── brush-erase.svg │ │ │ │ ├── custom.svg │ │ │ │ ├── delete_all.svg │ │ │ │ ├── displayReset.svg │ │ │ │ ├── docs.svg │ │ │ │ ├── doubleRight.svg │ │ │ │ ├── downArror.svg │ │ │ │ ├── download.svg │ │ │ │ ├── drag.svg │ │ │ │ ├── edge-stitch.svg │ │ │ │ ├── house.svg │ │ │ │ ├── img-broken.svg │ │ │ │ ├── imgSetting.svg │ │ │ │ ├── keyboard-down.svg │ │ │ │ ├── keyboard.svg │ │ │ │ ├── label.svg │ │ │ │ ├── layer.svg │ │ │ │ ├── logo.svg │ │ │ │ ├── magic-box.svg │ │ │ │ ├── magic-brush.svg │ │ │ │ ├── magic-click.svg │ │ │ │ ├── magic.svg │ │ │ │ ├── mask-ai.svg │ │ │ │ ├── mask.svg │ │ │ │ ├── mouse-left.svg │ │ │ │ ├── mouse-right.svg │ │ │ │ ├── palette.svg │ │ │ │ ├── pen-add.svg │ │ │ │ ├── pen-erase.svg │ │ │ │ ├── play-next.svg │ │ │ │ ├── play-pre.svg │ │ │ │ ├── play-stop.svg │ │ │ │ ├── play.svg │ │ │ │ ├── point.svg │ │ │ │ ├── polygon-ai.svg │ │ │ │ ├── polygon.svg │ │ │ │ ├── rectangle-ai.svg │ │ │ │ ├── rectangle.svg │ │ │ │ ├── redo.svg │ │ │ │ ├── remove-prompt.svg │ │ │ │ ├── repeat.svg │ │ │ │ ├── review.svg │ │ │ │ ├── segment-everything.svg │ │ │ │ ├── settings-sliders.svg │ │ │ │ ├── skeleton-ai.svg │ │ │ │ ├── skeleton.svg │ │ │ │ ├── text-prompt.svg │ │ │ │ ├── undo.svg │ │ │ │ ├── visual-prompt.svg │ │ │ │ └── zoomResize.svg │ │ │ ├── components │ │ │ │ ├── AttributeEditor │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── AttributesForm │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── CategoryCreator │ │ │ │ │ └── index.tsx │ │ │ │ ├── Classification │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── DisplaySettings │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── EditorStatus │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── FloatWrapper │ │ │ │ │ └── index.tsx │ │ │ │ ├── HighlightText │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ImageView │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── LabelSelector │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ModelSelectModal │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ModelSelector │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ObjectList │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── PointItem │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── PointsEditModal │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── PopoverMenu │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── SegConfirmModal │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ShortcutsInfo │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── SliderToolBar │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── SmartAnnotationControl │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── SubToolBar │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── TopPagination │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ └── TopTools │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ ├── constants │ │ │ │ ├── index.ts │ │ │ │ ├── render.ts │ │ │ │ └── shortcuts.ts │ │ │ ├── editor.tsx │ │ │ ├── hooks │ │ │ │ ├── useActions.tsx │ │ │ │ ├── useAiModels.ts │ │ │ │ ├── useAttributes.ts │ │ │ │ ├── useCanvasContainer.tsx │ │ │ │ ├── useCanvasRender.tsx │ │ │ │ ├── useColor.ts │ │ │ │ ├── useDataEffect.ts │ │ │ │ ├── useHistory.ts │ │ │ │ ├── useLabels.ts │ │ │ │ ├── useMouseCursor.ts │ │ │ │ ├── useMouseEvents.tsx │ │ │ │ ├── useObjects.ts │ │ │ │ ├── usePreviousState.ts │ │ │ │ ├── useShortcuts.ts │ │ │ │ ├── useSubtools.tsx │ │ │ │ ├── useToolActions.ts │ │ │ │ ├── useTopTools.tsx │ │ │ │ └── useTranslate.ts │ │ │ ├── index.less │ │ │ ├── index.tsx │ │ │ ├── preview.tsx │ │ │ ├── sevices │ │ │ │ └── index.ts │ │ │ ├── tools │ │ │ │ ├── base.ts │ │ │ │ ├── useMask.ts │ │ │ │ ├── useMatting.ts │ │ │ │ ├── usePoint.ts │ │ │ │ ├── usePolygon.ts │ │ │ │ ├── usePolyline.ts │ │ │ │ ├── useRectangle.ts │ │ │ │ └── useSkeleton.ts │ │ │ ├── type.ts │ │ │ ├── utils │ │ │ │ ├── base64.ts │ │ │ │ ├── color.ts │ │ │ │ ├── compute.ts │ │ │ │ ├── draw.ts │ │ │ │ └── verify.ts │ │ │ └── view.tsx │ │ ├── ColumnSettings │ │ │ ├── assets │ │ │ │ ├── minus.svg │ │ │ │ ├── plus.svg │ │ │ │ └── settings.svg │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── DynamicPagination │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── GlobalLoading │ │ │ ├── demo.tsx │ │ │ ├── index.md │ │ │ └── index.tsx │ │ ├── LangSelector │ │ │ ├── demo.tsx │ │ │ ├── index.less │ │ │ ├── index.md │ │ │ └── index.tsx │ │ ├── MobileAlert │ │ │ └── index.tsx │ │ ├── NotFoundTip │ │ │ └── index.tsx │ │ ├── QuickLabel │ │ │ ├── components │ │ │ │ ├── ImageFilter │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ ├── ImageList │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ │ └── QuickstartModal │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ ├── hooks │ │ │ │ └── useQuickLabelModel.ts │ │ │ ├── index.less │ │ │ ├── index.tsx │ │ │ ├── type.ts │ │ │ └── utils │ │ │ │ ├── adapter.ts │ │ │ │ └── idConverter.ts │ │ ├── RunningErrorTip │ │ │ ├── demo.tsx │ │ │ ├── index.md │ │ │ └── index.tsx │ │ ├── Upload │ │ │ ├── assets │ │ │ │ ├── checked.svg │ │ │ │ └── upload.svg │ │ │ ├── components │ │ │ │ └── FilePreviewList │ │ │ │ │ ├── index.less │ │ │ │ │ └── index.tsx │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── UploadPreAnno │ │ │ ├── assets │ │ │ │ └── upload_file.svg │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── index.ts │ │ └── locales │ │ │ ├── en-US.ts │ │ │ └── zh-CN.ts │ └── tsconfig.json ├── hooks │ ├── package.json │ ├── src │ │ ├── index.ts │ │ ├── usePageModelLifeCycle.ts │ │ └── useWindowResize.ts │ └── tsconfig.json └── utils │ ├── package.json │ ├── src │ ├── digit.ts │ ├── file.ts │ ├── index.ts │ ├── locale.ts │ └── url.ts │ └── tsconfig.json ├── pnpm-lock.yaml ├── pnpm-workspace.yaml ├── requirements-dev.txt ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── benchmark │ └── __init__.py ├── conftest.py ├── dds.test.yaml ├── integration │ ├── __init__.py │ └── test_label_project.py ├── resource │ └── image │ │ └── 1.jpg ├── seed.py └── unittest │ └── __init__.py ├── tsconfig.json └── typings.d.ts /.commitlintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["@commitlint/config-conventional"], 3 | "rules": { 4 | "type-enum": [2, "always", [ 5 | "build", "chore", "ci", "docs", "feat", "feature", "fix", "bugfix", "hotfix", 6 | "perf", "refactor", "revert", "style", "test", "update", "release", "delete" 7 | ]] 8 | } 9 | } -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # for git 2 | .git 3 | .github 4 | .gitignore 5 | 6 | # for dev 7 | .idea 8 | .husky 9 | .vscode 10 | .commitlintrc 11 | 12 | # for local cache 13 | op 14 | data 15 | dist 16 | docs 17 | node_modules 18 | deepdataspace.egg-info 19 | -------------------------------------------------------------------------------- /.github/workflows/sync-dev.yml: -------------------------------------------------------------------------------- 1 | name: Sync Dev to Private Repo 2 | on: 3 | push: 4 | branches: 5 | - dev 6 | jobs: 7 | merge-and-rebase: 8 | if: github.repository_owner == 'IDEA-Research' 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout Private Repo 12 | uses: actions/checkout@v2 13 | with: 14 | repository: deepdataspace/deepdataspace 15 | ref: dev 16 | token: ${{ secrets.PRIVATE_REPO_TOKEN }} 17 | 18 | - name: Add Public Remote 19 | run: | 20 | git remote add public https://github.com/IDEA-Research/deepdataspace.git 21 | 22 | - name: Rebase Private Repo 23 | run: | 24 | git pull public dev --rebase 25 | 26 | - name: Push Private Repo 27 | run: | 28 | git push 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # for local dev 2 | /op 3 | .idea 4 | .vscode 5 | dev.* 6 | venv 7 | /dev/* 8 | .DS_Store 9 | /config.py 10 | .python-version 11 | 12 | # for runtime file 13 | /data 14 | *.pyc 15 | *.log 16 | *.pid 17 | 18 | # for tests 19 | /htmlcov 20 | /.coverage 21 | /.coverage.* 22 | 23 | # for setup.py 24 | /dist 25 | /build 26 | /MANIFEST 27 | /deepdataspace.egg-info 28 | 29 | # for docs 30 | /docs 31 | 32 | # frontend 33 | node_modules 34 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npx --no -- commitlint --edit $1 -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | pnpm -r --stream run precommit 5 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry=https://registry.npmmirror.com 2 | strict-peer-dependencies=false 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18-alpine AS frontend-builder 2 | 3 | WORKDIR /frontend 4 | COPY . ./ 5 | RUN npm install -g pnpm@8.4.0 && \ 6 | pnpm install --frozen-lockfile && \ 7 | pnpm run build:app 8 | 9 | FROM python:3.10 10 | 11 | WORKDIR /dds/source 12 | COPY . ./ 13 | COPY --from=frontend-builder /frontend/packages/app/dist ./deepdataspace/server/static 14 | COPY --from=frontend-builder /frontend/packages/app/dist/index.html ./deepdataspace/server/templates/index.html 15 | RUN mkdir /dds/datasets && \ 16 | mkdir /dds/samples && \ 17 | python3 -m pip install -r requirements.txt && \ 18 | python3 -m pip cache purge 19 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include deepdataspace/server/static * 2 | recursive-include deepdataspace/server/templates * 3 | recursive-include deepdataspace/samples * 4 | global-exclude *.py[cod] 5 | global-exclude .DS_Store 6 | -------------------------------------------------------------------------------- /build_doc.sh: -------------------------------------------------------------------------------- 1 | sphinx-apidoc -o docs/api_reference deepdataspace/ 2 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: off 4 | patch: off 5 | 6 | flag_management: 7 | default_rules: 8 | carryforward: true 9 | individual_flags: 10 | - name: backend 11 | paths: 12 | - deepdataspace/ 13 | statuses: 14 | - type: project 15 | target: auto 16 | threshold: 1% 17 | - name: frontend 18 | paths: 19 | - packages/app/ 20 | statuses: 21 | - type: project 22 | target: auto 23 | threshold: 1% -------------------------------------------------------------------------------- /dds.py: -------------------------------------------------------------------------------- 1 | #! python 2 | """ 3 | dds.py 4 | 5 | This script starts the dds tool. 6 | 7 | Usage: dds.py [OPTIONS] [DATA_DIR] 8 | 9 | Options: 10 | --quickstart Quick start dds with sample datasets instead of a 11 | specified DATA_DIR, default false. This overwrites the 12 | DATA_DIR argument. 13 | -V, --verbose Display detailed logs on console, default false. 14 | -P, --public Enable public access from your network neighbors, default 15 | false. This sets the service host by an auto-detected 16 | outward IP address. 17 | --host TEXT Set the http service host. This overwrites the '--public' 18 | flag. 19 | --port TEXT Set the http service port, default 8765. 20 | --reload Auto reload service on code change, for development only. 21 | --configfile TEXT Load the target yaml file to initialize more 22 | configurations. The command line options take precedence 23 | of the config file. 24 | --help Show this message and exit. 25 | """ 26 | import os 27 | 28 | os.environ["DDS_STARTING"] = "1" 29 | 30 | from deepdataspace.scripts.start import start_dds 31 | 32 | 33 | def main(): 34 | start_dds() 35 | 36 | 37 | if __name__ == "__main__": 38 | main() 39 | -------------------------------------------------------------------------------- /ddsop.py: -------------------------------------------------------------------------------- 1 | #! python 2 | """ 3 | ddsop.py 4 | 5 | Usage: ddsop.py [OPTIONS] COMMAND [ARGS]... 6 | 7 | Options: 8 | --help Show this message and exit. 9 | 10 | Commands: 11 | delete_one Delete a dataset. 12 | import_all Trigger a background task of importing all datasets in a... 13 | import_coco Generate a coco meta file. 14 | import_one Trigger a background task of importing one dataset. 15 | lpexport Export the labels of a label project. 16 | migrate Run a migrate script. 17 | shell Enter a Python interpreter shell with all dds configs loaded. 18 | useradd Add a user by username. 19 | userban Ban a user by username. 20 | userdel Delete a user by username. 21 | useredit Edit user attributes for a user by username. 22 | userreset Reset password for a user by username. 23 | userunban Unban a user by username. 24 | """ 25 | 26 | from deepdataspace.scripts import ddsop 27 | 28 | if __name__ == "__main__": 29 | ddsop() 30 | -------------------------------------------------------------------------------- /deepdataspace/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The deepdataspace package. 3 | """ 4 | 5 | import logging 6 | import os 7 | import sys 8 | 9 | logging.basicConfig(stream=sys.stdout, level=logging.INFO) 10 | 11 | from deepdataspace.services import DDS 12 | from deepdataspace.services import config 13 | from deepdataspace.constants import RunningEnv 14 | 15 | env = os.environ.get("DDS_DEPLOY", RunningEnv.Local) 16 | if env == RunningEnv.Local and not config.load_all_env(): # this runs as early as possible 17 | if not os.environ.get("DDS_STARTING", None): 18 | msg = "\ndeepdataspace package is not available before dds services are started. \n" \ 19 | "You can start dds services by one of the two methods:\n" \ 20 | "1. CLI tool: dds start --quickstart\n" \ 21 | "2. Python API: from deepdataspace import DDS; DDS(quickstart=True).start()\n" 22 | logging.warning(msg) 23 | else: 24 | from deepdataspace.model import * 25 | -------------------------------------------------------------------------------- /deepdataspace/algos/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.algos 3 | 4 | This module provides kinds of algorithms for processing dataset. 5 | """ 6 | -------------------------------------------------------------------------------- /deepdataspace/io/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.io 3 | 4 | This module defines the common dataset IO interfaces. 5 | """ 6 | 7 | from deepdataspace.io.importer import import_dataset 8 | -------------------------------------------------------------------------------- /deepdataspace/model/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.model 3 | 4 | This module defines data models over mongodb. 5 | """ 6 | 7 | from deepdataspace.model.category import Category 8 | from deepdataspace.model.dataset import DataSet 9 | from deepdataspace.model.image import Image 10 | from deepdataspace.model.image import ImageModel 11 | from deepdataspace.model.label import Label 12 | from deepdataspace.model.label_task import LabelProject 13 | from deepdataspace.model.label_task import LabelTask 14 | from deepdataspace.model.label_task import LabelTaskImage 15 | from deepdataspace.model.label_task import LabelTaskImageModel 16 | from deepdataspace.model.label_task import ProjectRole 17 | from deepdataspace.model.label_task import TaskRole 18 | from deepdataspace.model.object import Object 19 | from deepdataspace.model.user import User 20 | from deepdataspace.model.user import UserToken 21 | -------------------------------------------------------------------------------- /deepdataspace/model/category.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.model.category 3 | 4 | The category model. 5 | """ 6 | 7 | from deepdataspace.model._base import BaseModel 8 | 9 | 10 | class Category(BaseModel): 11 | """ 12 | Category, or Class in some context, is the classification of objects in an image. 13 | 14 | Attributes: 15 | ----------- 16 | name: str 17 | The category name. 18 | id: str 19 | The category id. 20 | dataset_id: str 21 | The dataset id this category belongs to. 22 | """ 23 | 24 | @classmethod 25 | def get_collection(cls): 26 | """ 27 | Categories are stored in the "categories" collection. 28 | """ 29 | return cls.db["categories"] 30 | 31 | # the mandatory fields 32 | name: str # the category name 33 | id: str # category id 34 | dataset_id: str # which dateset this category belongs to 35 | -------------------------------------------------------------------------------- /deepdataspace/model/label.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.model.label 3 | 4 | The label model. 5 | """ 6 | 7 | from typing import List 8 | 9 | from deepdataspace.model._base import BaseModel 10 | 11 | 12 | class Label(BaseModel): 13 | """ 14 | | Label, or Label Set, or Prediction Set, is a set of predictions made to images of a dataset at the same time. 15 | | GroundTruth and UserAnnotation are special label sets. 16 | 17 | Attributes: 18 | ----------- 19 | name: str 20 | The label name. 21 | id: str 22 | The label id. 23 | type: str 24 | Is it a prediction? a GroundTruth? or a user annotation?, see :class:`deepdataspace.constants.LabelType`. 25 | dataset_id: str 26 | The dataset id this label belongs to. 27 | compare_precisions: list 28 | Pre-calculated thresh conf for comparing prediction to gt. 29 | clone_from_label: str 30 | Which label set this label is cloned from. 31 | """ 32 | 33 | @classmethod 34 | def get_collection(cls, *args, **kwargs): 35 | return cls.db["labels"] 36 | 37 | # the mandatory fields 38 | name: str # the label name 39 | 40 | # the optional fields 41 | id: str = "" # the label id 42 | type: str = "" # is it a prediction? a GroundTruth? or a user annotation? 43 | dataset_id: str = "" # which dataset this label belongs to 44 | compare_precisions: List = [] # pre-calculated thresh conf for comparing prediction to gt 45 | clone_from_label: str = "" # which label set this label is cloned from 46 | -------------------------------------------------------------------------------- /deepdataspace/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.plugins 3 | 4 | This module includes the plugable implementations of various dataset format. 5 | """ 6 | 7 | import os 8 | import sys 9 | import importlib 10 | 11 | 12 | def import_all_plugins(): 13 | """ 14 | Import all plugin modules in this directory without knowing their names. 15 | This function is called on django startup. 16 | Celery will also call this function on startup if DJANGO_SETTINGS_MODULE environment is set. 17 | """ 18 | 19 | module_name = sys.modules[__name__].__name__ 20 | module_dir = os.path.dirname(os.path.abspath(__file__)) 21 | 22 | items = os.listdir(module_dir) 23 | for item in items: 24 | item_path = os.path.join(module_dir, item) 25 | 26 | # skip non-module dir 27 | if os.path.isdir(item_path) and not os.path.exists(f"{item_path}/__init__.py"): 28 | continue 29 | 30 | # skip non-python file 31 | if os.path.isfile(item_path) and not item.endswith(".py"): 32 | continue 33 | 34 | # also skip __init__.py file 35 | if item == "__init__.py": 36 | continue 37 | 38 | importlib.import_module(f"{module_name}.{item}") 39 | -------------------------------------------------------------------------------- /deepdataspace/plugins/coco2017/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.plugins.coco2017 3 | 4 | This module supports coco2017 dataset format. 5 | """ 6 | 7 | from deepdataspace.plugins.coco2017.importer import COCO2017Importer 8 | -------------------------------------------------------------------------------- /deepdataspace/plugins/tsv/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.plugins.tsv 3 | 4 | This module supports tsv dataset format. 5 | """ 6 | 7 | from deepdataspace.plugins.tsv.importer import TSVImporter 8 | from deepdataspace.plugins.tsv.server import ReRankImagesByFlagsTaskView 9 | from deepdataspace.plugins.tsv.server import ReRankImagesByFlagsTasksView 10 | from deepdataspace.server.utils import route 11 | 12 | route("api/v1/tasks/rerank_by_flags")(ReRankImagesByFlagsTasksView), 13 | route("api/v1/tasks/rerank_by_flags/")(ReRankImagesByFlagsTaskView) 14 | -------------------------------------------------------------------------------- /deepdataspace/process/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.process 3 | 4 | This module defines common interface and implementations of processing a dataset. 5 | """ 6 | 7 | from deepdataspace.process.processor import process_dataset 8 | from deepdataspace.process.calculate_fnfp import FNFPCalculator 9 | -------------------------------------------------------------------------------- /deepdataspace/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.scripts 3 | This module adds convenient console commands for the dds services. 4 | """ 5 | import os 6 | 7 | import click 8 | 9 | from deepdataspace.constants import RunningEnv 10 | from deepdataspace.services import config 11 | 12 | env = os.environ.get("DDS_DEPLOY", RunningEnv.Local) 13 | 14 | 15 | @click.group 16 | def ddsop(): 17 | if env == RunningEnv.Local and not config.load_all_env(): # load the configurations generated by dds command 18 | exit(0) 19 | 20 | 21 | from deepdataspace.scripts.user_cmds import useradd 22 | from deepdataspace.scripts.user_cmds import userban 23 | from deepdataspace.scripts.user_cmds import userunban 24 | from deepdataspace.scripts.user_cmds import userreset 25 | from deepdataspace.scripts.enter_shell import enter_shell 26 | from deepdataspace.scripts.dataset_cmds import import_all 27 | from deepdataspace.scripts.dataset_cmds import import_one 28 | from deepdataspace.scripts.dataset_cmds import delete_one 29 | from deepdataspace.scripts.label_project_cmds import lp_export 30 | from deepdataspace.scripts.migrate import migrate 31 | from deepdataspace.scripts.dataset_cmds import import_coco -------------------------------------------------------------------------------- /deepdataspace/scripts/enter_shell.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.scripts.enter_shell 3 | 4 | This file adds a sub-command to ddsop command: ddsop shell. 5 | This sub-command will present user a Python interpreter shell with all configurations loaded. 6 | """ 7 | 8 | from deepdataspace.scripts import ddsop 9 | 10 | 11 | @ddsop.command("shell", help="Enter a Python interpreter shell with all dds configs loaded.") 12 | def enter_shell(): 13 | try: 14 | from django.core.management import execute_from_command_line 15 | except ImportError as exc: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) from exc 21 | 22 | run_cmd = ["manage.py", "shell"] 23 | execute_from_command_line(run_cmd) 24 | -------------------------------------------------------------------------------- /deepdataspace/scripts/migrate/2023053101.py: -------------------------------------------------------------------------------- 1 | """ 2 | Add image dirs to whitelist so that they can be accessed from the web API. 3 | """ 4 | 5 | import logging 6 | import os 7 | 8 | from deepdataspace.constants import RedisKey 9 | from deepdataspace.globals import Redis 10 | from deepdataspace.model import DataSet 11 | from deepdataspace.model import Image 12 | 13 | logger = logging.getLogger("scripts.migrate") 14 | 15 | 16 | def get_image_dirs(): 17 | image_dirs = set() 18 | filters = {"url": {"$regex": "^/files/local_files"}} 19 | for dataset in DataSet.find_many({}): 20 | ImageModel = Image(dataset.id) 21 | for image in ImageModel.find_many(filters): 22 | image_path = image.url.split("/") 23 | image_path = "/".join(image_path[7:]) 24 | image_dir = os.path.dirname(image_path) 25 | image_dirs.add(image_dir) 26 | 27 | return image_dirs 28 | 29 | 30 | def add_to_whitelist(image_dirs): 31 | for image_dir in image_dirs: 32 | if not Redis.sismember(RedisKey.DatasetImageDirs, image_dir): 33 | logger.info(f"Adding image dir to whitelist: {image_dir}") 34 | Redis.sadd(RedisKey.DatasetImageDirs, image_dir) 35 | 36 | 37 | def add_image_dirs_to_whitelist(): 38 | image_dirs = get_image_dirs() 39 | add_to_whitelist(image_dirs) 40 | 41 | 42 | def run(): 43 | add_image_dirs_to_whitelist() 44 | -------------------------------------------------------------------------------- /deepdataspace/scripts/migrate/2023061401.py: -------------------------------------------------------------------------------- 1 | """ 2 | Add cover to every dataset. 3 | """ 4 | 5 | import logging 6 | import os 7 | 8 | from deepdataspace.model import DataSet 9 | 10 | logger = logging.getLogger("scripts.migrate") 11 | 12 | 13 | def add_covers(): 14 | num = DataSet.count_num({}) 15 | logger.info(f"Adding covers to {num} dataset(s)...") 16 | 17 | datasets = DataSet.find_many({}) 18 | for idx, dataset in enumerate(datasets): 19 | dataset.add_cover(force_update=True) 20 | logger.info(f"[{idx + 1}/{num}]Added cover to dataset[{dataset.id}], cover_url={dataset.cover_url}") 21 | 22 | logger.info("Finished adding covers") 23 | 24 | 25 | def run(): 26 | add_covers() 27 | -------------------------------------------------------------------------------- /deepdataspace/scripts/migrate/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.scripts.migrate 3 | 4 | This file adds migrate sub-command to ddsop command. 5 | """ 6 | 7 | import importlib 8 | 9 | import click 10 | 11 | from deepdataspace.scripts import ddsop 12 | 13 | 14 | @ddsop.command("migrate", help="Run a migrate script.") 15 | @click.argument("script_name") 16 | def migrate(script_name): 17 | module_name = f"{__name__}.{script_name}" 18 | try: 19 | module = importlib.import_module(module_name) 20 | except ModuleNotFoundError: 21 | print(f"script [{script_name}] not found") 22 | exit(1) 23 | else: 24 | module.run() 25 | -------------------------------------------------------------------------------- /deepdataspace/server/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server 3 | 4 | The dds api server, implemented with django. 5 | """ 6 | -------------------------------------------------------------------------------- /deepdataspace/server/apps.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.apps 3 | 4 | The django start-up hook. 5 | """ 6 | 7 | from django.apps import AppConfig 8 | 9 | 10 | class DjangoApiConfig(AppConfig): 11 | default_auto_field = "django.db.models.BigAutoField" 12 | name = "deepdataspace.server" 13 | 14 | def ready(self): 15 | # This is for django. 16 | # But if DJANGO_SETTINGS_MODULE environment is set, celery will also set up django on start up, 17 | # which makes this effective for celery too. 18 | # See source code of .../site-packages/celery/fixups/django.py 19 | from deepdataspace.plugins import import_all_plugins 20 | 21 | import_all_plugins() 22 | -------------------------------------------------------------------------------- /deepdataspace/server/manage.py: -------------------------------------------------------------------------------- 1 | #! python 2 | import os 3 | import sys 4 | 5 | 6 | def main(): 7 | """Run administrative tasks.""" 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "deepdataspace.server.settings") 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError as exc: 12 | raise ImportError( 13 | "Couldn't import Django. Are you sure it's installed and " 14 | "available on your PYTHONPATH environment variable? Did you " 15 | "forget to activate a virtual environment?" 16 | ) from exc 17 | execute_from_command_line(sys.argv) 18 | 19 | 20 | if __name__ == "__main__": 21 | main() 22 | -------------------------------------------------------------------------------- /deepdataspace/server/middlewares/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.middlewares 3 | 4 | The django middlewares. 5 | """ 6 | 7 | from deepdataspace.server.middlewares.request_perf import RequestPerfMiddleware 8 | -------------------------------------------------------------------------------- /deepdataspace/server/middlewares/request_perf.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.middlewares.request_perf 3 | 4 | The request performance middleware. 5 | """ 6 | 7 | import time 8 | 9 | from django.utils.deprecation import MiddlewareMixin 10 | 11 | 12 | class RequestPerfMiddleware(MiddlewareMixin): 13 | def process_request(self, request): 14 | "Start time at request coming in" 15 | request.start_time = time.time() 16 | 17 | def process_response(self, request, response): 18 | "End of request, take time" 19 | total = time.time() - request.start_time 20 | 21 | # Add the header. 22 | response["X-Django-Cost"] = int(total * 1000) 23 | return response 24 | -------------------------------------------------------------------------------- /deepdataspace/server/resources/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.resources 3 | 4 | The RESTful resources. 5 | """ 6 | 7 | from deepdataspace.server.resources import files 8 | from deepdataspace.server.resources import api_v1 9 | -------------------------------------------------------------------------------- /deepdataspace/server/resources/files/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.resources.files 3 | 4 | Fake static file urls. 5 | Actually the file is dynamically generated according to the request. 6 | """ 7 | 8 | from django.urls import path 9 | 10 | from deepdataspace.server.resources.files.local_file import read_file 11 | from deepdataspace.server.resources.files.dataset_flags import dataset_flags 12 | 13 | urls = [ 14 | path("local_files/////", read_file), 15 | path("dataset_flags/.tsv", dataset_flags), 16 | ] 17 | -------------------------------------------------------------------------------- /deepdataspace/server/resources/files/dataset_flags.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.resources.files.dataset_flags 3 | 4 | This file add API to generate and return the flag file of a dataset. 5 | """ 6 | 7 | import json 8 | import tempfile 9 | 10 | from django.http.response import FileResponse 11 | from django.http.response import Http404 12 | 13 | from deepdataspace.model import DataSet 14 | from deepdataspace.model.image import Image 15 | 16 | 17 | def dataset_flags(request, dataset_id): 18 | """ 19 | Generate and return the flag file of the dataset. 20 | """ 21 | 22 | dataset = DataSet.find_one({"id": dataset_id}) 23 | if dataset is None: 24 | raise Http404() 25 | 26 | file_lines = [] # val.tsv {"time": 1664188933, "id": 0, "flag": 2} 27 | images = Image(dataset_id).find_many({}, sort=[("_id", 1)], to_dict=True) 28 | for image in images: 29 | flag_meta = {"time": image["flag_ts"], "id": image["id"], "flag": image["flag"]} 30 | line = f"{dataset.name}\t{json.dumps(flag_meta)}" 31 | file_lines.append(line) 32 | 33 | file_str = "\n".join(file_lines) 34 | tmp_file = tempfile.NamedTemporaryFile() 35 | tmp_file.name = f"{dataset.name}.flag.tsv" 36 | if dataset.group_name: 37 | tmp_file.name = f"{dataset.group_name}.{tmp_file.name}" 38 | tmp_file.write(file_str.encode("utf-8")) 39 | tmp_file.seek(0) 40 | 41 | response = FileResponse(tmp_file) 42 | response["Content-Type"] = "application/octet-stream" 43 | response["Content-Disposition"] = f'attachment;filename="{tmp_file.name}"' 44 | 45 | return response 46 | -------------------------------------------------------------------------------- /deepdataspace/server/static/9.9f27f1a0.async.js: -------------------------------------------------------------------------------- 1 | "use strict";(self.webpackChunkapp=self.webpackChunkapp||[]).push([[9],{67009:function(a,n,t){t.r(n),t.d(n,{default:function(){return s}});var e=t(58757),u=t(97375),E=t(35667);function s(){var O=(0,u.useOutletContext)();return(0,E.jsx)(u.Outlet,{context:O})}}}]); 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/deepdataspace/server/static/favicon.png -------------------------------------------------------------------------------- /deepdataspace/server/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__404.0f1cb5ec.async.js: -------------------------------------------------------------------------------- 1 | "use strict";(self.webpackChunkapp=self.webpackChunkapp||[]).push([[571],{12671:function(f,o,t){t.r(o),t.d(o,{default:function(){return e}});var v=t(58757),i=t(97375),s=t(12632),r=t(12562),u=t(53357),n=t(35667),a=function(){return(0,n.jsx)("div",{style:{position:"relative",height:"100vh",width:"100%",backgroundColor:"#fff",display:"flex",justifyContent:"center",alignItems:"center"},children:(0,n.jsx)(s.ZP,{status:"404",title:"404",subTitle:(0,u._w)("NotFoundTip.title"),extra:(0,n.jsx)(r.ZP,{type:"primary",onClick:function(){return i.history.push("/")},children:(0,u._w)("NotFoundTip.backHome")})})})},l=a,d=function(){return(0,n.jsx)(l,{})},e=d}}]); 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__Annotator__index.8750e2fd.async.js: -------------------------------------------------------------------------------- 1 | "use strict";(self.webpackChunkapp=self.webpackChunkapp||[]).push([[734],{25970:function(l,e,_){_.r(e);var a=_(63900),n=_.n(a),m=_(58757),s=_(28638),u=_(97375),o=_(35667),d=function(){var t=(0,u.useModel)("Annotator.model");return(0,o.jsx)(s.Z,n()({},t))};e.default=d}}]); 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__Lab__index.28ea9971.chunk.css: -------------------------------------------------------------------------------- 1 | .page___Lcm4P{background:#fafafa;min-height:100vh}.list___ohS5i{position:relative}.list___ohS5i .card___Dbi0J{overflow:hidden;width:300px}.list___ohS5i .card___Dbi0J img{display:block;width:100%}.list___ohS5i .card___Dbi0J .content___kIUmq{position:absolute;top:24px;left:28px;color:#fff}.list___ohS5i .card___Dbi0J .content___kIUmq .title___y_wsO{font-size:28px;font-weight:500}.list___ohS5i .card___Dbi0J .content___kIUmq .subTitle___bu4U_{font-size:16px} 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__Lab__index.5de6b129.async.js: -------------------------------------------------------------------------------- 1 | "use strict";(self.webpackChunkapp=self.webpackChunkapp||[]).push([[358],{23058:function(n,s,t){t.r(s),t.d(s,{default:function(){return m}});var f=t(58757),r=t(61845),i=t(37617),d=t(18624),o=t(97375),u=t(53357),l={page:"page___Lcm4P",list:"list___ohS5i",card:"card___Dbi0J",content:"content___kIUmq",title:"title___y_wsO",subTitle:"subTitle___bu4U_"},a=t(35667),v=function(){var x=(0,u.bU)(),c=x.localeText,g=c("lab.card.title"),h=c("lab.card.subTitle"),b=[{title:g,subTitle:h,image:t(62122),link:"/lab/datasets?type=flagtool"}];return(0,a.jsx)(r._z,{className:l.page,fixedHeader:!0,children:(0,a.jsx)(i.Z,{className:l.list,dataSource:b,renderItem:function(e){return(0,a.jsx)(i.Z.Item,{children:(0,a.jsxs)(d.Z,{hoverable:!0,className:l.card,onClick:function(){return o.history.push(e.link)},bodyStyle:{padding:0},children:[(0,a.jsx)("img",{src:e.image,alt:e.title}),(0,a.jsxs)("div",{className:l.content,children:[(0,a.jsx)("div",{className:l.title,children:e.title}),(0,a.jsx)("div",{className:l.subTitle,children:e.subTitle})]})]})})}})})},m=v},62122:function(n,s,t){n.exports=t.p+"static/flagtool.a991e5bd.png"}}]); 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__Login__index.26727079.chunk.css: -------------------------------------------------------------------------------- 1 | .page___movJn{height:100vh;background-image:linear-gradient(to right,#7a88ff,#7affaf)}.page___movJn .ant-pro-form-login-page-container{max-width:750px} 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__Project__Detail__index.9b99b574.chunk.css: -------------------------------------------------------------------------------- 1 | .tagsWrap___bmrPM{display:flex;flex-wrap:wrap;row-gap:8px}.container___SbJNH{position:relative;width:100%}.container___SbJNH .progress___E0MmQ{position:relative;display:flex;width:90%;height:8px;background-color:#e4e4e4;border-radius:100px;transition:all .4s cubic-bezier(.08,.82,.17,1) 0s;overflow:hidden}.container___SbJNH .progress___E0MmQ .progressItem___NiZAB{height:8px}.container___SbJNH .labels___TBmcS{position:relative;margin-top:8px;display:flex;flex-wrap:wrap;font-size:12px}.container___SbJNH .labels___TBmcS .split___NOm1V{margin:0 4px;color:#e4e4e4}.page___HrYAe{position:relative;height:calc(100vh - 56px);display:flex;flex-direction:column}.page___HrYAe .ant-pro-grid-content{position:relative;height:100%}.table___vpXyu{position:absolute;top:24px;left:24px;right:24px;bottom:24px}.table___vpXyu .ant-pro-card-body{padding-bottom:0}.table___vpXyu .actionCell___NNJhx{display:flex;align-items:center;justify-content:flex-start;gap:16px;flex-wrap:wrap} 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/p__Project__index.711b5411.chunk.css: -------------------------------------------------------------------------------- 1 | .tagsWrap___bmrPM{display:flex;flex-wrap:wrap;row-gap:8px}.page___VUB1h{position:relative;height:calc(100vh - 56px);display:flex;flex-direction:column}.page___VUB1h .ant-pro-grid-content{position:relative;height:100%}.table___BzPd6{position:absolute;inset:24px}.table___BzPd6 .ant-pro-card-body{padding-bottom:0}.table___BzPd6 .actionCell___IDtFO{display:flex;align-items:center;justify-content:flex-start;gap:16px;flex-wrap:wrap}.link___BBH6k{color:#389e0d}.input___Wk1gH{padding-top:8px} 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/static/annotate_coop.44ac433d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/deepdataspace/server/static/static/annotate_coop.44ac433d.png -------------------------------------------------------------------------------- /deepdataspace/server/static/static/annotate_quick.b8733d56.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/deepdataspace/server/static/static/annotate_quick.b8733d56.png -------------------------------------------------------------------------------- /deepdataspace/server/static/static/card_cover_4.b81bbca8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/deepdataspace/server/static/static/card_cover_4.b81bbca8.png -------------------------------------------------------------------------------- /deepdataspace/server/static/static/flagtool.a991e5bd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/deepdataspace/server/static/static/flagtool.a991e5bd.png -------------------------------------------------------------------------------- /deepdataspace/server/static/static/home_banner.0fcae71f.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/deepdataspace/server/static/static/home_banner.0fcae71f.png -------------------------------------------------------------------------------- /deepdataspace/server/static/t__plugin-layout__Layout.8f39539f.chunk.css: -------------------------------------------------------------------------------- 1 | @media screen and (max-width: 480px){.umi-plugin-layout-container{width:100%!important}.umi-plugin-layout-container>*{border-radius:0!important}}.umi-plugin-layout-menu .anticon{margin-right:8px}.umi-plugin-layout-menu .ant-dropdown-menu-item{min-width:160px}.umi-plugin-layout-right{display:flex!important;float:right;height:100%;margin-left:auto;overflow:hidden}.umi-plugin-layout-right .umi-plugin-layout-action{display:flex;align-items:center;height:100%;padding:0 12px;cursor:pointer;transition:all .3s}.umi-plugin-layout-right .umi-plugin-layout-action>i{color:#ffffffd9;vertical-align:middle}.umi-plugin-layout-right .umi-plugin-layout-action:hover,.umi-plugin-layout-right .umi-plugin-layout-action.opened{background:rgba(0,0,0,.025)}.umi-plugin-layout-right .umi-plugin-layout-search{padding:0 12px}.umi-plugin-layout-right .umi-plugin-layout-search:hover{background:transparent}.umi-plugin-layout-name{margin-left:8px} 2 | -------------------------------------------------------------------------------- /deepdataspace/server/static/wrappers__auth.d0b0c30b.async.js: -------------------------------------------------------------------------------- 1 | "use strict";(self.webpackChunkapp=self.webpackChunkapp||[]).push([[899],{8615:function(o,n,u){u.r(n);var e=u(97375),t=u(35667);n.default=function(){var s=(0,e.useModel)("user"),a=s.user,r=(0,e.useLocation)(),i=r.pathname;return a.isLogin===void 0?null:a.isLogin?(0,t.jsx)(e.Outlet,{}):(0,t.jsx)(e.Navigate,{to:"/login?redirect=".concat(i),replace:!0})}}}]); 2 | -------------------------------------------------------------------------------- /deepdataspace/server/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 | -------------------------------------------------------------------------------- /deepdataspace/server/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.urls 3 | 4 | The django server urls. 5 | """ 6 | 7 | from django.urls import include 8 | from django.urls import path 9 | from django.urls import re_path 10 | from django.views.generic.base import TemplateView 11 | 12 | from deepdataspace.server import resources 13 | from deepdataspace.server.views import index 14 | 15 | urlpatterns = [ 16 | path("", index), 17 | path("files/", include(resources.files.urls)), 18 | path("api/v1/", include(resources.api_v1.urls)), 19 | re_path(r"^page", TemplateView.as_view(template_name="index.html"), name="index"), 20 | ] 21 | -------------------------------------------------------------------------------- /deepdataspace/server/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.utils 3 | 4 | Convenient functions for django server. 5 | """ 6 | 7 | import inspect 8 | from functools import wraps 9 | 10 | from django.urls import path 11 | from rest_framework.views import APIView 12 | 13 | from deepdataspace.server.urls import urlpatterns 14 | 15 | 16 | def route(api_path: str): 17 | """ 18 | Register the decorated function or APIView class as the handler for given api path. 19 | """ 20 | 21 | path_record = {} 22 | 23 | def decorator(handler): 24 | registered = path_record.get(api_path, None) 25 | if registered is not None: 26 | raise RuntimeError(f"{api_path} is already registered by {registered}") 27 | 28 | path_record[api_path] = handler 29 | if inspect.isclass(handler) and issubclass(handler, APIView): 30 | pattern = path(api_path, handler.as_view()) 31 | else: 32 | pattern = path(api_path, handler) 33 | urlpatterns.append(pattern) 34 | 35 | @wraps(handler) 36 | def wrapper(*args, **kwargs): 37 | return handler(*args, **kwargs) 38 | 39 | return wrapper 40 | 41 | return decorator 42 | -------------------------------------------------------------------------------- /deepdataspace/server/views/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.views 3 | 4 | Server rendered views, only for index page. 5 | """ 6 | 7 | from deepdataspace.server.views.index import index 8 | -------------------------------------------------------------------------------- /deepdataspace/server/views/index.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.views.index 3 | 4 | The index page view. 5 | """ 6 | 7 | from django.shortcuts import redirect 8 | 9 | 10 | def index(request): 11 | return redirect("/page/") 12 | 13 | 14 | def sub_index(request, sub_path): 15 | return redirect("/page/") 16 | -------------------------------------------------------------------------------- /deepdataspace/server/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.server.wsgi 3 | 4 | The WSGI interface for django server. 5 | """ 6 | 7 | import os 8 | 9 | from django.core.wsgi import get_wsgi_application 10 | 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "deepdataspace.server.settings") 12 | 13 | app = get_wsgi_application() 14 | -------------------------------------------------------------------------------- /deepdataspace/services/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.services 3 | 4 | This module wraps all service components for dds and provides a common interface to control them. 5 | """ 6 | 7 | from deepdataspace.services.dds import DDS 8 | from deepdataspace.services.config import load_all_env 9 | -------------------------------------------------------------------------------- /deepdataspace/services/celery.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.services.celery 3 | 4 | The celery service manager. 5 | """ 6 | 7 | import os 8 | import sys 9 | from pathlib import Path 10 | 11 | from deepdataspace.services import config 12 | from deepdataspace.services.service import Service 13 | 14 | 15 | class Celery(Service): 16 | def __init__(self): 17 | python_path = sys.executable 18 | python_path = str(Path(os.path.abspath(python_path))) 19 | cmd_list = [ 20 | python_path, 21 | "-m", "celery", 22 | "-A", f"deepdataspace.task:app", 23 | "worker", 24 | "-l", "info", 25 | "-c", "1", 26 | f"--logfile={config.CELERY_LOG}", 27 | ] 28 | super(Celery, self).__init__("celery", cmd_list) 29 | 30 | self.cmd_id = " ".join(cmd_list[1:]) 31 | 32 | def start(self): 33 | try: 34 | self.start_process(self.cmd_list) 35 | except Exception as err: 36 | print(f"failed to start celery, err={err}") 37 | return False 38 | else: 39 | return True 40 | -------------------------------------------------------------------------------- /deepdataspace/services/dds.config.sample.yaml: -------------------------------------------------------------------------------- 1 | runtime_dir: # default to $HOME/.deepdataspace 2 | data_dir: # default to $HOME/.deepdataspace/datasets 3 | quick_start: false # download a small dataset for quick start 4 | verbose_log: false # display detailed logs 5 | 6 | db_engine: # sqlite3 or mysql 7 | db_host: 8 | db_port: # a number 9 | db_name: 10 | db_user: 11 | db_pass: 12 | 13 | redis_host: 14 | redis_port: # a number 15 | redis_pass: 16 | redis_dbname: # a number 17 | 18 | mongodb_host: 19 | mongodb_port: # a number 20 | mongodb_user: 21 | mongodb_pass: 22 | mongodb_dbname: 23 | 24 | django_host: 25 | django_port: # a number 26 | django_secret: 27 | django_reload: false 28 | 29 | celery_worker_pool: solo # solo or prefork, only solo works on Windows platform 30 | 31 | sentry_dsn: 32 | -------------------------------------------------------------------------------- /deepdataspace/services/sqlite.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.services.sqlite 3 | 4 | The sqlite db service manager. 5 | """ 6 | 7 | from deepdataspace.services import config 8 | from deepdataspace.services.config import log_and_save_env 9 | 10 | 11 | class SQLite: 12 | """ 13 | We don't need to 'start' sqlite, we just implement this to unify controller API. 14 | """ 15 | 16 | def start(self): 17 | log_and_save_env("DDS_DB_ENGIN", config.DB_ENGIN) 18 | log_and_save_env("DDS_DB_HOST", config.DB_HOST) 19 | log_and_save_env("DDS_DB_PORT", config.DB_PORT) 20 | log_and_save_env("DDS_DB_NAME", config.DB_NAME) 21 | log_and_save_env("DDS_DB_USER", config.DB_USER) 22 | log_and_save_env("DDS_DB_PASS", config.DB_PASS) 23 | return True 24 | -------------------------------------------------------------------------------- /deepdataspace/task/celery.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.task.celery 3 | 4 | The celery app initialization. 5 | """ 6 | 7 | from celery import Celery 8 | 9 | app = Celery("dds") 10 | app.config_from_object("deepdataspace.task.settings") 11 | -------------------------------------------------------------------------------- /deepdataspace/task/import_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.task.import_dataset 3 | 4 | Tasks about importing datasets. 5 | """ 6 | 7 | import logging 8 | 9 | from deepdataspace.io import import_dataset as imp_ds 10 | from deepdataspace.task.celery import app 11 | 12 | logger = logging.getLogger("celery") 13 | 14 | 15 | @app.task 16 | def import_dataset(target_path: str, enforce: bool = False): 17 | """ 18 | Import all datasets under target path. 19 | """ 20 | 21 | logger.info(f"import_dataset starts, target_path={target_path}, enforce={enforce}") 22 | return imp_ds(target_path, enforce) 23 | -------------------------------------------------------------------------------- /deepdataspace/task/ping.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.task.ping 3 | 4 | A simple ping task. 5 | """ 6 | 7 | import logging 8 | 9 | from deepdataspace.task.celery import app 10 | 11 | logger = logging.getLogger("celery") 12 | 13 | 14 | @app.task 15 | def ping(): 16 | return "pong" 17 | -------------------------------------------------------------------------------- /deepdataspace/task/process_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.task.process_dataset 3 | 4 | Task for processing datasets. 5 | """ 6 | 7 | import logging 8 | 9 | from deepdataspace.process import process_dataset as prc_ds 10 | from deepdataspace.task.celery import app 11 | 12 | logger = logging.getLogger("celery") 13 | 14 | 15 | @app.task 16 | def process_dataset(dataset_dir: str, enforce: bool = False, auto_triggered: bool = False): 17 | """ 18 | Process a dataset by all registered processors. 19 | """ 20 | 21 | logger.info(f"import_dataset starts, dataset_dir={dataset_dir}, enforce={enforce}") 22 | return prc_ds(dataset_dir, enforce, auto_triggered=auto_triggered) 23 | -------------------------------------------------------------------------------- /deepdataspace/task/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.task.settings 3 | 4 | The celery settings. 5 | """ 6 | 7 | from deepdataspace import environs 8 | 9 | broker_url = environs.CELERY_BROKER 10 | worker_pool = environs.CELERY_WORKER_POOL 11 | task_acks_late = True 12 | -------------------------------------------------------------------------------- /deepdataspace/utils/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module includes convenient functions 3 | """ 4 | -------------------------------------------------------------------------------- /deepdataspace/utils/network.py: -------------------------------------------------------------------------------- 1 | """ 2 | deepdataspace.utils.network 3 | 4 | Convenient functions about network. 5 | """ 6 | 7 | import os 8 | import shutil 9 | import socket 10 | 11 | import requests 12 | 13 | from deepdataspace.utils.string import gen_random_str 14 | 15 | 16 | def get_output_ip_address() -> str: 17 | """ 18 | Acquire the default output ip address of this machine. 19 | """ 20 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 21 | s.settimeout(5) 22 | s.connect(("114.114.114.114", 80)) 23 | ip = s.getsockname()[0] 24 | s.close() 25 | return ip 26 | 27 | 28 | def download_by_requests(url: str, path: str, timeout: int = 5): 29 | """ 30 | Download the specified url to the local path. 31 | 32 | :param url: the remote url to be downloaded 33 | :param path: the local path of download destination 34 | :param timeout: the timeout for download, in case of endless blocking 35 | 36 | This function downloads the url to a temporary path during the download procedure and 37 | move the temporary file to the destination path. 38 | This guarantees the integrity of the downloaded file. 39 | """ 40 | rsp = requests.get(url, timeout=timeout) 41 | 42 | tmp_file = f"{path}.{gen_random_str(6)}" 43 | os.makedirs(os.path.dirname(tmp_file), exist_ok=True) 44 | with open(tmp_file, "wb") as fp: 45 | fp.write(rsp.content) 46 | shutil.move(tmp_file, path) 47 | -------------------------------------------------------------------------------- /docker/init-dds.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import zipfile 5 | 6 | sys.path.append(os.path.dirname(os.path.abspath("__file__"))) 7 | 8 | from deepdataspace.utils.network import download_by_requests 9 | from deepdataspace.task import import_and_process_data_dir 10 | 11 | sample_dir = "/dds/samples" 12 | sample_file = f"{sample_dir}/dataset-samples.zip" 13 | 14 | 15 | def download_samples(): 16 | if os.path.exists(sample_file): 17 | print(f"Sample file {sample_file} already exists, skip downloading.") 18 | return False 19 | 20 | sample_url = "https://deepdataspace.oss-cn-shenzhen.aliyuncs.com/install_files/datasets/dataset-samples.zip" 21 | print(f"Downloading {sample_url} to {sample_file}") 22 | download_by_requests(sample_url, sample_file) 23 | 24 | with zipfile.ZipFile(sample_file, "r") as fp: 25 | fp.extractall(sample_dir) 26 | print(f"Extracted {sample_file} to {sample_dir}") 27 | 28 | 29 | def import_samples(): 30 | task_uuid = import_and_process_data_dir.apply_async(args=(sample_dir, False, True)) 31 | print(task_uuid) 32 | 33 | 34 | def main(): 35 | download_samples() 36 | import_samples() 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /docker/init-mongo.js: -------------------------------------------------------------------------------- 1 | db.getSiblingDB('admin').auth( 2 | 'mongodb', 3 | 'mongodb' 4 | ); 5 | db.createUser({ 6 | user: 'mongodb', 7 | pwd: 'mongodb', 8 | roles: ["readWrite"], 9 | }); 10 | -------------------------------------------------------------------------------- /packages/app/.env: -------------------------------------------------------------------------------- 1 | UMI_ENV=local 2 | API_PATH= 3 | MODEL_API_PATH=https://api.deepdataspace.com 4 | MODEL_API_TOKEN=dds-tool-free 5 | 6 | #sentry 7 | SENTRY_DSN= 8 | SENTRY_URL= 9 | SENTRY_ORG= 10 | SENTRY_TOKEN= 11 | 12 | # ga 13 | GA_KEY= 14 | 15 | # baidutongji 16 | BAIDU_KEY= -------------------------------------------------------------------------------- /packages/app/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | rules: { 3 | 'arrow-parens': 0, 4 | 'react/no-array-index-key': 0, 5 | 'react/sort-comp': 0, 6 | '@typescript-eslint/no-explicit-any': 1, 7 | '@typescript-eslint/no-empty-interface': 1, 8 | '@typescript-eslint/no-inferrable-types': 0, 9 | 'react/no-find-dom-node': 1, 10 | 'react/require-default-props': 0, 11 | 'no-confusing-arrow': 0, 12 | 'import/no-named-as-default-member': 0, 13 | 'jsx-a11y/label-has-for': 0, 14 | 'jsx-a11y/label-has-associated-control': 0, 15 | 'jsx-a11y/control-has-associated-label': 0, 16 | 'jsx-a11y/alt-text': 0, 17 | '@typescript-eslint/no-explicit-any': 0, 18 | }, 19 | extends: require.resolve('@umijs/max/eslint'), 20 | }; 21 | -------------------------------------------------------------------------------- /packages/app/.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /.env.local 3 | /.umirc.local.ts 4 | /config/config.local.ts 5 | /src/.umi 6 | /src/.umi-production 7 | /src/.umi-test 8 | /.umi 9 | /.umi-production 10 | /.umi-test 11 | /dist 12 | /dist.zip 13 | /public 14 | /.mfsu 15 | /coverage -------------------------------------------------------------------------------- /packages/app/.lintstagedrc: -------------------------------------------------------------------------------- 1 | { 2 | "*.{md,json}": ["prettier --cache --write"], 3 | "*.{js,jsx}": ["max lint --fix --eslint-only", "prettier --cache --write"], 4 | "*.{css,less}": [ 5 | "max lint --fix --stylelint-only", 6 | "prettier --cache --write" 7 | ], 8 | "*.ts?(x)": [ 9 | "max lint --fix --eslint-only", 10 | "prettier --cache --parser=typescript --write" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /packages/app/.npmrc: -------------------------------------------------------------------------------- 1 | registry=https://registry.npmmirror.com 2 | strict-peer-dependencies=false 3 | -------------------------------------------------------------------------------- /packages/app/.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | public 4 | .umi 5 | .umi-production 6 | pnpm-lock.yaml -------------------------------------------------------------------------------- /packages/app/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 80, 3 | "tabWidth": 2, 4 | "singleQuote": true, 5 | "trailingComma": "all", 6 | "proseWrap": "never", 7 | "overrides": [{ "files": ".prettierrc", "options": { "parser": "json" } }], 8 | "plugins": ["prettier-plugin-organize-imports", "prettier-plugin-packagejson"] 9 | } 10 | -------------------------------------------------------------------------------- /packages/app/.stylelintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | rules: { 3 | 'color-function-notation': 'legacy', 4 | 'alpha-value-notation': 'number', 5 | }, 6 | extends: require.resolve('@umijs/max/stylelint'), 7 | }; 8 | -------------------------------------------------------------------------------- /packages/app/jest.config.ts: -------------------------------------------------------------------------------- 1 | import { Config, configUmiAlias, createConfig } from '@umijs/max/test'; 2 | 3 | export default async () => { 4 | const config = createConfig({ 5 | target: 'browser', 6 | jsTransformer: 'esbuild', 7 | // config opts for esbuild , it will pass to esbuild directly 8 | jsTransformerOpts: { jsx: 'automatic' }, 9 | }); 10 | return (await configUmiAlias({ 11 | ...config, 12 | transform: { 13 | ...(config.transform || {}), 14 | '\\.(jpg|ico|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': 15 | require.resolve('.//tests/file-transform'), 16 | }, 17 | setupFilesAfterEnv: ['/tests/jest-setup.ts'], 18 | collectCoverageFrom: [ 19 | 'src/**/*.{ts,tsx,js,jsx}', 20 | '!src/.umi/**', 21 | '!src/.umi-test/**', 22 | '!src/.umi-production/**', 23 | // unnecessary test 24 | '!src/logs/**', 25 | '!src/services/**', 26 | '!src/types/**', 27 | ], 28 | // if you require some es-module npm package, please uncomment below line and insert your package name 29 | // transformIgnorePatterns: ['node_modules/(?!.*(lodash-es|your-es-pkg-name)/)'] 30 | coverageThreshold: { 31 | global: { 32 | lines: 1, 33 | }, 34 | }, 35 | maxWorkers: 8, 36 | })) as Config.InitialOptions; 37 | }; 38 | -------------------------------------------------------------------------------- /packages/app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "app", 3 | "author": "zhuyuanhao", 4 | "scripts": { 5 | "build": "max build", 6 | "build:inner": "max build", 7 | "dev": "max dev", 8 | "dev:build": "max dev & sleep 10; kill $!", 9 | "format": "prettier --cache --write .", 10 | "lint": "max lint", 11 | "precommit": "lint-staged --quiet", 12 | "preview": "PUBLIC_PATH=/page/ max build && max preview --port 8001", 13 | "setup": "max setup", 14 | "start": "npm run dev", 15 | "test": "jest --no-cache --coverage", 16 | "test:snapshot": "jest --no-cache --coverage --u" 17 | }, 18 | "dependencies": { 19 | "copy-to-clipboard": "^3.3.3", 20 | "dds-components": "workspace:^", 21 | "dds-hooks": "workspace:^", 22 | "dds-utils": "workspace:^", 23 | "react-masonry-component": "^6.3.0", 24 | "react-resize-detector": "^7.1.2" 25 | }, 26 | "devDependencies": {} 27 | } 28 | -------------------------------------------------------------------------------- /packages/app/src/assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/.gitkeep -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/annotate_coop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/annotate_coop.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/annotate_quick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/annotate_quick.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/card_cover_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/card_cover_0.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/card_cover_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/card_cover_1.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/card_cover_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/card_cover_2.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/card_cover_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/card_cover_3.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/card_cover_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/card_cover_4.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/card_cover_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/card_cover_5.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/cards/flagtool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/cards/flagtool.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/favicon.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/home_banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/home_banner.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/logo_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/logo_title.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/old_favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/old_favicon.png -------------------------------------------------------------------------------- /packages/app/src/assets/images/old_logo_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDEA-Research/deepdataspace/8c59f79d276c39f023359a53cc6c1228e535909d/packages/app/src/assets/images/old_logo_title.png -------------------------------------------------------------------------------- /packages/app/src/assets/svg/auto-awesome.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/auto-fix.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/classification.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/datasetDetection.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/datasetKeypoint.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/datasetMatting.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/datasetSegment.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/delete.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/download.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/edit.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/email-send.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/flag.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/hand.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/keypoints.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/maxSize.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/minSize.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/minus.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/multVisible.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/plus.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/random.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/rect.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/settings.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/square.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/app/src/assets/svg/visible.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /packages/app/src/components/CategoryFilter/index.less: -------------------------------------------------------------------------------- 1 | .wrapper { 2 | position: relative; 3 | display: flex; 4 | align-items: center; 5 | } 6 | -------------------------------------------------------------------------------- /packages/app/src/components/CategoryFilter/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useMemo } from 'react'; 2 | import { Select, SelectProps } from 'antd'; 3 | import { useLocale } from 'dds-utils/locale'; 4 | import styles from './index.less'; 5 | import { Category } from '@/types'; 6 | 7 | export interface IProps { 8 | categoryId?: string; 9 | categories: Category[]; 10 | onCategoryChange: (categoryId: string) => void; 11 | } 12 | 13 | const CategoryFilter: React.FC = (props) => { 14 | const { localeText } = useLocale(); 15 | const { categoryId, categories, onCategoryChange } = props; 16 | 17 | const options: SelectProps['options'] = useMemo(() => { 18 | return categories.map((item) => ({ 19 | label: item.name, 20 | value: item.id, 21 | })); 22 | }, [categories]); 23 | 24 | return ( 25 |
26 | {localeText('dataset.detail.category')}: 27 |