├── .gitignore
├── LICENSE
├── README.md
├── backend
├── .env
├── .gitignore
├── Dockerfile
├── README.md
├── app
│ ├── README.md
│ ├── __init__.py
│ ├── api
│ │ ├── __init__.py
│ │ ├── deps.py
│ │ ├── router.py
│ │ └── v1
│ │ │ ├── __init__.py
│ │ │ ├── const
│ │ │ └── common.py
│ │ │ ├── endpoints
│ │ │ ├── __init__.py
│ │ │ ├── file.py
│ │ │ ├── label_task.py
│ │ │ ├── operator
│ │ │ │ ├── __init__.py
│ │ │ │ ├── label_task.py
│ │ │ │ └── label_task_stat.py
│ │ │ ├── team.py
│ │ │ ├── team_invitation.py
│ │ │ ├── team_member.py
│ │ │ └── user.py
│ │ │ └── router.py
│ ├── client
│ │ ├── __init__.py
│ │ └── minio.py
│ ├── core
│ │ ├── __init__.py
│ │ ├── config.py
│ │ ├── exceptions.py
│ │ └── security.py
│ ├── crud
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── crud_data.py
│ │ ├── crud_file.py
│ │ ├── crud_label_task.py
│ │ ├── crud_record.py
│ │ ├── crud_team.py
│ │ ├── crud_team_invitation.py
│ │ └── crud_user.py
│ ├── db
│ │ ├── __init__.py
│ │ ├── init_db.py
│ │ └── session.py
│ ├── gunicorn_conf.py
│ ├── logger
│ │ ├── __init__.py
│ │ └── logger.py
│ ├── main.py
│ ├── middleware
│ │ ├── __init__.py
│ │ └── middleware.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── data.py
│ │ ├── file.py
│ │ ├── label_task.py
│ │ ├── record.py
│ │ ├── team.py
│ │ ├── team_invitation.py
│ │ └── user.py
│ ├── scheduler
│ │ ├── __init__.py
│ │ ├── init_scheduler.py
│ │ └── task.py
│ ├── schemas
│ │ ├── __init__.py
│ │ ├── data.py
│ │ ├── evaluation.py
│ │ ├── file.py
│ │ ├── message.py
│ │ ├── operator
│ │ │ ├── __init__.py
│ │ │ ├── stats.py
│ │ │ └── task.py
│ │ ├── record.py
│ │ ├── task.py
│ │ ├── team.py
│ │ ├── tool.py
│ │ └── user.py
│ ├── tests
│ │ └── __init__.py
│ ├── util
│ │ ├── __init__.py
│ │ └── stats.py
│ └── worker.py
├── migrations
│ └── 2023_0615_1112_mig_team_member.py
├── pdm.lock
├── pdm.toml
├── pyproject.toml
├── scripts
│ ├── start.sh
│ └── worker.sh
└── tests
│ └── test.py
├── docker-compose.yaml
├── frontend
├── .commitlintrc
├── .editorconfig
├── .eslintignore
├── .eslintrc.js
├── .husky
│ └── pre-commit
├── .npmrc
├── .prettierignore
├── .prettierrc.js
├── .stylelintrc.js
├── Dockerfile
├── README.md
├── index.html
├── mock
│ ├── auth
│ │ └── auth.ts
│ └── task
│ │ └── taskList.ts
├── nginx.conf
├── package.json
├── postcss.config.js
├── public
│ ├── favicon.svg
│ └── iconfont_3732290.js
├── rollup.config.ts
├── scripts
│ ├── bootstrap.js
│ ├── generate_css_variables_from_antd_theme_token.js
│ ├── generate_css_variables_from_antd_theme_token.ts
│ └── guide.js
├── src
│ ├── api
│ │ ├── errorCode.ts
│ │ ├── request.tsx
│ │ ├── team.ts
│ │ └── user.ts
│ ├── apps
│ │ ├── login
│ │ │ ├── App.tsx
│ │ │ ├── components
│ │ │ │ └── FullPageScroll
│ │ │ │ │ └── index.tsx
│ │ │ ├── index.html
│ │ │ ├── index.tsx
│ │ │ ├── pages
│ │ │ │ └── login
│ │ │ │ │ ├── bg.png
│ │ │ │ │ └── index.tsx
│ │ │ ├── readme.md
│ │ │ ├── routes.tsx
│ │ │ └── styles
│ │ │ │ └── index.css
│ │ ├── operator
│ │ │ ├── App.tsx
│ │ │ ├── README.md
│ │ │ ├── assets
│ │ │ │ ├── book.svg
│ │ │ │ ├── calendar-finished.svg
│ │ │ │ ├── calendar-progress.svg
│ │ │ │ ├── calendar-total.svg
│ │ │ │ ├── calendar.svg
│ │ │ │ ├── demo-conversation@1x.png
│ │ │ │ ├── demo-conversation@2x.png
│ │ │ │ ├── demo-question@1x.png
│ │ │ │ ├── demo-question@2x.png
│ │ │ │ ├── demo-reply@1x.png
│ │ │ │ ├── demo-reply@2x.png
│ │ │ │ ├── diff.png
│ │ │ │ ├── empty.svg
│ │ │ │ ├── grid1.png
│ │ │ │ ├── grid2.png
│ │ │ │ ├── grid3.png
│ │ │ │ ├── grid4.png
│ │ │ │ ├── logo.svg
│ │ │ │ ├── noAuth.svg
│ │ │ │ ├── title.svg
│ │ │ │ └── upload-cloud.svg
│ │ │ ├── components
│ │ │ │ ├── CopyTask.tsx
│ │ │ │ ├── CustomEmpty.tsx
│ │ │ │ ├── CustomFancy
│ │ │ │ │ ├── QuestionEditor
│ │ │ │ │ │ ├── Condition
│ │ │ │ │ │ │ ├── RecursiveCondition
│ │ │ │ │ │ │ │ └── index.tsx
│ │ │ │ │ │ │ ├── context.ts
│ │ │ │ │ │ │ └── index.tsx
│ │ │ │ │ │ ├── README.md
│ │ │ │ │ │ ├── TagSwitcher
│ │ │ │ │ │ │ └── index.tsx
│ │ │ │ │ │ ├── index.tsx
│ │ │ │ │ │ ├── svgs
│ │ │ │ │ │ │ ├── delete.svg
│ │ │ │ │ │ │ └── tree-switcher.svg
│ │ │ │ │ │ └── types.ts
│ │ │ │ │ └── utils.ts
│ │ │ │ ├── Help.tsx
│ │ │ │ ├── JsonlUpload
│ │ │ │ │ ├── index.tsx
│ │ │ │ │ └── jsonl.schema.json
│ │ │ │ ├── NoAuth.tsx
│ │ │ │ ├── PercentageCircle.tsx
│ │ │ │ ├── QueryBlock.tsx
│ │ │ │ ├── QueryForm.tsx
│ │ │ │ ├── QueryTable.tsx
│ │ │ │ ├── TableSelectedTips.tsx
│ │ │ │ └── ToolConfigUpload
│ │ │ │ │ ├── index.tsx
│ │ │ │ │ └── toolConfig.schema.json
│ │ │ ├── constant
│ │ │ │ ├── access.ts
│ │ │ │ ├── chineseCharMap.ts
│ │ │ │ └── query-key-factories
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── operator.ts
│ │ │ │ │ ├── task.ts
│ │ │ │ │ └── team.ts
│ │ │ ├── hooks
│ │ │ │ ├── useLockFn.ts
│ │ │ │ ├── useScrollFetch.ts
│ │ │ │ └── useUserInfo.ts
│ │ │ ├── index.html
│ │ │ ├── index.tsx
│ │ │ ├── layouts
│ │ │ │ ├── CustomPageContainer.tsx
│ │ │ │ ├── Main.tsx
│ │ │ │ └── index.css
│ │ │ ├── loaders
│ │ │ │ ├── task.loader.ts
│ │ │ │ └── team.loader.ts
│ │ │ ├── pages
│ │ │ │ ├── task.label.[id]
│ │ │ │ │ ├── Analyze
│ │ │ │ │ │ └── index.tsx
│ │ │ │ │ ├── DownloadRange.tsx
│ │ │ │ │ ├── QuickCreate.tsx
│ │ │ │ │ ├── index.tsx
│ │ │ │ │ ├── left.tsx
│ │ │ │ │ ├── right.tsx
│ │ │ │ │ └── users.tsx
│ │ │ │ ├── task.label.create
│ │ │ │ │ ├── basic.tsx
│ │ │ │ │ ├── context.ts
│ │ │ │ │ ├── index.tsx
│ │ │ │ │ ├── tool.tsx
│ │ │ │ │ └── utils.ts
│ │ │ │ ├── task.tsx
│ │ │ │ ├── users.operator
│ │ │ │ │ ├── Invite
│ │ │ │ │ │ └── index.tsx
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── users.team.[id]
│ │ │ │ │ └── index.tsx
│ │ │ │ └── users.team
│ │ │ │ │ ├── Edit
│ │ │ │ │ └── index.tsx
│ │ │ │ │ └── index.tsx
│ │ │ ├── routes.tsx
│ │ │ ├── services
│ │ │ │ ├── task.ts
│ │ │ │ ├── team.ts
│ │ │ │ └── types.ts
│ │ │ ├── styles
│ │ │ │ └── index.css
│ │ │ ├── utils
│ │ │ │ ├── bfsEach.ts
│ │ │ │ ├── downloadFromUrl.ts
│ │ │ │ ├── mapTree.ts
│ │ │ │ └── objectEach.ts
│ │ │ └── wrappers
│ │ │ │ ├── CheckChildRoute.tsx
│ │ │ │ ├── CheckUsersPagePermission.tsx
│ │ │ │ └── LoginCheck.tsx
│ │ └── supplier
│ │ │ ├── App.tsx
│ │ │ ├── README.md
│ │ │ ├── assets
│ │ │ ├── bg.png
│ │ │ ├── empty.png
│ │ │ ├── join.svg
│ │ │ ├── logo.svg
│ │ │ ├── preview.svg
│ │ │ ├── timeout.svg
│ │ │ └── title.svg
│ │ │ ├── components
│ │ │ ├── Copy
│ │ │ │ └── index.tsx
│ │ │ └── Empty
│ │ │ │ └── index.tsx
│ │ │ ├── constant
│ │ │ ├── access.ts
│ │ │ ├── operatorAccess.ts
│ │ │ ├── query-key-factories
│ │ │ │ ├── index.ts
│ │ │ │ ├── member.ts
│ │ │ │ └── task.ts
│ │ │ └── task.ts
│ │ │ ├── hooks
│ │ │ ├── useTaskData.ts
│ │ │ ├── useTaskParams.ts
│ │ │ └── useUserInfo.ts
│ │ │ ├── index.html
│ │ │ ├── index.tsx
│ │ │ ├── layouts
│ │ │ ├── CustomPageContainer.tsx
│ │ │ ├── Main.tsx
│ │ │ └── index.css
│ │ │ ├── pages
│ │ │ ├── 404.tsx
│ │ │ ├── join-team
│ │ │ │ └── index.tsx
│ │ │ ├── task.[id]
│ │ │ │ ├── Answer
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── AuditInfo
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── ChatBox
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── CheckTaskType
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── Countdown
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── CustomizeQuestion
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── CustomizeTextarea
│ │ │ │ │ ├── index.tsx
│ │ │ │ │ └── upload.ts
│ │ │ │ ├── DiffModal
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── Header
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── PluginSet
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── QuestionnaireSelect
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── TaskForm
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── Widget
│ │ │ │ │ └── index.tsx
│ │ │ │ ├── context.ts
│ │ │ │ ├── index.css
│ │ │ │ └── index.tsx
│ │ │ └── task
│ │ │ │ ├── Card
│ │ │ │ ├── index.module.css
│ │ │ │ └── index.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── routes.tsx
│ │ │ ├── services
│ │ │ ├── joinTeam.ts
│ │ │ └── task.ts
│ │ │ ├── styles
│ │ │ └── index.css
│ │ │ └── wrappers
│ │ │ ├── CheckPreviewAuth.tsx
│ │ │ └── CheckTaskRouter.tsx
│ ├── components
│ │ ├── AppContainer.tsx
│ │ ├── AppPanel
│ │ │ ├── arrow.svg
│ │ │ ├── index.tsx
│ │ │ ├── labelu.svg
│ │ │ ├── mineru.svg
│ │ │ ├── odl.svg
│ │ │ └── tool.svg
│ │ ├── Breadcrumb.tsx
│ │ ├── ErrorBoundary
│ │ │ └── index.tsx
│ │ ├── FancyGroup
│ │ │ └── index.tsx
│ │ ├── FancyInput
│ │ │ ├── README.md
│ │ │ ├── base
│ │ │ │ ├── Boolean.fancy.tsx
│ │ │ │ ├── Enum.fancy.tsx
│ │ │ │ ├── Number.fancy.tsx
│ │ │ │ └── String.fancy.tsx
│ │ │ ├── fancyInput.ts
│ │ │ ├── index.tsx
│ │ │ └── types.ts
│ │ ├── Help.tsx
│ │ ├── IconFont
│ │ │ └── index.tsx
│ │ ├── Markdown
│ │ │ ├── errorImage.png
│ │ │ └── index.tsx
│ │ ├── MemberInvite
│ │ │ └── index.tsx
│ │ ├── MessageBox
│ │ │ └── index.tsx
│ │ ├── RouterContainer.tsx
│ │ └── StaticAnt.tsx
│ ├── constant
│ │ ├── chat.ts
│ │ ├── query-key-factories
│ │ │ ├── index.ts
│ │ │ └── user.ts
│ │ ├── queryClient.tsx
│ │ └── team.ts
│ ├── hooks
│ │ ├── useLang.ts
│ │ └── useStoreIds.ts
│ ├── initialize.tsx
│ ├── loaders
│ │ └── sso.loader.ts
│ ├── locales
│ │ ├── en-US.ts
│ │ ├── index.ts
│ │ └── zh-CN.ts
│ ├── styles
│ │ ├── font
│ │ │ ├── iconfont.svg
│ │ │ ├── iconfont.ttf
│ │ │ ├── iconfont.woff
│ │ │ └── iconfont.woff2
│ │ ├── global-variables.css
│ │ ├── index.css
│ │ └── theme.json
│ ├── utils
│ │ ├── getUrlExtension.ts
│ │ ├── gid.ts
│ │ ├── parseDocumentType.ts
│ │ └── sso.ts
│ ├── vite-env.d.ts
│ └── wrappers
│ │ └── RequireSSO.tsx
├── tailwind.config.js
├── tsconfig.json
├── tsconfig.util.json
├── vite.config.dev.ts
└── vite.config.prod.ts
└── release-notes.md
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | frontend/node_modules
5 | .DS_Store
6 |
7 | # lock file
8 | frontend/yarn.lock
9 | frontend/index.html
10 | frontend/pnpm-lock.yaml
11 | gptcommit.toml
12 | .husky/prepare-commit-msg
13 |
14 | .idea
15 |
16 | # lock file
17 | yarn.lock
18 | # package-lock.json
19 | pnpm-lock.yaml
20 |
21 | # testing
22 | /coverage
23 |
24 | # production
25 | build
26 | dist
27 | es
28 |
29 | # misc
30 | .DS_Store
31 | .env.local
32 | .env.development.local
33 | .env.test.local
34 | .env.production.local
35 |
36 | npm-debug.log*
37 | yarn-debug.log*
38 | yarn-error.log*
39 | lerna-debug.log*
40 |
41 | /packages/server/public
42 |
43 | .vscode
--------------------------------------------------------------------------------
/backend/.env:
--------------------------------------------------------------------------------
1 | DEBUG = True
2 | ENVIRONMENT=local
3 |
4 | MINIO_ACCESS_KEY_ID = MekKrisWUnFFtsEk
5 | MINIO_ACCESS_KEY_SECRET = XK4uxD1czzYFJCRTcM70jVrchccBdy6C
6 | MINIO_ENDPOINT = localhost:9000
7 | MINIO_INTERNAL_ENDPOINT = minio:9000
8 | MINIO_BUCKET = label-llm-test
9 |
10 | MongoDB_DSN = mongodb://root:mypassword@mongo:27017
11 | MongoDB_DB_NAME = label_llm
12 |
13 |
14 | REDIS_DSN = redis://redis:6379/11
15 |
16 | SECRET_KEY="?*hsbRq5c9gpjBp~:oHU+7s8,I.67ewohfsib1=17dw@.q9r4Iidop:Oi_5oIYgw"
--------------------------------------------------------------------------------
/backend/Dockerfile:
--------------------------------------------------------------------------------
1 | # build stage
2 | FROM python:3.11 AS builder
3 |
4 | # set workdir
5 | WORKDIR /app
6 |
7 | # install PDM
8 | RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ && pip install -U pip setuptools wheel && pip install pdm
9 |
10 | # copy files
11 | COPY pyproject.toml pdm.lock README.md /app/
12 |
13 | # install dependencies and project into the local packages directory
14 | RUN pdm config pypi.url https://mirrors.aliyun.com/pypi/simple/ && mkdir __pypackages__ && pdm install --prod --no-lock --no-editable
15 |
16 |
17 | # run stage
18 | FROM python:3.11
19 |
20 | # set workdir
21 | WORKDIR /app
22 |
23 | # set env
24 | ARG APP_VERSION
25 | ENV PYTHONPATH=/app/pkgs
26 | ENV APP_VERSION ${APP_VERSION}
27 |
28 | RUN apt install libmagic1
29 | # retrieve packages from build stage
30 | COPY --from=builder /app/__pypackages__/3.11/lib /app/pkgs
31 |
32 | # copy files
33 | COPY ./ /app/
34 |
35 | # set entrypoint
36 | CMD [ "sh", "/app/scripts/start.sh" ]
--------------------------------------------------------------------------------
/backend/app/README.md:
--------------------------------------------------------------------------------
1 |
2 | ## 目录结构
3 | ```
4 | ├── api 路由
5 | ├── client http客户端
6 | ├── core 设置、异常等系统基础项
7 | ├── crud crud抽象
8 | ├── db 数据库连接、配置
9 | ├── gunicorn_conf.py gunicorn配置
10 | ├── __init__.py __init__.py
11 | ├── logger 日志配置
12 | ├── main.py 程序入口
13 | ├── middleware 中间件
14 | ├── models 数据库模型
15 | ├── scheduler 定时任务
16 | ├── schemas 业务模型
17 | ├── tests 测试
18 | └── util 工具
19 | ```
--------------------------------------------------------------------------------
/backend/app/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/opendatalab/LabelLLM/11f2a221f73c9625ec820bec8fc158c1ae16b8a9/backend/app/__init__.py
--------------------------------------------------------------------------------
/backend/app/api/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/opendatalab/LabelLLM/11f2a221f73c9625ec820bec8fc158c1ae16b8a9/backend/app/api/__init__.py
--------------------------------------------------------------------------------
/backend/app/api/deps.py:
--------------------------------------------------------------------------------
1 | from fastapi import Depends, Request
2 | from loguru._logger import Logger
3 |
4 | from fastapi import Depends
5 | from fastapi.security import APIKeyCookie
6 |
7 | from app import crud, schemas
8 | from app.core import exceptions
9 | from app.core.security import verify_access_token
10 |
11 | oauth2_scheme = APIKeyCookie(name="access_token")
12 |
13 |
14 | async def get_logger(
15 | request: Request,
16 | ) -> Logger:
17 | return request.state.logger
18 |
19 |
20 | async def get_current_user(token: str = Depends(oauth2_scheme)):
21 | try:
22 | payload = verify_access_token(token)
23 | username = payload.get("sub")
24 | if username is None:
25 | raise exceptions.TOKEN_INVALID
26 | user = await crud.user.query(name=username).first_or_none()
27 | if user is None:
28 | raise exceptions.USER_NOT_EXIST
29 | except Exception as e:
30 | raise e
31 | return user
32 |
33 |
34 | async def get_current_team(
35 | user: schemas.user.DoUser = Depends(get_current_user),
36 | ):
37 | teams = await crud.team.query(user_id=user.user_id).to_list()
38 |
39 | return teams
40 |
41 |
42 | # 用户是管理员或运营
43 | async def is_admin_or_operator(
44 | user: schemas.user.DoUser = Depends(get_current_user),
45 | ) -> None:
46 | if user.role not in [
47 | schemas.user.UserType.ADMIN,
48 | schemas.user.UserType.SUPER_ADMIN,
49 | ]:
50 | raise exceptions.USER_PERMISSION_DENIED
51 |
52 | return
53 |
--------------------------------------------------------------------------------
/backend/app/api/router.py:
--------------------------------------------------------------------------------
1 | from fastapi import APIRouter
2 |
3 | from .v1.router import v1_router
4 |
5 | router = APIRouter()
6 | router.include_router(v1_router)
7 |
8 |
9 | @router.get("/health")
10 | async def health():
11 | return {"status": "ok"}
12 |
--------------------------------------------------------------------------------
/backend/app/api/v1/__init__.py:
--------------------------------------------------------------------------------
1 | from .router import v1_router
2 |
--------------------------------------------------------------------------------
/backend/app/api/v1/const/common.py:
--------------------------------------------------------------------------------
1 | from uuid import UUID
2 |
3 | from app import schemas
4 | from app.schemas.message import MessageBase
5 |
6 |
7 | def get_audit_review_record() -> schemas.operator.task.RespPreviewRecord:
8 | return schemas.operator.task.RespPreviewRecord(
9 | data_id=UUID("00000000-0000-0000-0000-000000000000"),
10 | prompt="这是伪数据,仅用于预览!",
11 | conversation=[
12 | MessageBase(
13 | message_id=UUID("00000000-0000-0000-0000-000000000001"),
14 | parent_id=None,
15 | message_type="send",
16 | content="本数据仅用于预览!",
17 | ),
18 | MessageBase(
19 | message_id=UUID("00000000-0000-0000-0000-000000000002"),
20 | parent_id=UUID("00000000-0000-0000-0000-000000000001"),
21 | message_type="receive",
22 | content="本数据仅用于预览!",
23 | ),
24 | MessageBase(
25 | message_id=UUID("00000000-0000-0000-0000-000000000011"),
26 | parent_id=None,
27 | message_type="send",
28 | content="本数据仅用于预览!",
29 | ),
30 | MessageBase(
31 | message_id=UUID("00000000-0000-0000-0000-000000000012"),
32 | parent_id=UUID("00000000-0000-0000-0000-000000000011"),
33 | message_type="receive",
34 | content="本数据仅用于预览!",
35 | ),
36 | MessageBase(
37 | message_id=UUID("00000000-0000-0000-0000-000000000021"),
38 | parent_id=None,
39 | message_type="send",
40 | content="本数据仅用于预览!",
41 | ),
42 | MessageBase(
43 | message_id=UUID("00000000-0000-0000-0000-000000000022"),
44 | parent_id=UUID("00000000-0000-0000-0000-000000000021"),
45 | message_type="receive",
46 | content="本数据仅用于预览!",
47 | ),
48 | ],
49 | evaluation=schemas.evaluation.SingleEvaluation(
50 | message_evaluation=None,
51 | conversation_evaluation=None,
52 | questionnaire_evaluation=None,
53 | data_evaluation=None,
54 | ),
55 | reference_evaluation=None,
56 | label_user=None,
57 | )
58 |
--------------------------------------------------------------------------------
/backend/app/api/v1/endpoints/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/opendatalab/LabelLLM/11f2a221f73c9625ec820bec8fc158c1ae16b8a9/backend/app/api/v1/endpoints/__init__.py
--------------------------------------------------------------------------------
/backend/app/api/v1/endpoints/file.py:
--------------------------------------------------------------------------------
1 | import time
2 | from datetime import timedelta
3 | from pathlib import Path
4 | from io import BytesIO
5 |
6 | import magic
7 | import httpx
8 | from fastapi import APIRouter, Body, Depends, Response, UploadFile, File
9 |
10 | from app import crud, models, schemas
11 | from app.api import deps
12 | from app.client.minio import minio
13 | from app.core import exceptions
14 | from app.core.config import settings
15 | from app.db.session import redis_session
16 |
17 | router = APIRouter(prefix="/file", tags=["file"])
18 |
19 |
20 | @router.post(
21 | "/file_upload",
22 | summary="上传文件",
23 | description="上传文件",
24 | )
25 | async def file_upload(
26 | file: UploadFile,
27 | user: schemas.user.DoUser = Depends(deps.get_current_user),
28 | ):
29 | await redis_session.zremrangebyscore(
30 | f"file:upload:limit:{user.user_id}", 0, int(time.time()) - 60
31 | )
32 | if await redis_session.zcard(f"file:upload:limit:{user.user_id}") > 60:
33 | raise exceptions.FILE_UPLOAD_LIMIT
34 |
35 | db_file = await crud.file.create(
36 | obj_in=models.file.FileCreate(
37 | creator_id=user.user_id,
38 | )
39 | )
40 |
41 | data = await file.read()
42 |
43 | minio.client.put_object(
44 | settings.MINIO_BUCKET,
45 | f"{settings.ENVIRONMENT}/file_upload/{db_file.file_id}{Path(file.filename or '').suffix}",
46 | BytesIO(data),
47 | length=len(data),
48 | )
49 |
50 | return {
51 | "get_path": f"/api/v1/file/file_preview/{settings.ENVIRONMENT}/file_upload/{db_file.file_id}{Path(file.filename or '').suffix}",
52 | }
53 |
54 |
55 | @router.get(
56 | "/file_preview/{path:path}",
57 | summary="获取文件预览",
58 | description="获取文件预览",
59 | )
60 | async def file_preview(path: str):
61 | url = minio.client.presigned_get_object(
62 | settings.MINIO_BUCKET, path, expires=timedelta(days=1)
63 | )
64 | async with httpx.AsyncClient() as client:
65 | resp = await client.get(url)
66 | data = resp.content
67 | mimetype = magic.from_buffer(data[:1024], mime=True)
68 | return Response(data, media_type=mimetype)
69 |
--------------------------------------------------------------------------------
/backend/app/api/v1/endpoints/operator/__init__.py:
--------------------------------------------------------------------------------
1 | from fastapi import APIRouter, Depends
2 |
3 | from app.api import deps
4 |
5 | from . import label_task, label_task_stat
6 |
7 | router = APIRouter(
8 | prefix="/operator",
9 | tags=["operator"],
10 | dependencies=[Depends(deps.is_admin_or_operator)],
11 | )
12 | router.include_router(label_task.router)
13 | router.include_router(label_task_stat.router)
14 |
15 |
--------------------------------------------------------------------------------
/backend/app/api/v1/router.py:
--------------------------------------------------------------------------------
1 | from fastapi import APIRouter
2 |
3 | from .endpoints import (
4 | file,
5 | label_task,
6 | operator,
7 | team,
8 | team_invitation,
9 | team_member,
10 | user,
11 | )
12 |
13 | v1_router = APIRouter(prefix="/v1")
14 | v1_router.include_router(label_task.router)
15 | v1_router.include_router(team.router)
16 | v1_router.include_router(team_invitation.router)
17 | v1_router.include_router(team_member.router)
18 | v1_router.include_router(operator.router)
19 | v1_router.include_router(user.router)
20 | v1_router.include_router(file.router)
21 |
--------------------------------------------------------------------------------
/backend/app/client/__init__.py:
--------------------------------------------------------------------------------
1 | from . import minio
2 |
--------------------------------------------------------------------------------
/backend/app/client/minio.py:
--------------------------------------------------------------------------------
1 | from minio import Minio
2 |
3 | from app.core.config import settings
4 |
5 |
6 | class MinioClient:
7 | def __init__(
8 | self, ak: str, sk: str, endpoint: str, internal_endpoint: str, bucket: str
9 | ):
10 | self.client = Minio(internal_endpoint, access_key=ak, secret_key=sk, secure=False)
11 | self.bucket = bucket
12 | self.endpoint = endpoint
13 | self.internal_endpoint = internal_endpoint
14 |
15 | minio = MinioClient(
16 | settings.MINIO_ACCESS_KEY_ID,
17 | settings.MINIO_ACCESS_KEY_SECRET,
18 | settings.MINIO_ENDPOINT,
19 | settings.MINIO_INTERNAL_ENDPOINT,
20 | settings.MINIO_BUCKET,
21 | )
22 |
23 |
--------------------------------------------------------------------------------
/backend/app/core/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/opendatalab/LabelLLM/11f2a221f73c9625ec820bec8fc158c1ae16b8a9/backend/app/core/__init__.py
--------------------------------------------------------------------------------
/backend/app/core/config.py:
--------------------------------------------------------------------------------
1 | from pydantic import RedisDsn
2 | from pydantic_settings import BaseSettings
3 | import secrets
4 |
5 |
6 | class Settings(BaseSettings):
7 | # Debug Config
8 | DEBUG: bool = False
9 |
10 | # App Config
11 | APP_VERSION: str | None = None
12 | API_STR: str = "/api"
13 | ENVIRONMENT: str = "local"
14 |
15 | # MongoDB Config
16 | MongoDB_DSN: str = ""
17 | MongoDB_DB_NAME: str = ""
18 |
19 | # Redis Config
20 | REDIS_DSN: RedisDsn = RedisDsn("redis://localhost:16279/0") # type: ignore
21 |
22 | # Sentry Config
23 | SENTRY_DSN: str = ""
24 |
25 | # Minio Config
26 | MINIO_ACCESS_KEY_ID: str = ""
27 | MINIO_ACCESS_KEY_SECRET: str = ""
28 | MINIO_ENDPOINT: str = ""
29 | MINIO_INTERNAL_ENDPOINT: str = ""
30 | MINIO_BUCKET: str = ""
31 |
32 | SECRET_KEY: str = secrets.token_urlsafe(32)
33 | ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 7
34 |
35 | class Config:
36 | env_file = ".env"
37 |
38 |
39 | settings = Settings()
40 |
--------------------------------------------------------------------------------
/backend/app/core/security.py:
--------------------------------------------------------------------------------
1 | from datetime import datetime, timedelta, timezone
2 | from typing import Any
3 |
4 | from jose import jwt
5 | from passlib.context import CryptContext
6 |
7 | from app.core import exceptions
8 | from app.core.config import settings
9 |
10 | pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
11 |
12 |
13 | ALGORITHM = "HS256"
14 |
15 |
16 | def create_access_token(subject: str | Any, expires_delta: timedelta) -> str:
17 | expire = datetime.now(timezone.utc) + expires_delta
18 | to_encode = {"exp": expire, "sub": str(subject)}
19 | encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=ALGORITHM)
20 | return encoded_jwt
21 |
22 |
23 | def verify_password(plain_password: str, hashed_password: str) -> bool:
24 | return pwd_context.verify(plain_password, hashed_password)
25 |
26 |
27 | def get_password_hash(password: str) -> str:
28 | return pwd_context.hash(password)
29 |
30 |
31 | def verify_access_token(token: str):
32 | try:
33 | payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[ALGORITHM])
34 | return payload
35 | except Exception as e:
36 | raise exceptions.TOKEN_INVALID
37 |
--------------------------------------------------------------------------------
/backend/app/crud/__init__.py:
--------------------------------------------------------------------------------
1 | from .crud_data import data
2 | from .crud_file import file
3 | from .crud_label_task import label_task
4 | from .crud_record import record
5 | from .crud_team import team
6 | from .crud_team_invitation import team_invitation_link
7 | from .crud_user import user
8 |
--------------------------------------------------------------------------------
/backend/app/crud/crud_file.py:
--------------------------------------------------------------------------------
1 | from app.crud.base import CRUDBase
2 | from app.models.file import File, FileCreate, FileUpdate
3 |
4 |
5 | class CRUDFile(CRUDBase[File, FileCreate, FileUpdate]):
6 | ...
7 |
8 |
9 | file = CRUDFile(File)
10 |
--------------------------------------------------------------------------------
/backend/app/crud/crud_label_task.py:
--------------------------------------------------------------------------------
1 | import re
2 | from uuid import UUID
3 |
4 | from beanie.operators import ElemMatch, In, RegEx
5 |
6 | from app import schemas
7 | from app.crud.base import CRUDBase
8 | from app.models.label_task import LabelTask, LabelTaskCreate, LabelTaskUpdate
9 |
10 |
11 | class CRUDLabelTask(CRUDBase[LabelTask, LabelTaskCreate, LabelTaskUpdate]):
12 | def query(
13 | self,
14 | *,
15 | _id: list[str] | str | None = None,
16 | skip: int | None = None,
17 | limit: int | None = None,
18 | sort: str | list[str] | None = None,
19 | title: str | None = None,
20 | status: schemas.task.TaskStatus | list[schemas.task.TaskStatus] | None = None,
21 | team_id: UUID | list[UUID] | None = None,
22 | task_id: UUID | list[UUID] | None = None,
23 | creator_id: str | list[str] | None = None,
24 | ):
25 | query = super().query(_id=_id, sort=sort, skip=skip, limit=limit)
26 |
27 | if title is not None and len(title.strip()) > 0:
28 | title = title.strip()
29 | query = query.find(RegEx(field=self.model.title, pattern=re.escape(title), options="i")) # type: ignore
30 |
31 | if status is not None:
32 | if isinstance(status, list):
33 | query = query.find(In(self.model.status, status))
34 | else:
35 | query = query.find(self.model.status == status)
36 |
37 | if team_id is not None:
38 | if isinstance(team_id, list):
39 | query = query.find(ElemMatch(self.model.teams, {"$in": team_id}))
40 | else:
41 | query = query.find(ElemMatch(self.model.teams, {"$eq": team_id}))
42 |
43 | if task_id is not None:
44 | if isinstance(task_id, list):
45 | query = query.find(In(self.model.task_id, task_id))
46 | else:
47 | query = query.find(self.model.task_id == task_id)
48 |
49 | if creator_id is not None:
50 | if isinstance(creator_id, list):
51 | query = query.find(In(self.model.creator_id, creator_id))
52 | else:
53 | query = query.find(self.model.creator_id == creator_id)
54 |
55 | return query
56 |
57 |
58 | label_task = CRUDLabelTask(LabelTask)
59 |
--------------------------------------------------------------------------------
/backend/app/crud/crud_team.py:
--------------------------------------------------------------------------------
1 | from typing import Any
2 | from uuid import UUID
3 |
4 | from beanie.operators import Eq, In, RegEx
5 |
6 | from app.crud.base import CRUDBase
7 | from app.models.team import Team, TeamCreate, TeamUpdate
8 |
9 |
10 | class CRUDTeam(CRUDBase[Team, TeamCreate, TeamUpdate]):
11 | def query(
12 | self,
13 | *,
14 | _id: list[Any] | Any = None,
15 | skip: int | None = None,
16 | limit: int | None = None,
17 | sort: str | list[str] | None = None,
18 | user_id: list[str] | str | None = None,
19 | team_id: list[UUID] | UUID | None = None,
20 | name: str | None = None,
21 | ):
22 | query = super().query(_id=_id, skip=skip, limit=limit, sort=sort)
23 |
24 | if user_id is not None:
25 | if isinstance(user_id, list):
26 | query = query.find(In("users.user_id", user_id))
27 | else:
28 | query = query.find(Eq("users.user_id", user_id))
29 | if team_id is not None:
30 | if isinstance(team_id, list):
31 | query = query.find(In(self.model.team_id, team_id))
32 | else:
33 | query = query.find(self.model.team_id == team_id)
34 |
35 | if name is not None:
36 | query = query.find(RegEx(field=self.model.name, pattern=name)) # type: ignore
37 |
38 | return query
39 |
40 |
41 | team = CRUDTeam(Team)
42 |
43 |
44 |
--------------------------------------------------------------------------------
/backend/app/crud/crud_team_invitation.py:
--------------------------------------------------------------------------------
1 | from typing import Any
2 | from uuid import UUID
3 |
4 | from app.crud.base import CRUDBase
5 | from app.models.team_invitation import (
6 | TeamInvitationLink,
7 | TeamInvitationLinkCreate,
8 | TeamInvitationLinkUpdate,
9 | )
10 |
11 |
12 | class CRUDTeamInvitationLink(
13 | CRUDBase[TeamInvitationLink, TeamInvitationLinkCreate, TeamInvitationLinkUpdate]
14 | ):
15 | def query(
16 | self,
17 | *,
18 | _id: list[Any] | Any = None,
19 | skip: int | None = None,
20 | limit: int | None = None,
21 | sort: str | list[str] | None = None,
22 | link_id: UUID | None = None,
23 | ):
24 | query = super().query(_id=_id, sort=sort, skip=skip, limit=limit)
25 |
26 | if link_id is not None:
27 | query = query.find(self.model.link_id == link_id)
28 |
29 | return query
30 |
31 |
32 | team_invitation_link = CRUDTeamInvitationLink(TeamInvitationLink)
33 |
--------------------------------------------------------------------------------
/backend/app/crud/crud_user.py:
--------------------------------------------------------------------------------
1 | from typing import Any
2 |
3 | from beanie.operators import In, RegEx
4 |
5 | from app.crud.base import CRUDBase
6 | from app.models.user import User, UserCreate, UserUpdate
7 | from app.schemas.user import UserType
8 |
9 |
10 | class CRUDUser(CRUDBase[User, UserCreate, UserUpdate]):
11 | def query(
12 | self,
13 | *,
14 | _id: list[Any] | Any = None,
15 | skip: int | None = None,
16 | limit: int | None = None,
17 | sort: str | list[str] | None = None,
18 | user_id: list[str] | str | None = None,
19 | name: str | None = None,
20 | role: list[UserType] | UserType | None = None,
21 | password: list[str] | str | None = None
22 | ):
23 | query = super().query(_id=_id, sort=sort, skip=skip, limit=limit)
24 |
25 | if user_id is not None:
26 | if isinstance(user_id, list):
27 | query = query.find(In(self.model.user_id, user_id))
28 | else:
29 | query = query.find(self.model.user_id == user_id)
30 |
31 | if role:
32 | if isinstance(role, list):
33 | query = query.find(In(self.model.role, role))
34 | else:
35 | query.find(self.model.role==role)
36 |
37 | if password:
38 | if isinstance(password, list):
39 | query = query.find(In(self.model.password, password))
40 | else:
41 | query = query.find(self.model.password == password)
42 |
43 | if name:
44 | query = query.find(RegEx(field=self.model.name, pattern=name)) # type: ignore
45 |
46 | return query
47 |
48 |
49 | user = CRUDUser(User)
50 |
--------------------------------------------------------------------------------
/backend/app/db/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/opendatalab/LabelLLM/11f2a221f73c9625ec820bec8fc158c1ae16b8a9/backend/app/db/__init__.py
--------------------------------------------------------------------------------
/backend/app/db/init_db.py:
--------------------------------------------------------------------------------
1 | from beanie import init_beanie
2 |
3 | from app import crud, schemas
4 | from app.core.config import settings
5 | from app.db.session import mongo_session, redis_session
6 | from app.models.data import Data
7 | from app.models.file import File
8 | from app.models.label_task import LabelTask
9 | from app.models.record import Record
10 | from app.models.team import Team, TeamCreate
11 | from app.models.team_invitation import TeamInvitationLink
12 | from app.models.user import User
13 | from app.schemas.team import TeamMember, TeamMemberRole
14 |
15 |
16 | async def init_db():
17 | await init_beanie(
18 | database=mongo_session[settings.MongoDB_DB_NAME],
19 | document_models=[User, LabelTask, Data, Team, TeamInvitationLink, Record, File], # type: ignore
20 | )
21 |
22 | # 创建默认团队
23 | team = await crud.team.query(team_id=schemas.team.DEFAULT_TEAM_ID).first_or_none()
24 | if not team:
25 | team = await crud.team.create(
26 | obj_in=TeamCreate(
27 | team_id=schemas.team.DEFAULT_TEAM_ID,
28 | name="默认团队",
29 | owner="",
30 | owner_cellphone="",
31 | )
32 | )
33 | users = await crud.user.query().to_list()
34 | team.users = [
35 | TeamMember(user_id=user.user_id, name="", role=TeamMemberRole.USER)
36 | for user in users
37 | ]
38 | team.user_count = len(users)
39 | await team.save() # type: ignore
40 |
41 | await redis_session.ping()
42 |
43 |
44 | async def close_db():
45 | mongo_session.close()
46 | await redis_session.close()
47 |
--------------------------------------------------------------------------------
/backend/app/db/session.py:
--------------------------------------------------------------------------------
1 | from motor import motor_asyncio
2 | from redis.asyncio import Redis
3 |
4 | from app.core.config import settings
5 |
6 | # mongo session
7 | mongo_session = motor_asyncio.AsyncIOMotorClient(settings.MongoDB_DSN)
8 |
9 | # redis session
10 | redis_session = Redis.from_url(str(settings.REDIS_DSN))
11 |
12 |
--------------------------------------------------------------------------------
/backend/app/gunicorn_conf.py:
--------------------------------------------------------------------------------
1 | import json
2 | import multiprocessing
3 | import os
4 |
5 | workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1")
6 | max_workers_str = os.getenv("MAX_WORKERS")
7 | use_max_workers = None
8 | if max_workers_str:
9 | use_max_workers = int(max_workers_str)
10 | web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
11 |
12 | host = os.getenv("HOST", "0.0.0.0")
13 | port = os.getenv("PORT", "8080")
14 | bind_env = os.getenv("BIND", None)
15 | use_loglevel = os.getenv("LOG_LEVEL", "info")
16 | if bind_env:
17 | use_bind = bind_env
18 | else:
19 | use_bind = f"{host}:{port}"
20 |
21 | cores = multiprocessing.cpu_count()
22 | workers_per_core = float(workers_per_core_str)
23 | default_web_concurrency = workers_per_core * cores
24 | if web_concurrency_str:
25 | web_concurrency = int(web_concurrency_str)
26 | assert web_concurrency > 0
27 | else:
28 | web_concurrency = max(int(default_web_concurrency), 2)
29 | if use_max_workers:
30 | web_concurrency = min(web_concurrency, use_max_workers)
31 | accesslog_var = os.getenv("ACCESS_LOG", "-")
32 | use_accesslog = accesslog_var or None
33 | errorlog_var = os.getenv("ERROR_LOG", "-")
34 | use_errorlog = errorlog_var or None
35 | graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120")
36 | timeout_str = os.getenv("TIMEOUT", "120")
37 | keepalive_str = os.getenv("KEEP_ALIVE", "5")
38 |
39 | # Gunicorn config variables
40 | loglevel = use_loglevel
41 | workers = 1
42 | bind = use_bind
43 | errorlog = use_errorlog
44 | worker_tmp_dir = "/dev/shm"
45 | accesslog = use_accesslog
46 | graceful_timeout = int(graceful_timeout_str)
47 | timeout = int(timeout_str)
48 | keepalive = int(keepalive_str)
49 |
50 |
51 | # For debugging and testing
52 | log_data = {
53 | "loglevel": loglevel,
54 | "workers": workers,
55 | "bind": bind,
56 | "graceful_timeout": graceful_timeout,
57 | "timeout": timeout,
58 | "keepalive": keepalive,
59 | "errorlog": errorlog,
60 | "accesslog": accesslog,
61 | # Additional, non-gunicorn variables
62 | "workers_per_core": workers_per_core,
63 | "use_max_workers": use_max_workers,
64 | "host": host,
65 | "port": port,
66 | }
67 | print(json.dumps(log_data))
68 |
--------------------------------------------------------------------------------
/backend/app/logger/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/opendatalab/LabelLLM/11f2a221f73c9625ec820bec8fc158c1ae16b8a9/backend/app/logger/__init__.py
--------------------------------------------------------------------------------
/backend/app/logger/logger.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | from loguru import logger
4 |
5 |
6 | def init_logger():
7 | logger.remove()
8 |
9 | # scheduler log
10 | logger.add(
11 | sys.stdout,
12 | colorize=True,
13 | format="
暂无数据
13 |(fn: (...args: P) => Promise