├── dify_app.py ├── poetry.toml ├── .dockerignore ├── mypy.ini ├── app.py ├── pytest.ini ├── Dockerfile ├── .ruff.toml ├── app_factory.py ├── pyproject.toml ├── .env.example └── commands.py /dify_app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | 4 | class DifyApp(Flask): 5 | pass 6 | -------------------------------------------------------------------------------- /poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | create = true 4 | prefer-active-python = true -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .env 2 | *.env.* 3 | 4 | storage/privkeys/* 5 | 6 | # Logs 7 | logs 8 | *.log* 9 | 10 | # jetbrains 11 | .idea 12 | 13 | # venv 14 | .venv -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | warn_return_any = True 3 | warn_unused_configs = True 4 | check_untyped_defs = True 5 | exclude = (?x)( 6 | core/tools/provider/builtin/ 7 | | core/model_runtime/model_providers/ 8 | | tests/ 9 | | migrations/ 10 | ) -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | 5 | def is_db_command(): 6 | if len(sys.argv) > 1 and sys.argv[0].endswith("flask") and sys.argv[1] == "db": 7 | return True 8 | return False 9 | 10 | 11 | # create app 12 | if is_db_command(): 13 | from app_factory import create_migrations_app 14 | 15 | app = create_migrations_app() 16 | else: 17 | # It seems that JetBrains Python debugger does not work well with gevent, 18 | # so we need to disable gevent in debug mode. 19 | # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent. 20 | if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}: 21 | from gevent import monkey # type: ignore 22 | 23 | # gevent 24 | monkey.patch_all() 25 | 26 | from grpc.experimental import gevent as grpc_gevent # type: ignore 27 | 28 | # grpc gevent 29 | grpc_gevent.init_gevent() 30 | 31 | import psycogreen.gevent # type: ignore 32 | 33 | psycogreen.gevent.patch_psycopg() 34 | 35 | from app_factory import create_app 36 | 37 | app = create_app() 38 | celery = app.extensions["celery"] 39 | 40 | if __name__ == "__main__": 41 | app.run(host="0.0.0.0", port=5001) 42 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | env = 3 | ANTHROPIC_API_KEY = sk-ant-api11-IamNotARealKeyJustForMockTestKawaiiiiiiiiii-NotBaka-ASkksz 4 | AZURE_OPENAI_API_BASE = https://difyai-openai.openai.azure.com 5 | AZURE_OPENAI_API_KEY = xxxxb1707exxxxxxxxxxaaxxxxxf94 6 | CHATGLM_API_BASE = http://a.abc.com:11451 7 | CODE_EXECUTION_API_KEY = dify-sandbox 8 | CODE_EXECUTION_ENDPOINT = http://127.0.0.1:8194 9 | CODE_MAX_STRING_LENGTH = 80000 10 | FIRECRAWL_API_KEY = fc- 11 | FIREWORKS_API_KEY = fw_aaaaaaaaaaaaaaaaaaaa 12 | GOOGLE_API_KEY = abcdefghijklmnopqrstuvwxyz 13 | HUGGINGFACE_API_KEY = hf-awuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwu 14 | HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL = c 15 | HUGGINGFACE_TEXT2TEXT_GEN_ENDPOINT_URL = b 16 | HUGGINGFACE_TEXT_GEN_ENDPOINT_URL = a 17 | MIXEDBREAD_API_KEY = mk-aaaaaaaaaaaaaaaaaaaa 18 | MOCK_SWITCH = true 19 | NOMIC_API_KEY = nk-aaaaaaaaaaaaaaaaaaaa 20 | OPENAI_API_KEY = sk-IamNotARealKeyJustForMockTestKawaiiiiiiiiii 21 | TEI_EMBEDDING_SERVER_URL = http://a.abc.com:11451 22 | TEI_RERANK_SERVER_URL = http://a.abc.com:11451 23 | TEI_API_KEY = ttttttttttttttt 24 | UPSTAGE_API_KEY = up-aaaaaaaaaaaaaaaaaaaa 25 | VOYAGE_API_KEY = va-aaaaaaaaaaaaaaaaaaaa 26 | XINFERENCE_CHAT_MODEL_UID = chat 27 | XINFERENCE_EMBEDDINGS_MODEL_UID = embedding 28 | XINFERENCE_GENERATION_MODEL_UID = generate 29 | XINFERENCE_RERANK_MODEL_UID = rerank 30 | XINFERENCE_SERVER_URL = http://a.abc.com:11451 31 | GITEE_AI_API_KEY = aaaaaaaaaaaaaaaaaaaa 32 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # base image 2 | FROM python:3.12-slim-bookworm AS base 3 | 4 | WORKDIR /app/api 5 | 6 | # Install Poetry 7 | ENV POETRY_VERSION=1.8.4 8 | 9 | # if you located in China, you can use aliyun mirror to speed up 10 | # RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/ 11 | 12 | RUN pip install --no-cache-dir poetry==${POETRY_VERSION} 13 | 14 | # Configure Poetry 15 | ENV POETRY_CACHE_DIR=/tmp/poetry_cache 16 | ENV POETRY_NO_INTERACTION=1 17 | ENV POETRY_VIRTUALENVS_IN_PROJECT=true 18 | ENV POETRY_VIRTUALENVS_CREATE=true 19 | ENV POETRY_REQUESTS_TIMEOUT=15 20 | 21 | FROM base AS packages 22 | 23 | # if you located in China, you can use aliyun mirror to speed up 24 | # RUN sed -i 's@deb.debian.org@mirrors.aliyun.com@g' /etc/apt/sources.list.d/debian.sources 25 | 26 | RUN apt-get update \ 27 | && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev 28 | 29 | # Install Python dependencies 30 | COPY pyproject.toml poetry.lock ./ 31 | RUN poetry install --sync --no-cache --no-root 32 | 33 | # production stage 34 | FROM base AS production 35 | 36 | ENV FLASK_APP=app.py 37 | ENV EDITION=SELF_HOSTED 38 | ENV DEPLOY_ENV=PRODUCTION 39 | ENV CONSOLE_API_URL=http://127.0.0.1:5001 40 | ENV CONSOLE_WEB_URL=http://127.0.0.1:3000 41 | ENV SERVICE_API_URL=http://127.0.0.1:5001 42 | ENV APP_WEB_URL=http://127.0.0.1:3000 43 | 44 | EXPOSE 5001 45 | 46 | # set timezone 47 | ENV TZ=UTC 48 | 49 | WORKDIR /app/api 50 | 51 | RUN apt-get update \ 52 | && apt-get install -y --no-install-recommends curl nodejs libgmp-dev libmpfr-dev libmpc-dev \ 53 | # if you located in China, you can use aliyun mirror to speed up 54 | # && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list \ 55 | && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \ 56 | && apt-get update \ 57 | # For Security 58 | && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.19+dfsg-1 perl=5.40.0-8 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \ 59 | # install a chinese font to support the use of tools like matplotlib 60 | && apt-get install -y fonts-noto-cjk \ 61 | && apt-get autoremove -y \ 62 | && rm -rf /var/lib/apt/lists/* 63 | 64 | # Copy Python environment and packages 65 | ENV VIRTUAL_ENV=/app/api/.venv 66 | COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV} 67 | ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" 68 | 69 | # Download nltk data 70 | RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')" 71 | 72 | # Copy source code 73 | COPY . /app/api/ 74 | 75 | # Copy entrypoint 76 | COPY docker/entrypoint.sh /entrypoint.sh 77 | RUN chmod +x /entrypoint.sh 78 | 79 | 80 | ARG COMMIT_SHA 81 | ENV COMMIT_SHA=${COMMIT_SHA} 82 | 83 | ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] 84 | -------------------------------------------------------------------------------- /.ruff.toml: -------------------------------------------------------------------------------- 1 | exclude = [ 2 | "migrations/*", 3 | ] 4 | line-length = 120 5 | 6 | [format] 7 | quote-style = "double" 8 | 9 | [lint] 10 | preview = true 11 | select = [ 12 | "B", # flake8-bugbear rules 13 | "C4", # flake8-comprehensions 14 | "E", # pycodestyle E rules 15 | "F", # pyflakes rules 16 | "FURB", # refurb rules 17 | "I", # isort rules 18 | "N", # pep8-naming 19 | "PT", # flake8-pytest-style rules 20 | "PLC0208", # iteration-over-set 21 | "PLC2801", # unnecessary-dunder-call 22 | "PLC0414", # useless-import-alias 23 | "PLE0604", # invalid-all-object 24 | "PLE0605", # invalid-all-format 25 | "PLR0402", # manual-from-import 26 | "PLR1711", # useless-return 27 | "PLR1714", # repeated-equality-comparison 28 | "RUF013", # implicit-optional 29 | "RUF019", # unnecessary-key-check 30 | "RUF100", # unused-noqa 31 | "RUF101", # redirected-noqa 32 | "RUF200", # invalid-pyproject-toml 33 | "RUF022", # unsorted-dunder-all 34 | "S506", # unsafe-yaml-load 35 | "SIM", # flake8-simplify rules 36 | "TRY400", # error-instead-of-exception 37 | "TRY401", # verbose-log-message 38 | "UP", # pyupgrade rules 39 | "W191", # tab-indentation 40 | "W605", # invalid-escape-sequence 41 | ] 42 | 43 | ignore = [ 44 | "E402", # module-import-not-at-top-of-file 45 | "E711", # none-comparison 46 | "E712", # true-false-comparison 47 | "E721", # type-comparison 48 | "E722", # bare-except 49 | "E731", # lambda-assignment 50 | "F821", # undefined-name 51 | "F841", # unused-variable 52 | "FURB113", # repeated-append 53 | "FURB152", # math-constant 54 | "UP007", # non-pep604-annotation 55 | "UP032", # f-string 56 | "B005", # strip-with-multi-characters 57 | "B006", # mutable-argument-default 58 | "B007", # unused-loop-control-variable 59 | "B026", # star-arg-unpacking-after-keyword-arg 60 | "B904", # raise-without-from-inside-except 61 | "B905", # zip-without-explicit-strict 62 | "N806", # non-lowercase-variable-in-function 63 | "N815", # mixed-case-variable-in-class-scope 64 | "PT011", # pytest-raises-too-broad 65 | "SIM102", # collapsible-if 66 | "SIM103", # needless-bool 67 | "SIM105", # suppressible-exception 68 | "SIM107", # return-in-try-except-finally 69 | "SIM108", # if-else-block-instead-of-if-exp 70 | "SIM113", # enumerate-for-loop 71 | "SIM117", # multiple-with-statements 72 | "SIM210", # if-expr-with-true-false 73 | ] 74 | 75 | [lint.per-file-ignores] 76 | "__init__.py" = [ 77 | "F401", # unused-import 78 | "F811", # redefined-while-unused 79 | ] 80 | "configs/*" = [ 81 | "N802", # invalid-function-name 82 | ] 83 | "libs/gmpy2_pkcs10aep_cipher.py" = [ 84 | "N803", # invalid-argument-name 85 | ] 86 | "tests/*" = [ 87 | "F811", # redefined-while-unused 88 | ] 89 | 90 | [lint.pyflakes] 91 | allowed-unused-imports = [ 92 | "_pytest.monkeypatch", 93 | "tests.integration_tests", 94 | "tests.unit_tests", 95 | ] 96 | -------------------------------------------------------------------------------- /app_factory.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | from configs import dify_config 5 | from dify_app import DifyApp 6 | 7 | 8 | # ---------------------------- 9 | # Application Factory Function 10 | # ---------------------------- 11 | def create_flask_app_with_configs() -> DifyApp: 12 | """ 13 | create a raw flask app 14 | with configs loaded from .env file 15 | """ 16 | dify_app = DifyApp(__name__) 17 | dify_app.config.from_mapping(dify_config.model_dump()) 18 | 19 | return dify_app 20 | 21 | 22 | def create_app() -> DifyApp: 23 | start_time = time.perf_counter() 24 | app = create_flask_app_with_configs() 25 | initialize_extensions(app) 26 | end_time = time.perf_counter() 27 | if dify_config.DEBUG: 28 | logging.info(f"Finished create_app ({round((end_time - start_time) * 1000, 2)} ms)") 29 | return app 30 | 31 | 32 | def initialize_extensions(app: DifyApp): 33 | from extensions import ( 34 | ext_app_metrics, 35 | ext_blueprints, 36 | ext_celery, 37 | ext_code_based_extension, 38 | ext_commands, 39 | ext_compress, 40 | ext_database, 41 | ext_hosting_provider, 42 | ext_import_modules, 43 | ext_logging, 44 | ext_login, 45 | ext_mail, 46 | ext_migrate, 47 | ext_proxy_fix, 48 | ext_redis, 49 | ext_sentry, 50 | ext_set_secretkey, 51 | ext_storage, 52 | ext_timezone, 53 | ext_warnings, 54 | ) 55 | 56 | extensions = [ 57 | ext_timezone, 58 | ext_logging, 59 | ext_warnings, 60 | ext_import_modules, 61 | ext_set_secretkey, 62 | ext_compress, 63 | ext_code_based_extension, 64 | ext_database, 65 | ext_app_metrics, 66 | ext_migrate, 67 | ext_redis, 68 | ext_storage, 69 | ext_celery, 70 | ext_login, 71 | ext_mail, 72 | ext_hosting_provider, 73 | ext_sentry, 74 | ext_proxy_fix, 75 | ext_blueprints, 76 | ext_commands, 77 | ] 78 | for ext in extensions: 79 | short_name = ext.__name__.split(".")[-1] 80 | is_enabled = ext.is_enabled() if hasattr(ext, "is_enabled") else True 81 | if not is_enabled: 82 | if dify_config.DEBUG: 83 | logging.info(f"Skipped {short_name}") 84 | continue 85 | 86 | start_time = time.perf_counter() 87 | ext.init_app(app) 88 | end_time = time.perf_counter() 89 | if dify_config.DEBUG: 90 | logging.info(f"Loaded {short_name} ({round((end_time - start_time) * 1000, 2)} ms)") 91 | 92 | 93 | def create_migrations_app(): 94 | app = create_flask_app_with_configs() 95 | from extensions import ext_database, ext_migrate 96 | 97 | # Initialize only required extensions 98 | ext_database.init_app(app) 99 | ext_migrate.init_app(app) 100 | 101 | return app 102 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "dify-api" 3 | requires-python = ">=3.11,<3.13" 4 | 5 | [build-system] 6 | requires = ["poetry-core"] 7 | build-backend = "poetry.core.masonry.api" 8 | 9 | [tool.poetry] 10 | package-mode = false 11 | 12 | ############################################################ 13 | # [ Main ] Dependency group 14 | ############################################################ 15 | 16 | [tool.poetry.dependencies] 17 | anthropic = "~0.23.1" 18 | authlib = "1.3.1" 19 | azure-ai-inference = "~1.0.0b3" 20 | azure-ai-ml = "~1.20.0" 21 | azure-identity = "1.16.1" 22 | beautifulsoup4 = "4.12.2" 23 | boto3 = "1.35.74" 24 | bs4 = "~0.0.1" 25 | cachetools = "~5.3.0" 26 | celery = "~5.4.0" 27 | chardet = "~5.1.0" 28 | cohere = "~5.2.4" 29 | dashscope = { version = "~1.17.0", extras = ["tokenizer"] } 30 | fal-client = "0.5.6" 31 | flask = "~3.1.0" 32 | flask-compress = "~1.17" 33 | flask-cors = "~4.0.0" 34 | flask-login = "~0.6.3" 35 | flask-migrate = "~4.0.7" 36 | flask-restful = "~0.3.10" 37 | flask-sqlalchemy = "~3.1.1" 38 | gevent = "~24.11.1" 39 | gmpy2 = "~2.2.1" 40 | google-ai-generativelanguage = "0.6.9" 41 | google-api-core = "2.18.0" 42 | google-api-python-client = "2.90.0" 43 | google-auth = "2.29.0" 44 | google-auth-httplib2 = "0.2.0" 45 | google-cloud-aiplatform = "1.49.0" 46 | google-generativeai = "0.8.1" 47 | googleapis-common-protos = "1.63.0" 48 | gunicorn = "~23.0.0" 49 | httpx = { version = "~0.27.0", extras = ["socks"] } 50 | huggingface-hub = "~0.16.4" 51 | jieba = "0.42.1" 52 | langfuse = "~2.51.3" 53 | langsmith = "~0.1.77" 54 | mailchimp-transactional = "~1.0.50" 55 | markdown = "~3.5.1" 56 | nomic = "~3.1.2" 57 | novita-client = "~0.5.7" 58 | numpy = "~1.26.4" 59 | oci = "~2.135.1" 60 | openai = "~1.52.0" 61 | openpyxl = "~3.1.5" 62 | pandas = { version = "~2.2.2", extras = ["performance", "excel"] } 63 | pandas-stubs = "~2.2.3.241009" 64 | psycogreen = "~1.0.2" 65 | psycopg2-binary = "~2.9.6" 66 | pycryptodome = "3.19.1" 67 | pydantic = "~2.9.2" 68 | pydantic-settings = "~2.6.0" 69 | pydantic_extra_types = "~2.9.0" 70 | pyjwt = "~2.8.0" 71 | pypdfium2 = "~4.30.0" 72 | python = ">=3.11,<3.13" 73 | python-docx = "~1.1.0" 74 | python-dotenv = "1.0.1" 75 | pyyaml = "~6.0.1" 76 | readabilipy = "0.2.0" 77 | redis = { version = "~5.0.3", extras = ["hiredis"] } 78 | replicate = "~0.22.0" 79 | resend = "~0.7.0" 80 | sagemaker = "~2.231.0" 81 | scikit-learn = "~1.5.1" 82 | sentry-sdk = { version = "~1.44.1", extras = ["flask"] } 83 | sqlalchemy = "~2.0.29" 84 | starlette = "0.41.0" 85 | tencentcloud-sdk-python-hunyuan = "~3.0.1294" 86 | tiktoken = "~0.8.0" 87 | tokenizers = "~0.15.0" 88 | transformers = "~4.35.0" 89 | types-pytz = "~2024.2.0.20241003" 90 | unstructured = { version = "~0.16.1", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] } 91 | validators = "0.21.0" 92 | volcengine-python-sdk = {extras = ["ark"], version = "~1.0.98"} 93 | websocket-client = "~1.7.0" 94 | xinference-client = "0.15.2" 95 | yarl = "~1.18.3" 96 | youtube-transcript-api = "~0.6.2" 97 | zhipuai = "~2.1.5" 98 | # Before adding new dependency, consider place it in alphabet order (a-z) and suitable group. 99 | 100 | ############################################################ 101 | # [ Indirect ] dependency group 102 | # Related transparent dependencies with pinned version 103 | # required by main implementations 104 | ############################################################ 105 | [tool.poetry.group.indirect.dependencies] 106 | kaleido = "0.2.1" 107 | rank-bm25 = "~0.2.2" 108 | safetensors = "~0.4.3" 109 | 110 | ############################################################ 111 | # [ Tools ] dependency group 112 | ############################################################ 113 | [tool.poetry.group.tools.dependencies] 114 | arxiv = "2.1.0" 115 | cloudscraper = "1.2.71" 116 | duckduckgo-search = "~6.3.0" 117 | jsonpath-ng = "1.6.1" 118 | matplotlib = "~3.8.2" 119 | mplfonts = "~0.0.8" 120 | newspaper3k = "0.2.8" 121 | nltk = "3.9.1" 122 | numexpr = "~2.9.0" 123 | pydub = "~0.25.1" 124 | qrcode = "~7.4.2" 125 | twilio = "~9.0.4" 126 | vanna = { version = "0.7.5", extras = ["postgres", "mysql", "clickhouse", "duckdb", "oracle"] } 127 | wikipedia = "1.4.0" 128 | yfinance = "~0.2.40" 129 | 130 | ############################################################ 131 | # [ Storage ] dependency group 132 | # Required for storage clients 133 | ############################################################ 134 | [tool.poetry.group.storage.dependencies] 135 | azure-storage-blob = "12.13.0" 136 | bce-python-sdk = "~0.9.23" 137 | cos-python-sdk-v5 = "1.9.30" 138 | esdk-obs-python = "3.24.6.1" 139 | google-cloud-storage = "2.16.0" 140 | opendal = "~0.45.12" 141 | oss2 = "2.18.5" 142 | supabase = "~2.8.1" 143 | tos = "~2.7.1" 144 | 145 | ############################################################ 146 | # [ VDB ] dependency group 147 | # Required by vector store clients 148 | ############################################################ 149 | [tool.poetry.group.vdb.dependencies] 150 | alibabacloud_gpdb20160503 = "~3.8.0" 151 | alibabacloud_tea_openapi = "~0.3.9" 152 | chromadb = "0.5.20" 153 | clickhouse-connect = "~0.7.16" 154 | couchbase = "~4.3.0" 155 | elasticsearch = "8.14.0" 156 | opensearch-py = "2.4.0" 157 | oracledb = "~2.2.1" 158 | pgvecto-rs = { version = "~0.2.1", extras = ['sqlalchemy'] } 159 | pgvector = "0.2.5" 160 | pymilvus = "~2.5.0" 161 | pymochow = "1.3.1" 162 | pyobvector = "~0.1.6" 163 | qdrant-client = "1.7.3" 164 | tcvectordb = "1.3.2" 165 | tidb-vector = "0.0.9" 166 | upstash-vector = "0.6.0" 167 | volcengine-compat = "~1.0.156" 168 | weaviate-client = "~3.21.0" 169 | 170 | ############################################################ 171 | # [ Dev ] dependency group 172 | # Required for development and running tests 173 | ############################################################ 174 | [tool.poetry.group.dev] 175 | optional = true 176 | [tool.poetry.group.dev.dependencies] 177 | coverage = "~7.2.4" 178 | faker = "~32.1.0" 179 | mypy = "~1.13.0" 180 | pytest = "~8.3.2" 181 | pytest-benchmark = "~4.0.0" 182 | pytest-env = "~1.1.3" 183 | pytest-mock = "~3.14.0" 184 | 185 | ############################################################ 186 | # [ Lint ] dependency group 187 | # Required for code style linting 188 | ############################################################ 189 | [tool.poetry.group.lint] 190 | optional = true 191 | [tool.poetry.group.lint.dependencies] 192 | dotenv-linter = "~0.5.0" 193 | ruff = "~0.8.1" 194 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # Your App secret key will be used for securely signing the session cookie 2 | # Make sure you are changing this key for your deployment with a strong key. 3 | # You can generate a strong key using `openssl rand -base64 42`. 4 | # Alternatively you can set it with `SECRET_KEY` environment variable. 5 | SECRET_KEY= 6 | 7 | # Console API base URL 8 | CONSOLE_API_URL=http://127.0.0.1:5001 9 | CONSOLE_WEB_URL=http://127.0.0.1:3000 10 | 11 | # Service API base URL 12 | SERVICE_API_URL=http://127.0.0.1:5001 13 | 14 | # Web APP base URL 15 | APP_WEB_URL=http://127.0.0.1:3000 16 | 17 | # Files URL 18 | FILES_URL=http://127.0.0.1:5001 19 | 20 | # The time in seconds after the signature is rejected 21 | FILES_ACCESS_TIMEOUT=300 22 | 23 | # Access token expiration time in minutes 24 | ACCESS_TOKEN_EXPIRE_MINUTES=60 25 | 26 | # Refresh token expiration time in days 27 | REFRESH_TOKEN_EXPIRE_DAYS=30 28 | 29 | # celery configuration 30 | CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1 31 | 32 | # redis configuration 33 | REDIS_HOST=localhost 34 | REDIS_PORT=6379 35 | REDIS_USERNAME= 36 | REDIS_PASSWORD=difyai123456 37 | REDIS_USE_SSL=false 38 | REDIS_DB=0 39 | 40 | # redis Sentinel configuration. 41 | REDIS_USE_SENTINEL=false 42 | REDIS_SENTINELS= 43 | REDIS_SENTINEL_SERVICE_NAME= 44 | REDIS_SENTINEL_USERNAME= 45 | REDIS_SENTINEL_PASSWORD= 46 | REDIS_SENTINEL_SOCKET_TIMEOUT=0.1 47 | 48 | # redis Cluster configuration. 49 | REDIS_USE_CLUSTERS=false 50 | REDIS_CLUSTERS= 51 | REDIS_CLUSTERS_PASSWORD= 52 | 53 | # PostgreSQL database configuration 54 | DB_USERNAME=postgres 55 | DB_PASSWORD=difyai123456 56 | DB_HOST=localhost 57 | DB_PORT=5432 58 | DB_DATABASE=dify 59 | 60 | # Storage configuration 61 | # use for store upload files, private keys... 62 | # storage type: opendal, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase 63 | STORAGE_TYPE=opendal 64 | 65 | # Apache OpenDAL storage configuration, refer to https://github.com/apache/opendal 66 | OPENDAL_SCHEME=fs 67 | OPENDAL_FS_ROOT=storage 68 | 69 | # S3 Storage configuration 70 | S3_USE_AWS_MANAGED_IAM=false 71 | S3_ENDPOINT=https://your-bucket-name.storage.s3.cloudflare.com 72 | S3_BUCKET_NAME=your-bucket-name 73 | S3_ACCESS_KEY=your-access-key 74 | S3_SECRET_KEY=your-secret-key 75 | S3_REGION=your-region 76 | 77 | # Azure Blob Storage configuration 78 | AZURE_BLOB_ACCOUNT_NAME=your-account-name 79 | AZURE_BLOB_ACCOUNT_KEY=your-account-key 80 | AZURE_BLOB_CONTAINER_NAME=your-container-name 81 | AZURE_BLOB_ACCOUNT_URL=https://.blob.core.windows.net 82 | 83 | # Aliyun oss Storage configuration 84 | ALIYUN_OSS_BUCKET_NAME=your-bucket-name 85 | ALIYUN_OSS_ACCESS_KEY=your-access-key 86 | ALIYUN_OSS_SECRET_KEY=your-secret-key 87 | ALIYUN_OSS_ENDPOINT=your-endpoint 88 | ALIYUN_OSS_AUTH_VERSION=v1 89 | ALIYUN_OSS_REGION=your-region 90 | # Don't start with '/'. OSS doesn't support leading slash in object names. 91 | ALIYUN_OSS_PATH=your-path 92 | 93 | # Google Storage configuration 94 | GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name 95 | GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string 96 | 97 | # Tencent COS Storage configuration 98 | TENCENT_COS_BUCKET_NAME=your-bucket-name 99 | TENCENT_COS_SECRET_KEY=your-secret-key 100 | TENCENT_COS_SECRET_ID=your-secret-id 101 | TENCENT_COS_REGION=your-region 102 | TENCENT_COS_SCHEME=your-scheme 103 | 104 | # Huawei OBS Storage Configuration 105 | HUAWEI_OBS_BUCKET_NAME=your-bucket-name 106 | HUAWEI_OBS_SECRET_KEY=your-secret-key 107 | HUAWEI_OBS_ACCESS_KEY=your-access-key 108 | HUAWEI_OBS_SERVER=your-server-url 109 | 110 | # Baidu OBS Storage Configuration 111 | BAIDU_OBS_BUCKET_NAME=your-bucket-name 112 | BAIDU_OBS_SECRET_KEY=your-secret-key 113 | BAIDU_OBS_ACCESS_KEY=your-access-key 114 | BAIDU_OBS_ENDPOINT=your-server-url 115 | 116 | # OCI Storage configuration 117 | OCI_ENDPOINT=your-endpoint 118 | OCI_BUCKET_NAME=your-bucket-name 119 | OCI_ACCESS_KEY=your-access-key 120 | OCI_SECRET_KEY=your-secret-key 121 | OCI_REGION=your-region 122 | 123 | # Volcengine tos Storage configuration 124 | VOLCENGINE_TOS_ENDPOINT=your-endpoint 125 | VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name 126 | VOLCENGINE_TOS_ACCESS_KEY=your-access-key 127 | VOLCENGINE_TOS_SECRET_KEY=your-secret-key 128 | VOLCENGINE_TOS_REGION=your-region 129 | 130 | # Supabase Storage Configuration 131 | SUPABASE_BUCKET_NAME=your-bucket-name 132 | SUPABASE_API_KEY=your-access-key 133 | SUPABASE_URL=your-server-url 134 | 135 | # CORS configuration 136 | WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* 137 | CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* 138 | 139 | # Vector database configuration 140 | # support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase 141 | VECTOR_STORE=weaviate 142 | 143 | # Weaviate configuration 144 | WEAVIATE_ENDPOINT=http://localhost:8080 145 | WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih 146 | WEAVIATE_GRPC_ENABLED=false 147 | WEAVIATE_BATCH_SIZE=100 148 | 149 | # Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode 150 | QDRANT_URL=http://localhost:6333 151 | QDRANT_API_KEY=difyai123456 152 | QDRANT_CLIENT_TIMEOUT=20 153 | QDRANT_GRPC_ENABLED=false 154 | QDRANT_GRPC_PORT=6334 155 | 156 | #Couchbase configuration 157 | COUCHBASE_CONNECTION_STRING=127.0.0.1 158 | COUCHBASE_USER=Administrator 159 | COUCHBASE_PASSWORD=password 160 | COUCHBASE_BUCKET_NAME=Embeddings 161 | COUCHBASE_SCOPE_NAME=_default 162 | 163 | # Milvus configuration 164 | MILVUS_URI=http://127.0.0.1:19530 165 | MILVUS_TOKEN= 166 | MILVUS_USER=root 167 | MILVUS_PASSWORD=Milvus 168 | 169 | # MyScale configuration 170 | MYSCALE_HOST=127.0.0.1 171 | MYSCALE_PORT=8123 172 | MYSCALE_USER=default 173 | MYSCALE_PASSWORD= 174 | MYSCALE_DATABASE=default 175 | MYSCALE_FTS_PARAMS= 176 | 177 | # Relyt configuration 178 | RELYT_HOST=127.0.0.1 179 | RELYT_PORT=5432 180 | RELYT_USER=postgres 181 | RELYT_PASSWORD=postgres 182 | RELYT_DATABASE=postgres 183 | 184 | # Tencent configuration 185 | TENCENT_VECTOR_DB_URL=http://127.0.0.1 186 | TENCENT_VECTOR_DB_API_KEY=dify 187 | TENCENT_VECTOR_DB_TIMEOUT=30 188 | TENCENT_VECTOR_DB_USERNAME=dify 189 | TENCENT_VECTOR_DB_DATABASE=dify 190 | TENCENT_VECTOR_DB_SHARD=1 191 | TENCENT_VECTOR_DB_REPLICAS=2 192 | 193 | # ElasticSearch configuration 194 | ELASTICSEARCH_HOST=127.0.0.1 195 | ELASTICSEARCH_PORT=9200 196 | ELASTICSEARCH_USERNAME=elastic 197 | ELASTICSEARCH_PASSWORD=elastic 198 | 199 | # PGVECTO_RS configuration 200 | PGVECTO_RS_HOST=localhost 201 | PGVECTO_RS_PORT=5431 202 | PGVECTO_RS_USER=postgres 203 | PGVECTO_RS_PASSWORD=difyai123456 204 | PGVECTO_RS_DATABASE=postgres 205 | 206 | # PGVector configuration 207 | PGVECTOR_HOST=127.0.0.1 208 | PGVECTOR_PORT=5433 209 | PGVECTOR_USER=postgres 210 | PGVECTOR_PASSWORD=postgres 211 | PGVECTOR_DATABASE=postgres 212 | PGVECTOR_MIN_CONNECTION=1 213 | PGVECTOR_MAX_CONNECTION=5 214 | 215 | # Tidb Vector configuration 216 | TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com 217 | TIDB_VECTOR_PORT=4000 218 | TIDB_VECTOR_USER=xxx.root 219 | TIDB_VECTOR_PASSWORD=xxxxxx 220 | TIDB_VECTOR_DATABASE=dify 221 | 222 | # Tidb on qdrant configuration 223 | TIDB_ON_QDRANT_URL=http://127.0.0.1 224 | TIDB_ON_QDRANT_API_KEY=dify 225 | TIDB_ON_QDRANT_CLIENT_TIMEOUT=20 226 | TIDB_ON_QDRANT_GRPC_ENABLED=false 227 | TIDB_ON_QDRANT_GRPC_PORT=6334 228 | TIDB_PUBLIC_KEY=dify 229 | TIDB_PRIVATE_KEY=dify 230 | TIDB_API_URL=http://127.0.0.1 231 | TIDB_IAM_API_URL=http://127.0.0.1 232 | TIDB_REGION=regions/aws-us-east-1 233 | TIDB_PROJECT_ID=dify 234 | TIDB_SPEND_LIMIT=100 235 | 236 | # Chroma configuration 237 | CHROMA_HOST=127.0.0.1 238 | CHROMA_PORT=8000 239 | CHROMA_TENANT=default_tenant 240 | CHROMA_DATABASE=default_database 241 | CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider 242 | CHROMA_AUTH_CREDENTIALS=difyai123456 243 | 244 | # AnalyticDB configuration 245 | ANALYTICDB_KEY_ID=your-ak 246 | ANALYTICDB_KEY_SECRET=your-sk 247 | ANALYTICDB_REGION_ID=cn-hangzhou 248 | ANALYTICDB_INSTANCE_ID=gp-ab123456 249 | ANALYTICDB_ACCOUNT=testaccount 250 | ANALYTICDB_PASSWORD=testpassword 251 | ANALYTICDB_NAMESPACE=dify 252 | ANALYTICDB_NAMESPACE_PASSWORD=difypassword 253 | ANALYTICDB_HOST=gp-test.aliyuncs.com 254 | ANALYTICDB_PORT=5432 255 | ANALYTICDB_MIN_CONNECTION=1 256 | ANALYTICDB_MAX_CONNECTION=5 257 | 258 | # OpenSearch configuration 259 | OPENSEARCH_HOST=127.0.0.1 260 | OPENSEARCH_PORT=9200 261 | OPENSEARCH_USER=admin 262 | OPENSEARCH_PASSWORD=admin 263 | OPENSEARCH_SECURE=true 264 | 265 | # Baidu configuration 266 | BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287 267 | BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000 268 | BAIDU_VECTOR_DB_ACCOUNT=root 269 | BAIDU_VECTOR_DB_API_KEY=dify 270 | BAIDU_VECTOR_DB_DATABASE=dify 271 | BAIDU_VECTOR_DB_SHARD=1 272 | BAIDU_VECTOR_DB_REPLICAS=3 273 | 274 | # Upstash configuration 275 | UPSTASH_VECTOR_URL=your-server-url 276 | UPSTASH_VECTOR_TOKEN=your-access-token 277 | 278 | # ViKingDB configuration 279 | VIKINGDB_ACCESS_KEY=your-ak 280 | VIKINGDB_SECRET_KEY=your-sk 281 | VIKINGDB_REGION=cn-shanghai 282 | VIKINGDB_HOST=api-vikingdb.xxx.volces.com 283 | VIKINGDB_SCHEMA=http 284 | VIKINGDB_CONNECTION_TIMEOUT=30 285 | VIKINGDB_SOCKET_TIMEOUT=30 286 | 287 | # Lindorm configuration 288 | LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070 289 | LINDORM_USERNAME=admin 290 | LINDORM_PASSWORD=admin 291 | USING_UGC_INDEX=False 292 | 293 | # OceanBase Vector configuration 294 | OCEANBASE_VECTOR_HOST=127.0.0.1 295 | OCEANBASE_VECTOR_PORT=2881 296 | OCEANBASE_VECTOR_USER=root@test 297 | OCEANBASE_VECTOR_PASSWORD=difyai123456 298 | OCEANBASE_VECTOR_DATABASE=test 299 | OCEANBASE_MEMORY_LIMIT=6G 300 | 301 | 302 | # Upload configuration 303 | UPLOAD_FILE_SIZE_LIMIT=15 304 | UPLOAD_FILE_BATCH_LIMIT=5 305 | UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 306 | UPLOAD_VIDEO_FILE_SIZE_LIMIT=100 307 | UPLOAD_AUDIO_FILE_SIZE_LIMIT=50 308 | 309 | # Model configuration 310 | MULTIMODAL_SEND_FORMAT=base64 311 | PROMPT_GENERATION_MAX_TOKENS=512 312 | CODE_GENERATION_MAX_TOKENS=1024 313 | 314 | # Mail configuration, support: resend, smtp 315 | MAIL_TYPE= 316 | MAIL_DEFAULT_SEND_FROM=no-reply 317 | RESEND_API_KEY= 318 | RESEND_API_URL=https://api.resend.com 319 | # smtp configuration 320 | SMTP_SERVER=smtp.gmail.com 321 | SMTP_PORT=465 322 | SMTP_USERNAME=123 323 | SMTP_PASSWORD=abc 324 | SMTP_USE_TLS=true 325 | SMTP_OPPORTUNISTIC_TLS=false 326 | 327 | # Sentry configuration 328 | SENTRY_DSN= 329 | 330 | # DEBUG 331 | DEBUG=false 332 | SQLALCHEMY_ECHO=false 333 | 334 | # Notion import configuration, support public and internal 335 | NOTION_INTEGRATION_TYPE=public 336 | NOTION_CLIENT_SECRET=you-client-secret 337 | NOTION_CLIENT_ID=you-client-id 338 | NOTION_INTERNAL_SECRET=you-internal-secret 339 | 340 | ETL_TYPE=dify 341 | UNSTRUCTURED_API_URL= 342 | UNSTRUCTURED_API_KEY= 343 | SCARF_NO_ANALYTICS=true 344 | 345 | #ssrf 346 | SSRF_PROXY_HTTP_URL= 347 | SSRF_PROXY_HTTPS_URL= 348 | SSRF_DEFAULT_MAX_RETRIES=3 349 | SSRF_DEFAULT_TIME_OUT=5 350 | SSRF_DEFAULT_CONNECT_TIME_OUT=5 351 | SSRF_DEFAULT_READ_TIME_OUT=5 352 | SSRF_DEFAULT_WRITE_TIME_OUT=5 353 | 354 | BATCH_UPLOAD_LIMIT=10 355 | KEYWORD_DATA_SOURCE_TYPE=database 356 | 357 | # Workflow file upload limit 358 | WORKFLOW_FILE_UPLOAD_LIMIT=10 359 | 360 | # CODE EXECUTION CONFIGURATION 361 | CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194 362 | CODE_EXECUTION_API_KEY=dify-sandbox 363 | CODE_MAX_NUMBER=9223372036854775807 364 | CODE_MIN_NUMBER=-9223372036854775808 365 | CODE_MAX_STRING_LENGTH=80000 366 | TEMPLATE_TRANSFORM_MAX_LENGTH=80000 367 | CODE_MAX_STRING_ARRAY_LENGTH=30 368 | CODE_MAX_OBJECT_ARRAY_LENGTH=30 369 | CODE_MAX_NUMBER_ARRAY_LENGTH=1000 370 | 371 | # API Tool configuration 372 | API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 373 | API_TOOL_DEFAULT_READ_TIMEOUT=60 374 | 375 | # HTTP Node configuration 376 | HTTP_REQUEST_MAX_CONNECT_TIMEOUT=300 377 | HTTP_REQUEST_MAX_READ_TIMEOUT=600 378 | HTTP_REQUEST_MAX_WRITE_TIMEOUT=600 379 | HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 380 | HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 381 | 382 | # Respect X-* headers to redirect clients 383 | RESPECT_XFORWARD_HEADERS_ENABLED=false 384 | 385 | # Log file path 386 | LOG_FILE= 387 | # Log file max size, the unit is MB 388 | LOG_FILE_MAX_SIZE=20 389 | # Log file max backup count 390 | LOG_FILE_BACKUP_COUNT=5 391 | # Log dateformat 392 | LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S 393 | # Log Timezone 394 | LOG_TZ=UTC 395 | # Log format 396 | LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s 397 | 398 | # Indexing configuration 399 | INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 400 | 401 | # Workflow runtime configuration 402 | WORKFLOW_MAX_EXECUTION_STEPS=500 403 | WORKFLOW_MAX_EXECUTION_TIME=1200 404 | WORKFLOW_CALL_MAX_DEPTH=5 405 | WORKFLOW_PARALLEL_DEPTH_LIMIT=3 406 | MAX_VARIABLE_SIZE=204800 407 | 408 | # App configuration 409 | APP_MAX_EXECUTION_TIME=1200 410 | APP_MAX_ACTIVE_REQUESTS=0 411 | 412 | 413 | # Celery beat configuration 414 | CELERY_BEAT_SCHEDULER_TIME=1 415 | 416 | # Position configuration 417 | POSITION_TOOL_PINS= 418 | POSITION_TOOL_INCLUDES= 419 | POSITION_TOOL_EXCLUDES= 420 | 421 | POSITION_PROVIDER_PINS= 422 | POSITION_PROVIDER_INCLUDES= 423 | POSITION_PROVIDER_EXCLUDES= 424 | 425 | # Reset password token expiry minutes 426 | RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 427 | 428 | CREATE_TIDB_SERVICE_JOB_ENABLED=false 429 | 430 | # Maximum number of submitted thread count in a ThreadPool for parallel node execution 431 | MAX_SUBMIT_COUNT=100 432 | # Lockout duration in seconds 433 | LOGIN_LOCKOUT_DURATION=86400 -------------------------------------------------------------------------------- /commands.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | import logging 4 | import secrets 5 | from typing import Optional 6 | 7 | import click 8 | from flask import current_app 9 | from werkzeug.exceptions import NotFound 10 | 11 | from configs import dify_config 12 | from constants.languages import languages 13 | from core.rag.datasource.vdb.vector_factory import Vector 14 | from core.rag.datasource.vdb.vector_type import VectorType 15 | from core.rag.models.document import Document 16 | from events.app_event import app_was_created 17 | from extensions.ext_database import db 18 | from extensions.ext_redis import redis_client 19 | from libs.helper import email as email_validate 20 | from libs.password import hash_password, password_pattern, valid_password 21 | from libs.rsa import generate_key_pair 22 | from models import Tenant 23 | from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment 24 | from models.dataset import Document as DatasetDocument 25 | from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation 26 | from models.provider import Provider, ProviderModel 27 | from services.account_service import RegisterService, TenantService 28 | 29 | 30 | @click.command("reset-password", help="Reset the account password.") 31 | @click.option("--email", prompt=True, help="Account email to reset password for") 32 | @click.option("--new-password", prompt=True, help="New password") 33 | @click.option("--password-confirm", prompt=True, help="Confirm new password") 34 | def reset_password(email, new_password, password_confirm): 35 | """ 36 | Reset password of owner account 37 | Only available in SELF_HOSTED mode 38 | """ 39 | if str(new_password).strip() != str(password_confirm).strip(): 40 | click.echo(click.style("Passwords do not match.", fg="red")) 41 | return 42 | 43 | account = db.session.query(Account).filter(Account.email == email).one_or_none() 44 | 45 | if not account: 46 | click.echo(click.style("Account not found for email: {}".format(email), fg="red")) 47 | return 48 | 49 | try: 50 | valid_password(new_password) 51 | except: 52 | click.echo(click.style("Invalid password. Must match {}".format(password_pattern), fg="red")) 53 | return 54 | 55 | # generate password salt 56 | salt = secrets.token_bytes(16) 57 | base64_salt = base64.b64encode(salt).decode() 58 | 59 | # encrypt password with salt 60 | password_hashed = hash_password(new_password, salt) 61 | base64_password_hashed = base64.b64encode(password_hashed).decode() 62 | account.password = base64_password_hashed 63 | account.password_salt = base64_salt 64 | db.session.commit() 65 | click.echo(click.style("Password reset successfully.", fg="green")) 66 | 67 | 68 | @click.command("reset-email", help="Reset the account email.") 69 | @click.option("--email", prompt=True, help="Current account email") 70 | @click.option("--new-email", prompt=True, help="New email") 71 | @click.option("--email-confirm", prompt=True, help="Confirm new email") 72 | def reset_email(email, new_email, email_confirm): 73 | """ 74 | Replace account email 75 | :return: 76 | """ 77 | if str(new_email).strip() != str(email_confirm).strip(): 78 | click.echo(click.style("New emails do not match.", fg="red")) 79 | return 80 | 81 | account = db.session.query(Account).filter(Account.email == email).one_or_none() 82 | 83 | if not account: 84 | click.echo(click.style("Account not found for email: {}".format(email), fg="red")) 85 | return 86 | 87 | try: 88 | email_validate(new_email) 89 | except: 90 | click.echo(click.style("Invalid email: {}".format(new_email), fg="red")) 91 | return 92 | 93 | account.email = new_email 94 | db.session.commit() 95 | click.echo(click.style("Email updated successfully.", fg="green")) 96 | 97 | 98 | @click.command( 99 | "reset-encrypt-key-pair", 100 | help="Reset the asymmetric key pair of workspace for encrypt LLM credentials. " 101 | "After the reset, all LLM credentials will become invalid, " 102 | "requiring re-entry." 103 | "Only support SELF_HOSTED mode.", 104 | ) 105 | @click.confirmation_option( 106 | prompt=click.style( 107 | "Are you sure you want to reset encrypt key pair? This operation cannot be rolled back!", fg="red" 108 | ) 109 | ) 110 | def reset_encrypt_key_pair(): 111 | """ 112 | Reset the encrypted key pair of workspace for encrypt LLM credentials. 113 | After the reset, all LLM credentials will become invalid, requiring re-entry. 114 | Only support SELF_HOSTED mode. 115 | """ 116 | if dify_config.EDITION != "SELF_HOSTED": 117 | click.echo(click.style("This command is only for SELF_HOSTED installations.", fg="red")) 118 | return 119 | 120 | tenants = db.session.query(Tenant).all() 121 | for tenant in tenants: 122 | if not tenant: 123 | click.echo(click.style("No workspaces found. Run /install first.", fg="red")) 124 | return 125 | 126 | tenant.encrypt_public_key = generate_key_pair(tenant.id) 127 | 128 | db.session.query(Provider).filter(Provider.provider_type == "custom", Provider.tenant_id == tenant.id).delete() 129 | db.session.query(ProviderModel).filter(ProviderModel.tenant_id == tenant.id).delete() 130 | db.session.commit() 131 | 132 | click.echo( 133 | click.style( 134 | "Congratulations! The asymmetric key pair of workspace {} has been reset.".format(tenant.id), 135 | fg="green", 136 | ) 137 | ) 138 | 139 | 140 | @click.command("vdb-migrate", help="Migrate vector db.") 141 | @click.option("--scope", default="all", prompt=False, help="The scope of vector database to migrate, Default is All.") 142 | def vdb_migrate(scope: str): 143 | if scope in {"knowledge", "all"}: 144 | migrate_knowledge_vector_database() 145 | if scope in {"annotation", "all"}: 146 | migrate_annotation_vector_database() 147 | 148 | 149 | def migrate_annotation_vector_database(): 150 | """ 151 | Migrate annotation datas to target vector database . 152 | """ 153 | click.echo(click.style("Starting annotation data migration.", fg="green")) 154 | create_count = 0 155 | skipped_count = 0 156 | total_count = 0 157 | page = 1 158 | while True: 159 | try: 160 | # get apps info 161 | apps = ( 162 | App.query.filter(App.status == "normal") 163 | .order_by(App.created_at.desc()) 164 | .paginate(page=page, per_page=50) 165 | ) 166 | except NotFound: 167 | break 168 | 169 | page += 1 170 | for app in apps: 171 | total_count = total_count + 1 172 | click.echo( 173 | f"Processing the {total_count} app {app.id}. " + f"{create_count} created, {skipped_count} skipped." 174 | ) 175 | try: 176 | click.echo("Creating app annotation index: {}".format(app.id)) 177 | app_annotation_setting = ( 178 | db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app.id).first() 179 | ) 180 | 181 | if not app_annotation_setting: 182 | skipped_count = skipped_count + 1 183 | click.echo("App annotation setting disabled: {}".format(app.id)) 184 | continue 185 | # get dataset_collection_binding info 186 | dataset_collection_binding = ( 187 | db.session.query(DatasetCollectionBinding) 188 | .filter(DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id) 189 | .first() 190 | ) 191 | if not dataset_collection_binding: 192 | click.echo("App annotation collection binding not found: {}".format(app.id)) 193 | continue 194 | annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all() 195 | dataset = Dataset( 196 | id=app.id, 197 | tenant_id=app.tenant_id, 198 | indexing_technique="high_quality", 199 | embedding_model_provider=dataset_collection_binding.provider_name, 200 | embedding_model=dataset_collection_binding.model_name, 201 | collection_binding_id=dataset_collection_binding.id, 202 | ) 203 | documents = [] 204 | if annotations: 205 | for annotation in annotations: 206 | document = Document( 207 | page_content=annotation.question, 208 | metadata={"annotation_id": annotation.id, "app_id": app.id, "doc_id": annotation.id}, 209 | ) 210 | documents.append(document) 211 | 212 | vector = Vector(dataset, attributes=["doc_id", "annotation_id", "app_id"]) 213 | click.echo(f"Migrating annotations for app: {app.id}.") 214 | 215 | try: 216 | vector.delete() 217 | click.echo(click.style(f"Deleted vector index for app {app.id}.", fg="green")) 218 | except Exception as e: 219 | click.echo(click.style(f"Failed to delete vector index for app {app.id}.", fg="red")) 220 | raise e 221 | if documents: 222 | try: 223 | click.echo( 224 | click.style( 225 | f"Creating vector index with {len(documents)} annotations for app {app.id}.", 226 | fg="green", 227 | ) 228 | ) 229 | vector.create(documents) 230 | click.echo(click.style(f"Created vector index for app {app.id}.", fg="green")) 231 | except Exception as e: 232 | click.echo(click.style(f"Failed to created vector index for app {app.id}.", fg="red")) 233 | raise e 234 | click.echo(f"Successfully migrated app annotation {app.id}.") 235 | create_count += 1 236 | except Exception as e: 237 | click.echo( 238 | click.style( 239 | "Error creating app annotation index: {} {}".format(e.__class__.__name__, str(e)), fg="red" 240 | ) 241 | ) 242 | continue 243 | 244 | click.echo( 245 | click.style( 246 | f"Migration complete. Created {create_count} app annotation indexes. Skipped {skipped_count} apps.", 247 | fg="green", 248 | ) 249 | ) 250 | 251 | 252 | def migrate_knowledge_vector_database(): 253 | """ 254 | Migrate vector database datas to target vector database . 255 | """ 256 | click.echo(click.style("Starting vector database migration.", fg="green")) 257 | create_count = 0 258 | skipped_count = 0 259 | total_count = 0 260 | vector_type = dify_config.VECTOR_STORE 261 | upper_collection_vector_types = { 262 | VectorType.MILVUS, 263 | VectorType.PGVECTOR, 264 | VectorType.RELYT, 265 | VectorType.WEAVIATE, 266 | VectorType.ORACLE, 267 | VectorType.ELASTICSEARCH, 268 | } 269 | lower_collection_vector_types = { 270 | VectorType.ANALYTICDB, 271 | VectorType.CHROMA, 272 | VectorType.MYSCALE, 273 | VectorType.PGVECTO_RS, 274 | VectorType.TIDB_VECTOR, 275 | VectorType.OPENSEARCH, 276 | VectorType.TENCENT, 277 | VectorType.BAIDU, 278 | VectorType.VIKINGDB, 279 | VectorType.UPSTASH, 280 | VectorType.COUCHBASE, 281 | VectorType.OCEANBASE, 282 | } 283 | page = 1 284 | while True: 285 | try: 286 | datasets = ( 287 | Dataset.query.filter(Dataset.indexing_technique == "high_quality") 288 | .order_by(Dataset.created_at.desc()) 289 | .paginate(page=page, per_page=50) 290 | ) 291 | except NotFound: 292 | break 293 | 294 | page += 1 295 | for dataset in datasets: 296 | total_count = total_count + 1 297 | click.echo( 298 | f"Processing the {total_count} dataset {dataset.id}. {create_count} created, {skipped_count} skipped." 299 | ) 300 | try: 301 | click.echo("Creating dataset vector database index: {}".format(dataset.id)) 302 | if dataset.index_struct_dict: 303 | if dataset.index_struct_dict["type"] == vector_type: 304 | skipped_count = skipped_count + 1 305 | continue 306 | collection_name = "" 307 | dataset_id = dataset.id 308 | if vector_type in upper_collection_vector_types: 309 | collection_name = Dataset.gen_collection_name_by_id(dataset_id) 310 | elif vector_type == VectorType.QDRANT: 311 | if dataset.collection_binding_id: 312 | dataset_collection_binding = ( 313 | db.session.query(DatasetCollectionBinding) 314 | .filter(DatasetCollectionBinding.id == dataset.collection_binding_id) 315 | .one_or_none() 316 | ) 317 | if dataset_collection_binding: 318 | collection_name = dataset_collection_binding.collection_name 319 | else: 320 | raise ValueError("Dataset Collection Binding not found") 321 | else: 322 | collection_name = Dataset.gen_collection_name_by_id(dataset_id) 323 | 324 | elif vector_type in lower_collection_vector_types: 325 | collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower() 326 | else: 327 | raise ValueError(f"Vector store {vector_type} is not supported.") 328 | 329 | index_struct_dict = {"type": vector_type, "vector_store": {"class_prefix": collection_name}} 330 | dataset.index_struct = json.dumps(index_struct_dict) 331 | vector = Vector(dataset) 332 | click.echo(f"Migrating dataset {dataset.id}.") 333 | 334 | try: 335 | vector.delete() 336 | click.echo( 337 | click.style(f"Deleted vector index {collection_name} for dataset {dataset.id}.", fg="green") 338 | ) 339 | except Exception as e: 340 | click.echo( 341 | click.style( 342 | f"Failed to delete vector index {collection_name} for dataset {dataset.id}.", fg="red" 343 | ) 344 | ) 345 | raise e 346 | 347 | dataset_documents = ( 348 | db.session.query(DatasetDocument) 349 | .filter( 350 | DatasetDocument.dataset_id == dataset.id, 351 | DatasetDocument.indexing_status == "completed", 352 | DatasetDocument.enabled == True, 353 | DatasetDocument.archived == False, 354 | ) 355 | .all() 356 | ) 357 | 358 | documents = [] 359 | segments_count = 0 360 | for dataset_document in dataset_documents: 361 | segments = ( 362 | db.session.query(DocumentSegment) 363 | .filter( 364 | DocumentSegment.document_id == dataset_document.id, 365 | DocumentSegment.status == "completed", 366 | DocumentSegment.enabled == True, 367 | ) 368 | .all() 369 | ) 370 | 371 | for segment in segments: 372 | document = Document( 373 | page_content=segment.content, 374 | metadata={ 375 | "doc_id": segment.index_node_id, 376 | "doc_hash": segment.index_node_hash, 377 | "document_id": segment.document_id, 378 | "dataset_id": segment.dataset_id, 379 | }, 380 | ) 381 | 382 | documents.append(document) 383 | segments_count = segments_count + 1 384 | 385 | if documents: 386 | try: 387 | click.echo( 388 | click.style( 389 | f"Creating vector index with {len(documents)} documents of {segments_count}" 390 | f" segments for dataset {dataset.id}.", 391 | fg="green", 392 | ) 393 | ) 394 | vector.create(documents) 395 | click.echo(click.style(f"Created vector index for dataset {dataset.id}.", fg="green")) 396 | except Exception as e: 397 | click.echo(click.style(f"Failed to created vector index for dataset {dataset.id}.", fg="red")) 398 | raise e 399 | db.session.add(dataset) 400 | db.session.commit() 401 | click.echo(f"Successfully migrated dataset {dataset.id}.") 402 | create_count += 1 403 | except Exception as e: 404 | db.session.rollback() 405 | click.echo( 406 | click.style("Error creating dataset index: {} {}".format(e.__class__.__name__, str(e)), fg="red") 407 | ) 408 | continue 409 | 410 | click.echo( 411 | click.style( 412 | f"Migration complete. Created {create_count} dataset indexes. Skipped {skipped_count} datasets.", fg="green" 413 | ) 414 | ) 415 | 416 | 417 | @click.command("convert-to-agent-apps", help="Convert Agent Assistant to Agent App.") 418 | def convert_to_agent_apps(): 419 | """ 420 | Convert Agent Assistant to Agent App. 421 | """ 422 | click.echo(click.style("Starting convert to agent apps.", fg="green")) 423 | 424 | proceeded_app_ids = [] 425 | 426 | while True: 427 | # fetch first 1000 apps 428 | sql_query = """SELECT a.id AS id FROM apps a 429 | INNER JOIN app_model_configs am ON a.app_model_config_id=am.id 430 | WHERE a.mode = 'chat' 431 | AND am.agent_mode is not null 432 | AND ( 433 | am.agent_mode like '%"strategy": "function_call"%' 434 | OR am.agent_mode like '%"strategy": "react"%' 435 | ) 436 | AND ( 437 | am.agent_mode like '{"enabled": true%' 438 | OR am.agent_mode like '{"max_iteration": %' 439 | ) ORDER BY a.created_at DESC LIMIT 1000 440 | """ 441 | 442 | with db.engine.begin() as conn: 443 | rs = conn.execute(db.text(sql_query)) 444 | 445 | apps = [] 446 | for i in rs: 447 | app_id = str(i.id) 448 | if app_id not in proceeded_app_ids: 449 | proceeded_app_ids.append(app_id) 450 | app = db.session.query(App).filter(App.id == app_id).first() 451 | if app is not None: 452 | apps.append(app) 453 | 454 | if len(apps) == 0: 455 | break 456 | 457 | for app in apps: 458 | click.echo("Converting app: {}".format(app.id)) 459 | 460 | try: 461 | app.mode = AppMode.AGENT_CHAT.value 462 | db.session.commit() 463 | 464 | # update conversation mode to agent 465 | db.session.query(Conversation).filter(Conversation.app_id == app.id).update( 466 | {Conversation.mode: AppMode.AGENT_CHAT.value} 467 | ) 468 | 469 | db.session.commit() 470 | click.echo(click.style("Converted app: {}".format(app.id), fg="green")) 471 | except Exception as e: 472 | click.echo(click.style("Convert app error: {} {}".format(e.__class__.__name__, str(e)), fg="red")) 473 | 474 | click.echo(click.style("Conversion complete. Converted {} agent apps.".format(len(proceeded_app_ids)), fg="green")) 475 | 476 | 477 | @click.command("add-qdrant-doc-id-index", help="Add Qdrant doc_id index.") 478 | @click.option("--field", default="metadata.doc_id", prompt=False, help="Index field , default is metadata.doc_id.") 479 | def add_qdrant_doc_id_index(field: str): 480 | click.echo(click.style("Starting Qdrant doc_id index creation.", fg="green")) 481 | vector_type = dify_config.VECTOR_STORE 482 | if vector_type != "qdrant": 483 | click.echo(click.style("This command only supports Qdrant vector store.", fg="red")) 484 | return 485 | create_count = 0 486 | 487 | try: 488 | bindings = db.session.query(DatasetCollectionBinding).all() 489 | if not bindings: 490 | click.echo(click.style("No dataset collection bindings found.", fg="red")) 491 | return 492 | import qdrant_client 493 | from qdrant_client.http.exceptions import UnexpectedResponse 494 | from qdrant_client.http.models import PayloadSchemaType 495 | 496 | from core.rag.datasource.vdb.qdrant.qdrant_vector import QdrantConfig 497 | 498 | for binding in bindings: 499 | if dify_config.QDRANT_URL is None: 500 | raise ValueError("Qdrant URL is required.") 501 | qdrant_config = QdrantConfig( 502 | endpoint=dify_config.QDRANT_URL, 503 | api_key=dify_config.QDRANT_API_KEY, 504 | root_path=current_app.root_path, 505 | timeout=dify_config.QDRANT_CLIENT_TIMEOUT, 506 | grpc_port=dify_config.QDRANT_GRPC_PORT, 507 | prefer_grpc=dify_config.QDRANT_GRPC_ENABLED, 508 | ) 509 | try: 510 | client = qdrant_client.QdrantClient(**qdrant_config.to_qdrant_params()) 511 | # create payload index 512 | client.create_payload_index(binding.collection_name, field, field_schema=PayloadSchemaType.KEYWORD) 513 | create_count += 1 514 | except UnexpectedResponse as e: 515 | # Collection does not exist, so return 516 | if e.status_code == 404: 517 | click.echo(click.style(f"Collection not found: {binding.collection_name}.", fg="red")) 518 | continue 519 | # Some other error occurred, so re-raise the exception 520 | else: 521 | click.echo( 522 | click.style( 523 | f"Failed to create Qdrant index for collection: {binding.collection_name}.", fg="red" 524 | ) 525 | ) 526 | 527 | except Exception as e: 528 | click.echo(click.style("Failed to create Qdrant client.", fg="red")) 529 | 530 | click.echo(click.style(f"Index creation complete. Created {create_count} collection indexes.", fg="green")) 531 | 532 | 533 | @click.command("create-tenant", help="Create account and tenant.") 534 | @click.option("--email", prompt=True, help="Tenant account email.") 535 | @click.option("--name", prompt=True, help="Workspace name.") 536 | @click.option("--language", prompt=True, help="Account language, default: en-US.") 537 | def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None): 538 | """ 539 | Create tenant account 540 | """ 541 | if not email: 542 | click.echo(click.style("Email is required.", fg="red")) 543 | return 544 | 545 | # Create account 546 | email = email.strip() 547 | 548 | if "@" not in email: 549 | click.echo(click.style("Invalid email address.", fg="red")) 550 | return 551 | 552 | account_name = email.split("@")[0] 553 | 554 | if language not in languages: 555 | language = "en-US" 556 | 557 | # Validates name encoding for non-Latin characters. 558 | name = name.strip().encode("utf-8").decode("utf-8") if name else None 559 | 560 | # generate random password 561 | new_password = secrets.token_urlsafe(16) 562 | 563 | # register account 564 | account = RegisterService.register( 565 | email=email, 566 | name=account_name, 567 | password=new_password, 568 | language=language, 569 | create_workspace_required=False, 570 | ) 571 | TenantService.create_owner_tenant_if_not_exist(account, name) 572 | 573 | click.echo( 574 | click.style( 575 | "Account and tenant created.\nAccount: {}\nPassword: {}".format(email, new_password), 576 | fg="green", 577 | ) 578 | ) 579 | 580 | 581 | @click.command("upgrade-db", help="Upgrade the database") 582 | def upgrade_db(): 583 | click.echo("Preparing database migration...") 584 | lock = redis_client.lock(name="db_upgrade_lock", timeout=60) 585 | if lock.acquire(blocking=False): 586 | try: 587 | click.echo(click.style("Starting database migration.", fg="green")) 588 | 589 | # run db migration 590 | import flask_migrate # type: ignore 591 | 592 | flask_migrate.upgrade() 593 | 594 | click.echo(click.style("Database migration successful!", fg="green")) 595 | 596 | except Exception as e: 597 | logging.exception("Failed to execute database migration") 598 | finally: 599 | lock.release() 600 | else: 601 | click.echo("Database migration skipped") 602 | 603 | 604 | @click.command("fix-app-site-missing", help="Fix app related site missing issue.") 605 | def fix_app_site_missing(): 606 | """ 607 | Fix app related site missing issue. 608 | """ 609 | click.echo(click.style("Starting fix for missing app-related sites.", fg="green")) 610 | 611 | failed_app_ids = [] 612 | while True: 613 | sql = """select apps.id as id from apps left join sites on sites.app_id=apps.id 614 | where sites.id is null limit 1000""" 615 | with db.engine.begin() as conn: 616 | rs = conn.execute(db.text(sql)) 617 | 618 | processed_count = 0 619 | for i in rs: 620 | processed_count += 1 621 | app_id = str(i.id) 622 | 623 | if app_id in failed_app_ids: 624 | continue 625 | 626 | try: 627 | app = db.session.query(App).filter(App.id == app_id).first() 628 | if not app: 629 | print(f"App {app_id} not found") 630 | continue 631 | 632 | tenant = app.tenant 633 | if tenant: 634 | accounts = tenant.get_accounts() 635 | if not accounts: 636 | print("Fix failed for app {}".format(app.id)) 637 | continue 638 | 639 | account = accounts[0] 640 | print("Fixing missing site for app {}".format(app.id)) 641 | app_was_created.send(app, account=account) 642 | except Exception as e: 643 | failed_app_ids.append(app_id) 644 | click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red")) 645 | logging.exception(f"Failed to fix app related site missing issue, app_id: {app_id}") 646 | continue 647 | 648 | if not processed_count: 649 | break 650 | 651 | click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green")) 652 | --------------------------------------------------------------------------------