├── backend ├── vectors │ ├── __init__.py │ ├── vector_store.py │ ├── vector_chroma.py │ └── translate_wrapper.py ├── docker_build.sh ├── models │ ├── __init__.py │ ├── task_sql.py │ ├── task_doc.py │ ├── task_table.py │ ├── project.py │ ├── task_column.py │ ├── job.py │ ├── definition_rule.py │ ├── definition_doc.py │ ├── definition_relation.py │ ├── task.py │ ├── definition_table.py │ ├── definition_column.py │ └── base.py ├── .cursorrules ├── utils │ ├── __init__.py │ ├── utils.py │ ├── schemas.py │ ├── prompt_util.py │ └── structure_util.py ├── routes │ ├── __init__.py │ ├── test.py │ └── project.py ├── services │ ├── __init__.py │ ├── openai_service.py │ ├── translate_service.py │ ├── job_service.py │ └── project_service.py ├── docker_run.sh ├── dto │ ├── selected_column_dto.py │ ├── disable_table_query.py │ ├── definition_rule_dto.py │ ├── definition_doc_query_result_dto.py │ ├── refresh_index_query_dto.py │ ├── job_dto.py │ ├── ai_comment_dto.py │ ├── update_ddl_by_query_dto.py │ ├── learn_result_dto.py │ ├── project_settings_dto.py │ ├── schema_dto.py │ ├── gen_ai_comments_dto.py │ ├── update_task_query.py │ └── task_dto.py ├── assets │ ├── demo_tables.csv │ └── demo_columns.csv ├── Dockerfile ├── requirements.txt ├── jobs │ ├── __init__.py │ ├── job_vector_db.py │ └── job_job.py ├── .env.template ├── wsgi.py ├── prompt_templates │ ├── gen_ai_comments.mustache │ ├── optimize_question.mustache │ ├── gen_sql.mustache │ ├── gen_related_columns.mustache │ └── learn.mustache ├── .gitignore ├── gunicorn.conf.py ├── enums.py ├── base_enum.py ├── vector_stores.py ├── start.sh ├── logger.py ├── database.py └── app.py ├── frontend ├── src │ ├── vite-env.d.ts │ ├── index.css │ ├── typings.d.ts │ ├── store │ │ ├── hooks.ts │ │ ├── slices │ │ │ ├── schemaSlice.ts │ │ │ ├── ddlSlice.ts │ │ │ ├── taskSlice.ts │ │ │ └── appSlice.ts │ │ └── index.ts │ ├── main.tsx │ ├── components │ │ ├── records │ │ │ ├── refs │ │ │ │ ├── DDLRefsModal.css │ │ │ │ ├── AICommentModal.tsx │ │ │ │ ├── RelatedColumnsRefs.tsx │ │ │ │ └── DocRefs.tsx │ │ │ ├── GenerationRecords.tsx │ │ │ ├── OptimizedQuestion.tsx │ │ │ ├── QuestionSupplement.tsx │ │ │ ├── SqlFeedback.tsx │ │ │ ├── QuestionList.tsx │ │ │ ├── Refs.tsx │ │ │ └── MainContent.tsx │ │ ├── ddl │ │ │ ├── DDL.tsx │ │ │ ├── UploadDDLModal.tsx │ │ │ ├── RelationEditModal.tsx │ │ │ ├── TableList.tsx │ │ │ ├── QueryImportTab.tsx │ │ │ └── FileImportTab.tsx │ │ ├── LanguageSwitcher.tsx │ │ ├── AppSidebar.tsx │ │ └── ProjectLayout.tsx │ ├── utils │ │ ├── stringUtils.ts │ │ ├── bizUtil.ts │ │ └── learnUtil.ts │ ├── i18n │ │ └── i18n.ts │ ├── hooks │ │ ├── useEnvService.ts │ │ └── useAppService.ts │ ├── App.tsx │ └── consts.ts ├── public │ ├── tables_template.csv │ └── columns_template.csv ├── postcss.config.js ├── tsconfig.json ├── openapitools.json ├── .cursorrules ├── vite.config.ts ├── getOpenapi.cjs ├── index.html ├── .gitignore ├── tailwind.config.js ├── tsconfig.node.json ├── eslint.config.js ├── tsconfig.app.json └── package.json ├── .gitignore ├── README_cn.md └── README.md /backend/vectors/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /backend/docker_build.sh: -------------------------------------------------------------------------------- 1 | # build docker image 2 | docker build -t sqlwise . -------------------------------------------------------------------------------- /backend/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Empty file to make models a Python package 2 | -------------------------------------------------------------------------------- /backend/.cursorrules: -------------------------------------------------------------------------------- 1 | This project use python+flask+apiflask+psycopg2-binary+SQLAlchemy -------------------------------------------------------------------------------- /backend/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Empty file to mark the directory as a Python package 2 | -------------------------------------------------------------------------------- /frontend/public/tables_template.csv: -------------------------------------------------------------------------------- 1 | TABLE_NAME,TABLE_COMMENT 2 | user,User Table 3 | -------------------------------------------------------------------------------- /backend/routes/__init__.py: -------------------------------------------------------------------------------- 1 | # Empty file, used to mark the directory as a Python package 2 | -------------------------------------------------------------------------------- /backend/services/__init__.py: -------------------------------------------------------------------------------- 1 | # empty file, used to mark the directory as a Python package 2 | -------------------------------------------------------------------------------- /frontend/src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /frontend/public/columns_template.csv: -------------------------------------------------------------------------------- 1 | TABLE_NAME,COLUMN_NAME,COLUMN_TYPE,COLUMN_COMMENT 2 | user,id,varchar(36),User ID 3 | -------------------------------------------------------------------------------- /frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } -------------------------------------------------------------------------------- /backend/docker_run.sh: -------------------------------------------------------------------------------- 1 | # start docker container 2 | docker run \ 3 | -it \ 4 | -p 8000:8000 \ 5 | --env-file .env.template \ 6 | --name sqlwise \ 7 | sqlwise -------------------------------------------------------------------------------- /backend/dto/selected_column_dto.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class SelectedColumnDTO: 5 | table: str 6 | columns: list[str] -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { "path": "./tsconfig.app.json" }, 5 | { "path": "./tsconfig.node.json" } 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /backend/dto/disable_table_query.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class DisableTableQueryDTO: 5 | project_id: int 6 | table: str 7 | disabled: bool -------------------------------------------------------------------------------- /frontend/openapitools.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", 3 | "spaces": 2, 4 | "generator-cli": { 5 | "version": "7.9.0" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /backend/dto/definition_rule_dto.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class DefinitionRuleDTO: 5 | id: int 6 | name: str 7 | content: str 8 | def_selected: bool 9 | disabled: bool -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .DS_Store 3 | .env 4 | dist/ 5 | *.log 6 | .idea/ 7 | .vscode/ 8 | *.pyc 9 | __pycache__/ 10 | venv/ 11 | .env.local 12 | .env.development.local 13 | .env.test.local 14 | .env.production.local 15 | -------------------------------------------------------------------------------- /backend/dto/definition_doc_query_result_dto.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class DefinitionDocQueryResultDTO: 5 | id: int 6 | def_doc: str 7 | def_selected: bool 8 | disabled: bool -------------------------------------------------------------------------------- /frontend/.cursorrules: -------------------------------------------------------------------------------- 1 | Project Name: SQLWise 2 | Project Description: Use AI to generate SQL. 3 | Project Framework: typescript+vite+react+tailwindcss+flowbite-react+react-icons+react-toastify 4 | Import path use @ instead of /src directory -------------------------------------------------------------------------------- /frontend/src/typings.d.ts: -------------------------------------------------------------------------------- 1 | type TableCellType = 'new' | 'newOverride' | 'stay' | 'modify' | 'none'; 2 | type ColumnCellType = 'new' | 'newOverride' | 'stay' | 'modify' | 'none'; 3 | type RelationCellType = 'new' | 'stay' | 'modify' | 'none'; 4 | -------------------------------------------------------------------------------- /backend/assets/demo_tables.csv: -------------------------------------------------------------------------------- 1 | TABLE_NAME,TABLE_COMMENT 2 | user,User Table 3 | product,Product Table 4 | category,Product Category Table 5 | order,Order Table 6 | order_item,Order Items Table 7 | payment,Payment Table 8 | address,User Address Table 9 | -------------------------------------------------------------------------------- /frontend/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react-swc' 3 | 4 | // https://vite.dev/config/ 5 | export default defineConfig({ 6 | resolve: { 7 | alias: { 8 | '@': '/src' 9 | } 10 | }, 11 | plugins: [react()], 12 | }) 13 | -------------------------------------------------------------------------------- /backend/dto/refresh_index_query_dto.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class RefreshIndexQueryDTO: 5 | project_id: int 6 | refresh_table: bool = False 7 | refresh_column: bool = False 8 | refresh_doc: bool = False 9 | refresh_sql: bool = False 10 | -------------------------------------------------------------------------------- /frontend/src/store/hooks.ts: -------------------------------------------------------------------------------- 1 | import { TypedUseSelectorHook, useDispatch, useSelector } from 'react-redux'; 2 | import type { RootState, AppDispatch } from './index'; 3 | 4 | export const useAppDispatch = () => useDispatch(); 5 | export const useAppSelector: TypedUseSelectorHook = useSelector; -------------------------------------------------------------------------------- /frontend/getOpenapi.cjs: -------------------------------------------------------------------------------- 1 | // Get the http://localhost:8000/openapi.json file and write it to openapi.json 2 | 3 | const axios = require('axios') 4 | const fs = require('fs') 5 | 6 | axios.get('http://127.0.0.1:8000/api/openapi.json').then(res => { 7 | fs.writeFileSync('openapi.json', JSON.stringify(res.data)) 8 | }) 9 | 10 | fs.rmSync('src/api-docs', { recursive: true, force: true }) 11 | -------------------------------------------------------------------------------- /backend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12 2 | 3 | # set working directory 4 | WORKDIR /app 5 | 6 | # set environment variable 7 | ENV PYTHONUNBUFFERED=1 8 | 9 | # copy dependency files 10 | COPY requirements.txt . 11 | 12 | # install python dependencies 13 | RUN pip3 install --no-cache-dir -r requirements.txt 14 | 15 | # copy application code 16 | COPY . . 17 | 18 | # expose port 19 | EXPOSE 8000 20 | 21 | CMD ["./start.sh"] -------------------------------------------------------------------------------- /frontend/src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom/client' 3 | import { Provider } from 'react-redux' 4 | import { store } from './store' 5 | import App from './App' 6 | import './index.css' 7 | 8 | ReactDOM.createRoot(document.getElementById('root')!).render( 9 | 10 | 11 | 12 | 13 | , 14 | ) 15 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | SQLWise 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | # OpenAPI generated files 27 | openapi.json 28 | src/api-docs -------------------------------------------------------------------------------- /backend/dto/job_dto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from datetime import datetime 3 | 4 | @dataclass 5 | class JobDTO: 6 | id: int 7 | version: int 8 | task_id: int 9 | project_id: int 10 | job_type: str 11 | job_data: dict 12 | job_status: str 13 | job_type_display_name: str 14 | job_status_display_name: str 15 | error_message: str 16 | created_at: datetime 17 | updated_at: datetime 18 | job_cost_time: int -------------------------------------------------------------------------------- /backend/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==3.0.2 2 | Flask-APScheduler==1.13.1 3 | python-dotenv==1.0.1 4 | flask-smorest==0.45.0 5 | flask-cors==4.0.0 6 | psycopg2-binary==2.9.9 7 | SQLAlchemy==2.0.27 8 | gunicorn==23.0.0 9 | chromadb==0.5.18 10 | openai==1.54.4 11 | sqlparse==0.5.2 12 | dataclasses-json==0.6.7 13 | marshmallow-dataclass==8.7.1 14 | marshmallow-sqlalchemy==1.1.0 15 | marshmallow==3.23.1 16 | Flask-SQLAlchemy==3.1.1 17 | requests==2.31.0 18 | chevron==0.14.0 19 | httpx==0.27.2 -------------------------------------------------------------------------------- /backend/dto/ai_comment_dto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | from dataclasses_json import dataclass_json 4 | from dto.learn_result_dto import TableRelationDTO 5 | 6 | @dataclass_json 7 | @dataclass 8 | class ColumnAICommentDTO: 9 | col: str 10 | comment: str 11 | 12 | @dataclass_json 13 | @dataclass 14 | class UpdateAICommentDTO: 15 | project_id: int 16 | table: str 17 | comment: str 18 | columns: List[ColumnAICommentDTO] 19 | relations: List[TableRelationDTO] -------------------------------------------------------------------------------- /backend/dto/update_ddl_by_query_dto.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class UpdateDDLByQueryTableDTO: 5 | table: str 6 | comment: str | None = None 7 | 8 | @dataclass 9 | class UpdateDDLByQueryColumnDTO: 10 | table: str 11 | column: str 12 | type: str 13 | comment: str | None = None 14 | 15 | @dataclass 16 | class UpdateDDLByQueryDTO: 17 | project_id: int 18 | tables: list[UpdateDDLByQueryTableDTO] 19 | columns: list[UpdateDDLByQueryColumnDTO] 20 | -------------------------------------------------------------------------------- /backend/utils/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def extract_json(text: str) -> dict: 4 | """ 5 | Extract JSON from text 6 | """ 7 | text = text.strip() 8 | if text.startswith('```json'): 9 | return json.loads(text[len('```json'):-len('```')]) 10 | return json.loads(text) 11 | 12 | def reverse_relation_type(relation_type: str) -> str: 13 | """Reverse relation type""" 14 | return { 15 | '1-1': '1-1', 16 | '1-n': 'n-1', 17 | 'n-1': '1-n', 18 | 'n-n': 'n-n' 19 | }[relation_type] -------------------------------------------------------------------------------- /backend/jobs/__init__.py: -------------------------------------------------------------------------------- 1 | from flask_apscheduler import APScheduler 2 | import os 3 | 4 | scheduler = APScheduler() 5 | 6 | def init_scheduler(app): 7 | # Apply scheduler configuration 8 | app.config['SCHEDULER_API_ENABLED'] = True 9 | app.config['SCHEDULER_TIMEZONE'] = "Asia/Shanghai" 10 | 11 | scheduler.init_app(app) 12 | 13 | # Import jobs module to register tasks 14 | from . import job_job 15 | from . import job_vector_db 16 | 17 | scheduler.start() 18 | app.logger.info("Scheduler started successfully") 19 | -------------------------------------------------------------------------------- /frontend/src/components/records/refs/DDLRefsModal.css: -------------------------------------------------------------------------------- 1 | .table-row-enter { 2 | opacity: 0; 3 | transform: translateY(-10px); 4 | } 5 | 6 | .table-row-enter-active { 7 | opacity: 1; 8 | transform: translateY(0); 9 | transition: opacity 500ms ease-in-out, transform 500ms ease-in-out; 10 | } 11 | 12 | .table-row-exit { 13 | opacity: 1; 14 | transform: translateY(0); 15 | } 16 | 17 | .table-row-exit-active { 18 | opacity: 0; 19 | transform: translateY(-10px); 20 | transition: opacity 500ms ease-in-out, transform 500ms ease-in-out; 21 | } 22 | -------------------------------------------------------------------------------- /backend/.env.template: -------------------------------------------------------------------------------- 1 | # database 2 | DATABASE_URL=sqlite:///sqlwise.db 3 | # DATABASE_URL=postgresql://postgres:password@localhost:5432/sqlwise 4 | 5 | # chroma 6 | CHROMA_HOST=localhost 7 | CHROMA_PORT=8051 8 | 9 | # openai 10 | OPENAI_API_KEY=sk-xxx 11 | OPENAI_API_BASE=https://api.openai.com/v1 12 | OPENAI_API_MODEL=gpt-4o-mini 13 | OPENAI_API_TEMPERATURE=0.0 14 | 15 | # azure translator(optional, recommended for non-English languages!) 16 | # AZURE_TRANSLATOR_KEY=xxx 17 | # AZURE_TRANSLATOR_ENDPOINT=https://api.cognitive.microsofttranslator.com 18 | # AZURE_TRANSLATOR_LOCATION=global -------------------------------------------------------------------------------- /frontend/tailwind.config.js: -------------------------------------------------------------------------------- 1 | const flowbite = require("flowbite-react/tailwind"); 2 | 3 | /** @type {import('tailwindcss').Config} */ 4 | export default { 5 | darkMode: 'class', 6 | content: [ 7 | "./index.html", 8 | "./src/**/*.{js,ts,jsx,tsx}", 9 | flowbite.content(), 10 | ], 11 | theme: { 12 | extend: {}, 13 | }, 14 | plugins: [ 15 | flowbite.plugin(), 16 | require('@tailwindcss/typography'), 17 | ], 18 | safelist: [ 19 | 'grid-cols-1', 20 | 'grid-cols-2', 21 | 'grid-cols-3', 22 | 'grid-cols-4', 23 | 'grid-cols-5', 24 | ], 25 | } -------------------------------------------------------------------------------- /backend/dto/learn_result_dto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | 4 | @dataclass 5 | class TableDescDTO: 6 | table: str 7 | desc: str 8 | 9 | @dataclass 10 | class ColumnDescDTO: 11 | table: str 12 | column: str 13 | desc: str 14 | 15 | @dataclass 16 | class TableRelationDTO: 17 | table1: str 18 | column1: str 19 | table2: str 20 | column2: str 21 | relation_type: str 22 | 23 | @dataclass 24 | class LearnResultDTO: 25 | tables: List[TableDescDTO] 26 | columns: List[ColumnDescDTO] 27 | relations: List[TableRelationDTO] 28 | -------------------------------------------------------------------------------- /frontend/src/store/slices/schemaSlice.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import { Schema } from '../../api-docs'; 3 | 4 | interface SchemaState { 5 | schema?: Schema; 6 | } 7 | 8 | const initialState: SchemaState = { 9 | }; 10 | 11 | export const schemaSlice = createSlice({ 12 | name: 'schema', 13 | initialState, 14 | reducers: { 15 | setSchema: (state, action: PayloadAction) => { 16 | state.schema = action.payload; 17 | }, 18 | }, 19 | }); 20 | 21 | export const { setSchema } = schemaSlice.actions; 22 | export default schemaSlice.reducer; -------------------------------------------------------------------------------- /frontend/src/store/slices/ddlSlice.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from "@reduxjs/toolkit"; 2 | 3 | interface DDLState { 4 | selectedTable: string; 5 | } 6 | 7 | const initialState: DDLState = { 8 | selectedTable: '', 9 | }; 10 | 11 | export const ddlSlice = createSlice({ 12 | name: 'ddl', 13 | initialState, 14 | reducers: { 15 | setSelectedTable: (state, action: PayloadAction) => { 16 | state.selectedTable = action.payload; 17 | }, 18 | }, 19 | }); 20 | 21 | export const { setSelectedTable } = ddlSlice.actions; 22 | export default ddlSlice.reducer; -------------------------------------------------------------------------------- /frontend/src/store/index.ts: -------------------------------------------------------------------------------- 1 | import { configureStore } from '@reduxjs/toolkit'; 2 | import schemaReducer from './slices/schemaSlice'; 3 | import recordsReducer from './slices/recordsSlice'; 4 | import taskReducer from './slices/taskSlice'; 5 | import ddlReducer from './slices/ddlSlice'; 6 | import appReducer from './slices/appSlice'; 7 | 8 | export const store = configureStore({ 9 | reducer: { 10 | schema: schemaReducer, 11 | records: recordsReducer, 12 | task: taskReducer, 13 | ddl: ddlReducer, 14 | app: appReducer, 15 | }, 16 | }); 17 | 18 | export type RootState = ReturnType; 19 | export type AppDispatch = typeof store.dispatch; -------------------------------------------------------------------------------- /frontend/src/components/records/GenerationRecords.tsx: -------------------------------------------------------------------------------- 1 | import { QuestionList } from './QuestionList'; 2 | import { MainContent } from './MainContent'; 3 | import useTask from '@/hooks/useTask'; 4 | import { useEffect } from 'react'; 5 | 6 | export function GenerationRecords() { 7 | const { refreshQuestions } = useTask(); 8 | 9 | // Initial fetch questions 10 | useEffect(() => { 11 | refreshQuestions() 12 | }, [refreshQuestions]) 13 | 14 | return ( 15 |
16 | {/* Left list */} 17 | 18 | {/* Right content */} 19 | 20 |
21 | ); 22 | } -------------------------------------------------------------------------------- /backend/vectors/vector_store.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | class VectorStore(ABC): 4 | """Abstract base class for vector databases""" 5 | 6 | @abstractmethod 7 | def add_document(self, document, metadata, doc_id): 8 | """Add document to vector storage""" 9 | pass 10 | 11 | @abstractmethod 12 | def query_documents(self, query_text, n_results=1): 13 | """Query documents""" 14 | pass 15 | 16 | @abstractmethod 17 | def delete_documents(self, where): 18 | """Delete documents""" 19 | pass 20 | 21 | @abstractmethod 22 | def clear_collection(self): 23 | """Clear collection""" 24 | pass -------------------------------------------------------------------------------- /frontend/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", 4 | "target": "ES2022", 5 | "lib": ["ES2023"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "Bundler", 11 | "allowImportingTsExtensions": true, 12 | "isolatedModules": true, 13 | "moduleDetection": "force", 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "noFallthroughCasesInSwitch": true, 21 | "noUncheckedSideEffectImports": true 22 | }, 23 | "include": ["vite.config.ts"] 24 | } 25 | -------------------------------------------------------------------------------- /backend/dto/project_settings_dto.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class ProjectSettingsDTO: 5 | name: str 6 | description: str 7 | db_type: str 8 | db_version: str 9 | 10 | vector_waiting_table_count: int 11 | vector_waiting_column_count: int 12 | vector_waiting_doc_count: int 13 | vector_waiting_task_count: int 14 | 15 | definition_doc_count: int 16 | definition_rule_count: int 17 | definition_table_count: int 18 | definition_column_count: int 19 | definition_relation_count: int 20 | 21 | task_count: int 22 | task_doc_count: int 23 | task_sql_count: int 24 | task_table_count: int 25 | task_column_count: int 26 | job_count: int 27 | -------------------------------------------------------------------------------- /frontend/src/utils/stringUtils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * VSCode-style fuzzy string matching 3 | * Supports matching across underscores and case-insensitive matching 4 | */ 5 | export const fuzzyMatch = (pattern: string, str: string): boolean => { 6 | pattern = pattern.toLowerCase(); 7 | str = str.toLowerCase(); 8 | 9 | let patternIdx = 0; 10 | let strIdx = 0; 11 | 12 | while (patternIdx < pattern.length && strIdx < str.length) { 13 | // Support matching across underscores 14 | if (pattern[patternIdx] === str[strIdx] || 15 | (str[strIdx] === '_' && pattern[patternIdx] === str[strIdx + 1])) { 16 | patternIdx++; 17 | } 18 | strIdx++; 19 | } 20 | 21 | return patternIdx === pattern.length; 22 | }; -------------------------------------------------------------------------------- /backend/dto/schema_dto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List 3 | from dto.definition_rule_dto import DefinitionRuleDTO 4 | 5 | @dataclass 6 | class DefinitionTableDTO: 7 | table: str 8 | comment: str 9 | ai_comment: str 10 | disabled: bool 11 | 12 | @dataclass 13 | class DefinitionColumnDTO: 14 | table: str 15 | type: str 16 | column: str 17 | comment: str 18 | ai_comment: str 19 | 20 | @dataclass 21 | class DefinitionRelationDTO: 22 | table1: str 23 | column1: str 24 | table2: str 25 | column2: str 26 | relation_type: str 27 | 28 | @dataclass 29 | class SchemaDTO: 30 | tables: List[DefinitionTableDTO] 31 | columns: List[DefinitionColumnDTO] 32 | relations: List[DefinitionRelationDTO] 33 | rules: List[DefinitionRuleDTO] -------------------------------------------------------------------------------- /backend/wsgi.py: -------------------------------------------------------------------------------- 1 | from app import app 2 | from jobs import init_scheduler 3 | 4 | def on_post_fork(server, worker): 5 | """This runs in each worker process.""" 6 | from database import db 7 | 8 | with app.app_context(): 9 | if hasattr(db, 'engine'): 10 | # 断开继承自主进程的数据库连接 11 | db.engine.dispose() 12 | 13 | def on_when_ready(server): 14 | """This runs in the master process before spawning workers.""" 15 | app.logger.info('Initializing scheduler in master process') 16 | with app.app_context(): 17 | try: 18 | init_scheduler(app) 19 | app.logger.info('Scheduler initialized successfully') 20 | except Exception as e: 21 | app.logger.error(f'Failed to initialize scheduler: {str(e)}') 22 | 23 | if __name__ == "__main__": 24 | app.run() -------------------------------------------------------------------------------- /backend/prompt_templates/gen_ai_comments.mustache: -------------------------------------------------------------------------------- 1 | 2 | You are a database expert, skilled at generating AI comments for tables and columns based on provided table information. 3 | 4 | 5 | 6 | {{{tableStr}}} 7 |
8 | 9 | 10 | Please generate AI comments for tables and columns based on the table information. 11 | 12 | 13 | 14 | Please output JSON result in the following format, only including comments for tables and columns: 15 | 16 | ```json 17 | { 18 | "table": { 19 | "t": "Table Name", 20 | "v": "Table Comment", 21 | "cols": [ 22 | { 23 | "c": "Column Name", 24 | "v": "Column Comment" 25 | } 26 | ] 27 | } 28 | } 29 | ``` 30 | 31 | -------------------------------------------------------------------------------- /backend/dto/gen_ai_comments_dto.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from marshmallow_dataclass import dataclass 3 | 4 | @dataclass 5 | class GenAICommentsColumnDTO: 6 | """ 7 | Column information DTO 8 | """ 9 | column: str 10 | type: str 11 | comment: str 12 | 13 | @dataclass 14 | class GenAICommentsTableDTO: 15 | """ 16 | Table information DTO 17 | """ 18 | table: str 19 | comment: str 20 | columns: List[GenAICommentsColumnDTO] 21 | 22 | @dataclass 23 | class GenAICommentsResponseColumnDTO: 24 | """Column information for AI generated comments""" 25 | column: str 26 | comment: str 27 | 28 | @dataclass 29 | class GenAICommentsResponseDTO: 30 | """Response DTO for AI generated comments""" 31 | table: str 32 | comment: str 33 | columns: list[GenAICommentsResponseColumnDTO] 34 | -------------------------------------------------------------------------------- /backend/models/task_sql.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | 5 | class TaskSQL(ProjectBaseModel): 6 | """Task model: which SQL records are selected""" 7 | __tablename__ = 'task_sql' 8 | 9 | task_id = db.Column(db.Integer, nullable=False, comment='Task ID') 10 | sql_id = db.Column(db.Integer, nullable=False, comment='SQL record ID') 11 | 12 | __table_args__ = ( 13 | db.Index('ix_task_sql_task_id', 'task_id'), 14 | ) 15 | 16 | class TaskSQLSchema(SQLAlchemyAutoSchema): 17 | class Meta: 18 | model = TaskSQL 19 | load_instance = True 20 | include_relationships = True 21 | sqla_session = db.session 22 | 23 | task_sql_schema = TaskSQLSchema() 24 | task_sqls_schema = TaskSQLSchema(many=True) 25 | -------------------------------------------------------------------------------- /frontend/src/i18n/i18n.ts: -------------------------------------------------------------------------------- 1 | import i18n from 'i18next'; 2 | import { initReactI18next } from 'react-i18next'; 3 | 4 | import enUS from './en-US.json'; 5 | import zhCN from './zh-CN.json'; 6 | 7 | // Get saved language settings from localStorage 8 | const savedLanguage = localStorage.getItem('language') || 'zh-CN'; 9 | 10 | i18n 11 | .use(initReactI18next) 12 | .init({ 13 | resources: { 14 | 'en-US': enUS, 15 | 'zh-CN': zhCN, 16 | }, 17 | lng: savedLanguage, // Use saved language settings 18 | fallbackLng: 'en-US', 19 | interpolation: { 20 | escapeValue: false, 21 | }, 22 | }); 23 | 24 | // Add language switching function 25 | export const changeLanguage = (language: string) => { 26 | i18n.changeLanguage(language); 27 | localStorage.setItem('language', language); 28 | }; 29 | 30 | export default i18n; 31 | -------------------------------------------------------------------------------- /backend/models/task_doc.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | 5 | class TaskDoc(ProjectBaseModel): 6 | """Task model: which documents are selected""" 7 | __tablename__ = 'task_doc' 8 | 9 | task_id = db.Column(db.Integer, nullable=False, comment='Task ID') 10 | doc_id = db.Column(db.Integer, nullable=False, comment='Document definition ID') 11 | 12 | __table_args__ = ( 13 | db.Index('ix_task_doc_task_id', 'task_id'), 14 | ) 15 | 16 | class TaskDocSchema(SQLAlchemyAutoSchema): 17 | class Meta: 18 | model = TaskDoc 19 | load_instance = True 20 | include_relationships = True 21 | sqla_session = db.session 22 | 23 | task_doc_schema = TaskDocSchema() 24 | task_docs_schema = TaskDocSchema(many=True) 25 | -------------------------------------------------------------------------------- /backend/dto/update_task_query.py: -------------------------------------------------------------------------------- 1 | from marshmallow_dataclass import dataclass 2 | 3 | @dataclass 4 | class UpdateTaskColumnQueryDTO: 5 | table: str 6 | columns: list[str] 7 | 8 | @dataclass 9 | class UpdateTaskQueryDTO: 10 | task_id: int 11 | 12 | question: str 13 | question_supplement: str 14 | options: dict 15 | rules: list[int] 16 | doc_ids: list[int] 17 | sql_ids: list[int] 18 | columns: list[UpdateTaskColumnQueryDTO] 19 | sql: str 20 | 21 | question_modified: bool | None = False 22 | question_supplement_modified: bool | None = False 23 | options_modified: bool | None = False 24 | rules_modified: bool | None = False 25 | doc_ids_modified: bool | None = False 26 | sql_ids_modified: bool | None = False 27 | columns_modified: bool | None = False 28 | sql_modified: bool | None = False 29 | -------------------------------------------------------------------------------- /backend/models/task_table.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | 5 | class TaskTable(ProjectBaseModel): 6 | """Task model: which tables are selected""" 7 | __tablename__ = 'task_table' 8 | 9 | task_id = db.Column(db.Integer, nullable=False, comment='Task ID') 10 | table_name = db.Column(db.String(100), nullable=False, comment='Table name') 11 | 12 | __table_args__ = ( 13 | db.Index('ix_task_table_task_id', 'task_id'), 14 | ) 15 | 16 | class TaskTableSchema(SQLAlchemyAutoSchema): 17 | class Meta: 18 | model = TaskTable 19 | load_instance = True 20 | include_relationships = True 21 | sqla_session = db.session 22 | 23 | task_table_schema = TaskTableSchema() 24 | task_tables_schema = TaskTableSchema(many=True) 25 | -------------------------------------------------------------------------------- /frontend/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import reactHooks from 'eslint-plugin-react-hooks' 4 | import reactRefresh from 'eslint-plugin-react-refresh' 5 | import tseslint from 'typescript-eslint' 6 | 7 | export default tseslint.config( 8 | { ignores: ['dist'] }, 9 | { 10 | extends: [js.configs.recommended, ...tseslint.configs.recommended], 11 | files: ['**/*.{ts,tsx}'], 12 | languageOptions: { 13 | ecmaVersion: 2020, 14 | globals: globals.browser, 15 | }, 16 | plugins: { 17 | 'react-hooks': reactHooks, 18 | 'react-refresh': reactRefresh, 19 | }, 20 | rules: { 21 | ...reactHooks.configs.recommended.rules, 22 | 'react-refresh/only-export-components': [ 23 | 'warn', 24 | { allowConstantExport: true }, 25 | ], 26 | }, 27 | }, 28 | ) 29 | -------------------------------------------------------------------------------- /backend/.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Virtual Environment 24 | venv/ 25 | env/ 26 | ENV/ 27 | .env 28 | 29 | # IDE 30 | .idea/ 31 | .vscode/ 32 | *.swp 33 | *.swo 34 | .DS_Store 35 | 36 | # Database 37 | instance/ 38 | *.db 39 | *.sqlite3 40 | *.sqlite 41 | 42 | # Logs 43 | *.log 44 | logs/ 45 | 46 | # Local development settings 47 | .env.local 48 | .env.development.local 49 | .env.test.local 50 | .env.production.local 51 | 52 | # Coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .coverage 56 | .coverage.* 57 | .cache 58 | nosetests.xml 59 | coverage.xml 60 | *.cover 61 | 62 | # Gunicorn 63 | gunicorn.pid 64 | 65 | # Temp data 66 | temp-data/ -------------------------------------------------------------------------------- /frontend/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", 4 | "target": "ES2020", 5 | "useDefineForClassFields": true, 6 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 7 | "module": "ESNext", 8 | "skipLibCheck": true, 9 | 10 | /* Bundler mode */ 11 | "moduleResolution": "Bundler", 12 | "allowImportingTsExtensions": true, 13 | "isolatedModules": true, 14 | "moduleDetection": "force", 15 | "noEmit": true, 16 | "jsx": "react-jsx", 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "noUncheckedSideEffectImports": true, 24 | 25 | /* Path Aliases */ 26 | "baseUrl": ".", 27 | "paths": { 28 | "@/*": ["src/*"] 29 | } 30 | }, 31 | "include": ["src"] 32 | } 33 | -------------------------------------------------------------------------------- /backend/models/project.py: -------------------------------------------------------------------------------- 1 | from models.base import BaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema, auto_field 4 | 5 | class Project(BaseModel): 6 | """Project model""" 7 | __tablename__ = 'project' 8 | 9 | name = db.Column(db.String(20), comment='Project name') 10 | description = db.Column(db.String(200), comment='Project description') 11 | db_type = db.Column(db.String(20), comment='Database type') 12 | db_version = db.Column(db.String(255), comment='Database version information') 13 | cur_version = db.Column(db.Integer, default=1, comment='Current index version') 14 | 15 | class ProjectSchema(SQLAlchemyAutoSchema): 16 | class Meta: 17 | model = Project 18 | load_instance = True 19 | include_relationships = True 20 | sqla_session = db.session 21 | 22 | project_schema = ProjectSchema() 23 | projects_schema = ProjectSchema(many=True) 24 | -------------------------------------------------------------------------------- /frontend/src/components/ddl/DDL.tsx: -------------------------------------------------------------------------------- 1 | import { useAppSelector } from '@/store/hooks'; 2 | import { TableList } from './TableList'; 3 | import TableCommentEditor from '@/components/ddl/TableCommentEditor'; 4 | 5 | export function DDL() { 6 | const selectedTable = useAppSelector(state => state.ddl.selectedTable); 7 | 8 | const handleConfirm = () => { 9 | // Handle logic after confirmation 10 | }; 11 | 12 | return ( 13 |
14 | {/* Left list */} 15 | 16 | {/* Right content */} 17 |
18 | {selectedTable && ( 19 | 23 | )} 24 |
25 |
26 | ); 27 | } -------------------------------------------------------------------------------- /backend/models/task_column.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | 5 | class TaskColumn(ProjectBaseModel): 6 | """Task model: which columns are selected""" 7 | __tablename__ = 'task_column' 8 | 9 | task_id = db.Column(db.Integer, nullable=False, comment='Task ID') 10 | table_name = db.Column(db.String(100), nullable=False, comment='Table name') 11 | column_name = db.Column(db.String(100), nullable=False, comment='Column name') 12 | 13 | __table_args__ = ( 14 | db.Index('ix_task_column_task_id', 'task_id'), 15 | ) 16 | 17 | class TaskColumnSchema(SQLAlchemyAutoSchema): 18 | class Meta: 19 | model = TaskColumn 20 | load_instance = True 21 | include_relationships = True 22 | sqla_session = db.session 23 | 24 | task_column_schema = TaskColumnSchema() 25 | task_columns_schema = TaskColumnSchema(many=True) 26 | -------------------------------------------------------------------------------- /backend/gunicorn.conf.py: -------------------------------------------------------------------------------- 1 | # Gunicorn configuration file 2 | import multiprocessing 3 | import os 4 | from wsgi import on_post_fork, on_when_ready 5 | 6 | # Get current directory 7 | current_dir = os.path.dirname(os.path.abspath(__file__)) 8 | 9 | # Bind IP and port 10 | bind = "0.0.0.0:8000" 11 | 12 | # Number of worker processes 13 | workers = multiprocessing.cpu_count() * 2 + 1 14 | 15 | # Worker mode 16 | worker_class = "sync" 17 | 18 | # Maximum number of concurrent clients 19 | worker_connections = 2000 20 | 21 | # Process ID file 22 | pidfile = os.path.join(current_dir, "gunicorn.pid") 23 | 24 | # Access and error logs 25 | accesslog = os.path.join(current_dir, "logs/access.log") 26 | errorlog = os.path.join(current_dir, "logs/error.log") 27 | 28 | # Log level 29 | loglevel = "info" 30 | 31 | # Prevent multiple scheduler instances 32 | preload_app = True 33 | 34 | # Keep workers alive for long-running tasks 35 | timeout = 300 36 | 37 | # 确保启用钩子函数 38 | post_fork = on_post_fork 39 | when_ready = on_when_ready 40 | -------------------------------------------------------------------------------- /backend/enums.py: -------------------------------------------------------------------------------- 1 | from base_enum import BaseEnum 2 | 3 | class JobType(BaseEnum): 4 | """Job type enumeration""" 5 | GEN_RELATED_COLUMNS = ('gen_related_columns', 'Generate Related Columns') 6 | MATCH_DOC = ('match_doc', 'Match Document') 7 | MATCH_SQL_LOG = ('match_sql_log', 'Match SQL Log') 8 | MATCH_DDL = ('match_ddl', 'Match DDL') 9 | GENERATE_SQL = ('generate_sql', 'Generate SQL') 10 | LEARN_FROM_SQL = ('learn_from_sql', 'Learn') 11 | 12 | class JobStatus(BaseEnum): 13 | """Job status enumeration""" 14 | INIT = ('init', 'Initial') # Initial state 15 | RUNNING = ('running', 'Running') # In progress 16 | SUCCESS = ('success', 'Success') # Successful 17 | FAIL = ('fail', 'Failed') # Failed 18 | CANCELED = ('canceled', 'Canceled') # Canceled 19 | 20 | class DbType(BaseEnum): 21 | """Database type enumeration""" 22 | SQLITE = ('sqlite', 'SQLite') 23 | MYSQL = ('mysql', 'MySQL') 24 | POSTGRESQL = ('postgresql', 'PostgreSQL') 25 | SQLSERVER = ('sqlserver', 'SQLServer') 26 | -------------------------------------------------------------------------------- /backend/models/job.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | from enums import JobStatus 5 | 6 | class Job(ProjectBaseModel): 7 | """Task model""" 8 | __tablename__ = 'job' 9 | 10 | task_id = db.Column(db.Integer, nullable=False, comment='Task ID') 11 | job_type = db.Column(db.String(20), nullable=False, comment='Task type') 12 | job_data = db.Column(db.JSON, comment='Task data') 13 | job_status = db.Column(db.String(20), nullable=False, default=JobStatus.INIT.value, comment='Task status') 14 | job_cost_time = db.Column(db.Integer, nullable=False, default=0, comment='Task cost time, unit: ms') 15 | error_message = db.Column(db.Text, comment='Error message') 16 | 17 | class JobSchema(SQLAlchemyAutoSchema): 18 | class Meta: 19 | model = Job 20 | load_instance = True 21 | include_relationships = True 22 | sqla_session = db.session 23 | 24 | job_schema = JobSchema() 25 | jobs_schema = JobSchema(many=True) 26 | -------------------------------------------------------------------------------- /backend/models/definition_rule.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | 5 | class DefinitionRule(ProjectBaseModel): 6 | """Rule definition model""" 7 | __tablename__ = 'definition_rule' 8 | 9 | name = db.Column(db.String(100), nullable=False, comment='Rule name') 10 | content = db.Column(db.Text, nullable=False, comment='Rule content') 11 | def_selected = db.Column(db.Boolean, default=False, comment='Whether default selected') 12 | disabled = db.Column(db.Boolean, default=False, comment='Whether disabled') 13 | 14 | __table_args__ = ( 15 | db.Index('idx_definition_rule_def_selected', 'def_selected'), 16 | ) 17 | 18 | class DefinitionRuleSchema(SQLAlchemyAutoSchema): 19 | class Meta: 20 | model = DefinitionRule 21 | load_instance = True 22 | include_relationships = True 23 | sqla_session = db.session 24 | 25 | definition_rule_schema = DefinitionRuleSchema() 26 | definition_rules_schema = DefinitionRuleSchema(many=True) -------------------------------------------------------------------------------- /backend/vectors/vector_chroma.py: -------------------------------------------------------------------------------- 1 | import chromadb 2 | from vectors.vector_store import VectorStore 3 | 4 | class ChromaDBHandler(VectorStore): 5 | def __init__(self, host, port, collection_name): 6 | self.client = chromadb.HttpClient(host=host, port=port) 7 | self.collection = self.client.get_or_create_collection(name=collection_name) 8 | print(f"Initialized ChromaDBHandler for collection: {collection_name}") 9 | 10 | def add_document(self, document, metadata, doc_id): 11 | self.collection.upsert( 12 | documents=[document], 13 | metadatas=[metadata], 14 | ids=[doc_id] 15 | ) 16 | 17 | def query_documents(self, query_text, n_results=1, where=None): 18 | return self.collection.query( 19 | query_texts=[query_text], 20 | n_results=n_results, 21 | where=where 22 | ) 23 | 24 | def delete_documents(self, where): 25 | self.collection.delete(where=where) 26 | 27 | def clear_collection(self): 28 | self.collection.delete(ids=self.collection.get()["ids"]) -------------------------------------------------------------------------------- /backend/models/definition_doc.py: -------------------------------------------------------------------------------- 1 | from models.base import ProjectBaseModel 2 | from database import db 3 | from marshmallow_sqlalchemy import SQLAlchemyAutoSchema 4 | 5 | class DefinitionDoc(ProjectBaseModel): 6 | """Document definition model""" 7 | __tablename__ = 'definition_doc' 8 | 9 | def_doc = db.Column(db.Text, nullable=False, comment='Document content') 10 | def_selected = db.Column(db.Boolean, default=False, comment='Whether default selected') 11 | def_waiting = db.Column(db.Boolean, default=False, comment='Whether waiting for building') 12 | disabled = db.Column(db.Boolean, default=False, comment='Whether disabled') 13 | 14 | # Non-unique index: def_selected 15 | __table_args__ = (db.Index('idx_definition_doc_def_selected', 'def_selected'),) 16 | 17 | class DefinitionDocSchema(SQLAlchemyAutoSchema): 18 | class Meta: 19 | model = DefinitionDoc 20 | load_instance = True 21 | include_relationships = True 22 | sqla_session = db.session 23 | 24 | definition_doc_schema = DefinitionDocSchema() 25 | definition_docs_schema = DefinitionDocSchema(many=True) -------------------------------------------------------------------------------- /backend/dto/task_dto.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from datetime import datetime 3 | from dto.job_dto import JobDTO 4 | from dto.learn_result_dto import LearnResultDTO 5 | from marshmallow_dataclass import dataclass 6 | 7 | @dataclass 8 | class TaskTableDTO: 9 | table_name: str 10 | 11 | @dataclass 12 | class TaskColumnDTO: 13 | table_name: str 14 | column_name: str 15 | 16 | @dataclass 17 | class TaskDocDTO: 18 | doc_id: int 19 | def_doc: str 20 | 21 | @dataclass 22 | class TaskSQLDTO: 23 | task_id: int 24 | question: str 25 | sql: str 26 | 27 | @dataclass 28 | class TaskDTO: 29 | id: int 30 | project_id: int 31 | version: int 32 | question: str 33 | question_supplement: str 34 | options: dict 35 | rules: List[int] 36 | related_columns: str 37 | sql: str 38 | sql_right: bool 39 | sql_refer: bool 40 | learn_result: LearnResultDTO 41 | created_at: datetime 42 | updated_at: datetime 43 | tables: List[TaskTableDTO] 44 | columns: List[TaskColumnDTO] 45 | docs: List[TaskDocDTO] 46 | sqls: List[TaskSQLDTO] 47 | jobs: List[JobDTO] -------------------------------------------------------------------------------- /backend/base_enum.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | class BaseEnum(Enum): 4 | """Base enumeration class providing common methods""" 5 | def __init__(self, value, display_name): 6 | self._value_ = value 7 | self.display_name = display_name 8 | 9 | @property 10 | def value(self): 11 | """Get the status value""" 12 | return self._value_ 13 | 14 | @classmethod 15 | def values(cls): 16 | """Get a list of all status values""" 17 | return [item.value for item in cls] 18 | 19 | @classmethod 20 | def names(cls): 21 | """Get a list of all status names""" 22 | return [item.display_name for item in cls] 23 | 24 | @classmethod 25 | def get_by_value(cls, value): 26 | """Get enum value by value""" 27 | for item in cls: 28 | if item.value == value: 29 | return item 30 | return None 31 | 32 | @classmethod 33 | def get_display_name_by_value(cls, value): 34 | """Get status name by value""" 35 | item = cls.get_by_value(value) 36 | return item.display_name if item else None -------------------------------------------------------------------------------- /backend/prompt_templates/optimize_question.mustache: -------------------------------------------------------------------------------- 1 | ## Context 2 | User Question: 3 | 4 | ```txt 5 | {{{question}}} 6 | ``` 7 | 8 | ## Instructions 9 | 10 | Please optimize the expression to be more standardized, clear, and unambiguous, eliminating ambiguity, and making it easier to generate SQL statements. If there are contradictions or ambiguities in the statement, please understand the user's intention and fix them: 11 | 12 | ### Optimization Example 13 | 14 | Example 1: 15 | Original Question: Query the number of orders per day, aggregated by customer 16 | Optimized: Please count the total number of orders per customer per day, including date, customer ID, and order count. Results should be sorted by date and customer ID. 17 | 18 | Example 2: 19 | Original Question: Query items with sales greater than 1000 20 | Optimized: Please query information about items with a cumulative sales total exceeding 1000 yuan, including item ID, item name, and sales total. Results should be sorted in descending order by sales total. 21 | 22 | ## Output Format 23 | Output in JSON format, as follows: 24 | 25 | ```json 26 | { 27 | "result": "Optimized Question" 28 | } 29 | ``` -------------------------------------------------------------------------------- /frontend/src/components/records/OptimizedQuestion.tsx: -------------------------------------------------------------------------------- 1 | import { Button } from 'flowbite-react' 2 | import { IoCheckmark, IoClose } from 'react-icons/io5' 3 | import { useTranslation } from 'react-i18next' 4 | 5 | interface OptimizedQuestionProps { 6 | optimizedQuestion: string 7 | onAccept: () => void 8 | onReject: () => void 9 | } 10 | 11 | export function OptimizedQuestion({ optimizedQuestion, onAccept, onReject }: OptimizedQuestionProps) { 12 | const { t } = useTranslation() 13 | 14 | return ( 15 |
16 |
{optimizedQuestion}
17 |
18 | 26 | 33 |
34 |
35 | ) 36 | } -------------------------------------------------------------------------------- /frontend/src/components/ddl/UploadDDLModal.tsx: -------------------------------------------------------------------------------- 1 | import { Modal, Tabs } from 'flowbite-react'; 2 | import { useTranslation } from 'react-i18next'; 3 | import { FileImportTab } from './FileImportTab'; 4 | import { QueryImportTab } from './QueryImportTab'; 5 | 6 | interface UploadDDLModalProps { 7 | show: boolean; 8 | onClose: () => void; 9 | } 10 | 11 | export function UploadDDLModal({ show, onClose }: UploadDDLModalProps) { 12 | const { t } = useTranslation(); 13 | 14 | const handleClose = () => { 15 | onClose(); 16 | }; 17 | 18 | return ( 19 | 20 | {t('ddl.uploadDDLFiles')} 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | ); 33 | } -------------------------------------------------------------------------------- /backend/vector_stores.py: -------------------------------------------------------------------------------- 1 | import os 2 | from vectors.vector_chroma import ChromaDBHandler 3 | from vectors.translate_wrapper import TranslateWrapper 4 | 5 | table_def_store = TranslateWrapper( 6 | vector_store=ChromaDBHandler( 7 | host=os.getenv("CHROMA_HOST", "localhost"), 8 | port=int(os.getenv("CHROMA_PORT", "8000")), 9 | collection_name="table_def" 10 | ), 11 | target_language='en' 12 | ) 13 | 14 | column_def_store = TranslateWrapper( 15 | vector_store=ChromaDBHandler( 16 | host=os.getenv("CHROMA_HOST", "localhost"), 17 | port=int(os.getenv("CHROMA_PORT", "8000")), 18 | collection_name="column_def" 19 | ), 20 | target_language='en' 21 | ) 22 | 23 | doc_def_store = TranslateWrapper( 24 | vector_store=ChromaDBHandler( 25 | host=os.getenv("CHROMA_HOST", "localhost"), 26 | port=int(os.getenv("CHROMA_PORT", "8000")), 27 | collection_name="doc_def" 28 | ), 29 | target_language='en' 30 | ) 31 | 32 | sql_log_store = TranslateWrapper( 33 | vector_store=ChromaDBHandler( 34 | host=os.getenv("CHROMA_HOST", "localhost"), 35 | port=int(os.getenv("CHROMA_PORT", "8000")), 36 | collection_name="sql_log" 37 | ), 38 | target_language='en' 39 | ) -------------------------------------------------------------------------------- /frontend/src/utils/bizUtil.ts: -------------------------------------------------------------------------------- 1 | import { TaskColumnDTO } from "@/api-docs/api" 2 | import { SelectedColumn } from "@/store/slices/recordsSlice" 3 | import { Schema } from "@/api-docs" 4 | 5 | export const getSelectedColumns = (columns: TaskColumnDTO[] | undefined, schema: Schema) => { 6 | const selectedColumns: SelectedColumn[] = [] 7 | const tableMap = new Map() 8 | 9 | // Group columns by table name 10 | columns?.forEach(column => { 11 | const tableName = column.table_name! 12 | const columnName = column.column_name! 13 | 14 | if (!tableMap.has(tableName)) { 15 | tableMap.set(tableName, []) 16 | } 17 | tableMap.get(tableName)!.push(columnName) 18 | }) 19 | 20 | // Sort columns by schema.columns 21 | tableMap.forEach((columns, table) => { 22 | columns.sort((a, b) => schema!.columns!.findIndex(c => c.table === table && c.column === a) - schema!.columns!.findIndex(c => c.table === table && c.column === b)) 23 | }) 24 | 25 | // Convert map to selectedColumns array 26 | tableMap.forEach((columns, table) => { 27 | selectedColumns.push({ 28 | table, 29 | columns 30 | }) 31 | }) 32 | 33 | return selectedColumns 34 | } -------------------------------------------------------------------------------- /backend/utils/schemas.py: -------------------------------------------------------------------------------- 1 | from marshmallow import Schema, fields 2 | from marshmallow_dataclass import class_schema 3 | 4 | class MessageResponseSchema(Schema): 5 | message = fields.Str(description='Return message') 6 | 7 | class PaginationQuerySchema(Schema): 8 | page = fields.Int(load_default=1, description="Page number") 9 | per_page = fields.Int(load_default=20, description="Number of items per page") 10 | 11 | class ProjectIdQuerySchema(Schema): 12 | project_id = fields.Int(description="Project ID") 13 | 14 | class PaginationBaseSchema(Schema): 15 | """Base schema for pagination responses""" 16 | items = fields.List(fields.Nested(Schema), description='List of items') 17 | total = fields.Int(description='Total number of items') 18 | page = fields.Int(description='Current page number') 19 | per_page = fields.Int(description='Number of items per page') 20 | pages = fields.Int(description='Total number of pages') 21 | 22 | class PaginationSchema: 23 | """Generic pagination schema factory""" 24 | @classmethod 25 | def create(cls, nested_schema, name): 26 | class_dict = { 27 | '__name__': name, 28 | 'items': fields.List(fields.Nested(nested_schema)) 29 | } 30 | return type(name, (PaginationBaseSchema,), class_dict) -------------------------------------------------------------------------------- /frontend/src/store/slices/taskSlice.ts: -------------------------------------------------------------------------------- 1 | import { createSlice, PayloadAction } from '@reduxjs/toolkit'; 2 | import { Job, TaskDTO, TaskQuestion } from '../../api-docs'; 3 | 4 | interface TaskState { 5 | questions: TaskQuestion[] 6 | 7 | taskId?: number 8 | task?: TaskDTO 9 | 10 | jobs?: Job[] 11 | currentJob?: Job 12 | } 13 | 14 | const initialState: TaskState = { 15 | questions: [] 16 | }; 17 | 18 | export const taskSlice = createSlice({ 19 | name: 'task', 20 | initialState, 21 | reducers: { 22 | setTaskId: (state, action: PayloadAction) => { 23 | state.taskId = action.payload; 24 | }, 25 | setTask: (state, action: PayloadAction) => { 26 | state.task = action.payload; 27 | }, 28 | setJobs: (state, action: PayloadAction) => { 29 | state.jobs = action.payload; 30 | }, 31 | setCurrentJob: (state, action: PayloadAction) => { 32 | state.currentJob = action.payload; 33 | }, 34 | setQuestions: (state, action: PayloadAction) => { 35 | state.questions = action.payload; 36 | }, 37 | }, 38 | }); 39 | 40 | export const { setTaskId, setTask, setJobs, setCurrentJob, setQuestions } = taskSlice.actions; 41 | export default taskSlice.reducer; 42 | -------------------------------------------------------------------------------- /frontend/src/hooks/useEnvService.ts: -------------------------------------------------------------------------------- 1 | import { useEffect } from 'react' 2 | import { useAppDispatch, useAppSelector } from '../store/hooks' 3 | import { setInitEnvData } from '../store/slices/appSlice' 4 | import { useAsyncEffect } from 'ahooks' 5 | 6 | const useEnvService = () => { 7 | const dispatch = useAppDispatch() 8 | const envReady = useAppSelector(state => state.app.envReady) 9 | const envData = useAppSelector(state => state.app.envData) 10 | 11 | // Initial read envData 12 | useEffect(() => { 13 | const savedEnvData = localStorage.getItem('envData') 14 | let parsedData = undefined 15 | if (savedEnvData) { 16 | try { 17 | parsedData = JSON.parse(savedEnvData) 18 | } catch (error) { 19 | console.error('Failed to parse envData from localStorage:', error) 20 | } 21 | } 22 | 23 | dispatch(setInitEnvData(parsedData)) 24 | console.log('[envData] loaded:', parsedData) 25 | }, []) 26 | 27 | // If envData changes, save it to localStorage 28 | useAsyncEffect(async () => { 29 | if (envReady) { 30 | console.log('[envData] saved:', envData) 31 | localStorage.setItem('envData', JSON.stringify(envData)) 32 | } 33 | }, [envData]) 34 | } 35 | 36 | export default useEnvService -------------------------------------------------------------------------------- /frontend/src/components/records/QuestionSupplement.tsx: -------------------------------------------------------------------------------- 1 | import { Textarea } from 'flowbite-react'; 2 | import { FC } from 'react'; 3 | import { useTranslation } from 'react-i18next'; 4 | 5 | interface QuestionSupplementProps { 6 | value: string; 7 | onChange: (value: string) => void; 8 | placeholder?: string; 9 | disabled?: boolean; 10 | } 11 | 12 | const QuestionSupplement: FC = ({ 13 | value, 14 | onChange, 15 | placeholder, 16 | disabled = false, 17 | }) => { 18 | const { t } = useTranslation(); 19 | 20 | return ( 21 |
22 | 25 |