├── server ├── app │ ├── __init__.py │ ├── crud │ │ ├── __init__.py │ │ ├── crud_user_database.py │ │ ├── crud_table_column.py │ │ ├── crud_embedding.py │ │ ├── crud_query.py │ │ ├── crud_user.py │ │ └── crud_database_table.py │ ├── db │ │ ├── __init__.py │ │ ├── base.py │ │ ├── base_class.py │ │ └── session.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_services │ │ │ ├── __init__.py │ │ │ ├── test_openai_service.py │ │ │ ├── test_database_details.py │ │ │ └── test_embeddings_service.py │ │ ├── test_utilities │ │ │ └── test_fernet_manager.py │ │ ├── test_crud │ │ │ ├── test_crud_user.py │ │ │ ├── test_crud_embedding.py │ │ │ ├── test_crud_table_column.py │ │ │ └── test_crud_user_database.py │ │ ├── test_models │ │ │ ├── test_user_model.py │ │ │ ├── test_query_model.py │ │ │ ├── test_table_column_model.py │ │ │ ├── test_user_database_model.py │ │ │ ├── test_database_table_model.py │ │ │ └── test_embedding_model.py │ │ ├── test_schemas │ │ │ ├── test_user_database_schema.py │ │ │ ├── test_table_column_schema.py │ │ │ ├── test_database_table_schema.py │ │ │ ├── test_user_schema.py │ │ │ ├── test_embedding_schema.py │ │ │ └── test_query_schema.py │ │ ├── test_api │ │ │ └── v1 │ │ │ │ ├── test_authentication.py │ │ │ │ ├── test_dependencies.py │ │ │ │ └── test_query_executor.py │ │ ├── test_clients │ │ │ └── test_openai_client.py │ │ └── conftest.py │ ├── api │ │ └── v1 │ │ │ ├── __init__.py │ │ │ ├── routes.py │ │ │ ├── dependencies.py │ │ │ ├── authentication.py │ │ │ ├── query_executor.py │ │ │ └── query.py │ ├── schemas │ │ ├── __init__.py │ │ ├── table_column.py │ │ ├── user_database.py │ │ ├── database_table.py │ │ ├── user.py │ │ ├── query.py │ │ └── embedding.py │ ├── services │ │ ├── __init__.py │ │ ├── query_service.py │ │ ├── database_details.py │ │ ├── openai_service.py │ │ └── embeddings_service.py │ ├── constants.py │ ├── models │ │ ├── timestamp_base.py │ │ ├── query.py │ │ ├── database_table.py │ │ ├── table_column.py │ │ ├── user_database.py │ │ ├── embedding.py │ │ └── user.py │ ├── core │ │ └── log_config.py │ ├── main.py │ ├── user_utils.py │ ├── utilities │ │ └── fernet_manager.py │ └── clients │ │ └── openai_client.py ├── .dockerignore ├── alembic │ ├── README │ ├── script.py.mako │ ├── versions │ │ ├── af7438a7b330_rename_query_histories_to_queries.py │ │ ├── 794012702ac2_add_text_node_field_to_database_tables.py │ │ ├── 68b6ee05672c_add_connection_string_in_user_databases.py │ │ ├── 14c2ea88a9d6_add_user_database_id_column_to_.py │ │ ├── 7b50a83a45cd_create_user_database_id_and_name_unique_.py │ │ ├── da86d1e90141_add_table_user_databases.py │ │ ├── 3fbd94be5602_add_table_database_tables.py │ │ ├── baec336a3603_add_table_table_columns.py │ │ ├── cd660fe23c72_add_table_embeddings.py │ │ ├── 33633a5ee909_create_query_history_table.py │ │ ├── bf224d84fb4d_create_users_table.py │ │ └── 45e10e526767_add_timestamps_to_tables.py │ └── env.py ├── image.png ├── .coveragerc ├── requirements.txt ├── .env ├── Dockerfile ├── extract_schema.sh ├── output_schema.txt ├── scripts │ └── create_user.py └── alembic.ini ├── client ├── .env ├── public │ ├── MQLAI.png │ ├── eye_close.png │ ├── eye_open.png │ ├── loading.gif │ ├── database_3.jpg │ └── database_4.png ├── app │ ├── components │ │ ├── toast.ts │ │ ├── loader.tsx │ │ ├── errorBox.tsx │ │ ├── downloadCSV.tsx │ │ ├── navbar.tsx │ │ ├── fileUploader.tsx │ │ ├── codeBlock.tsx │ │ ├── databaseCard.tsx │ │ ├── schemaUploader.tsx │ │ ├── databaseInfo.tsx │ │ └── header.tsx │ ├── providers │ │ └── queryResultProvider │ │ │ ├── actions.ts │ │ │ ├── queryResultContext.ts │ │ │ ├── reducer.ts │ │ │ └── index.tsx │ ├── utils │ │ ├── helper.js │ │ ├── constant.js │ │ └── routes.js │ ├── (routes) │ │ ├── (protected) │ │ │ ├── layout.tsx │ │ │ ├── query │ │ │ │ └── page.tsx │ │ │ ├── databases │ │ │ │ └── [id] │ │ │ │ │ └── page.tsx │ │ │ ├── home │ │ │ │ └── page.tsx │ │ │ └── add-database │ │ │ │ └── page.tsx │ │ ├── layout.tsx │ │ └── (public) │ │ │ └── login │ │ │ └── page.tsx │ ├── viewControllers │ │ ├── databaseInfoViewController.ts │ │ ├── addDatabaseViewController.ts │ │ ├── genericViewController.ts │ │ ├── queryResultViewController.ts │ │ ├── homeAccountsViewController.ts │ │ ├── databaseViewController.ts │ │ ├── databaseCardViewController.ts │ │ ├── codeBlockViewController.ts │ │ ├── loginViewController.ts │ │ ├── uploadSchemaViewController.ts │ │ ├── connectDatabaseViewController.ts │ │ ├── homeViewController.ts │ │ └── chatViewController.ts │ ├── types.ts │ ├── styles │ │ └── globals.css │ └── lib │ │ └── service.js ├── next.config.js ├── postcss.config.js ├── .dockerignore ├── Dockerfile ├── README.md ├── middleware.ts ├── tailwind.config.ts ├── tsconfig.json └── package.json ├── storage ├── init.sql └── Dockerfile ├── .gitignore ├── docker-compose.yml ├── LICENSE ├── CODE_OF_CONDUCT.md ├── setup.sh ├── Makefile └── test_data └── elearning_schema /server/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/app/crud/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/app/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/app/api/v1/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/app/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/app/services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/.dockerignore: -------------------------------------------------------------------------------- 1 | **/*__pycache__ 2 | -------------------------------------------------------------------------------- /server/app/tests/test_services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /client/.env: -------------------------------------------------------------------------------- 1 | NEXT_PUBLIC_API_URL=http://localhost:8000/api/v1 -------------------------------------------------------------------------------- /server/alembic/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /server/app/constants.py: -------------------------------------------------------------------------------- 1 | DQL = { 2 | "Statement_Type_Keyowrd": "select", 3 | } -------------------------------------------------------------------------------- /server/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/server/image.png -------------------------------------------------------------------------------- /client/public/MQLAI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/client/public/MQLAI.png -------------------------------------------------------------------------------- /server/app/db/base.py: -------------------------------------------------------------------------------- 1 | from app.db.base_class import Base 2 | from app.models.user import User 3 | -------------------------------------------------------------------------------- /client/public/eye_close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/client/public/eye_close.png -------------------------------------------------------------------------------- /client/public/eye_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/client/public/eye_open.png -------------------------------------------------------------------------------- /client/public/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/client/public/loading.gif -------------------------------------------------------------------------------- /client/app/components/toast.ts: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | export { ToastContainer } from "react-toastify"; 4 | -------------------------------------------------------------------------------- /client/public/database_3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/client/public/database_3.jpg -------------------------------------------------------------------------------- /client/public/database_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shurutech/mql/HEAD/client/public/database_4.png -------------------------------------------------------------------------------- /client/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = {}; 3 | 4 | module.exports = nextConfig; 5 | -------------------------------------------------------------------------------- /client/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /server/app/db/base_class.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import DeclarativeBase 2 | 3 | 4 | class Base(DeclarativeBase): 5 | pass 6 | -------------------------------------------------------------------------------- /client/.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .dockerignore 3 | Dockerfile 4 | .gitignore 5 | 6 | # dependencies 7 | 8 | node_modules 9 | .next 10 | -------------------------------------------------------------------------------- /server/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = ./app 3 | omit = 4 | ./app/tests/* 5 | *__pycache__* 6 | 7 | [html] 8 | directory = coverage_report 9 | -------------------------------------------------------------------------------- /client/app/providers/queryResultProvider/actions.ts: -------------------------------------------------------------------------------- 1 | export const SET_RESULT = 'SET_RESULT'; 2 | 3 | export const SET_PAGE = 'SET_PAGE'; 4 | 5 | export const SET_ROWS_PER_PAGE = 'SET_ROWS_PER_PAGE'; -------------------------------------------------------------------------------- /client/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18-alpine 2 | 3 | WORKDIR /mql-web 4 | 5 | COPY package*.json ./ 6 | 7 | RUN npm install 8 | 9 | COPY . . 10 | 11 | RUN npm run build 12 | 13 | CMD ["npm", "run", "start"] -------------------------------------------------------------------------------- /server/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2 2 | fastapi[all] 3 | sqlalchemy 4 | pgvector 5 | alembic 6 | bcrypt 7 | httpx 8 | pytest 9 | pytest-mock 10 | python-jose[cryptography] 11 | black 12 | openai 13 | pytest-cov 14 | -------------------------------------------------------------------------------- /client/app/utils/helper.js: -------------------------------------------------------------------------------- 1 | export const handleDate = (date) => { 2 | const d = new Date(date); 3 | const month = d.toLocaleString("default", { month: "short" }); 4 | const day = d.getDate(); 5 | const year = d.getFullYear(); 6 | return `${day} ${month} ${year}`; 7 | }; 8 | -------------------------------------------------------------------------------- /client/app/providers/queryResultProvider/queryResultContext.ts: -------------------------------------------------------------------------------- 1 | import { createContext } from "react"; 2 | 3 | const QueryResultContext = createContext<{ 4 | state: QueryResultState; 5 | dispatch: React.Dispatch; 6 | } | null>(null); 7 | 8 | export default QueryResultContext; -------------------------------------------------------------------------------- /client/app/utils/constant.js: -------------------------------------------------------------------------------- 1 | export const COMMAND_DOWNLOAD_SCRIPT = 2 | "curl -o script.sh https://storage.googleapis.com/analytics-script-bucket/extract_schema.sh"; 3 | export const COMMAND_RUN_SCRIPT = `chmod +x ./script.sh && ./script.sh `; -------------------------------------------------------------------------------- /server/app/schemas/table_column.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID 2 | from pydantic import BaseModel, ConfigDict 3 | 4 | 5 | class TableColumn(BaseModel): 6 | name: str 7 | data_type: str 8 | database_table_id: UUID 9 | 10 | model_config = ConfigDict(from_attributes=True) 11 | -------------------------------------------------------------------------------- /server/app/db/session.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.orm import sessionmaker 3 | from dotenv import load_dotenv 4 | import os 5 | 6 | load_dotenv() 7 | 8 | engine = create_engine(os.getenv("DATABASE_URL"), echo=True) 9 | 10 | sessionLocal = sessionmaker(engine) 11 | -------------------------------------------------------------------------------- /client/app/(routes)/(protected)/layout.tsx: -------------------------------------------------------------------------------- 1 | import Navbar from "@/app/components/navbar"; 2 | 3 | export default function RootLayout({ 4 | children, 5 | }: { 6 | children: React.ReactNode; 7 | }) { 8 | return ( 9 |
10 | 11 | {children} 12 |
13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /server/.env: -------------------------------------------------------------------------------- 1 | DATABASE_URL="postgresql://shuru:password@postgres:5432/mql" 2 | TEST_DATABASE_URL="postgresql://shuru:password@localhost:5432/mql_test" 3 | 4 | 5 | SECRET_KEY="this-is-shuru-internal-project" 6 | ALGORITHM="HS256" 7 | ACCESS_TOKEN_EXPIRE_MINUTES=60 8 | ENCRYPTION_SALT="$2b$12$K5Ute4TtBvOpHbGnQkA8gO" 9 | -------------------------------------------------------------------------------- /storage/init.sql: -------------------------------------------------------------------------------- 1 | CREATE ROLE shuru WITH superuser; 2 | 3 | ALTER ROLE shuru WITH LOGIN PASSWORD 'password'; 4 | 5 | CREATE DATABASE mql WITH OWNER shuru; 6 | 7 | CREATE DATABASE mql_test WITH OWNER shuru; 8 | 9 | \c mql 10 | 11 | CREATE EXTENSION vector; 12 | 13 | \c mql_test 14 | 15 | CREATE EXTENSION vector; -------------------------------------------------------------------------------- /server/app/models/timestamp_base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, DateTime, func 2 | from app.db.base_class import Base 3 | 4 | 5 | class TimestampBase(Base): 6 | __abstract__ = True 7 | created_at = Column(DateTime, default=func.now()) 8 | updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) 9 | -------------------------------------------------------------------------------- /server/app/schemas/user_database.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, ConfigDict 2 | from uuid import UUID 3 | from typing import Optional 4 | 5 | 6 | class UserDatabase(BaseModel): 7 | name: str 8 | user_id: UUID 9 | connection_string: Optional[str] = None 10 | 11 | model_config = ConfigDict(from_attributes=True) 12 | -------------------------------------------------------------------------------- /server/app/schemas/database_table.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from uuid import UUID 3 | from pydantic import BaseModel, ConfigDict 4 | 5 | 6 | class DatabaseTable(BaseModel): 7 | name: str 8 | user_database_id: UUID 9 | text_node: Optional[str] = None 10 | 11 | model_config = ConfigDict(from_attributes=True) 12 | -------------------------------------------------------------------------------- /server/app/schemas/user.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from uuid import UUID 3 | from pydantic import BaseModel, EmailStr, ConfigDict 4 | 5 | 6 | class User(BaseModel): 7 | name: str 8 | email: EmailStr 9 | password: str 10 | id: Optional[UUID] = None 11 | 12 | model_config = ConfigDict(from_attributes=True) 13 | -------------------------------------------------------------------------------- /server/app/schemas/query.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, ConfigDict 2 | from uuid import UUID 3 | from typing import Optional 4 | 5 | 6 | class Query(BaseModel): 7 | id: Optional[UUID] = None 8 | nl_query: str 9 | sql_query: Optional[str] = None 10 | user_database_id: UUID 11 | 12 | model_config = ConfigDict(from_attributes=True) 13 | -------------------------------------------------------------------------------- /server/app/schemas/embedding.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | from uuid import UUID 3 | from pydantic import BaseModel, ConfigDict 4 | 5 | 6 | class Embedding(BaseModel): 7 | id: Optional[UUID] = None 8 | embeddings_vector: List[float] 9 | database_table_id: UUID 10 | user_database_id: UUID 11 | 12 | model_config = ConfigDict(from_attributes=True) 13 | -------------------------------------------------------------------------------- /client/app/utils/routes.js: -------------------------------------------------------------------------------- 1 | export const LOGIN = "/login"; 2 | export const DATABASES = "/databases"; 3 | export const CONNECT_DATABASE = "/connect-database"; 4 | export const UPLOAD_DATABASE = "/upload-database-schema"; 5 | export const QUERIES = "/queries"; 6 | export const QUERY_EXECUTION = "/sql-data"; 7 | export const SCHEMA_SYNC = "/schema-sync"; 8 | export const DELETE_DATABASE = "/delete-database"; 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | server/virtual_env 2 | *__pycache__ 3 | .vscode 4 | server/env/ 5 | server/.pytest_cache 6 | server/.coverage 7 | server/coverage_report/ 8 | 9 | client/node_modules 10 | client/.next 11 | client/.pnp 12 | client/.pnp.js 13 | client/build 14 | client/out 15 | client/coverage 16 | client/.DS_Store 17 | *.pem 18 | client/npm-debug.log* 19 | client/yarn-debug.log* 20 | client/yarn-error.log* 21 | *.tsbuildinfo 22 | client/next-env.d.ts 23 | .DS_Store 24 | env 25 | -------------------------------------------------------------------------------- /client/app/(routes)/(protected)/query/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import ChatInterface from "@/app/components/chatInterface"; 4 | import React from "react"; 5 | import { useSearchParams } from 'next/navigation' 6 | 7 | const Query:React.FC = () => { 8 | const params = useSearchParams(); 9 | const dbId = params.get('db_id')?.toString()!; 10 | return ( 11 |
12 | 13 |
14 | ); 15 | }; 16 | 17 | export default Query; 18 | -------------------------------------------------------------------------------- /server/app/api/v1/routes.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | 3 | from app.api.v1 import authentication, databases, query, query_executor 4 | 5 | api_router = APIRouter() 6 | api_router.include_router(authentication.router, prefix="/api/v1", tags=["authentication"]) 7 | api_router.include_router(databases.router, prefix="/api/v1", tags=["databases"]) 8 | api_router.include_router(query.router, prefix="/api/v1", tags=["query"]) 9 | api_router.include_router(query_executor.router, prefix="/api/v1", tags=["sql-data"]) 10 | -------------------------------------------------------------------------------- /client/app/viewControllers/databaseInfoViewController.ts: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | 3 | const useDatabaseInfoViewController = () => { 4 | const [activeIndex, setActiveIndex] = useState(null); 5 | 6 | const handleAccordionToggle = (index: number) => { 7 | setActiveIndex((prevIndex) => (prevIndex === index ? null : index)); 8 | }; 9 | 10 | return { 11 | activeIndex, 12 | handleAccordionToggle, 13 | } 14 | } 15 | 16 | export default useDatabaseInfoViewController; -------------------------------------------------------------------------------- /client/app/types.ts: -------------------------------------------------------------------------------- 1 | type DataValue = string | number | boolean; 2 | 3 | type Column = string; 4 | 5 | type Row = DataValue[]; 6 | 7 | type QueryResult = { 8 | column_names: Column[]; 9 | rows: Row[]; 10 | }; 11 | 12 | type Action = { type: 'SET_RESULT'; payload: QueryResult } 13 | | { type: 'SET_PAGE'; payload: number } 14 | | { type: 'SET_ROWS_PER_PAGE'; payload: number }; 15 | 16 | type QueryResultState = { 17 | queryResult: QueryResult; 18 | currentPage: number; 19 | rowsPerPage: number; 20 | }; 21 | 22 | -------------------------------------------------------------------------------- /client/app/(routes)/(protected)/databases/[id]/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import DatabaseInfo from "@/app/components/databaseInfo"; 4 | import useDatabaseViewController from "@/app/viewControllers/databaseViewController"; 5 | import React from "react"; 6 | 7 | 8 | 9 | const DatabaseView:React.FC = () => { 10 | const { 11 | database 12 | } = useDatabaseViewController(); 13 | 14 | return ( 15 |
16 | 17 |
18 | ); 19 | }; 20 | 21 | export default DatabaseView; 22 | -------------------------------------------------------------------------------- /client/app/components/loader.tsx: -------------------------------------------------------------------------------- 1 | const Loader = () => { 2 | return ( 3 |
7 | 8 | Loading... 9 | 10 |
11 | ); 12 | }; 13 | 14 | export default Loader; 15 | -------------------------------------------------------------------------------- /server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11 2 | 3 | RUN apt-get update 4 | 5 | RUN apt-get install -y python3-pip 6 | 7 | ENV OPENAI_API_KEY 8 | 9 | WORKDIR /mql 10 | 11 | COPY requirements.txt /mql/requirements.txt 12 | 13 | RUN pip install --upgrade -r /mql/requirements.txt 14 | 15 | COPY .env /mql/.env 16 | 17 | COPY ./app ./app 18 | 19 | COPY ./scripts ./scripts 20 | 21 | COPY ./alembic ./alembic 22 | 23 | COPY ./alembic.ini /mql/alembic.ini 24 | 25 | CMD ["/bin/bash", "-c", "alembic upgrade head; uvicorn app.main:app --host 0.0.0.0 --port 8000"] 26 | -------------------------------------------------------------------------------- /client/app/components/errorBox.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | type Props = { 4 | errorMessage: string; 5 | } 6 | const ErrorBox = ({errorMessage}: Props):React.JSX.Element => { 7 | return ( 8 |
9 |
10 | Error! 11 | {errorMessage} 12 |
13 |
14 | ) 15 | } 16 | 17 | export default ErrorBox; -------------------------------------------------------------------------------- /server/extract_schema.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DBNAME=$1 4 | USER=$2 5 | OUTPUT_FILE=${3:-output.txt} 6 | 7 | exec > $OUTPUT_FILE 8 | 9 | if [[ -z "$DBNAME" || -z "$USER" ]]; then 10 | echo "Usage: $0 dbname username" 11 | exit 1 12 | fi 13 | 14 | TABLES=$(psql -U $USER -d $DBNAME -t -c "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'") 15 | 16 | for TABLE in $TABLES; do 17 | echo "Table: $TABLE" 18 | psql -U $USER -d $DBNAME -t -c "SELECT column_name || ' | ' || data_type FROM information_schema.columns WHERE table_name = '$TABLE'" 19 | echo 20 | done 21 | -------------------------------------------------------------------------------- /client/app/providers/queryResultProvider/reducer.ts: -------------------------------------------------------------------------------- 1 | import { SET_PAGE, SET_RESULT, SET_ROWS_PER_PAGE } from "./actions"; 2 | 3 | const reducer: React.Reducer = (state, action) => { 4 | switch (action.type) { 5 | case SET_RESULT: 6 | return { ...state, queryResult: action.payload }; 7 | case SET_PAGE: 8 | return { ...state, currentPage: action.payload }; 9 | case SET_ROWS_PER_PAGE: 10 | return { ...state, rowsPerPage: action.payload, currentPage: 0 }; 11 | default: 12 | return state; 13 | } 14 | }; 15 | 16 | export default reducer; -------------------------------------------------------------------------------- /server/alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade() -> None: 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade() -> None: 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /server/alembic/versions/af7438a7b330_rename_query_histories_to_queries.py: -------------------------------------------------------------------------------- 1 | """Rename query_histories to queries 2 | 3 | Revision ID: af7438a7b330 4 | Revises: 68b6ee05672c 5 | Create Date: 2024-03-28 16:38:38.987895 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'af7438a7b330' 14 | down_revision = '68b6ee05672c' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.rename_table('query_histories', 'queries') 21 | 22 | 23 | def downgrade() -> None: 24 | op.rename_table('queries', 'query_histories') 25 | -------------------------------------------------------------------------------- /server/app/services/query_service.py: -------------------------------------------------------------------------------- 1 | 2 | class QueryService: 3 | def convert_to_2d_array(self,json_data): 4 | column_names = [] 5 | 6 | rows = [] 7 | 8 | for row_data in json_data: 9 | if not column_names: 10 | column_names = list(row_data.keys()) 11 | 12 | row_values = [row_data[column_name] for column_name in column_names] 13 | rows.append(row_values) 14 | 15 | result = { 16 | 'column_names': column_names, 17 | 'rows': rows 18 | } 19 | 20 | return result 21 | 22 | 23 | 24 | query_service = QueryService() -------------------------------------------------------------------------------- /server/app/tests/test_utilities/test_fernet_manager.py: -------------------------------------------------------------------------------- 1 | from app.utilities.fernet_manager import FernetManager 2 | import pytest 3 | 4 | @pytest.fixture 5 | def fernet_manager() -> FernetManager: 6 | return FernetManager("password") 7 | 8 | def test_generate_key(fernet_manager: FernetManager): 9 | key = fernet_manager.generate_key() 10 | assert key is not None 11 | assert isinstance(key, bytes) 12 | assert len(key) == 44 13 | 14 | def test_encrypt_decrypt(fernet_manager: FernetManager): 15 | data = "Hello, World!" 16 | encrypted_data = fernet_manager.encrypt(data) 17 | assert fernet_manager.decrypt(encrypted_data) == data -------------------------------------------------------------------------------- /client/README.md: -------------------------------------------------------------------------------- 1 | # Project Setup Guide 2 | 3 | ## Installation 4 | 5 | 1. **Install Nodejs** 6 | 7 | You can download the Node from official website: 8 | [https://nodejs.org/en](https://nodejs.org/en) 9 | 10 | or install using brew: 11 | 12 | ```bash 13 | brew install node@18 14 | ``` 15 | 16 | ## Running the Project 17 | 18 | 1. Clone the Repo 19 | 2. Go to the directory 20 | 3. Install the dependencies: 21 | 22 | ```bash 23 | npm install 24 | ``` 25 | 26 | 4. Run the development server 27 | 28 | ```bash 29 | npm run dev 30 | ``` 31 | 32 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 33 | -------------------------------------------------------------------------------- /server/app/core/log_config.py: -------------------------------------------------------------------------------- 1 | log_config = { 2 | "version": 1, 3 | "disable_existing_loggers": False, 4 | "formatters": { 5 | "default": { 6 | "()": "uvicorn.logging.DefaultFormatter", 7 | "fmt": "%(asctime)s | %(name)s | %(levelprefix)s | %(message)s", 8 | "datefmt": "%Y-%m-%d %H:%M:%S", 9 | }, 10 | }, 11 | "handlers": { 12 | "default": { 13 | "formatter": "default", 14 | "class": "logging.StreamHandler", 15 | "stream": "ext://sys.stderr", 16 | }, 17 | }, 18 | "loggers": { 19 | "mql": {"handlers": ["default"], "level": "INFO"}, 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /server/alembic/versions/794012702ac2_add_text_node_field_to_database_tables.py: -------------------------------------------------------------------------------- 1 | """add text_node field to database_tables 2 | 3 | Revision ID: 794012702ac2 4 | Revises: cd660fe23c72 5 | Create Date: 2023-08-11 17:02:36.631364 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "794012702ac2" 14 | down_revision = "cd660fe23c72" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("database_tables", sa.Column("text_node", sa.Text(), nullable=True)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("database_tables", "text_node") 25 | -------------------------------------------------------------------------------- /server/alembic/versions/68b6ee05672c_add_connection_string_in_user_databases.py: -------------------------------------------------------------------------------- 1 | """add connection_string in user_databases 2 | 3 | Revision ID: 68b6ee05672c 4 | Revises: 33633a5ee909 5 | Create Date: 2023-09-13 13:40:27.700989 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '68b6ee05672c' 14 | down_revision = '33633a5ee909' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("user_databases", sa.Column("connection_string", sa.Text(), nullable=True)) 21 | 22 | 23 | def downgrade() -> None: 24 | op.drop_column("user_databases", "connection_string") 25 | 26 | -------------------------------------------------------------------------------- /server/output_schema.txt: -------------------------------------------------------------------------------- 1 | Table: embeddings 2 | embedding | USER-DEFINED 3 | table_name | character varying 4 | 5 | 6 | Table: alembic_version 7 | version_num | character varying 8 | 9 | 10 | Table: users 11 | id | uuid 12 | name | character varying 13 | email | character varying 14 | hashed_password | character varying 15 | 16 | 17 | Table: user_databases 18 | id | uuid 19 | user_id | uuid 20 | name | character varying 21 | 22 | 23 | Table: database_tables 24 | id | uuid 25 | user_database_id | uuid 26 | name | character varying 27 | 28 | 29 | Table: table_columns 30 | id | uuid 31 | database_table_id | uuid 32 | name | character varying 33 | data_type | character varying 34 | 35 | 36 | -------------------------------------------------------------------------------- /client/app/viewControllers/addDatabaseViewController.ts: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | 3 | interface UseAddDatabaseViewController { 4 | isConnectVisible: boolean; 5 | setIsConnectVisible: React.Dispatch>; 6 | toggleComponent: () => void; 7 | } 8 | 9 | const useAddDatabaseViewController = (): UseAddDatabaseViewController => { 10 | 11 | const [isConnectVisible, setIsConnectVisible] = useState(true); 12 | 13 | const toggleComponent = () => setIsConnectVisible(!isConnectVisible); 14 | 15 | 16 | return { 17 | isConnectVisible, 18 | setIsConnectVisible, 19 | toggleComponent 20 | }; 21 | }; 22 | 23 | export default useAddDatabaseViewController; 24 | -------------------------------------------------------------------------------- /client/middleware.ts: -------------------------------------------------------------------------------- 1 | import { NextResponse } from "next/server"; 2 | import type { NextRequest } from "next/server"; 3 | 4 | const protectedPaths = ["/home", "/add-database", "/database", "/query"]; 5 | 6 | function isProtectedPath(path: string) { 7 | for (const protectedPath of protectedPaths) { 8 | if (path.startsWith(protectedPath)) { 9 | return true; 10 | } 11 | } 12 | return false; 13 | } 14 | 15 | export function middleware(request: NextRequest) { 16 | const path = request.nextUrl.pathname; 17 | const token = request.cookies.get("token")?.value 18 | if(isProtectedPath(path) && !token) { 19 | return NextResponse.redirect(new URL("/login", request.nextUrl)); 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /server/alembic/versions/14c2ea88a9d6_add_user_database_id_column_to_.py: -------------------------------------------------------------------------------- 1 | """add user_database_id column to embeddings table 2 | 3 | Revision ID: 14c2ea88a9d6 4 | Revises: 794012702ac2 5 | Create Date: 2023-08-11 18:43:53.565859 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "14c2ea88a9d6" 14 | down_revision = "794012702ac2" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column( 21 | "embeddings", sa.Column("user_database_id", sa.UUID(), nullable=False) 22 | ) 23 | 24 | 25 | def downgrade() -> None: 26 | op.drop_column("embeddings", "user_database_id") 27 | -------------------------------------------------------------------------------- /client/app/(routes)/layout.tsx: -------------------------------------------------------------------------------- 1 | import { ToastContainer } from "../components/toast"; 2 | import type { Metadata } from "next"; 3 | import "react-loading-skeleton/dist/skeleton.css"; 4 | import "react-toastify/dist/ReactToastify.css"; 5 | import "../styles/globals.css"; 6 | import appText from "../assets/strings"; 7 | 8 | export const metadata: Metadata = { 9 | title: appText.metadata.title, 10 | description: appText.metadata.description, 11 | } 12 | 13 | export default function RootLayout({ 14 | children, 15 | }: { 16 | children: React.ReactNode; 17 | }) { 18 | return ( 19 | 20 | 21 | {children} 22 | 23 | 24 | 25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /client/app/providers/queryResultProvider/index.tsx: -------------------------------------------------------------------------------- 1 | import { ReactElement, useReducer } from "react"; 2 | import reducer from "./reducer"; 3 | import QueryResultContext from "./queryResultContext"; 4 | 5 | type QueryResultProviderProps = { 6 | result: QueryResult; 7 | children: ReactElement; 8 | } 9 | 10 | export default function QueryResultProvider({ result, children }: QueryResultProviderProps) { 11 | const [state, dispatch] = useReducer(reducer, { 12 | queryResult: result, 13 | currentPage: 0, 14 | rowsPerPage: 10, 15 | }); 16 | 17 | return ( 18 | 19 | {children} 20 | 21 | ) 22 | } -------------------------------------------------------------------------------- /server/alembic/versions/7b50a83a45cd_create_user_database_id_and_name_unique_.py: -------------------------------------------------------------------------------- 1 | """create_user_database_id and name_unique_constraint_as_unique 2 | 3 | Revision ID: 7b50a83a45cd 4 | Revises: af7438a7b330 5 | Create Date: 2024-05-23 02:19:43.747312 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '7b50a83a45cd' 14 | down_revision = 'af7438a7b330' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.create_unique_constraint( 21 | "user_id_and_name_unique_constraint", 22 | "user_databases", 23 | ["user_id", "name"], 24 | ) 25 | 26 | 27 | def downgrade() -> None: 28 | op.drop_constraint( 29 | "user_database_id_and_name_unique_constraint", "user_databases", type_="unique" 30 | ) -------------------------------------------------------------------------------- /server/alembic/versions/da86d1e90141_add_table_user_databases.py: -------------------------------------------------------------------------------- 1 | """add table user-databases 2 | 3 | Revision ID: da86d1e90141 4 | Revises: bf224d84fb4d 5 | Create Date: 2023-08-01 12:44:01.275690 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "da86d1e90141" 14 | down_revision = "bf224d84fb4d" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.create_table( 21 | "user_databases", 22 | sa.Column("id", sa.UUID(), nullable=False), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("user_id", sa.UUID(), nullable=False), 25 | sa.PrimaryKeyConstraint("id"), 26 | ) 27 | op.create_index(op.f("ix_user_databases_id"), "user_databases", ["id"], unique=True) 28 | -------------------------------------------------------------------------------- /client/tailwind.config.ts: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | const { fontFamily } = require("tailwindcss/defaultTheme"); 3 | 4 | module.exports = { 5 | content: [ 6 | "./pages/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./components/**/*.{js,ts,jsx,tsx,mdx}", 8 | "./app/**/*.{js,ts,jsx,tsx,mdx}", 9 | ], 10 | theme: { 11 | extend: { 12 | boxShadow: { 13 | custom_shadow: "0 3px 10px rgb(0,0,0,0.2)", 14 | }, 15 | colors: { 16 | mqlBlue: { 17 | 100: "#1D85FF", 18 | 80: "#4A9DFF", 19 | }, 20 | }, 21 | backgroundImage: { 22 | "gradient-radial": "radial-gradient(var(--tw-gradient-stops))", 23 | "gradient-conic": 24 | "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))", 25 | }, 26 | }, 27 | }, 28 | plugins: [], 29 | }; 30 | -------------------------------------------------------------------------------- /server/app/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from app.api.v1.routes import api_router 3 | from fastapi.middleware.cors import CORSMiddleware 4 | from logging.config import dictConfig 5 | import logging 6 | from app.core.log_config import log_config 7 | 8 | dictConfig(log_config) 9 | 10 | app = FastAPI() 11 | 12 | logger = logging.getLogger("mql") 13 | 14 | 15 | @app.get("/ping") 16 | async def ping_health(): 17 | logger.info("Ping health check") 18 | return {"message": "OK"} 19 | 20 | 21 | @app.get("/") 22 | async def root(): 23 | return {"message": "Hello World"} 24 | 25 | 26 | app.add_middleware( 27 | CORSMiddleware, 28 | allow_origins=["*"], 29 | allow_credentials=True, 30 | allow_methods=["*"], 31 | allow_headers=["*"], 32 | expose_headers=["x-auth-token"], 33 | ) 34 | 35 | app.include_router(api_router) 36 | -------------------------------------------------------------------------------- /server/alembic/versions/3fbd94be5602_add_table_database_tables.py: -------------------------------------------------------------------------------- 1 | """add table database-tables 2 | 3 | Revision ID: 3fbd94be5602 4 | Revises: da86d1e90141 5 | Create Date: 2023-08-01 12:46:25.628021 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "3fbd94be5602" 14 | down_revision = "da86d1e90141" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.create_table( 21 | "database_tables", 22 | sa.Column("id", sa.UUID(), nullable=False), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("user_database_id", sa.UUID(), nullable=False), 25 | sa.PrimaryKeyConstraint("id"), 26 | ) 27 | op.create_index( 28 | op.f("ix_database_tables_id"), "database_tables", ["id"], unique=True 29 | ) 30 | -------------------------------------------------------------------------------- /server/app/user_utils.py: -------------------------------------------------------------------------------- 1 | # app/user_utils.py 2 | from app.crud.crud_user import crud_user 3 | from app.schemas.user import User 4 | from app.db.session import sessionLocal 5 | import logging 6 | 7 | def create_default_user(): 8 | db = sessionLocal() 9 | try: 10 | name = "Admin" 11 | default_email = "admin@example.com" 12 | user = crud_user.get_by_email(db, default_email) 13 | if user is None: 14 | password = "admin" 15 | crud_user.create(db, User(name=name, email=default_email, password=password)) 16 | logging.info("Default user created.") 17 | else: 18 | logging.info("Default user already exists.") 19 | except Exception as e: 20 | logging.error(f"Failed to initialize database: {e}") 21 | finally: 22 | db.close() 23 | 24 | if __name__ == "__main__": 25 | create_default_user() -------------------------------------------------------------------------------- /client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "strict": true, 12 | "noEmit": true, 13 | "esModuleInterop": true, 14 | "module": "esnext", 15 | "moduleResolution": "bundler", 16 | "resolveJsonModule": true, 17 | "isolatedModules": true, 18 | "jsx": "preserve", 19 | "incremental": true, 20 | "plugins": [ 21 | { 22 | "name": "next" 23 | } 24 | ], 25 | "paths": { 26 | "@/*": [ 27 | "./*" 28 | ] 29 | }, 30 | "forceConsistentCasingInFileNames": true 31 | }, 32 | "include": [ 33 | "next-env.d.ts", 34 | "**/*.ts", 35 | "**/*.tsx", 36 | ".next/types/**/*.ts", 37 | ], 38 | "exclude": [ 39 | "node_modules" 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /server/alembic/versions/baec336a3603_add_table_table_columns.py: -------------------------------------------------------------------------------- 1 | """add table table-columns 2 | 3 | Revision ID: baec336a3603 4 | Revises: 3fbd94be5602 5 | Create Date: 2023-08-01 12:55:28.697091 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "baec336a3603" 14 | down_revision = "3fbd94be5602" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.create_table( 21 | "table_columns", 22 | sa.Column("id", sa.UUID(), nullable=False), 23 | sa.Column("name", sa.String(), nullable=False), 24 | sa.Column("data_type", sa.String(), nullable=False), 25 | sa.Column("database_table_id", sa.UUID(), nullable=False), 26 | sa.PrimaryKeyConstraint("id"), 27 | ) 28 | op.create_index(op.f("ix_table_columns_id"), "table_columns", ["id"], unique=True) 29 | -------------------------------------------------------------------------------- /server/app/models/query.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import String, UUID 3 | from sqlalchemy.orm import mapped_column 4 | from app.models.timestamp_base import TimestampBase 5 | 6 | 7 | class Query(TimestampBase): 8 | __tablename__ = "queries" 9 | 10 | id = mapped_column( 11 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 12 | ) 13 | nl_query = mapped_column(String, nullable=False) 14 | sql_query = mapped_column(String, nullable=True) 15 | user_database_id = mapped_column(UUID, nullable=False) 16 | 17 | def as_dict(self) -> dict: 18 | return { 19 | "nl_query": self.nl_query, 20 | "sql_query": self.sql_query, 21 | "user_database_id": str(self.user_database_id), 22 | "id": str(self.id), 23 | "created_at": self.created_at.isoformat(), 24 | "updated_at": self.updated_at.isoformat(), 25 | } 26 | -------------------------------------------------------------------------------- /server/app/models/database_table.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import String, UUID, Text 3 | from sqlalchemy.orm import mapped_column 4 | from app.models.timestamp_base import TimestampBase 5 | 6 | 7 | class DatabaseTable(TimestampBase): 8 | __tablename__ = "database_tables" 9 | 10 | id = mapped_column( 11 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 12 | ) 13 | name = mapped_column(String, nullable=False) 14 | text_node = mapped_column(Text, nullable=True) 15 | user_database_id = mapped_column(UUID, nullable=False) 16 | 17 | def as_dict(self) -> dict: 18 | return { 19 | "name": self.name, 20 | "user_database_id": str(self.user_database_id), 21 | "id": str(self.id), 22 | "text_node": self.text_node, 23 | "created_at": self.created_at.isoformat(), 24 | "updated_at": self.updated_at.isoformat(), 25 | } 26 | -------------------------------------------------------------------------------- /server/app/models/table_column.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import String, UUID 3 | from sqlalchemy.orm import mapped_column 4 | 5 | from app.models.timestamp_base import TimestampBase 6 | 7 | 8 | class TableColumn(TimestampBase): 9 | __tablename__ = "table_columns" 10 | 11 | id = mapped_column( 12 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 13 | ) 14 | name = mapped_column(String, nullable=False) 15 | data_type = mapped_column(String, nullable=False) 16 | database_table_id = mapped_column(UUID, nullable=False) 17 | 18 | def as_dict(self) -> dict: 19 | return { 20 | "name": self.name, 21 | "data_type": self.data_type, 22 | "database_table_id": str(self.database_table_id), 23 | "id": str(self.id), 24 | "created_at": self.created_at.isoformat(), 25 | "updated_at": self.updated_at.isoformat(), 26 | } 27 | -------------------------------------------------------------------------------- /server/app/models/user_database.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import String, UUID 3 | from sqlalchemy.orm import mapped_column 4 | from app.models.timestamp_base import TimestampBase 5 | 6 | 7 | class UserDatabase(TimestampBase): 8 | __tablename__ = "user_databases" 9 | 10 | id = mapped_column( 11 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 12 | ) 13 | name = mapped_column(String, nullable=False) 14 | user_id = mapped_column(UUID, nullable=False) 15 | connection_string = mapped_column(String, nullable=True) 16 | 17 | def as_dict(self) -> dict: 18 | return { 19 | "name": self.name, 20 | "user_id": str(self.user_id), 21 | "connection_string": True if self.connection_string else False, 22 | "id": str(self.id), 23 | "created_at": self.created_at.isoformat(), 24 | "updated_at": self.updated_at.isoformat(), 25 | } 26 | -------------------------------------------------------------------------------- /storage/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official PostgreSQL image as the base image 2 | FROM ankane/pgvector:latest 3 | 4 | # Install necessary packages and generate locale 5 | RUN apt-get update && apt-get upgrade -y && apt-get install -y locales locales-all && \ 6 | locale-gen en_US.UTF-8 && \ 7 | update-locale LANG=en_US.UTF-8 8 | 9 | # Debug: List generated locales 10 | RUN locale -a 11 | 12 | # Set environment variables for locale 13 | ENV LANG en_US.UTF-8 14 | ENV LANGUAGE en_US:en 15 | ENV LC_ALL en_US.UTF-8 16 | 17 | COPY ./init.sql /docker-entrypoint-initdb.d/ 18 | 19 | # Install build dependencies and cleanup in a single RUN to reduce image layers and size 20 | RUN apt-get install -y \ 21 | build-essential \ 22 | postgresql-server-dev-all \ 23 | git && \ 24 | apt-get remove -y build-essential postgresql-server-dev-all git && apt-get clean && \ 25 | rm -rf /var/lib/apt/lists/* 26 | 27 | # Expose the PostgreSQL port 28 | EXPOSE 5432 29 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | frontend: 4 | build: 5 | context: ./client 6 | ports: 7 | - "3000:3000" 8 | networks: 9 | - mql-net 10 | 11 | postgres: 12 | build: 13 | context: ./storage 14 | environment: 15 | PGUSER: postgres 16 | POSTGRES_PASSWORD: password 17 | healthcheck: 18 | test: ["CMD-SHELL", "pg_isready", "-U", "postgres", "-d", "mql"] 19 | interval: 10s 20 | timeout: 60s 21 | retries: 5 22 | start_period: 80s 23 | volumes: 24 | - postgres-data:/var/lib/postgresql/data 25 | networks: 26 | - mql-net 27 | 28 | backend: 29 | build: 30 | context: ./server 31 | ports: 32 | - "8000:8000" 33 | networks: 34 | - mql-net 35 | depends_on: 36 | postgres: 37 | condition: service_healthy 38 | 39 | networks: 40 | mql-net: 41 | name: mql-database-network 42 | 43 | volumes: 44 | postgres-data: 45 | -------------------------------------------------------------------------------- /server/alembic/versions/cd660fe23c72_add_table_embeddings.py: -------------------------------------------------------------------------------- 1 | """add table embeddings 2 | 3 | Revision ID: cd660fe23c72 4 | Revises: 45e10e526767 5 | Create Date: 2023-08-10 12:17:25.657217 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from pgvector.sqlalchemy import Vector 11 | 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "cd660fe23c72" 15 | down_revision = "45e10e526767" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.create_table( 22 | "embeddings", 23 | sa.Column("id", sa.UUID(), nullable=False), 24 | sa.Column("embeddings_vector", Vector(1536), nullable=False), 25 | sa.Column("database_table_id", sa.UUID(), nullable=False), 26 | sa.Column("created_at", sa.DateTime(), nullable=False), 27 | sa.Column("updated_at", sa.DateTime(), nullable=False), 28 | sa.PrimaryKeyConstraint("id"), 29 | ) 30 | 31 | 32 | def downgrade() -> None: 33 | pass 34 | -------------------------------------------------------------------------------- /client/app/viewControllers/genericViewController.ts: -------------------------------------------------------------------------------- 1 | import Cookies from "js-cookie"; 2 | import { usePathname } from "next/navigation"; 3 | import { useState } from "react"; 4 | import appText from "../assets/strings"; 5 | 6 | type NavigationItem = { 7 | name: string; 8 | href: string; 9 | }; 10 | 11 | const useGenericViewController = () => { 12 | const [mobileMenuOpen, setMobileMenuOpen] = useState(false); 13 | const pathname = usePathname(); 14 | const text = appText.header; 15 | 16 | const headerNavigation:NavigationItem[] = [ 17 | { name: text.features, href: "/#features" }, 18 | { name: text.steps, href: "/#steps" }, 19 | ]; 20 | 21 | const logout = () => { 22 | Cookies.set("token", ""); 23 | window.location.href = "/"; 24 | }; 25 | 26 | return { 27 | logout, 28 | headerNavigation, 29 | setMobileMenuOpen, 30 | mobileMenuOpen, 31 | pathname 32 | }; 33 | } 34 | 35 | export default useGenericViewController; -------------------------------------------------------------------------------- /server/app/models/embedding.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import UUID 3 | from sqlalchemy.orm import mapped_column 4 | from app.models.timestamp_base import TimestampBase 5 | from pgvector.sqlalchemy import Vector 6 | 7 | 8 | class Embedding(TimestampBase): 9 | __tablename__ = "embeddings" 10 | id = mapped_column( 11 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 12 | ) 13 | embeddings_vector = mapped_column(Vector, nullable=False) 14 | database_table_id = mapped_column(UUID, nullable=False) 15 | user_database_id = mapped_column(UUID, nullable=False) 16 | 17 | def as_dict(self) -> dict: 18 | return { 19 | "id": str(self.id), 20 | "embeddings_vector": self.embeddings_vector, 21 | "database_table_id": str(self.database_table_id), 22 | "user_database_id": str(self.user_database_id), 23 | "created_at": self.created_at.isoformat(), 24 | "updated_at": self.updated_at.isoformat(), 25 | } 26 | -------------------------------------------------------------------------------- /server/alembic/versions/33633a5ee909_create_query_history_table.py: -------------------------------------------------------------------------------- 1 | """Create query history table 2 | 3 | Revision ID: 33633a5ee909 4 | Revises: 14c2ea88a9d6 5 | Create Date: 2023-08-14 13:43:54.954424 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "33633a5ee909" 14 | down_revision = "14c2ea88a9d6" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.create_table( 21 | "query_histories", 22 | sa.Column("id", sa.UUID(), nullable=False), 23 | sa.Column("nl_query", sa.String(), nullable=False), 24 | sa.Column("sql_query", sa.String(), nullable=True), 25 | sa.Column("user_database_id", sa.UUID(), nullable=False), 26 | sa.Column("created_at", sa.DateTime(), nullable=False), 27 | sa.Column("updated_at", sa.DateTime(), nullable=False), 28 | sa.PrimaryKeyConstraint("id", name=op.f("pk_query_histories")), 29 | ) 30 | 31 | 32 | def downgrade() -> None: 33 | pass 34 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mql-web", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@headlessui/react": "^1.7.17", 13 | "@heroicons/react": "^2.0.18", 14 | "@types/js-cookie": "^3.0.3", 15 | "@types/react-dom": "18.2.7", 16 | "autoprefixer": "10.4.14", 17 | "axios": "^1.4.0", 18 | "debug": "^4.3.4", 19 | "js-cookie": "^3.0.5", 20 | "next": "13.4.12", 21 | "postcss": "8.4.27", 22 | "react": "18.2.0", 23 | "react-ace": "^11.0.1", 24 | "react-dom": "18.2.0", 25 | "react-icons": "^4.10.1", 26 | "react-loading-skeleton": "^3.3.1", 27 | "react-toastify": "^9.1.3", 28 | "supports-color": "^8.1.1", 29 | "tailwindcss": "3.3.3" 30 | }, 31 | "devDependencies": { 32 | "@types/node": "20.5.6", 33 | "@types/react": "18.2.21", 34 | "@types/react-syntax-highlighter": "^15.5.7", 35 | "typescript": "5.2.2" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /client/app/components/downloadCSV.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | type QueryResultTableProps = { 4 | result: QueryResult; 5 | }; 6 | 7 | const DownloadCSV = ({ result }: QueryResultTableProps) => { 8 | const downloadCSV = () => { 9 | if (!result) return; 10 | 11 | const columnNames = result.column_names; 12 | const rows = result.rows; 13 | 14 | let csvContent = "data:text/csv;charset=utf-8,"; 15 | csvContent += columnNames.join(",") + "\n"; 16 | rows.forEach((row: any) => { 17 | csvContent += row.join(",") + "\n"; 18 | }); 19 | 20 | const encodedUri = encodeURI(csvContent); 21 | const link = document.createElement("a"); 22 | link.setAttribute("href", encodedUri); 23 | const time = new Date().getTime(); 24 | link.setAttribute("download", `query_result${time}.csv`); 25 | document.body.appendChild(link); 26 | link.click(); 27 | }; 28 | 29 | return ; 30 | }; 31 | 32 | export default DownloadCSV; 33 | -------------------------------------------------------------------------------- /server/app/models/user.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy import String, UUID 3 | from sqlalchemy.orm import mapped_column 4 | from app.models.timestamp_base import TimestampBase 5 | 6 | 7 | class User(TimestampBase): 8 | __tablename__ = "users" 9 | 10 | id = mapped_column( 11 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 12 | ) 13 | name = mapped_column(String, nullable=False) 14 | email = mapped_column(String, unique=True, nullable=False) 15 | hashed_password = mapped_column(String, nullable=False) 16 | _hashed_key = None 17 | 18 | @property 19 | def hashed_key(self): 20 | return self._hashed_key 21 | 22 | @hashed_key.setter 23 | def hashed_key(self, value): 24 | self._hashed_key = value 25 | 26 | 27 | def as_dict(self) -> dict: 28 | return { 29 | "name": self.name, 30 | "email": self.email, 31 | "id": str(self.id), 32 | "created_at": self.created_at.isoformat(), 33 | "updated_at": self.updated_at.isoformat(), 34 | "hashed_key": self.hashed_key, 35 | } 36 | -------------------------------------------------------------------------------- /server/app/utilities/fernet_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | import base64 3 | 4 | from cryptography.fernet import Fernet 5 | from cryptography.hazmat.primitives import hashes 6 | from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC 7 | import logging 8 | logger = logging.getLogger("analytics") 9 | 10 | 11 | class FernetManager: 12 | def __init__(self,password: str): 13 | self.encryption_salt = os.getenv("ENCRYPTION_SALT") 14 | self.password = password 15 | self.key = self.generate_key() 16 | self.fernet = Fernet(self.key) 17 | 18 | def generate_key(self) -> bytes: 19 | kdf = PBKDF2HMAC( 20 | algorithm=hashes.SHA256(), 21 | length=32, 22 | salt=self.encryption_salt.encode(), 23 | iterations=100000, 24 | ) 25 | key = base64.urlsafe_b64encode(kdf.derive(self.password.encode())) 26 | return key 27 | 28 | def encrypt(self, data: str) -> str: 29 | return self.fernet.encrypt(data.encode()).decode() 30 | 31 | def decrypt(self, data: str) -> str: 32 | return self.fernet.decrypt(data.encode()).decode() -------------------------------------------------------------------------------- /client/app/viewControllers/queryResultViewController.ts: -------------------------------------------------------------------------------- 1 | import { useContext } from "react"; 2 | import QueryResultContext from "../providers/queryResultProvider/queryResultContext"; 3 | 4 | export default function useQueryResultViewController() { 5 | const context = useContext(QueryResultContext); 6 | 7 | if(!context) { 8 | throw new Error('useQueryResult must be used within a QueryResultProvider'); 9 | } 10 | 11 | const { state, dispatch } = context; 12 | 13 | const { queryResult, currentPage, rowsPerPage } = state; 14 | 15 | const startIndex = currentPage * rowsPerPage; 16 | const endIndex = startIndex + rowsPerPage; 17 | const paginatedRows = queryResult.rows.slice(startIndex, endIndex); 18 | 19 | return { 20 | columns: queryResult.column_names, 21 | rows: paginatedRows, 22 | currentPage, 23 | rowsPerPage, 24 | totalRows: queryResult.rows.length, 25 | setPage: (page: number) => dispatch({ type: 'SET_PAGE', payload: page }), 26 | setRowsPerPage: (rowsPerPage: number) => dispatch({ type: 'SET_ROWS_PER_PAGE', payload: rowsPerPage }), 27 | }; 28 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Shuru Technologies 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Introduction 4 | 5 | We are committed to providing a welcoming and inclusive environment for all participants, regardless of gender, sexual orientation, disability, ethnicity, religion, or any other personal characteristic. We value open and respectful communication and expect all contributors, maintainers, and users of this repository to abide by this Code of Conduct. 6 | 7 | ## Expected Behavior 8 | 9 | When participating in this repository, we expect everyone to: 10 | 11 | - Be respectful and considerate of others' perspectives and experiences. 12 | - Use inclusive language and avoid derogatory or discriminatory comments or behavior. 13 | - Be open to constructive feedback and engage in healthy discussions. 14 | - Exercise empathy and understanding towards fellow contributors. 15 | - Be mindful of the impact of your words and actions on others. 16 | 17 | ## Unacceptable Behavior 18 | 19 | The following behaviors are considered unacceptable and will not be tolerated: 20 | 21 | - Harassment, discrimination, or intimidation in any form. 22 | - Offensive, derogatory, or inappropriate comments or content. 23 | -------------------------------------------------------------------------------- /server/app/tests/test_crud/test_crud_user.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import Session 2 | from app.models.user import User as UserModel 3 | from app.crud.crud_user import CRUDUser 4 | from app.schemas.user import User as UserSchema 5 | 6 | 7 | def test_get_by_email(db: Session, valid_user: UserSchema) -> None: 8 | crud_user = CRUDUser() 9 | crud_user.create(db=db, user_obj=valid_user) 10 | result = crud_user.get_by_email(db=db, email=valid_user.email) 11 | 12 | assert result is not None 13 | assert result.name == valid_user.name 14 | assert result.email == valid_user.email 15 | assert ( 16 | db.query(UserModel).filter(UserModel.email == result.email).first() is not None 17 | ) 18 | 19 | 20 | def test_get_by_id(db: Session, valid_user_model: UserModel) -> None: 21 | crud_user = CRUDUser() 22 | result = crud_user.get_by_id(db=db, id=valid_user_model.id) 23 | 24 | assert result is not None 25 | assert result.name == valid_user_model.name 26 | assert result.email == valid_user_model.email 27 | assert ( 28 | db.query(UserModel).filter(UserModel.email == result.email).first() is not None 29 | ) 30 | -------------------------------------------------------------------------------- /client/app/viewControllers/homeAccountsViewController.ts: -------------------------------------------------------------------------------- 1 | import { getAllDatabase } from "@/app/lib/service"; 2 | import { useEffect, useState } from "react"; 3 | import { toast } from "react-toastify"; 4 | import appText from "../assets/strings"; 5 | 6 | type Database = { 7 | id: string; 8 | name: string; 9 | connection_string: boolean; 10 | created_at: string; 11 | }; 12 | 13 | const useHomeAccountsViewController = () => { 14 | const [databases, setDatabases] = useState([]); 15 | const [refresh, setRefresh] = useState(false); 16 | 17 | useEffect(() => { 18 | const fetchAllDB = async () => { 19 | try { 20 | const response = await getAllDatabase(); 21 | setDatabases(response.data.data.user_databases); 22 | } catch (error) { 23 | toast.error(appText.toast.errGeneric); 24 | } 25 | }; 26 | fetchAllDB(); 27 | }, [refresh]); 28 | 29 | const refreshDatabases = () => { 30 | setRefresh(!refresh); 31 | } 32 | 33 | return { databases, refreshDatabases }; 34 | } 35 | 36 | export default useHomeAccountsViewController; -------------------------------------------------------------------------------- /server/alembic/versions/bf224d84fb4d_create_users_table.py: -------------------------------------------------------------------------------- 1 | """create users table 2 | 3 | Revision ID: bf224d84fb4d 4 | Revises: 5 | Create Date: 2023-07-27 16:52:52.016522 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "bf224d84fb4d" 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "users", 23 | sa.Column("id", sa.UUID(), nullable=False), 24 | sa.Column("name", sa.String(), nullable=False), 25 | sa.Column("email", sa.String(), nullable=False), 26 | sa.Column("hashed_password", sa.String(), nullable=False), 27 | sa.PrimaryKeyConstraint("id"), 28 | sa.UniqueConstraint("email"), 29 | ) 30 | op.create_index(op.f("ix_users_id"), "users", ["id"], unique=True) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade() -> None: 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.drop_index(op.f("ix_users_id"), table_name="users") 37 | op.drop_table("users") 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /server/app/tests/test_models/test_user_model.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import Session 2 | from app.models.user import User as UserModel 3 | 4 | 5 | def test_as_dict_method(valid_user_model: UserModel) -> None: 6 | result = valid_user_model.as_dict() 7 | 8 | assert "id" in result 9 | assert isinstance(result["id"], str) 10 | assert result == { 11 | "name": valid_user_model.name, 12 | "email": valid_user_model.email, 13 | "id": result["id"], 14 | "created_at": result["created_at"], 15 | "updated_at": result["updated_at"], 16 | "hashed_key": valid_user_model.hashed_key, 17 | } 18 | 19 | 20 | def test_timestamp_on_create(valid_user_model: UserModel) -> None: 21 | assert valid_user_model.created_at is not None 22 | assert valid_user_model.updated_at is not None 23 | assert valid_user_model.created_at == valid_user_model.updated_at 24 | 25 | 26 | def test_timestamp_on_update(valid_user_model: UserModel, db: Session) -> None: 27 | valid_user_model.name = "test_user_updated" 28 | db.commit() 29 | assert valid_user_model.created_at is not None 30 | assert valid_user_model.updated_at is not None 31 | assert valid_user_model.created_at != valid_user_model.updated_at 32 | -------------------------------------------------------------------------------- /client/app/viewControllers/databaseViewController.ts: -------------------------------------------------------------------------------- 1 | import { useParams } from "next/navigation"; 2 | import { useEffect, useState } from "react"; 3 | import { getDatabase } from "@/app/lib/service"; 4 | import { toast } from "react-toastify"; 5 | import appText from "../assets/strings"; 6 | 7 | type Database = { 8 | database_id: string; 9 | database_name: string; 10 | database_tables: { 11 | table_id: string; 12 | table_name: string; 13 | table_columns: { 14 | column_id: string; 15 | column_name: string; 16 | column_type: string; 17 | }[]; 18 | }[]; 19 | }; 20 | 21 | const useDatabaseViewController = () => { 22 | const { id } = useParams(); 23 | const [database, setDatabase] = useState({} as Database); 24 | 25 | useEffect(() => { 26 | const fetchDB = async () => { 27 | try { 28 | const res = await getDatabase(id); 29 | setDatabase(res.data.data); 30 | } catch (error) { 31 | toast.error(appText.toast.errGeneric); 32 | } 33 | }; 34 | fetchDB(); 35 | }, []); 36 | 37 | return { 38 | database 39 | } 40 | } 41 | 42 | export default useDatabaseViewController; 43 | 44 | -------------------------------------------------------------------------------- /server/app/crud/crud_user_database.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID 2 | from sqlalchemy.orm import Session 3 | from app.models.user_database import UserDatabase as UserDatabaseModel 4 | from app.schemas.user_database import UserDatabase as UserDatabaseSchema 5 | 6 | 7 | class CRUDUserDatabase: 8 | def create( 9 | self, db: Session, user_database_obj: UserDatabaseSchema 10 | ) -> UserDatabaseModel: 11 | user_database = UserDatabaseModel( 12 | name=user_database_obj.name, 13 | user_id=user_database_obj.user_id, 14 | connection_string=user_database_obj.connection_string, 15 | ) 16 | db.add(user_database) 17 | db.flush() 18 | return user_database 19 | 20 | def get_by_user_id(self, db: Session, user_id: UUID) -> UserDatabaseModel: 21 | return db.query(UserDatabaseModel).filter(UserDatabaseModel.user_id == user_id) 22 | 23 | def get_by_id(self, db: Session, id: UUID) -> UserDatabaseModel: 24 | return db.query(UserDatabaseModel).filter(UserDatabaseModel.id == id).first() 25 | 26 | def delete_by_id(self, db: Session, id: UUID) -> None: 27 | db.query(UserDatabaseModel).filter(UserDatabaseModel.id == id).delete() 28 | 29 | crud_user_database = CRUDUserDatabase() 30 | -------------------------------------------------------------------------------- /client/app/viewControllers/databaseCardViewController.ts: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { syncSchema, deleteDatabase } from "@/app/lib/service"; 3 | import { toast } from "react-toastify"; 4 | 5 | const useDatabaseCardViewController = (id: string, refreshDatabases: ()=> void) => { 6 | const [syncDbLoader, setSyncDbLoader] = useState(false); 7 | const [deleteDbLoader, setDeleteDbLoader] = useState(false); 8 | const syncDb = async () => { 9 | setSyncDbLoader(true); 10 | const formData = new FormData(); 11 | formData.append("database_id", id); 12 | try { 13 | await syncSchema(formData); 14 | toast.success("Database synced successfully"); 15 | } catch (error) { 16 | toast.error("Error syncing database"); 17 | } 18 | setSyncDbLoader(false); 19 | }; 20 | 21 | const deleteDb = async () => { 22 | setDeleteDbLoader(true); 23 | try{ 24 | await deleteDatabase(id); 25 | toast.success("Database deleted successfully"); 26 | refreshDatabases(); 27 | } 28 | catch(error){ 29 | toast.error("Error deleting database"); 30 | } 31 | setDeleteDbLoader(false); 32 | }; 33 | 34 | return { syncDbLoader, syncDb , deleteDbLoader, deleteDb}; 35 | }; 36 | export default useDatabaseCardViewController; 37 | -------------------------------------------------------------------------------- /client/app/viewControllers/codeBlockViewController.ts: -------------------------------------------------------------------------------- 1 | import React, { useState } from "react"; 2 | 3 | type Props = { 4 | codeString: string; 5 | setSql: (sql: string) => void; 6 | handleQueryResponse: () => void; 7 | }; 8 | 9 | const useCodeBlockViewController = ({ handleQueryResponse, codeString, setSql}: Props) => { 10 | const [copySuccess, setCopySuccess] = useState(false); 11 | const [executeSuccess, setExecuteSuccess] = useState(false); 12 | const [query, setQuery] = useState(codeString); 13 | 14 | const handleQueryChange = (query: string) => { 15 | setExecuteSuccess(false); 16 | setQuery(query); 17 | setSql(query); 18 | } 19 | const handleCopyClick = () => { 20 | navigator.clipboard.writeText(codeString); 21 | setCopySuccess(true); 22 | setTimeout(() => setCopySuccess(false), 1500); 23 | }; 24 | 25 | const handleExecuteClick = async () => { 26 | if(!executeSuccess){ 27 | setExecuteSuccess(true); 28 | handleQueryResponse(); 29 | } 30 | } 31 | 32 | return { 33 | copySuccess, 34 | handleCopyClick, 35 | executeSuccess, 36 | query, 37 | handleQueryChange, 38 | handleExecuteClick 39 | } 40 | } 41 | 42 | export default useCodeBlockViewController; -------------------------------------------------------------------------------- /client/app/viewControllers/loginViewController.ts: -------------------------------------------------------------------------------- 1 | import { toast } from "react-toastify"; 2 | import { login } from "@/app/lib/service"; 3 | import Cookies from "js-cookie"; 4 | import { useRouter } from "next/navigation"; 5 | import { useState } from "react"; 6 | import appText from "../assets/strings"; 7 | 8 | const useLoginViewController = () => { 9 | const { push } = useRouter(); 10 | const [email, setEmail] = useState(""); 11 | const [password, setPassword] = useState(""); 12 | 13 | const handleLogin = async (e: React.ChangeEvent) => { 14 | e.preventDefault(); 15 | try { 16 | const formData = new FormData(); 17 | formData.append("email", email); 18 | formData.append("password", password); 19 | const res = await login(formData); 20 | if (res.status === 200) { 21 | Cookies.set("token", res.headers["x-auth-token"]); 22 | toast.success(appText.toast.loginSuccess); 23 | push("/home"); 24 | } 25 | } catch (error: any) { 26 | toast.error(error.detail); 27 | } 28 | }; 29 | 30 | return { 31 | handleLogin, 32 | setEmail, 33 | setPassword, 34 | email, 35 | password, 36 | } 37 | } 38 | 39 | export default useLoginViewController; -------------------------------------------------------------------------------- /client/app/styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --foreground-rgb: 0, 0, 0; 7 | --background-start-rgb: 214, 219, 220; 8 | --background-end-rgb: 255, 255, 255; 9 | } 10 | 11 | @media (prefers-color-scheme: dark) { 12 | :root { 13 | --foreground-rgb: 255, 255, 255; 14 | --background-start-rgb: 0, 0, 0; 15 | --background-end-rgb: 0, 0, 0; 16 | } 17 | } 18 | 19 | /* body { 20 | color: rgb(var(--foreground-rgb)); 21 | background: linear-gradient( 22 | to bottom, 23 | transparent, 24 | rgb(var(--background-end-rgb)) 25 | ) 26 | rgb(var(--background-start-rgb)); 27 | } */ 28 | 29 | .db-img { 30 | filter: drop-shadow(10px 10px 10px rgba(0, 0, 0, 0.5)); 31 | } 32 | 33 | .custom-scrollbar::-webkit-scrollbar, 34 | .custom-scrollbar::-webkit-scrollbar-thumb { 35 | width: 26px; 36 | border-radius: 13px; 37 | background-clip: padding-box; 38 | border: 10px solid transparent; 39 | color: rgba(0, 0, 0, 0.5); 40 | } 41 | 42 | .custom-scrollbar::-webkit-scrollbar-thumb { 43 | box-shadow: inset 0 0 0 10px; 44 | } 45 | 46 | .no-scrollbar::-webkit-scrollbar { 47 | display: none; 48 | } 49 | 50 | /* Hide scrollbar for IE, Edge and Firefox */ 51 | .no-scrollbar { 52 | -ms-overflow-style: none; /* IE and Edge */ 53 | scrollbar-width: none; /* Firefox */ 54 | } 55 | -------------------------------------------------------------------------------- /client/app/(routes)/(protected)/home/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import useHomeAccountsViewController from "@/app/viewControllers/homeAccountsViewController"; 4 | import appText from "@/app/assets/strings"; 5 | import DatabaseCard from "@/app/components/databaseCard"; 6 | import React from "react"; 7 | 8 | const Home:React.FC = () => { 9 | const { databases,refreshDatabases } = useHomeAccountsViewController(); 10 | const text = appText.homeDatabases; 11 | 12 | return ( 13 |
14 |
15 |
16 |

17 | {text.databases} 18 |

19 | 23 |
24 |
25 |
26 | {databases.length ? ( 27 | databases?.map((db, idx) => ( 28 | 29 | )) 30 | ) : ( 31 |

{text.noDatabase}

32 | )} 33 |
34 |
35 | ); 36 | }; 37 | 38 | export default Home; 39 | -------------------------------------------------------------------------------- /server/alembic/versions/45e10e526767_add_timestamps_to_tables.py: -------------------------------------------------------------------------------- 1 | """add timestamps to tables 2 | 3 | Revision ID: 45e10e526767 4 | Revises: baec336a3603 5 | Create Date: 2023-08-09 10:56:40.156275 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "45e10e526767" 14 | down_revision = "baec336a3603" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.add_column("users", sa.Column("created_at", sa.DateTime(), nullable=False)) 21 | op.add_column("users", sa.Column("updated_at", sa.DateTime(), nullable=False)) 22 | op.add_column( 23 | "user_databases", sa.Column("created_at", sa.DateTime(), nullable=False) 24 | ) 25 | op.add_column( 26 | "user_databases", sa.Column("updated_at", sa.DateTime(), nullable=False) 27 | ) 28 | op.add_column( 29 | "database_tables", sa.Column("created_at", sa.DateTime(), nullable=False) 30 | ) 31 | op.add_column( 32 | "database_tables", sa.Column("updated_at", sa.DateTime(), nullable=False) 33 | ) 34 | op.add_column( 35 | "table_columns", sa.Column("created_at", sa.DateTime(), nullable=False) 36 | ) 37 | op.add_column( 38 | "table_columns", sa.Column("updated_at", sa.DateTime(), nullable=False) 39 | ) 40 | 41 | 42 | def downgrade() -> None: 43 | pass 44 | -------------------------------------------------------------------------------- /server/app/crud/crud_table_column.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID 2 | from sqlalchemy.orm import Session 3 | from app.models.table_column import TableColumn as TableColumnModel 4 | from app.schemas.table_column import TableColumn as TableColumnSchema 5 | 6 | 7 | class CRUDTableColumn: 8 | def create( 9 | self, db: Session, table_column_obj: TableColumnSchema 10 | ) -> TableColumnModel: 11 | table_column = TableColumnModel( 12 | name=table_column_obj.name, 13 | data_type=table_column_obj.data_type, 14 | database_table_id=table_column_obj.database_table_id, 15 | ) 16 | db.add(table_column) 17 | db.flush() 18 | return table_column 19 | 20 | def get_by_database_table_id( 21 | self, db: Session, database_table_id: UUID 22 | ) -> TableColumnModel: 23 | return db.query(TableColumnModel).filter( 24 | TableColumnModel.database_table_id == database_table_id 25 | ) 26 | 27 | def delete_by_database_table_id(self, db: Session, database_table_id: UUID) -> None: 28 | db.query(TableColumnModel).filter( 29 | TableColumnModel.database_table_id == database_table_id 30 | ).delete() 31 | db.commit() 32 | 33 | def delete_by_database_table_id(self, db: Session, database_table_id: UUID) -> None: 34 | db.query(TableColumnModel).filter(TableColumnModel.database_table_id == database_table_id).delete() 35 | 36 | crud_table_column = CRUDTableColumn() 37 | -------------------------------------------------------------------------------- /client/app/viewControllers/uploadSchemaViewController.ts: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { useRouter } from "next/navigation"; 3 | import { toast } from "react-toastify"; 4 | import { uploadSchema } from "@/app/lib/service"; 5 | import appText from "../assets/strings"; 6 | 7 | const useUploadSchemaViewController = () => { 8 | const [file, setFile] = useState(""); 9 | const [databaseName, setDatabaseName] = useState(""); 10 | const [showLoader, setShowLoader] = useState(false); 11 | 12 | const { push } = useRouter(); 13 | 14 | const handleFileChange = (event: React.ChangeEvent) => { 15 | const selectedFile = event.target.files[0]; 16 | setFile(selectedFile); 17 | }; 18 | 19 | const handleUpload = async () => { 20 | setShowLoader(true); 21 | try { 22 | const formData = new FormData(); 23 | formData.append("database_name", databaseName); 24 | formData.append("file", file); 25 | 26 | const res = await uploadSchema(formData); 27 | toast.success(appText.toast.uploadSuccess); 28 | push("/home"); 29 | setShowLoader(false); 30 | } catch (error) { 31 | setShowLoader(false); 32 | toast.error(appText.toast.errGeneric); 33 | } 34 | }; 35 | 36 | return { 37 | file, 38 | setFile, 39 | databaseName, 40 | setDatabaseName, 41 | showLoader, 42 | setShowLoader, 43 | handleFileChange, 44 | handleUpload, 45 | } 46 | 47 | } 48 | 49 | export default useUploadSchemaViewController; -------------------------------------------------------------------------------- /server/app/services/database_details.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import Session 2 | from app.crud.crud_database_table import crud_database_table 3 | from app.crud.crud_table_column import crud_table_column 4 | from app.crud.crud_user_database import crud_user_database 5 | 6 | 7 | class DatabaseDetails: 8 | def fetch_database_details(self, database_id: str, db: Session): 9 | database_name = crud_user_database.get_by_id(db, database_id).name 10 | content = { 11 | "database_id": database_id, 12 | "database_name": database_name, 13 | "database_tables": [], 14 | } 15 | tables = crud_database_table.get_by_user_database_id(db, database_id) 16 | for table in tables: 17 | content["database_tables"].append( 18 | { 19 | "table_id": str(table.id), 20 | "table_name": table.name, 21 | "table_columns": [], 22 | } 23 | ) 24 | columns = crud_table_column.get_by_database_table_id(db, table.id) 25 | for column in columns: 26 | content["database_tables"][-1]["table_columns"].append( 27 | { 28 | "column_id": str(column.id), 29 | "column_name": column.name, 30 | "column_type": column.data_type, 31 | } 32 | ) 33 | return content 34 | 35 | database_details = DatabaseDetails() 36 | -------------------------------------------------------------------------------- /server/app/crud/crud_embedding.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from uuid import UUID 3 | from sqlalchemy.orm import Session 4 | from app.models.embedding import Embedding as EmbeddingModel 5 | from app.schemas.embedding import Embedding as EmbeddingSchema 6 | 7 | 8 | class CRUDEmbedding: 9 | def create(self, db: Session, embedding_obj: EmbeddingSchema) -> EmbeddingModel: 10 | embedding = EmbeddingModel( 11 | embeddings_vector=embedding_obj.embeddings_vector, 12 | database_table_id=embedding_obj.database_table_id, 13 | user_database_id=embedding_obj.user_database_id, 14 | ) 15 | db.add(embedding) 16 | db.commit() 17 | db.refresh(embedding) 18 | return embedding 19 | 20 | def get_by_user_database_id( 21 | self, db: Session, user_database_id: UUID 22 | ) -> List[EmbeddingModel]: 23 | return db.query(EmbeddingModel).filter( 24 | EmbeddingModel.user_database_id == user_database_id 25 | ) 26 | 27 | def get_closest_embeddings_by_database_id( 28 | self, query_embedding: list[float], database_id: UUID, db: Session 29 | ) -> List[str]: 30 | return ( 31 | db.query(EmbeddingModel.database_table_id) 32 | .filter(EmbeddingModel.user_database_id == database_id) 33 | .order_by(EmbeddingModel.embeddings_vector.l2_distance(query_embedding)) 34 | .limit(5) 35 | ) 36 | 37 | def delete_by_database_id(self, db: Session, database_id: UUID) -> None: 38 | db.query(EmbeddingModel).filter(EmbeddingModel.user_database_id == database_id).delete() 39 | 40 | 41 | crud_embedding = CRUDEmbedding() 42 | -------------------------------------------------------------------------------- /server/app/tests/test_schemas/test_user_database_schema.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | from uuid import uuid4 4 | from app.schemas.user_database import UserDatabase 5 | 6 | 7 | def test_user_database_schema_with_valid_data() -> None: 8 | name = "Test Database" 9 | user_id = uuid4() 10 | connection_string = "connection string" 11 | 12 | user_database = UserDatabase(name=name, user_id=user_id, connection_string=connection_string) 13 | 14 | assert user_database.name == name 15 | assert user_database.user_id == user_id 16 | assert user_database.connection_string == connection_string 17 | 18 | 19 | def test_user_database_schema_with_missing_name() -> None: 20 | user_id = uuid4() 21 | 22 | with pytest.raises(ValidationError): 23 | UserDatabase(name=None, user_id=user_id) 24 | 25 | 26 | def test_user_database_schema_with_invalid_name() -> None: 27 | name = 123 28 | user_id = uuid4() 29 | 30 | with pytest.raises(ValidationError): 31 | UserDatabase(name=name, user_id=user_id) 32 | 33 | 34 | def test_user_database_schema_with_missing_user_id() -> None: 35 | name = "Test Database" 36 | 37 | with pytest.raises(ValidationError): 38 | UserDatabase(name=name, user_id=None) 39 | 40 | 41 | def test_user_database_schema_with_invalid_user_id() -> None: 42 | name = "Test Database" 43 | user_id = "invalid_uuid" 44 | 45 | with pytest.raises(ValidationError): 46 | UserDatabase(name=name, user_id=user_id) 47 | 48 | def test_user_database_schema_with_invalid_connection_string() -> None: 49 | name = "Test Database" 50 | user_id = uuid4() 51 | connection_string = 123 52 | 53 | with pytest.raises(ValidationError): 54 | UserDatabase(name=name, user_id=user_id, connection_string=connection_string) 55 | -------------------------------------------------------------------------------- /server/app/tests/test_services/test_openai_service.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import patch 3 | from app.services.openai_service import openai_service as OpenAIService 4 | from app.clients.openai_client import OpenAIClient 5 | from unittest.mock import MagicMock 6 | 7 | mock_choice = MagicMock() 8 | mock_choice.message.content = "some_generated_sql_query;" 9 | mock_chat_completion_object = MagicMock() 10 | mock_chat_completion_object.choices = [mock_choice] 11 | 12 | 13 | 14 | mock_openai_chat_response = { 15 | "id": "chatcmpl-7mdUAqgxJlviahl2xORk7wLSDmtN6", 16 | "object": "chat.completion", 17 | "created": 1691825882, 18 | "model": "gpt-4-0613", 19 | "choices": [ 20 | { 21 | "index": 0, 22 | "message": { 23 | "role": "assistant", 24 | "content": "some_generated_sql_query;\nSQLResult: sample_sql_result\n\nAnswer: sample_answer.", 25 | }, 26 | "finish_reason": "stop", 27 | } 28 | ], 29 | "usage": {"prompt_tokens": 75, "completion_tokens": 39, "total_tokens": 114}, 30 | } 31 | 32 | 33 | def mock_chat_response(*args, **kwargs) -> dict: 34 | return mock_openai_chat_response 35 | 36 | 37 | @pytest.fixture 38 | def openai_service() -> OpenAIService: 39 | return OpenAIService 40 | 41 | 42 | def test_text_2_sql_query(openai_service) -> None: 43 | query_str = "user_query" 44 | relevant_table_schema = "relevant_table_schema_from_db" 45 | dialect = "postgresql" 46 | 47 | with patch.object(OpenAIClient, 'get_chat_response', return_value={'response': mock_chat_completion_object}): 48 | 49 | sql_query = openai_service.text_2_sql_query( 50 | query_str, relevant_table_schema, dialect 51 | ) 52 | 53 | assert sql_query == "some_generated_sql_query;" 54 | -------------------------------------------------------------------------------- /server/app/crud/crud_query.py: -------------------------------------------------------------------------------- 1 | from uuid import UUID 2 | from sqlalchemy.orm import Session 3 | from app.models.query import Query as QueryModel 4 | from app.schemas.query import Query as QuerySchema 5 | 6 | 7 | class CRUDQuery: 8 | def create( 9 | self, db: Session, query_obj: QuerySchema 10 | ) -> QueryModel: 11 | query = QueryModel( 12 | nl_query=query_obj.nl_query, 13 | user_database_id=query_obj.user_database_id, 14 | sql_query=query_obj.sql_query, 15 | ) 16 | db.add(query) 17 | db.commit() 18 | db.refresh(query) 19 | return query 20 | 21 | def get_by_id(self, db: Session, id: UUID) -> QueryModel: 22 | return db.query(QueryModel).filter(QueryModel.id == id).first() 23 | 24 | def get_by_database_id( 25 | self, db: Session, user_database_id: UUID 26 | ) -> QueryModel: 27 | return db.query(QueryModel).filter( 28 | QueryModel.user_database_id == user_database_id 29 | ) 30 | 31 | def get_by_datatbase_id_where_sql_query_not_null( 32 | self, db: Session, user_database_id: UUID 33 | ) -> QueryModel: 34 | return ( 35 | db.query(QueryModel) 36 | .filter(QueryModel.user_database_id == user_database_id) 37 | .filter(QueryModel.sql_query != None) 38 | ) 39 | 40 | def insert_sql_query_by_id( 41 | self, db: Session, id: UUID, sql_query: str 42 | ) -> QueryModel: 43 | query = ( 44 | db.query(QueryModel).filter(QueryModel.id == id).first() 45 | ) 46 | if query.sql_query is not None: 47 | raise ValueError("SQL query already exists") 48 | query.sql_query = sql_query 49 | db.commit() 50 | return query 51 | 52 | 53 | crud_query = CRUDQuery() 54 | -------------------------------------------------------------------------------- /server/app/crud/crud_user.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from uuid import UUID 3 | from sqlalchemy.orm import Session 4 | from bcrypt import hashpw, gensalt, checkpw 5 | 6 | from app.models.user import User as UserModel 7 | from app.schemas.user import User 8 | import os 9 | 10 | 11 | def hash_password(password: str) -> str: 12 | hashed_password = hashpw(password.encode("utf-8"), gensalt()) 13 | return hashed_password.decode("utf-8") 14 | 15 | def hash_key(key: str) -> str: 16 | salt = os.getenv("ENCRYPTION_SALT") 17 | if salt is not None: 18 | salt = salt.encode("utf-8") 19 | hashed_key = hashpw(key.encode("utf-8"), salt) 20 | return hashed_key.decode("utf-8") 21 | 22 | 23 | class CRUDUser: 24 | def create(self, db: Session, user_obj: User) -> UserModel: 25 | user = UserModel( 26 | email=user_obj.email, 27 | hashed_password=hash_password(user_obj.password), 28 | name=user_obj.name, 29 | hashed_key=hash_key(user_obj.password), 30 | ) 31 | db.add(user) 32 | db.commit() 33 | db.refresh(user) 34 | return user 35 | 36 | def get_by_id(self, db: Session, id: UUID) -> Optional[UserModel]: 37 | return db.query(UserModel).filter(UserModel.id == id).first() 38 | 39 | def get_by_email(self, db: Session, email: str) -> Optional[UserModel]: 40 | return db.query(UserModel).filter(UserModel.email == email).first() 41 | 42 | def authenticate( 43 | self, db: Session, email: str, password: str 44 | ) -> Optional[UserModel]: 45 | user = self.get_by_email(db, email=email) 46 | if not user: 47 | return None 48 | if not checkpw(password.encode("utf-8"), user.hashed_password.encode("utf-8")): 49 | return None 50 | return user 51 | 52 | 53 | crud_user = CRUDUser() 54 | -------------------------------------------------------------------------------- /client/app/components/navbar.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import Link from "next/link"; 4 | import useGenericViewController from "../viewControllers/genericViewController"; 5 | import appText from "../assets/strings"; 6 | import React from "react"; 7 | 8 | const Navbar:React.FC = () => { 9 | const { 10 | logout 11 | } = useGenericViewController(); 12 | 13 | const text = appText.headerNavbar; 14 | 15 | return ( 16 |
17 |
18 |
19 |
20 |
21 | Logo 22 | 29 |
30 |
31 |
32 | 36 | {text.addDatabase} 37 | 38 | 44 |
45 |
46 |
47 |
48 |
49 |
50 | ); 51 | }; 52 | 53 | export default Navbar; 54 | -------------------------------------------------------------------------------- /server/app/tests/test_api/v1/test_authentication.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from app.crud.crud_user import crud_user 3 | from app.schemas.user import User as UserSchema 4 | from sqlalchemy.orm import Session 5 | 6 | 7 | def test_login_user(client: TestClient, db: Session, valid_user: UserSchema) -> None: 8 | crud_user.create(db, valid_user) 9 | response = client.post( 10 | "/api/v1/login", 11 | headers={"Content-Type": "application/x-www-form-urlencoded"}, 12 | data={"email": valid_user.email, "password": valid_user.password}, 13 | ) 14 | assert response.status_code == 200 15 | assert response.headers["x-auth-token"] is not None 16 | assert response.json() == { 17 | "message": "Login successfully", 18 | "data": {"name": valid_user.name, "email": valid_user.email} 19 | } 20 | 21 | 22 | def test_login_user_not_found(client: TestClient, valid_user: UserSchema) -> None: 23 | response = client.post( 24 | "/api/v1/login", 25 | headers={"Content-Type": "application/x-www-form-urlencoded"}, 26 | data={"email": valid_user.email, "password": valid_user.password}, 27 | ) 28 | assert response.status_code == 404 29 | assert response.headers.get("x-auth-token") is None 30 | assert response.json() == {"detail": "User Not Found"} 31 | 32 | 33 | def test_login_user_incorrect_password( 34 | client: TestClient, db: Session, valid_user: UserSchema 35 | ) -> None: 36 | crud_user.create(db, valid_user) 37 | response = client.post( 38 | "/api/v1/login", 39 | headers={"Content-Type": "application/x-www-form-urlencoded"}, 40 | data={"email": valid_user.email, "password": "wrong"}, 41 | ) 42 | assert response.status_code == 401 43 | assert response.headers.get("x-auth-token") is None 44 | assert response.json() == {"detail": "Incorrect Password"} 45 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RED='\033[0;31m' 4 | GREEN='\033[0;32m' 5 | YELLOW='\033[0;33m' 6 | RESET='\033[0m' 7 | 8 | # Prerequisites 9 | echo -e "${GREEN}Checking prerequisites...${RESET}" 10 | 11 | # Check for Node.js 12 | if ! command -v node &> /dev/null; then 13 | echo -e "${RED}Node.js is not installed. Please install it before running this script.${RESET}" 14 | exit 1 15 | fi 16 | 17 | # Check for Python 18 | if ! command -v python3 &> /dev/null; then 19 | echo -e "${RED}Python is not installed. Please install it before running this script.${RESET}" 20 | exit 1 21 | fi 22 | 23 | # Check for PostgreSQL 24 | if ! command -v psql &> /dev/null; then 25 | echo -e "${RED}PostgreSQL is not installed. Please install it before running this script.${RESET}" 26 | exit 1 27 | fi 28 | 29 | # Check for pgvector extension 30 | if ! psql -d mql -c "SELECT 'pgvector is installed';" &> /dev/null; then 31 | echo -e "${RED}pgvector extension is not installed. Please install it by following the instructions at [https://github.com/ankane/pgvector](https://github.com/ankane/pgvector).${RESET}" 32 | exit 1 33 | fi 34 | 35 | # Install backend dependencies 36 | echo -e "${GREEN}Installing backend dependencies...${RESET}" 37 | cd server 38 | pip3 install -r requirements.txt 39 | cd .. 40 | 41 | # Install frontend dependencies 42 | echo -e "${GREEN}Installing frontend dependencies...${RESET}" 43 | cd client 44 | npm install 45 | cd .. 46 | 47 | # Configure and start the application 48 | echo -e "${GREEN}Configuring and starting the application...${RESET}" 49 | 50 | cd server 51 | alembic upgrade head 52 | uvicorn app.main:app --reload & 53 | backend_pid=$! 54 | cd .. 55 | 56 | cd client 57 | npm run dev & 58 | frontend_pid=$! 59 | cd .. 60 | 61 | echo -e "${GREEN}Setup complete! You can now access your application on http://localhost:3000.${RESET}" 62 | 63 | wait $frontend_pid 64 | wait $backend_pid 65 | 66 | 67 | -------------------------------------------------------------------------------- /client/app/viewControllers/connectDatabaseViewController.ts: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | import { useRouter } from "next/navigation"; 3 | import { toast } from "react-toastify"; 4 | import { connectDatabase } from "@/app/lib/service"; 5 | import appText from "../assets/strings"; 6 | 7 | const useConnectDatabaseViewController = () => { 8 | const [databaseName, setDatabaseName] = useState(""); 9 | const [databaseUser, setDatabaseUser] = useState(""); 10 | const [databasePassword, setDatabasePassword] = useState(""); 11 | const [databaseHost, setDatabaseHost] = useState(""); 12 | const [databasePort, setDatabasePort] = useState(""); 13 | const [showLoader, setShowLoader] = useState(false); 14 | 15 | const { push } = useRouter(); 16 | 17 | 18 | const handleConnectDatabase = async (e: React.ChangeEvent) => { 19 | e.preventDefault(); 20 | try { 21 | setShowLoader(true); 22 | const formData = new FormData(); 23 | formData.append("database_name", databaseName); 24 | formData.append("database_user", databaseUser); 25 | formData.append("database_password", databasePassword); 26 | formData.append("database_host", databaseHost); 27 | formData.append("database_port", databasePort); 28 | const res = await connectDatabase(formData); 29 | toast.success(appText.toast.connectedSuccess); 30 | push("/home"); 31 | setShowLoader(false); 32 | } catch (error) { 33 | setShowLoader(false); 34 | toast.error(appText.toast.errGeneric); 35 | } 36 | }; 37 | 38 | return { 39 | databaseName, 40 | setDatabaseName, 41 | databaseUser, 42 | setDatabaseUser, 43 | databasePassword, 44 | setDatabasePassword, 45 | databaseHost, 46 | setDatabaseHost, 47 | databasePort, 48 | setDatabasePort, 49 | showLoader, 50 | setShowLoader, 51 | handleConnectDatabase, 52 | } 53 | 54 | } 55 | 56 | export default useConnectDatabaseViewController; -------------------------------------------------------------------------------- /server/app/tests/test_models/test_query_model.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.models.query import Query 4 | 5 | 6 | def test_as_dict_method(db: Session) -> None: 7 | user_database_id = uuid.uuid4() 8 | query_instance = Query( 9 | nl_query="test_nl_query", 10 | sql_query="test_sql_query", 11 | user_database_id=user_database_id, 12 | ) 13 | db.add(query_instance) 14 | db.commit() 15 | result = query_instance.as_dict() 16 | 17 | assert "id" in result 18 | assert isinstance(result["id"], str) 19 | assert isinstance(result["user_database_id"], str) 20 | assert result == { 21 | "nl_query": "test_nl_query", 22 | "sql_query": "test_sql_query", 23 | "user_database_id": str(user_database_id), 24 | "id": result["id"], 25 | "created_at": result["created_at"], 26 | "updated_at": result["updated_at"], 27 | } 28 | 29 | 30 | def test_timestamp_on_create(db: Session) -> None: 31 | user_database_id = uuid.uuid4() 32 | query_instance = Query( 33 | nl_query="test_nl_query", 34 | sql_query="test_sql_query", 35 | user_database_id=user_database_id, 36 | ) 37 | 38 | db.add(query_instance) 39 | db.commit() 40 | 41 | assert query_instance.created_at is not None 42 | assert query_instance.updated_at is not None 43 | assert query_instance.created_at == query_instance.updated_at 44 | 45 | 46 | def test_timestamp_on_update(db: Session) -> None: 47 | user_database_id = uuid.uuid4() 48 | query_instance = Query( 49 | nl_query="test_nl_query", 50 | sql_query="test_sql_query", 51 | user_database_id=user_database_id, 52 | ) 53 | db.add(query_instance) 54 | db.commit() 55 | query_instance.nl_query = "test_nl_query_updated" 56 | db.commit() 57 | assert query_instance.created_at is not None 58 | assert query_instance.updated_at is not None 59 | assert query_instance.created_at != query_instance.updated_at 60 | -------------------------------------------------------------------------------- /client/app/components/fileUploader.tsx: -------------------------------------------------------------------------------- 1 | import Image from "next/image"; 2 | import appText from "../assets/strings"; 3 | import React from "react"; 4 | 5 | type Props = { 6 | file: File | null; 7 | handleFileChange: (e: React.ChangeEvent) => void; 8 | databaseName: string; 9 | handleUpload: () => void; 10 | setDatabaseName: (name: string) => void; 11 | showLoader: boolean; 12 | }; 13 | 14 | const FileUploader = ({ 15 | file, 16 | handleFileChange, 17 | databaseName, 18 | handleUpload, 19 | setDatabaseName, 20 | showLoader, 21 | }: Props):React.JSX.Element => { 22 | 23 | const text = appText.uploadDatabaseSchema; 24 | 25 | return ( 26 |
27 |
28 | 31 | 32 |
33 |
34 | 37 | setDatabaseName(e.target.value)} 41 | className="mb-4 border border-gray-300 px-4 py-2 rounded-full" 42 | /> 43 |
44 | 45 |
46 | 53 | {showLoader && ( 54 | loading 55 | )} 56 |
57 |
58 | ); 59 | }; 60 | 61 | export default FileUploader; 62 | -------------------------------------------------------------------------------- /server/app/crud/crud_database_table.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from uuid import UUID 3 | from sqlalchemy.orm import Session 4 | from app.models.database_table import DatabaseTable as DatabaseTableModel 5 | from app.schemas.database_table import DatabaseTable as DatabaseTableSchema 6 | 7 | 8 | class CRUDDatabaseTable: 9 | def create( 10 | self, db: Session, database_table_obj: DatabaseTableSchema 11 | ) -> DatabaseTableModel: 12 | database_table = DatabaseTableModel( 13 | name=database_table_obj.name, 14 | text_node=database_table_obj.text_node, 15 | user_database_id=database_table_obj.user_database_id, 16 | ) 17 | db.add(database_table) 18 | db.flush() 19 | return database_table 20 | 21 | def get_by_user_database_id( 22 | self, db: Session, user_database_id: UUID 23 | ) -> DatabaseTableModel: 24 | return db.query(DatabaseTableModel).filter( 25 | DatabaseTableModel.user_database_id == user_database_id 26 | ) 27 | 28 | def get_by_id(self, db: Session, id: UUID) -> DatabaseTableModel: 29 | return db.query(DatabaseTableModel).filter(DatabaseTableModel.id == id).first() 30 | 31 | def get_by_ids( 32 | self, db: Session, table_ids: List[UUID] 33 | ) -> List[DatabaseTableModel]: 34 | return db.query(DatabaseTableModel).filter(DatabaseTableModel.id.in_(table_ids)) 35 | 36 | def upsert_text_node_by_id( 37 | self, db: Session, table_id: UUID, text_node: str 38 | ) -> None: 39 | database_table = ( 40 | db.query(DatabaseTableModel) 41 | .filter(DatabaseTableModel.id == table_id) 42 | .first() 43 | ) 44 | database_table.text_node = text_node 45 | db.commit() 46 | 47 | def delete_by_user_database_id(self, db: Session, user_database_id: UUID) -> None: 48 | db.query(DatabaseTableModel).filter(DatabaseTableModel.user_database_id == user_database_id).delete() 49 | 50 | crud_database_table = CRUDDatabaseTable() 51 | -------------------------------------------------------------------------------- /server/app/api/v1/dependencies.py: -------------------------------------------------------------------------------- 1 | from typing import Generator 2 | from fastapi import Depends, HTTPException, status 3 | from fastapi.security import HTTPBearer 4 | from sqlalchemy.orm import Session 5 | from app.models.user import User as UserModel 6 | from app.crud.crud_user import crud_user 7 | from app.db.session import sessionLocal 8 | from datetime import datetime, timedelta 9 | from dotenv import load_dotenv 10 | from jose import jwt 11 | from jose.exceptions import JWTError, ExpiredSignatureError 12 | import os 13 | 14 | load_dotenv() 15 | 16 | 17 | def get_db() -> Generator: 18 | db = sessionLocal() 19 | try: 20 | yield db 21 | finally: 22 | db.close() 23 | 24 | 25 | def create_access_token(data: dict, expires_delta: timedelta | None = None) -> str: 26 | to_encode = data.copy() 27 | if expires_delta: 28 | expire = datetime.utcnow() + expires_delta 29 | else: 30 | expire = datetime.utcnow() + timedelta( 31 | minutes=os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES") 32 | ) 33 | to_encode.update({"exp": expire}) 34 | encoded_jwt = jwt.encode( 35 | to_encode, os.getenv("SECRET_KEY"), algorithm=os.getenv("ALGORITHM") 36 | ) 37 | return encoded_jwt 38 | 39 | 40 | def validate_access_token(token: str) -> dict | bool: 41 | try: 42 | payload = jwt.decode( 43 | token, os.getenv("SECRET_KEY"), algorithms=[os.getenv("ALGORITHM")] 44 | ) 45 | return payload 46 | except ExpiredSignatureError: 47 | return False 48 | except JWTError: 49 | return False 50 | 51 | 52 | def get_current_user( 53 | credentials: str = Depends(HTTPBearer()), db: Session = Depends(get_db) 54 | ) -> UserModel: 55 | payload = validate_access_token(credentials.credentials) 56 | 57 | if not payload: 58 | raise HTTPException( 59 | status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid Token" 60 | ) 61 | user = crud_user.get_by_email(db, payload["email"]) 62 | user.hashed_key = payload["hashed_key"] 63 | return user 64 | -------------------------------------------------------------------------------- /client/app/components/codeBlock.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import useCodeBlockViewController from "../viewControllers/codeBlockViewController"; 4 | import AceEditor from 'react-ace'; 5 | import 'ace-builds/src-noconflict/mode-sql'; 6 | import 'ace-builds/src-noconflict/theme-monokai'; 7 | import React from "react"; 8 | 9 | type Props = { 10 | handleQueryResponse: () => void; 11 | codeString: string; 12 | setSql : (sql: string) => void; 13 | executeFlag?: boolean; 14 | }; 15 | 16 | const CodeBlock = ({ handleQueryResponse , codeString, setSql , executeFlag=true}: Props):React.JSX.Element => { 17 | 18 | const { 19 | handleCopyClick, 20 | copySuccess, 21 | executeSuccess, 22 | query, 23 | handleQueryChange, 24 | handleExecuteClick 25 | } = useCodeBlockViewController({ codeString, setSql, handleQueryResponse}); 26 | 27 | return ( 28 |
29 | 47 | 48 |
49 | 50 | {executeFlag && } 56 | 62 |
63 |
64 | ); 65 | }; 66 | 67 | export default CodeBlock; 68 | -------------------------------------------------------------------------------- /client/app/viewControllers/homeViewController.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArrowTrendingUpIcon, 3 | BoltIcon, 4 | LightBulbIcon, 5 | TableCellsIcon, 6 | } from "@heroicons/react/24/outline"; 7 | 8 | import { 9 | ChatBubbleOvalLeftEllipsisIcon, 10 | CircleStackIcon, 11 | QuestionMarkCircleIcon, 12 | } from "@heroicons/react/24/solid"; 13 | 14 | import { cookies } from "next/headers"; 15 | import appText from "../assets/strings"; 16 | 17 | 18 | 19 | const useHomeViewController = () => { 20 | 21 | const token = cookies().get("token")?.value 22 | const text = appText.home; 23 | 24 | const features = [ 25 | { 26 | name: text.insightsEmpowered, 27 | description: text.insightsEmpoweredDescription, 28 | icon: LightBulbIcon, 29 | }, 30 | { 31 | name: text.simplifiedQueries, 32 | description: 33 | text.simplifiedQueriesDescription, 34 | icon: TableCellsIcon, 35 | }, 36 | { 37 | name: text.powerfulDecisions, 38 | description: 39 | text.powerfulDecisionsDescription, 40 | icon: BoltIcon, 41 | }, 42 | { 43 | name: text.dataEmpowerment, 44 | description: 45 | text.dataEmpowermentDescription, 46 | icon: ArrowTrendingUpIcon, 47 | }, 48 | ]; 49 | 50 | const setupSteps = [ 51 | { 52 | name: text.setTheStage, 53 | description: text.setTheStageDescription, 54 | icon: CircleStackIcon, 55 | }, 56 | { 57 | name: text.askAway, 58 | description: text.askAwayDescription, 59 | icon: QuestionMarkCircleIcon, 60 | }, 61 | { 62 | name: text.queryDelivered, 63 | description: text.queryDeliveredDescription, 64 | icon: ChatBubbleOvalLeftEllipsisIcon, 65 | }, 66 | ]; 67 | 68 | return { 69 | features, 70 | setupSteps, 71 | token, 72 | }; 73 | } 74 | 75 | export default useHomeViewController; -------------------------------------------------------------------------------- /server/app/tests/test_models/test_table_column_model.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.models.table_column import TableColumn 4 | 5 | 6 | def test_as_dict_method(db: Session) -> None: 7 | database_table_id = uuid.uuid4() 8 | table_column_instance = TableColumn( 9 | name="test_column", 10 | data_type="Character Varying", 11 | database_table_id=database_table_id, 12 | ) 13 | db.add(table_column_instance) 14 | db.commit() 15 | result = table_column_instance.as_dict() 16 | assert "id" in result 17 | assert isinstance(result["id"], str) 18 | assert isinstance(result["database_table_id"], str) 19 | assert result == { 20 | "name": "test_column", 21 | "data_type": "Character Varying", 22 | "database_table_id": str(database_table_id), 23 | "id": result["id"], 24 | "created_at": result["created_at"], 25 | "updated_at": result["updated_at"], 26 | } 27 | 28 | 29 | def test_timestamp_on_create(db: Session) -> None: 30 | database_table_id = uuid.uuid4() 31 | table_column_instance = TableColumn( 32 | name="test_column", 33 | data_type="Character Varying", 34 | database_table_id=database_table_id, 35 | ) 36 | db.add(table_column_instance) 37 | db.commit() 38 | assert table_column_instance.created_at is not None 39 | assert table_column_instance.updated_at is not None 40 | assert table_column_instance.created_at == table_column_instance.updated_at 41 | 42 | 43 | def test_timestamp_on_update(db: Session) -> None: 44 | database_table_id = uuid.uuid4() 45 | table_column_instance = TableColumn( 46 | name="test_column", 47 | data_type="Character Varying", 48 | database_table_id=database_table_id, 49 | ) 50 | db.add(table_column_instance) 51 | db.commit() 52 | table_column_instance.name = "test_column_updated" 53 | db.commit() 54 | assert table_column_instance.created_at is not None 55 | assert table_column_instance.updated_at is not None 56 | assert table_column_instance.created_at != table_column_instance.updated_at 57 | -------------------------------------------------------------------------------- /server/app/tests/test_clients/test_openai_client.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import patch 3 | from app.clients.openai_client import openai_client as OpenAIClient 4 | 5 | mock_openai_embedding_response = { 6 | "data": [ 7 | {"embedding": [0.1, 0.2, 0.3]}, 8 | {"embedding": [0.4, 0.5, 0.6]}, 9 | ] 10 | } 11 | 12 | mock_chat_response = { 13 | 'id': 'chatcmpl-99TiMJDtVEyznHcIbhatdpz9mYbzV', 14 | 'choices': [ 15 | { 16 | 'finish_reason': 'stop', 17 | 'index': 0, 18 | 'message': { 19 | 'content': "\n\nHello there, I am fine.\nhow may I assist you today?", 20 | 'role': 'system', 21 | } 22 | } 23 | ], 24 | 'created': 1712046202, 25 | 'model': 'gpt-4-0613', 26 | 'object': 'chat.completion', 27 | 'usage': { 28 | 'completion_tokens': 64, 29 | 'prompt_tokens': 356, 30 | 'total_tokens': 420 31 | } 32 | } 33 | 34 | 35 | def mock_embedding_create(*args, **kwargs) -> dict: 36 | return mock_openai_embedding_response 37 | 38 | 39 | def mock_chat_response(*args, **kwargs) -> dict: 40 | return mock_openai_chat_response 41 | 42 | 43 | @pytest.fixture 44 | def openai_client() -> OpenAIClient: 45 | return OpenAIClient 46 | 47 | 48 | def test_get_embeddings(openai_client) -> None: 49 | nodes = ["Hello", "World"] 50 | 51 | with patch.object(OpenAIClient, 'get_embeddings', return_value=[ 52 | [0.1, 0.2, 0.3], 53 | [0.4, 0.5, 0.6], 54 | ]) as mock_method: 55 | embeddings = openai_client.get_embeddings(nodes) 56 | 57 | assert embeddings == [ 58 | [0.1, 0.2, 0.3], 59 | [0.4, 0.5, 0.6], 60 | ] 61 | 62 | 63 | def test_get_chat_response(openai_client): 64 | messages = [{"role": "system", "content": "Hello, How are you?"}] 65 | 66 | with patch.object(OpenAIClient, 'get_chat_response', return_value={'response': mock_chat_response}) as mock_method: 67 | chat_response = openai_client.get_chat_response(messages) 68 | 69 | expected_response = {'response': mock_chat_response} 70 | 71 | assert chat_response == expected_response, "The chat response did not match the expected output." 72 | -------------------------------------------------------------------------------- /server/app/api/v1/authentication.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Form, status, HTTPException, Depends 2 | from fastapi.responses import JSONResponse 3 | from typing import Annotated 4 | from pydantic import EmailStr 5 | from sqlalchemy.orm import Session 6 | from app.api.v1.dependencies import get_db, create_access_token 7 | from app.crud.crud_user import crud_user, hash_key 8 | from datetime import timedelta 9 | import logging 10 | 11 | 12 | router = APIRouter() 13 | logger = logging.getLogger("mql") 14 | 15 | 16 | @router.post("/login") 17 | async def login( 18 | email: Annotated[EmailStr, Form()], 19 | password: Annotated[str, Form()], 20 | db: Session = Depends(get_db), 21 | ) -> JSONResponse: 22 | try: 23 | user = crud_user.get_by_email(db, email) 24 | if not user: 25 | logger.info("User not found for email {}".format(email)) 26 | raise HTTPException( 27 | status_code=status.HTTP_404_NOT_FOUND, detail="User Not Found" 28 | ) 29 | if not crud_user.authenticate(db, email, password): 30 | logger.info("Incorrect password for email {}".format(email)) 31 | raise HTTPException( 32 | status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect Password" 33 | ) 34 | password = hash_key(password) 35 | token = create_access_token( 36 | data={"id": str(user.id), "email": user.email, "name": user.name, "hashed_key": password}, 37 | expires_delta=timedelta(minutes=60), 38 | ) 39 | except HTTPException as e: 40 | raise HTTPException(status_code=e.status_code, detail=e.detail) 41 | except Exception as e: 42 | logger.error("User login failed for email {}. Error is {}".format(email, e)) 43 | raise HTTPException( 44 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 45 | detail="Internal server error", 46 | ) 47 | 48 | return JSONResponse( 49 | content={ 50 | "message": "Login successfully", 51 | "data": {"name": user.name, "email": user.email}, 52 | }, 53 | headers={"x-auth-token": token}, 54 | status_code=status.HTTP_200_OK, 55 | ) 56 | -------------------------------------------------------------------------------- /client/app/lib/service.js: -------------------------------------------------------------------------------- 1 | import { CONNECT_DATABASE, DATABASES, DELETE_DATABASE, LOGIN, QUERIES, QUERY_EXECUTION, UPLOAD_DATABASE, SCHEMA_SYNC } from "@/app/utils/routes"; 2 | import axios from "axios"; 3 | import Cookies from "js-cookie"; 4 | import { toast } from "react-toastify"; 5 | import appText from "../assets/strings"; 6 | 7 | axios.defaults.baseURL = process.env.NEXT_PUBLIC_API_URL; 8 | 9 | axios.interceptors.response.use( 10 | (response) => response, 11 | async (error) => { 12 | const errCode = error.response?.status; 13 | if (errCode === 401 && error.response?.data?.detail !== "Incorrect Password") { 14 | toast.error(appText.toast.errSessionExpired); 15 | setTimeout(() => { 16 | Cookies.set("token", ""); 17 | window.location.href = "/login"; 18 | }, 2000); 19 | } 20 | return Promise.reject(error); 21 | } 22 | ); 23 | 24 | axios.interceptors.request.use((request) => { 25 | const token = Cookies.get("token"); 26 | if (token) { 27 | request.headers.Authorization = `Bearer ${token}`; 28 | } 29 | return request; 30 | }); 31 | 32 | const handleRequest = async (method, url, data) => { 33 | try { 34 | const res = await axios[method](url, data); 35 | return res; 36 | } catch (err) { 37 | if (err.response) throw err.response.data; 38 | throw err.message; 39 | } 40 | }; 41 | 42 | export const login = (data) => handleRequest('post', LOGIN, data); 43 | export const getAllDatabase = () => handleRequest('get', DATABASES); 44 | export const getDatabase = (id) => handleRequest('get', `${DATABASES}/${id}`); 45 | export const connectDatabase = (data) => handleRequest('post', CONNECT_DATABASE, data); 46 | export const uploadSchema = (formData) => handleRequest('post', UPLOAD_DATABASE, formData); 47 | export const askQuery = (payload) => handleRequest('post', QUERIES, payload); 48 | export const getQueries = (dbId) => handleRequest('get', `${QUERIES}?db_id=${dbId}`); 49 | export const getQuery = ({ id }) => handleRequest('get', `${QUERIES}/${id}`); 50 | export const executeQuery = (payload) => handleRequest('get', `${QUERY_EXECUTION}?db_id=${payload.db_id}&sql_query=${payload.sql_query}`); 51 | export const syncSchema = (payload) => handleRequest('post', SCHEMA_SYNC, payload); 52 | export const deleteDatabase = (id) => handleRequest('delete', `${DELETE_DATABASE}/${id}`); 53 | -------------------------------------------------------------------------------- /client/app/(routes)/(protected)/add-database/page.tsx: -------------------------------------------------------------------------------- 1 | // Use client directive indicates this file should only run in the client-side environment. 2 | "use client"; 3 | import Image from "next/image"; 4 | import useAddDatabaseViewController from "@/app/viewControllers/addDatabaseViewController"; 5 | import appText from "@/app/assets/strings"; 6 | import DatabaseConnector from "@/app/components/databaseConnector"; 7 | import UploadDatabaseSchema from "@/app/components/schemaUploader"; 8 | import React, { MutableRefObject, useEffect, useRef } from "react"; 9 | 10 | 11 | type AddDatabaseViewController = { 12 | isConnectVisible: boolean; 13 | toggleComponent: () => void; 14 | }; 15 | 16 | const AddDatabase:React.FC = () => { 17 | const { 18 | isConnectVisible, 19 | toggleComponent 20 | }:AddDatabaseViewController = useAddDatabaseViewController(); 21 | 22 | const text = appText.addDatabase; 23 | 24 | const titleRef = useRef(null) 25 | 26 | useEffect(()=>{ 27 | const timer = setTimeout(()=>{ 28 | titleRef?.current?.scrollIntoView({behavior:'smooth'}); 29 | },1000) 30 | return ()=>clearTimeout(timer) 31 | },[]) 32 | 33 | return ( 34 |
35 |
36 |
37 |
38 |

{text?.title}

39 |

40 | {text?.description}
41 |

42 |
43 |
44 |
45 | database 52 |
53 |
54 |
55 | {isConnectVisible ? ( 56 | 57 | ) : ( 58 | 59 | )} 60 |
61 |
62 | ); 63 | }; 64 | 65 | export default AddDatabase; 66 | -------------------------------------------------------------------------------- /server/app/tests/test_api/v1/test_dependencies.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | import pytest 4 | from jose import jwt 5 | from sqlalchemy.orm import Session 6 | from fastapi import HTTPException 7 | from datetime import timedelta, datetime 8 | from fastapi.security import HTTPAuthorizationCredentials 9 | from app.models.user import User as UserModel 10 | from app.api.v1.dependencies import ( 11 | validate_access_token, 12 | get_current_user, 13 | ) 14 | 15 | 16 | def is_valid_uuid(id_to_test) -> bool: 17 | try: 18 | uuid.UUID(str(id_to_test)) 19 | return True 20 | except ValueError: 21 | return False 22 | 23 | 24 | def test_validate_access_token_with_valid_token( 25 | valid_user_model: UserModel, valid_jwt: str 26 | ) -> None: 27 | payload = validate_access_token(valid_jwt) 28 | assert payload["name"] == valid_user_model.name 29 | assert payload["email"] == valid_user_model.email 30 | 31 | 32 | def test_validate_access_token_with_expired_token() -> None: 33 | expired_data = { 34 | "name": "test", 35 | "email": "test@test.com", 36 | "exp": datetime.utcnow() - timedelta(minutes=30), 37 | } 38 | expired_token = jwt.encode( 39 | expired_data, os.getenv("SECRET_KEY"), algorithm=os.getenv("ALGORITHM") 40 | ) 41 | expired_payload = validate_access_token(expired_token) 42 | assert expired_payload is False 43 | 44 | 45 | def test_validate_access_token_with_invalid_token() -> None: 46 | invalid_token = "this-is-not-a-valid-token" 47 | invalid_payload = validate_access_token(invalid_token) 48 | assert invalid_payload is False 49 | 50 | 51 | def test_get_current_user_with_valid_token( 52 | valid_user_model: UserModel, auth_bearer: HTTPAuthorizationCredentials, db: Session 53 | ) -> None: 54 | user = get_current_user(auth_bearer, db) 55 | assert user.name == valid_user_model.name 56 | assert user.email == valid_user_model.email 57 | assert is_valid_uuid(user.id) 58 | 59 | 60 | def test_get_current_user_with_invalid_token( 61 | auth_bearer_with_invalid_token: HTTPAuthorizationCredentials, db: Session 62 | ) -> None: 63 | with pytest.raises(HTTPException) as e: 64 | get_current_user(auth_bearer_with_invalid_token, db) 65 | assert e.value.status_code == 401 66 | assert e.value.detail == "Invalid Token" 67 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Variables 2 | DOCKER_COMPOSE = docker-compose 3 | 4 | # Colors 5 | COLOR_RESET = \033[0m 6 | COLOR_BOLD = \033[1m 7 | COLOR_GREEN = \033[32m 8 | COLOR_YELLOW = \033[33m 9 | 10 | # Check if Docker is installed 11 | DOCKER_COMPOSE_INSTALLED := $(shell command -v docker-compose 2> /dev/null) 12 | DOCKER_INSTALLED := $(shell command -v docker 2> /dev/null) 13 | 14 | # Targets 15 | install: 16 | ifndef DOCKER_COMPOSE_INSTALLED 17 | $(error "docker-compose is not installed. Please install docker-compose and try again.") 18 | endif 19 | ifndef DOCKER_INSTALLED 20 | $(error "docker is not installed. Please install docker and try again.") 21 | endif 22 | @echo "$(COLOR_BOLD)=== Putting the services down (if already running) ===$(COLOR_RESET)" 23 | $(DOCKER_COMPOSE) down 24 | @echo "$(COLOR_BOLD)=== Building services ===$(COLOR_RESET)" 25 | $(DOCKER_COMPOSE) build --no-cache 26 | $(DOCKER_COMPOSE) up -d 27 | @echo "$(COLOR_BOLD)=== Waiting for services to start (~20 seconds) ===$(COLOR_RESET)" 28 | @sleep 20 29 | @$(MAKE) create-default-user 30 | @echo "$(COLOR_BOLD)=== Default Login Credentials >>> Email -> admin@example.com , Password -> admin ===$(COLOR_RESET)" 31 | @echo "$(COLOR_BOLD)=== Installation completed ===$(COLOR_RESET)" 32 | @echo "$(COLOR_BOLD)=== 🔥🔥 You can now access the dashboard at -> http://localhost:3000 ===$(COLOR_RESET)" 33 | @echo "$(COLOR_BOLD)=== Enjoy! ===$(COLOR_RESET)" 34 | 35 | create-default-user: 36 | @echo "$(COLOR_BOLD)Creating default user...$(COLOR_RESET)" 37 | $(DOCKER_COMPOSE) exec -T backend python3 -m app.user_utils 38 | @echo "$(COLOR_BOLD)Default user creation attempted. Check logs for details.$(COLOR_RESET)" 39 | 40 | down: 41 | $(DOCKER_COMPOSE) down --remove-orphans 42 | 43 | restart: 44 | @echo "$(COLOR_BOLD)=== Checking for changes and rebuilding if necessary ===$(COLOR_RESET)" 45 | $(DOCKER_COMPOSE) up -d --build 46 | @echo "$(COLOR_BOLD)=== Restart completed ===$(COLOR_RESET)" 47 | @echo "$(COLOR_BOLD)=== 🔥🔥 You can now access the dashboard at -> http://localhost:3000 ===$(COLOR_RESET)" 48 | @echo "$(COLOR_BOLD)=== Enjoy! ===$(COLOR_RESET)" 49 | 50 | up: 51 | @echo "$(COLOR_BOLD)=== Starting all services ===$(COLOR_RESET)" 52 | $(DOCKER_COMPOSE) up -d 53 | @echo "$(COLOR_BOLD)=== All services are up and running ===$(COLOR_RESET)" 54 | 55 | logs: 56 | $(DOCKER_COMPOSE) logs -f 57 | 58 | .PHONY: install down -------------------------------------------------------------------------------- /server/scripts/create_user.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.orm import sessionmaker 3 | from dotenv import load_dotenv 4 | import os 5 | from uuid import UUID 6 | import getpass 7 | from bcrypt import hashpw, gensalt 8 | import uuid 9 | from sqlalchemy import String, UUID 10 | from sqlalchemy.orm import mapped_column 11 | from sqlalchemy import Column, DateTime, func 12 | from sqlalchemy.orm import DeclarativeBase 13 | import logging 14 | 15 | def hash_password(password: str) -> str: 16 | hashed_password = hashpw(password.encode("utf-8"), gensalt()) 17 | return hashed_password.decode("utf-8") 18 | 19 | class Base(DeclarativeBase): 20 | pass 21 | 22 | 23 | class TimestampBase(Base): 24 | __abstract__ = True 25 | created_at = Column(DateTime, default=func.now()) 26 | updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) 27 | 28 | 29 | class User(TimestampBase): 30 | __tablename__ = "users" 31 | 32 | id = mapped_column( 33 | UUID, primary_key=True, unique=True, index=True, default=uuid.uuid4 34 | ) 35 | name = mapped_column(String, nullable=False) 36 | email = mapped_column(String, unique=True, nullable=False) 37 | hashed_password = mapped_column(String, nullable=False) 38 | 39 | def as_dict(self) -> dict: 40 | return { 41 | "name": self.name, 42 | "email": self.email, 43 | "id": str(self.id), 44 | "created_at": self.created_at.isoformat(), 45 | "updated_at": self.updated_at.isoformat(), 46 | } 47 | 48 | 49 | load_dotenv() 50 | 51 | engine = create_engine(os.getenv("DATABASE_URL"), echo=True) 52 | 53 | SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) 54 | 55 | 56 | email = input("Enter your email: ") 57 | password = getpass.getpass("Enter your password: ") 58 | name = input("Enter your name: ") 59 | 60 | user = SessionLocal().query(User).filter(User.email == email).first() 61 | 62 | if user: 63 | logging.getLogger("mql").info("User already exists with email {}".format(email)) 64 | exit() 65 | 66 | with SessionLocal() as session: 67 | user = User(email=email, hashed_password= hash_password(password=password), name=name) 68 | 69 | session.add(user) 70 | session.commit() 71 | session.refresh(user) 72 | session.close() 73 | 74 | 75 | print("User created successfully!!") 76 | -------------------------------------------------------------------------------- /server/app/tests/test_models/test_user_database_model.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.models.user_database import UserDatabase 4 | 5 | 6 | def test_as_dict_method(db: Session) -> None: 7 | user_id = uuid.uuid4() 8 | user_database_instance = UserDatabase(name="test_database", user_id=user_id,connection_string="connection_string") 9 | db.add(user_database_instance) 10 | db.commit() 11 | result = user_database_instance.as_dict() 12 | 13 | assert "id" in result 14 | assert isinstance(result["id"], str) 15 | assert isinstance(result["user_id"], str) 16 | assert result == { 17 | "name": "test_database", 18 | "user_id": str(user_id), 19 | "connection_string": True, 20 | "id": result["id"], 21 | "created_at": result["created_at"], 22 | "updated_at": result["updated_at"], 23 | } 24 | 25 | 26 | def test_timestamp_on_create(db: Session) -> None: 27 | user_id = uuid.uuid4() 28 | user_database_instance = UserDatabase(name="test_database", user_id=user_id) 29 | 30 | db.add(user_database_instance) 31 | db.commit() 32 | 33 | assert user_database_instance.created_at is not None 34 | assert user_database_instance.updated_at is not None 35 | assert user_database_instance.created_at == user_database_instance.updated_at 36 | 37 | 38 | def test_timestamp_on_update(db: Session) -> None: 39 | user_id = uuid.uuid4() 40 | user_database_instance = UserDatabase(name="test_database", user_id=user_id) 41 | 42 | db.add(user_database_instance) 43 | db.commit() 44 | user_database_instance.name = "test_database_updated" 45 | db.commit() 46 | assert user_database_instance.created_at is not None 47 | assert user_database_instance.updated_at is not None 48 | assert user_database_instance.created_at != user_database_instance.updated_at 49 | 50 | def test_connection_string(db: Session) -> None: 51 | user_id = uuid.uuid4() 52 | connection_string = "connection_string" 53 | user_database_instance = UserDatabase(name="test_database", user_id=user_id,connection_string=connection_string) 54 | 55 | db.add(user_database_instance) 56 | db.commit() 57 | result = user_database_instance.as_dict() 58 | print("result is",result) 59 | 60 | assert "connection_string" in result 61 | assert user_database_instance.connection_string == connection_string 62 | -------------------------------------------------------------------------------- /server/app/tests/test_schemas/test_table_column_schema.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | from uuid import uuid4 4 | from app.schemas.table_column import TableColumn 5 | 6 | 7 | def test_table_column_schema_with_valid_data() -> None: 8 | name = "Test Column" 9 | data_type = "Character Varying" 10 | database_table_id = uuid4() 11 | 12 | table_column = TableColumn( 13 | name=name, data_type=data_type, database_table_id=database_table_id 14 | ) 15 | 16 | assert table_column.name == name 17 | assert table_column.data_type == data_type 18 | assert table_column.database_table_id == database_table_id 19 | 20 | 21 | def test_table_column_schema_with_missing_name() -> None: 22 | data_type = "Character Varying" 23 | database_table_id = uuid4() 24 | 25 | with pytest.raises(ValidationError): 26 | TableColumn(name=None, data_type=data_type, database_table_id=database_table_id) 27 | 28 | 29 | def test_table_column_schema_with_invalid_name() -> None: 30 | name = 123 31 | data_type = "Character Varying" 32 | database_table_id = uuid4() 33 | 34 | with pytest.raises(ValidationError): 35 | TableColumn(name=name, data_type=data_type, database_table_id=database_table_id) 36 | 37 | 38 | def test_table_column_schema_with_missing_data_type() -> None: 39 | name = "Test Column" 40 | database_table_id = uuid4() 41 | 42 | with pytest.raises(ValidationError): 43 | TableColumn(name=name, data_type=None, database_table_id=database_table_id) 44 | 45 | 46 | def test_table_column_schema_with_invalid_data_type() -> None: 47 | name = "Test Column" 48 | data_type = 123 49 | database_table_id = uuid4() 50 | 51 | with pytest.raises(ValidationError): 52 | TableColumn(name=name, data_type=data_type, database_table_id=database_table_id) 53 | 54 | 55 | def test_table_column_schema_with_missing_database_table_id() -> None: 56 | name = "Test Column" 57 | data_type = "Character Varying" 58 | 59 | with pytest.raises(ValidationError): 60 | TableColumn(name=name, data_type=data_type, database_table_id=None) 61 | 62 | 63 | def test_table_column_schema_with_invalid_database_table_id() -> None: 64 | name = "Test Column" 65 | data_type = "Character Varying" 66 | database_table_id = "invalid_uuid" 67 | 68 | with pytest.raises(ValidationError): 69 | TableColumn(name=name, data_type=data_type, database_table_id=database_table_id) 70 | -------------------------------------------------------------------------------- /server/app/services/openai_service.py: -------------------------------------------------------------------------------- 1 | from app.clients.openai_client import openai_client 2 | 3 | 4 | class OpenAIService: 5 | def __init__(self, DEFAULT_TEXT_TO_SQL_TMPL: str | None = None) -> None: 6 | self.__DEFAULT_TEXT_TO_SQL_TMPL = ( 7 | "Given an input question, first create a syntactically correct {dialect} " 8 | "query to run, then look at the results of the query and return the answer. " 9 | "You can order the results by a relevant column to return the most " 10 | "interesting examples in the database.\n" 11 | "Never query for all the columns from a specific table, only ask for a " 12 | "few relevant columns given the question.\n" 13 | "Pay attention to use only the column names that you can see in the schema " 14 | "description. " 15 | "Be careful to not query for columns that do not exist. " 16 | "Pay attention to which column is in which table. " 17 | "Also, qualify column names with the table name when needed.\n" 18 | "Use the following format:\n" 19 | "Question: Question here\n" 20 | "SQLQuery: SQL Query to run\n" 21 | "SQLResult: Result of the SQLQuery\n" 22 | "Answer: Final answer here\n" 23 | "Only use the tables listed below.\n" 24 | "{schema}\n" 25 | "Question: {query_str}\n" 26 | "SQLQuery: " 27 | ) 28 | 29 | def text_2_sql_query( 30 | self, query_str: str, relevant_table_schema: str, dialect: str = "postgresql" 31 | ) -> str: 32 | formatted_text_2_sql_prompt = self.__DEFAULT_TEXT_TO_SQL_TMPL.format( 33 | query_str=query_str, schema=relevant_table_schema, dialect=dialect 34 | ) 35 | openai_response = openai_client.get_chat_response( 36 | messages=[ 37 | {"role": "system", "content": formatted_text_2_sql_prompt}, 38 | ], 39 | ) 40 | chat_completion_object = openai_response['response'] 41 | response = chat_completion_object.choices[0].message.content 42 | 43 | text2sql_response_copy = response 44 | sql_result_start = text2sql_response_copy.find("SQLResult:") 45 | if sql_result_start != -1: 46 | text2sql_response_copy = text2sql_response_copy[:sql_result_start] 47 | sql_query = text2sql_response_copy.strip() 48 | return sql_query 49 | 50 | 51 | openai_service = OpenAIService() 52 | -------------------------------------------------------------------------------- /server/app/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from typing import Generator 3 | from sqlalchemy import create_engine 4 | from sqlalchemy.orm import Session 5 | from fastapi.testclient import TestClient 6 | from app.crud.crud_user import crud_user 7 | from app.db.session import sessionmaker 8 | from dotenv import load_dotenv 9 | from app.main import app 10 | from app.api.v1.dependencies import create_access_token, get_db 11 | from app.db.base_class import Base 12 | import os 13 | import pytest 14 | from fastapi.security import HTTPAuthorizationCredentials 15 | 16 | from app.schemas.user import User as UserSchema 17 | from app.models.user import User as UserModel 18 | 19 | load_dotenv() 20 | 21 | engine = create_engine(os.getenv("TEST_DATABASE_URL")) 22 | 23 | sessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) 24 | 25 | 26 | def override_get_db() -> Generator: 27 | try: 28 | db = sessionLocal() 29 | yield db 30 | finally: 31 | db.close() 32 | 33 | 34 | @pytest.fixture(scope="function") 35 | def db() -> Generator: 36 | Base.metadata.drop_all(bind=engine) 37 | Base.metadata.create_all(bind=engine) 38 | yield from override_get_db() 39 | 40 | 41 | @pytest.fixture(scope="function") 42 | def client() -> Generator: 43 | Base.metadata.drop_all(bind=engine) 44 | Base.metadata.create_all(bind=engine) 45 | app.dependency_overrides[get_db] = override_get_db 46 | with TestClient(app) as c: 47 | yield c 48 | 49 | 50 | @pytest.fixture(scope="function") 51 | def valid_user() -> Generator: 52 | email = "test@test.com" 53 | name = "test" 54 | password = "test" 55 | yield UserSchema(name=name, email=email, password=password) 56 | 57 | 58 | @pytest.fixture(scope="function") 59 | def valid_user_model(db: Session, valid_user: UserSchema) -> Generator: 60 | user = crud_user.create(db, valid_user) 61 | yield user 62 | 63 | 64 | @pytest.fixture(scope="function") 65 | def valid_jwt(valid_user_model: UserModel) -> Generator: 66 | data = valid_user_model.as_dict() 67 | token = create_access_token(data, timedelta(minutes=5)) 68 | yield token 69 | 70 | 71 | @pytest.fixture(scope="function") 72 | def auth_bearer(valid_jwt: str) -> Generator: 73 | yield HTTPAuthorizationCredentials(scheme="Bearer", credentials=valid_jwt) 74 | 75 | 76 | @pytest.fixture(scope="function") 77 | def auth_bearer_with_invalid_token() -> Generator: 78 | yield HTTPAuthorizationCredentials(scheme="Bearer", credentials="invalid-token") 79 | -------------------------------------------------------------------------------- /server/alembic/env.py: -------------------------------------------------------------------------------- 1 | from logging.config import fileConfig 2 | 3 | from sqlalchemy import engine_from_config 4 | from sqlalchemy import pool 5 | 6 | from alembic import context 7 | import os 8 | from dotenv import load_dotenv 9 | 10 | load_dotenv() 11 | 12 | # this is the Alembic Config object, which provides 13 | # access to the values within the .ini file in use. 14 | config = context.config 15 | config.set_main_option("sqlalchemy.url", os.getenv("DATABASE_URL")) 16 | 17 | # Interpret the config file for Python logging. 18 | # This line sets up loggers basically. 19 | if config.config_file_name is not None: 20 | fileConfig(config.config_file_name) 21 | 22 | # add your model's MetaData object here 23 | # for 'autogenerate' support 24 | # from myapp import mymodel 25 | # target_metadata = mymodel.Base.metadata 26 | from app.db.base import Base 27 | 28 | target_metadata = Base.metadata 29 | 30 | # other values from the config, defined by the needs of env.py, 31 | # can be acquired: 32 | # my_important_option = config.get_main_option("my_important_option") 33 | # ... etc. 34 | 35 | 36 | def run_migrations_offline() -> None: 37 | """Run migrations in 'offline' mode. 38 | 39 | This configures the context with just a URL 40 | and not an Engine, though an Engine is acceptable 41 | here as well. By skipping the Engine creation 42 | we don't even need a DBAPI to be available. 43 | 44 | Calls to context.execute() here emit the given string to the 45 | script output. 46 | 47 | """ 48 | url = config.get_main_option("sqlalchemy.url") 49 | context.configure( 50 | url=url, 51 | target_metadata=target_metadata, 52 | literal_binds=True, 53 | dialect_opts={"paramstyle": "named"}, 54 | ) 55 | 56 | with context.begin_transaction(): 57 | context.run_migrations() 58 | 59 | 60 | def run_migrations_online() -> None: 61 | """Run migrations in 'online' mode. 62 | 63 | In this scenario we need to create an Engine 64 | and associate a connection with the context. 65 | 66 | """ 67 | connectable = engine_from_config( 68 | config.get_section(config.config_ini_section, {}), 69 | prefix="sqlalchemy.", 70 | poolclass=pool.NullPool, 71 | ) 72 | 73 | with connectable.connect() as connection: 74 | context.configure(connection=connection, target_metadata=target_metadata) 75 | 76 | with context.begin_transaction(): 77 | context.run_migrations() 78 | 79 | 80 | if context.is_offline_mode(): 81 | run_migrations_offline() 82 | else: 83 | run_migrations_online() 84 | -------------------------------------------------------------------------------- /server/app/tests/test_crud/test_crud_embedding.py: -------------------------------------------------------------------------------- 1 | from app.crud.crud_embedding import CRUDEmbedding 2 | from uuid import uuid4 3 | from sqlalchemy.orm import Session 4 | from app.schemas.embedding import Embedding as EmbeddingSchema 5 | from app.models.embedding import Embedding as EmbeddingModel 6 | 7 | 8 | def test_create_embedding(db: Session) -> None: 9 | crud_embedding = CRUDEmbedding() 10 | embedding_obj = EmbeddingSchema( 11 | embeddings_vector=[1, 2, 3, 4, 5], 12 | database_table_id=uuid4(), 13 | user_database_id=uuid4(), 14 | ) 15 | 16 | result = crud_embedding.create(db=db, embedding_obj=embedding_obj) 17 | 18 | assert result is not None 19 | assert result.embeddings_vector is not None 20 | assert result.database_table_id == embedding_obj.database_table_id 21 | assert result.user_database_id == embedding_obj.user_database_id 22 | assert ( 23 | db.query(EmbeddingModel).filter(EmbeddingModel.id == result.id).first() 24 | is not None 25 | ) 26 | 27 | 28 | def test_get_closest_embeddings_by_database_id(db: Session) -> None: 29 | crud_embedding = CRUDEmbedding() 30 | embedding_obj = EmbeddingSchema( 31 | embeddings_vector=[1, 2, 3, 4, 5], 32 | database_table_id=uuid4(), 33 | user_database_id=uuid4(), 34 | ) 35 | result = crud_embedding.create(db=db, embedding_obj=embedding_obj) 36 | 37 | result = crud_embedding.get_closest_embeddings_by_database_id( 38 | query_embedding=[1, 2, 3, 4, 5], 39 | database_id=embedding_obj.user_database_id, 40 | db=db, 41 | ) 42 | assert result is not None 43 | 44 | def delete_by_database_id(db: Session) -> None: 45 | crud_embedding = CRUDEmbedding() 46 | embedding_obj = EmbeddingSchema( 47 | embeddings_vector=[1, 2, 3, 4, 5], 48 | database_table_id=uuid4(), 49 | user_database_id=uuid4(), 50 | ) 51 | result = crud_embedding.create(db=db, embedding_obj=embedding_obj) 52 | 53 | crud_embedding.delete_by_database_id(db=db, database_id=embedding_obj.user_database_id) 54 | result = crud_embedding.get_by_user_database_id( 55 | db=db, user_database_id=embedding_obj.user_database_id 56 | ) 57 | assert result.count() == 0 58 | assert result[0].embeddings_vector == embedding_obj.embeddings_vector 59 | assert result[0].database_table_id == embedding_obj.database_table_id 60 | assert result[0].user_database_id == embedding_obj.user_database_id 61 | assert ( 62 | db.query(EmbeddingModel).filter(EmbeddingModel.id == result[0].id).first() 63 | is not None 64 | ) -------------------------------------------------------------------------------- /server/app/tests/test_services/test_database_details.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import patch 3 | from fastapi.testclient import TestClient 4 | from sqlalchemy.orm import Session 5 | from app.models.user import User as UserModel 6 | from app.services.database_details import DatabaseDetails 7 | from app.crud.crud_user_database import CRUDUserDatabase 8 | 9 | 10 | def mock_background_task(*args, **kwargs) -> None: 11 | return None 12 | 13 | 14 | @pytest.fixture 15 | def database_details() -> DatabaseDetails: 16 | return DatabaseDetails() 17 | 18 | 19 | def test_fetch_database_details( 20 | client: TestClient, 21 | db: Session, 22 | valid_user_model: UserModel, 23 | valid_jwt: str, 24 | database_details: DatabaseDetails, 25 | ) -> None: 26 | crud_user_database = CRUDUserDatabase() 27 | 28 | headers = {"Authorization": f"Bearer {valid_jwt}"} 29 | 30 | with patch( 31 | "app.services.embeddings_service.embeddings_service.create_embeddings", 32 | side_effect=mock_background_task, 33 | ): 34 | with open("server/output_schema.txt", "rb") as file: 35 | response = client.post( 36 | "/api/v1/upload-database-schema", 37 | files={"file": file}, 38 | data={"database_name": "Test"}, 39 | headers=headers, 40 | ) 41 | 42 | assert response.status_code == 202 43 | 44 | user_database = crud_user_database.get_by_user_id( 45 | db=db, user_id=valid_user_model.id 46 | )[0] 47 | assert user_database is not None 48 | assert user_database.user_id == valid_user_model.id 49 | assert user_database.name == "Test" 50 | 51 | content = database_details.fetch_database_details( 52 | database_id=str(user_database.id), db=db 53 | ) 54 | 55 | assert content["database_id"] == str(user_database.id) 56 | assert content["database_name"] == user_database.name 57 | assert len(content["database_tables"]) == 6 58 | assert set([table["table_name"] for table in content["database_tables"]]) == set( 59 | [ 60 | "users", 61 | "embeddings", 62 | "alembic_version", 63 | "user_databases", 64 | "database_tables", 65 | "table_columns", 66 | ] 67 | ) 68 | 69 | for table in content["database_tables"]: 70 | assert table["table_id"] is not None 71 | assert len(table["table_columns"]) > 0 72 | for column in table["table_columns"]: 73 | assert column["column_id"] is not None 74 | assert column["column_name"] is not None 75 | assert column["column_type"] is not None 76 | -------------------------------------------------------------------------------- /server/app/tests/test_schemas/test_database_table_schema.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | from uuid import uuid4 4 | from app.schemas.database_table import DatabaseTable 5 | 6 | 7 | def test_database_table_schema_with_valid_data() -> None: 8 | name = "Test Table" 9 | user_database_id = uuid4() 10 | 11 | database_table = DatabaseTable(name=name, user_database_id=user_database_id) 12 | 13 | assert database_table.name == name 14 | assert database_table.user_database_id == user_database_id 15 | 16 | 17 | def test_database_table_schema_with_missing_name() -> None: 18 | user_database_id = uuid4() 19 | 20 | with pytest.raises(ValidationError): 21 | DatabaseTable(name=None, user_database_id=user_database_id) 22 | 23 | 24 | def test_database_table_schema_with_invalid_name() -> None: 25 | name = 123 26 | user_database_id = uuid4() 27 | 28 | with pytest.raises(ValidationError): 29 | DatabaseTable(name=name, user_database_id=user_database_id) 30 | 31 | 32 | def test_database_table_schema_with_missing_user_database_id() -> None: 33 | name = "Test Table" 34 | 35 | with pytest.raises(ValidationError): 36 | DatabaseTable(name=name, user_database_id=None) 37 | 38 | 39 | def test_database_table_schema_with_invalid_user_database_id() -> None: 40 | name = "Test Table" 41 | user_database_id = "invalid_uuid" 42 | 43 | with pytest.raises(ValidationError): 44 | DatabaseTable(name=name, user_database_id=user_database_id) 45 | 46 | 47 | def test_database_table_schema_with_missing_text_node() -> None: 48 | name = "Test Table" 49 | user_database_id = uuid4() 50 | 51 | database_table = DatabaseTable(name=name, user_database_id=user_database_id) 52 | 53 | assert database_table.name == name 54 | assert database_table.user_database_id == user_database_id 55 | assert database_table.text_node is None 56 | 57 | 58 | def test_database_table_schema_with_invalid_text_node() -> None: 59 | name = "Test Table" 60 | user_database_id = uuid4() 61 | text_node = 123 62 | 63 | with pytest.raises(ValidationError): 64 | DatabaseTable(name=name, user_database_id=user_database_id, text_node=text_node) 65 | 66 | 67 | def test_database_table_schema_with_valid_text_node() -> None: 68 | name = "Test Table" 69 | user_database_id = uuid4() 70 | text_node = "text_node" 71 | 72 | database_table = DatabaseTable( 73 | name=name, user_database_id=user_database_id, text_node=text_node 74 | ) 75 | 76 | assert database_table.name == name 77 | assert database_table.user_database_id == user_database_id 78 | assert database_table.text_node == text_node 79 | -------------------------------------------------------------------------------- /client/app/components/databaseCard.tsx: -------------------------------------------------------------------------------- 1 | import { ArrowPathIcon, TrashIcon } from "@heroicons/react/24/solid"; 2 | import { handleDate } from "@/app/utils/helper"; 3 | import Link from "next/link"; 4 | import appText from "@/app/assets/strings"; 5 | import useDatabaseCardViewController from "../viewControllers/databaseCardViewController"; 6 | import React from "react"; 7 | 8 | type Database = { 9 | id: string; 10 | name: string; 11 | connection_string: boolean; 12 | created_at: string; 13 | }; 14 | type Props = { 15 | db: Database; 16 | refreshDatabases: () => void; 17 | }; 18 | 19 | const DatabaseCard = ({ db,refreshDatabases }: Props):React.JSX.Element =>{ 20 | const text = appText.homeDatabases; 21 | const { syncDbLoader, syncDb, deleteDb, deleteDbLoader} = useDatabaseCardViewController(db.id, refreshDatabases); 22 | return ( 23 |
26 |
27 |

{db.name}

28 |
29 | 30 | {db.connection_string &&
31 |
32 | 33 |
34 |
} 35 |
36 | 37 |
38 |
39 |
40 |
41 |

42 | {text.added} {handleDate(db.created_at)} 43 |

44 |
45 | 49 | {text.viewDb} 50 | 51 | 55 | { 56 | localStorage.setItem("selectedDb", db.name) 57 | localStorage.setItem("selectedDbExecutable", JSON.stringify(db.connection_string)) 58 | } 59 | } 60 | > 61 | {text.askQuery} 62 | 63 |
64 |
65 |
66 | ) 67 | } 68 | export default DatabaseCard; -------------------------------------------------------------------------------- /server/app/tests/test_models/test_database_table_model.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.models.database_table import DatabaseTable 4 | 5 | 6 | def test_as_dict_method_with_not_null_text_node(db: Session) -> None: 7 | user_database_id = uuid.uuid4() 8 | database_table_instance = DatabaseTable( 9 | name="test_table", text_node="text_node", user_database_id=user_database_id 10 | ) 11 | db.add(database_table_instance) 12 | db.commit() 13 | result = database_table_instance.as_dict() 14 | assert "id" in result 15 | assert isinstance(result["id"], str) 16 | assert isinstance(result["user_database_id"], str) 17 | assert result == { 18 | "name": "test_table", 19 | "user_database_id": str(user_database_id), 20 | "id": result["id"], 21 | "text_node": "text_node", 22 | "created_at": result["created_at"], 23 | "updated_at": result["updated_at"], 24 | } 25 | 26 | 27 | def test_as_dict_method_with_null_text_node(db: Session) -> None: 28 | user_database_id = uuid.uuid4() 29 | database_table_instance = DatabaseTable( 30 | name="test_table", user_database_id=user_database_id 31 | ) 32 | db.add(database_table_instance) 33 | db.commit() 34 | result = database_table_instance.as_dict() 35 | assert "id" in result 36 | assert isinstance(result["id"], str) 37 | assert result == { 38 | "name": "test_table", 39 | "user_database_id": str(user_database_id), 40 | "id": result["id"], 41 | "text_node": None, 42 | "created_at": result["created_at"], 43 | "updated_at": result["updated_at"], 44 | } 45 | 46 | 47 | def test_timestamp_on_create(db: Session) -> None: 48 | user_database_id = uuid.uuid4() 49 | database_table_instance = DatabaseTable( 50 | name="test_table", user_database_id=user_database_id 51 | ) 52 | db.add(database_table_instance) 53 | db.commit() 54 | assert database_table_instance.created_at is not None 55 | assert database_table_instance.updated_at is not None 56 | assert database_table_instance.created_at == database_table_instance.updated_at 57 | 58 | 59 | def test_timestamp_on_update(db: Session) -> None: 60 | user_database_id = uuid.uuid4() 61 | database_table_instance = DatabaseTable( 62 | name="test_table", user_database_id=user_database_id 63 | ) 64 | db.add(database_table_instance) 65 | db.commit() 66 | database_table_instance.name = "test_table_updated" 67 | db.commit() 68 | assert database_table_instance.created_at is not None 69 | assert database_table_instance.updated_at is not None 70 | assert database_table_instance.created_at != database_table_instance.updated_at 71 | -------------------------------------------------------------------------------- /server/app/tests/test_models/test_embedding_model.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.models.embedding import Embedding 4 | 5 | 6 | def test_as_dict_method(db: Session) -> None: 7 | embedding_id = uuid.uuid4() 8 | table_id = uuid.uuid4() 9 | database_id = uuid.uuid4() 10 | embedding_instance = Embedding( 11 | id=embedding_id, 12 | embeddings_vector=[1.0, 2.0, 3.0], 13 | database_table_id=table_id, 14 | user_database_id=database_id, 15 | ) 16 | 17 | db.add(embedding_instance) 18 | db.commit() 19 | 20 | db_result = db.query(Embedding).first() 21 | assert db_result is not None 22 | 23 | result_dict = db_result.as_dict() 24 | assert isinstance(result_dict["id"], str) 25 | assert isinstance(result_dict["database_table_id"], str) 26 | assert isinstance(result_dict["user_database_id"], str) 27 | assert result_dict["id"] == str(embedding_id) 28 | assert list(result_dict["embeddings_vector"]) == [1.0, 2.0, 3.0] 29 | assert result_dict["database_table_id"] == str(table_id) 30 | assert result_dict["user_database_id"] == str(database_id) 31 | assert result_dict["created_at"] == db_result.created_at.isoformat() 32 | assert result_dict["updated_at"] == db_result.updated_at.isoformat() 33 | 34 | 35 | def test_timestamp_on_create(db: Session) -> None: 36 | embedding_id = uuid.uuid4() 37 | table_id = uuid.uuid4() 38 | database_id = uuid.uuid4() 39 | embedding_instance = Embedding( 40 | id=embedding_id, 41 | embeddings_vector=[1.0, 2.0, 3.0], 42 | database_table_id=table_id, 43 | user_database_id=database_id, 44 | ) 45 | 46 | db.add(embedding_instance) 47 | db.commit() 48 | 49 | db_result = db.query(Embedding).first() 50 | assert db_result is not None 51 | 52 | assert db_result.created_at is not None 53 | assert db_result.updated_at is not None 54 | assert db_result.created_at == db_result.updated_at 55 | 56 | 57 | def test_timestamp_on_update(db: Session) -> None: 58 | embedding_id = uuid.uuid4() 59 | table_id = uuid.uuid4() 60 | database_id = uuid.uuid4() 61 | embedding_instance = Embedding( 62 | id=embedding_id, 63 | embeddings_vector=[1.0, 2.0, 3.0], 64 | database_table_id=table_id, 65 | user_database_id=database_id, 66 | ) 67 | 68 | db.add(embedding_instance) 69 | db.commit() 70 | 71 | db_result = db.query(Embedding).first() 72 | assert db_result is not None 73 | 74 | db_result.embeddings_vector = [4.0, 5.0, 6.0] 75 | db.commit() 76 | 77 | assert db_result.created_at is not None 78 | assert db_result.updated_at is not None 79 | assert db_result.created_at != db_result.updated_at 80 | -------------------------------------------------------------------------------- /server/app/tests/test_services/test_embeddings_service.py: -------------------------------------------------------------------------------- 1 | from app.crud.crud_embedding import CRUDEmbedding 2 | import pytest 3 | import numpy 4 | from unittest.mock import patch 5 | from fastapi.testclient import TestClient 6 | from sqlalchemy.orm import Session 7 | from app.models.user import User as UserModel 8 | from app.crud.crud_user_database import CRUDUserDatabase 9 | from app.services.embeddings_service import EmbeddingsService 10 | from app.clients.openai_client import OpenAIClient 11 | 12 | 13 | mock_openai_embedding_response = { 14 | "data": [ 15 | [0.1], [0.2], [0.3], [0.4], [0.5], [0.6], [0.7], [0.8], [0.9], [0.1], [0.11], [0.12], [0.13], [0.14], [0.15], [0.16], [0.17], [0.18], 16 | ] 17 | } 18 | 19 | 20 | def mock_embedding_create(*args, **kwargs) -> dict: 21 | return mock_openai_embedding_response 22 | 23 | 24 | def mock_background_task(*args, **kwargs) -> None: 25 | return None 26 | 27 | 28 | @pytest.fixture 29 | def embeddings_service() -> EmbeddingsService: 30 | return EmbeddingsService() 31 | 32 | 33 | def test_create_embeddings( 34 | client: TestClient, 35 | db: Session, 36 | valid_user_model: UserModel, 37 | valid_jwt: str, 38 | embeddings_service: EmbeddingsService, 39 | ) -> None: 40 | crud_user_database = CRUDUserDatabase() 41 | crud_embedding = CRUDEmbedding() 42 | headers = {"Authorization": f"Bearer {valid_jwt}"} 43 | 44 | with patch( 45 | "app.services.embeddings_service.embeddings_service.create_embeddings", 46 | side_effect=mock_background_task, 47 | ): 48 | with open("server/output_schema.txt", "rb") as file: 49 | response = client.post( 50 | "/api/v1/upload-database-schema", 51 | files={"file": file}, 52 | data={"database_name": "Test"}, 53 | headers=headers, 54 | ) 55 | 56 | assert response.status_code == 202 57 | 58 | user_database = crud_user_database.get_by_user_id( 59 | db=db, user_id=valid_user_model.id 60 | )[0] 61 | assert user_database is not None 62 | assert user_database.user_id == valid_user_model.id 63 | assert user_database.name == "Test" 64 | 65 | with patch.object(OpenAIClient, 'get_embeddings', return_value=mock_openai_embedding_response['data']) as mock_method: 66 | 67 | embeddings_service.create_embeddings(db=db, database_id=str(user_database.id)) 68 | 69 | result = crud_embedding.get_by_user_database_id( 70 | db=db, user_database_id=user_database.id 71 | ) 72 | 73 | assert result.count() == 6 74 | for index in range(0, 6): 75 | assert result[index].embeddings_vector is not None 76 | assert result[index].database_table_id is not None 77 | assert result[index].user_database_id == user_database.id 78 | -------------------------------------------------------------------------------- /server/app/tests/test_crud/test_crud_table_column.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.crud.crud_table_column import CRUDTableColumn 4 | from app.schemas.table_column import TableColumn as TableColumnSchema 5 | from app.models.table_column import TableColumn as TableColumnModel 6 | 7 | 8 | def test_create_table_column(db: Session) -> None: 9 | crud_table_column = CRUDTableColumn() 10 | table_column_obj = TableColumnSchema( 11 | name="test_column", data_type="int", database_table_id=uuid.uuid4() 12 | ) 13 | result = crud_table_column.create(db=db, table_column_obj=table_column_obj) 14 | assert result.name == table_column_obj.name 15 | assert result.data_type == table_column_obj.data_type 16 | assert result.database_table_id == table_column_obj.database_table_id 17 | assert ( 18 | db.query(TableColumnModel).filter(TableColumnModel.id == result.id).first() 19 | is not None 20 | ) 21 | 22 | 23 | def test_get_by_database_table_id(db: Session) -> None: 24 | crud_table_column = CRUDTableColumn() 25 | table_column_obj = TableColumnSchema( 26 | name="test_column", data_type="int", database_table_id=uuid.uuid4() 27 | ) 28 | crud_table_column.create(db=db, table_column_obj=table_column_obj) 29 | result = crud_table_column.get_by_database_table_id( 30 | db=db, database_table_id=table_column_obj.database_table_id 31 | ) 32 | assert result.count() == 1 33 | assert result[0].name == table_column_obj.name 34 | assert result[0].data_type == table_column_obj.data_type 35 | assert result[0].database_table_id == table_column_obj.database_table_id 36 | assert ( 37 | db.query(TableColumnModel).filter(TableColumnModel.id == result[0].id).first() 38 | is not None 39 | ) 40 | 41 | 42 | def test_get_by_database_table_id_not_found(db: Session) -> None: 43 | crud_table_column = CRUDTableColumn() 44 | table_column_obj = TableColumnSchema( 45 | name="test_column", data_type="int", database_table_id=uuid.uuid4() 46 | ) 47 | crud_table_column.create(db=db, table_column_obj=table_column_obj) 48 | result = crud_table_column.get_by_database_table_id( 49 | db=db, database_table_id=uuid.uuid4() 50 | ) 51 | assert result.count() == 0 52 | 53 | def test_delete_by_database_table_id(db:Session)->None: 54 | crud_table_column = CRUDTableColumn() 55 | table_column_obj = TableColumnSchema( 56 | name="test_column", data_type="int", database_table_id=uuid.uuid4() 57 | ) 58 | crud_table_column.create(db=db, table_column_obj=table_column_obj) 59 | crud_table_column.delete_by_database_table_id(db=db, database_table_id=table_column_obj.database_table_id) 60 | result = crud_table_column.get_by_database_table_id( 61 | db=db, database_table_id=table_column_obj.database_table_id 62 | ) 63 | assert result.count() == 0 64 | -------------------------------------------------------------------------------- /server/app/tests/test_schemas/test_user_schema.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | from uuid import uuid4 4 | from app.schemas.user import User as UserSchema 5 | 6 | 7 | def test_user_schema_with_valid_data(valid_user: UserSchema) -> None: 8 | id = uuid4() 9 | user = UserSchema( 10 | id=id, 11 | name=valid_user.name, 12 | email=valid_user.email, 13 | password=valid_user.password, 14 | ) 15 | 16 | assert user.id == id 17 | assert user.name == valid_user.name 18 | assert user.email == valid_user.email 19 | assert user.password == valid_user.password 20 | 21 | 22 | def test_user_schema_with_missing_id(valid_user: UserSchema) -> None: 23 | user = UserSchema( 24 | name=valid_user.name, email=valid_user.email, password=valid_user.password 25 | ) 26 | 27 | assert user.name == valid_user.name 28 | assert user.email == valid_user.email 29 | assert user.password == valid_user.password 30 | assert user.id is None 31 | 32 | 33 | def test_user_schema_with_missing_name(valid_user: UserSchema) -> None: 34 | with pytest.raises(ValidationError): 35 | UserSchema(name=None, email=valid_user.email, password=valid_user.password) 36 | 37 | 38 | def test_user_schema_with_missing_email(valid_user: UserSchema) -> None: 39 | with pytest.raises(ValidationError): 40 | UserSchema(name=valid_user.name, email=None, password=valid_user.password) 41 | 42 | 43 | def test_user_schema_with_missing_password(valid_user: UserSchema) -> None: 44 | with pytest.raises(ValidationError): 45 | UserSchema(name=valid_user.name, email=valid_user.email, password=None) 46 | 47 | 48 | def test_user_schema_with_invalid_name(valid_user: UserSchema) -> None: 49 | id = uuid4() 50 | name = 123 51 | 52 | with pytest.raises(ValidationError): 53 | UserSchema( 54 | id=id, name=name, email=valid_user.email, password=valid_user.password 55 | ) 56 | 57 | 58 | def test_user_schema_with_invalid_email(valid_user: UserSchema) -> None: 59 | id = uuid4() 60 | email = "testemail.com" 61 | 62 | with pytest.raises(ValidationError): 63 | UserSchema( 64 | id=id, name=valid_user.name, email=email, password=valid_user.password 65 | ) 66 | 67 | 68 | def test_user_schema_with_invalid_password(valid_user: UserSchema) -> None: 69 | id = uuid4() 70 | password = 123 71 | 72 | with pytest.raises(ValidationError): 73 | UserSchema( 74 | id=id, name=valid_user.name, email=valid_user.email, password=password 75 | ) 76 | 77 | 78 | def test_user_schema_with_invalid_user_id(valid_user: UserSchema) -> None: 79 | id = "invalid_uuid" 80 | 81 | with pytest.raises(ValidationError): 82 | UserSchema( 83 | id=id, 84 | name=valid_user.name, 85 | email=valid_user.email, 86 | password=valid_user.password, 87 | ) 88 | -------------------------------------------------------------------------------- /server/app/tests/test_schemas/test_embedding_schema.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | from pydantic import ValidationError 3 | import pytest 4 | from app.schemas.embedding import Embedding as EmbeddingSchema 5 | 6 | 7 | def test_embedding_schema_with_valid_data(): 8 | id = uuid4() 9 | database_table_id = uuid4() 10 | user_database_id = uuid4() 11 | embedding_vector = [1.0, 2.3, 3.5] 12 | valid_embedding = EmbeddingSchema( 13 | id=id, 14 | embeddings_vector=embedding_vector, 15 | database_table_id=database_table_id, 16 | user_database_id=user_database_id, 17 | ) 18 | 19 | assert valid_embedding.id == id 20 | assert valid_embedding.embeddings_vector == embedding_vector 21 | assert valid_embedding.database_table_id == database_table_id 22 | assert valid_embedding.user_database_id == user_database_id 23 | 24 | 25 | def test_embedding_schema_with_missing_id(): 26 | database_table_id = uuid4() 27 | user_database_id = uuid4() 28 | embedding_vector = [1.0, 2.3, 3.5] 29 | valid_embedding = EmbeddingSchema( 30 | embeddings_vector=embedding_vector, 31 | database_table_id=database_table_id, 32 | user_database_id=user_database_id, 33 | ) 34 | 35 | assert valid_embedding.embeddings_vector == embedding_vector 36 | assert valid_embedding.database_table_id == database_table_id 37 | assert valid_embedding.user_database_id == user_database_id 38 | assert valid_embedding.id is None 39 | 40 | 41 | def test_embedding_schema_with_missing_database_table_id(): 42 | id = uuid4() 43 | user_database_id = uuid4() 44 | embedding_vector = [1.0, 2.3, 3.5] 45 | 46 | with pytest.raises(ValidationError): 47 | EmbeddingSchema( 48 | id=id, embeddings_vector=embedding_vector, user_database_id=user_database_id 49 | ) 50 | 51 | 52 | def test_embedding_schema_with_missing_user_database_id(): 53 | id = uuid4() 54 | database_table_id = uuid4() 55 | embedding_vector = [1.0, 2.3, 3.5] 56 | 57 | with pytest.raises(ValidationError): 58 | EmbeddingSchema( 59 | id=id, 60 | embeddings_vector=embedding_vector, 61 | database_table_id=database_table_id, 62 | ) 63 | 64 | 65 | def test_embedding_schema_with_missing_embedding(): 66 | id = uuid4() 67 | database_table_id = uuid4() 68 | user_database_id = uuid4() 69 | 70 | with pytest.raises(ValidationError): 71 | EmbeddingSchema( 72 | id=id, 73 | database_table_id=database_table_id, 74 | user_database_id=user_database_id, 75 | ) 76 | 77 | 78 | def test_embedding_schema_with_invalid_embedding(): 79 | id = uuid4() 80 | database_table_id = uuid4() 81 | user_database_id = uuid4() 82 | embeddings_vector = "invalid_embedding" 83 | 84 | with pytest.raises(ValidationError): 85 | EmbeddingSchema( 86 | id=id, 87 | embeddings_vector=embeddings_vector, 88 | database_table_id=database_table_id, 89 | user_database_id=user_database_id, 90 | ) 91 | -------------------------------------------------------------------------------- /server/app/services/embeddings_service.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import Session 2 | from app.clients.openai_client import openai_client 3 | from app.crud.crud_database_table import crud_database_table 4 | from app.services.database_details import database_details 5 | from app.crud.crud_embedding import crud_embedding 6 | from app.schemas.embedding import Embedding as EmbeddingSchema 7 | 8 | 9 | class EmbeddingsService: 10 | def __create_node_for_table(self, table_details: dict): 11 | template = ( 12 | "Schema of table {table_name}:\n" 13 | "Table '{table_name}' has columns: {columns} " 14 | ) 15 | columns = [] 16 | for column in table_details["table_columns"]: 17 | columns.append(f"{column['column_name']} ({column['column_type']})") 18 | column_str = ", ".join(columns) 19 | text_node = template.format( 20 | table_name=table_details["table_name"], columns=column_str 21 | ) 22 | return text_node 23 | 24 | def create_embeddings(self, database_id: str, db: Session): 25 | try: 26 | db_details = database_details.fetch_database_details(database_id, db) 27 | text_nodes_of_tables = dict() 28 | for table_details in db_details["database_tables"]: 29 | text_node = self.__create_node_for_table(table_details) 30 | crud_database_table.upsert_text_node_by_id( 31 | db, table_details["table_id"], text_node 32 | ) 33 | text_nodes_of_tables[table_details["table_id"]] = text_node 34 | 35 | table_embeddings = openai_client.get_embeddings( 36 | list(text_nodes_of_tables.values()) 37 | ) 38 | 39 | for idx, table_id in enumerate(text_nodes_of_tables.keys()): 40 | embedding_vector = table_embeddings[idx] 41 | 42 | crud_embedding.create( 43 | db, 44 | EmbeddingSchema( 45 | embeddings_vector=embedding_vector, 46 | database_table_id=table_id, 47 | user_database_id=database_id, 48 | ), 49 | ) 50 | except Exception as e: 51 | print(e) 52 | raise e 53 | 54 | def get_relevant_tables_for_query( 55 | self, query_embedding: list[float], database_id: str, db: Session 56 | ): 57 | try: 58 | closest_database_table_ids = ( 59 | crud_embedding.get_closest_embeddings_by_database_id( 60 | query_embedding, database_id, db 61 | ) 62 | ) 63 | closest_database_tables = [ 64 | database_table.text_node 65 | for database_table in crud_database_table.get_by_ids( 66 | db, closest_database_table_ids 67 | ) 68 | ] 69 | return closest_database_tables 70 | except Exception as e: 71 | print(e) 72 | raise e 73 | 74 | 75 | embeddings_service = EmbeddingsService() 76 | -------------------------------------------------------------------------------- /client/app/components/schemaUploader.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import CodeBlock from "@/app/components/codeBlock"; 4 | import FileUploader from "@/app/components/fileUploader"; 5 | import { COMMAND_RUN_SCRIPT, COMMAND_DOWNLOAD_SCRIPT } from "@/app/utils/constant"; 6 | import useUploadSchemaViewController from "@/app/viewControllers/uploadSchemaViewController"; 7 | import appText from "@/app/assets/strings"; 8 | import React from "react"; 9 | 10 | 11 | const UploadDatabaseSchema = ({ onToggle }: { onToggle: any }):React.JSX.Element => { 12 | const { 13 | file, 14 | databaseName, 15 | setDatabaseName, 16 | showLoader, 17 | handleFileChange, 18 | handleUpload, 19 | } = useUploadSchemaViewController(); 20 | 21 | const text = appText.uploadDatabaseSchema; 22 | 23 | return ( 24 |
25 |
26 |
27 |
{text.steps}
28 |
29 | {text.or}{' '} 30 | 36 | {' '}{text.yourDatabase} 37 |
38 |
39 |
{text.information}
40 |
    41 |
  • 42 |
    43 |

    44 | {text.firstStep} 45 |

    46 |
    47 | null} handleQueryResponse={()=>null} executeFlag={false}/> 48 |
    49 |
    50 |
    51 |

    52 | {text.secondStep} 53 |

    54 |
    55 | null} handleQueryResponse={()=>null} executeFlag={false}/> 56 |
    57 |
    58 |
    59 |

    60 | {text.thirdStep} 61 |

    62 |
    63 | 71 |
    72 |
    73 |
    • 74 |
75 |
76 |
77 | ); 78 | }; 79 | 80 | export default UploadDatabaseSchema; 81 | -------------------------------------------------------------------------------- /server/app/tests/test_crud/test_crud_user_database.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from sqlalchemy.orm import Session 3 | from app.crud.crud_user_database import CRUDUserDatabase 4 | from app.schemas.user_database import UserDatabase as UserDatabaseSchema 5 | from app.models.user_database import UserDatabase as UserDatabaseModel 6 | 7 | 8 | def test_create_user_database(db: Session) -> None: 9 | crud_user_database = CRUDUserDatabase() 10 | user_database_obj = UserDatabaseSchema(name="test_database", user_id=uuid.uuid4()) 11 | 12 | result = crud_user_database.create(db=db, user_database_obj=user_database_obj) 13 | 14 | assert result.name == user_database_obj.name 15 | assert result.user_id == user_database_obj.user_id 16 | assert ( 17 | db.query(UserDatabaseModel).filter(UserDatabaseModel.id == result.id).first() 18 | is not None 19 | ) 20 | 21 | 22 | def test_get_by_user_id(db: Session) -> None: 23 | crud_user_database = CRUDUserDatabase() 24 | user_database_obj = UserDatabaseSchema(name="test_database", user_id=uuid.uuid4()) 25 | crud_user_database.create(db=db, user_database_obj=user_database_obj) 26 | 27 | result = crud_user_database.get_by_user_id(db=db, user_id=user_database_obj.user_id) 28 | 29 | assert result.count() == 1 30 | assert result[0].name == user_database_obj.name 31 | assert result[0].user_id == user_database_obj.user_id 32 | assert ( 33 | db.query(UserDatabaseModel).filter(UserDatabaseModel.id == result[0].id).first() 34 | is not None 35 | ) 36 | 37 | 38 | def test_get_by_user_id_not_found(db: Session) -> None: 39 | crud_user_database = CRUDUserDatabase() 40 | user_database_obj = UserDatabaseSchema(name="test_database", user_id=uuid.uuid4()) 41 | crud_user_database.create(db=db, user_database_obj=user_database_obj) 42 | 43 | result = crud_user_database.get_by_user_id(db=db, user_id=uuid.uuid4()) 44 | 45 | assert result.count() == 0 46 | 47 | 48 | def test_get_by_id(db: Session) -> None: 49 | crud_user_database = CRUDUserDatabase() 50 | user_database_obj = UserDatabaseSchema(name="test_database", user_id=uuid.uuid4()) 51 | user_database = crud_user_database.create( 52 | db=db, user_database_obj=user_database_obj 53 | ) 54 | 55 | result = crud_user_database.get_by_id(db=db, id=user_database.id) 56 | 57 | assert result is not None 58 | assert result.name == user_database_obj.name 59 | assert result.user_id == user_database_obj.user_id 60 | assert result.id == user_database.id 61 | assert ( 62 | db.query(UserDatabaseModel).filter(UserDatabaseModel.id == result.id).first() 63 | is not None 64 | ) 65 | 66 | def test_delete_by_id(db: Session) -> None: 67 | crud_user_database = CRUDUserDatabase() 68 | user_database_obj = UserDatabaseSchema(name="test_database", user_id=uuid.uuid4()) 69 | user_database = crud_user_database.create( 70 | db=db, user_database_obj=user_database_obj 71 | ) 72 | 73 | crud_user_database.delete_by_id(db=db, id=user_database.id) 74 | result = crud_user_database.get_by_user_id(db=db, user_id=user_database_obj.user_id) 75 | assert result.count() == 0 -------------------------------------------------------------------------------- /server/app/tests/test_api/v1/test_query_executor.py: -------------------------------------------------------------------------------- 1 | from fastapi.testclient import TestClient 2 | from sqlalchemy.orm import Session 3 | from app.models.user import User as UserModel 4 | from app.crud.crud_user_database import crud_user_database 5 | from app.schemas.user_database import UserDatabase as UserDatabaseSchema 6 | from app.utilities.fernet_manager import FernetManager 7 | import logging 8 | 9 | logger = logging.getLogger("mql") 10 | 11 | def test_query_executor( 12 | client: TestClient, db: Session, valid_jwt: str, valid_user_model: UserModel 13 | ) -> None: 14 | headers = {"Authorization": f"Bearer {valid_jwt}"} 15 | fernet_manager = FernetManager(valid_user_model.hashed_key) 16 | connection_string = fernet_manager.encrypt("postgresql://shuru:password@postgres:5432/mql_test") 17 | database = crud_user_database.create( 18 | db=db, 19 | user_database_obj=UserDatabaseSchema( 20 | name="Test", 21 | user_id=valid_user_model.id, 22 | connection_string=connection_string 23 | ), 24 | ) 25 | db.commit() 26 | 27 | response = client.get( 28 | f"api/v1/sql-data?db_id={database.id}&sql_query=SELECT name, email FROM users limit 1;", 29 | headers=headers, 30 | ) 31 | 32 | assert response.status_code == 200 33 | assert response.json() == { 34 | "message": "Query executed successfully", 35 | "data": { 36 | "query_result": { 37 | "column_names": ["name", "email"], 38 | "rows": [ 39 | ["test", "test@test.com"], 40 | ] 41 | }, 42 | "sql_query": "SELECT name, email FROM users limit 1;" 43 | } 44 | } 45 | 46 | def test_query_executor_invalid_query( 47 | client: TestClient, db: Session, valid_jwt: str, valid_user_model: UserModel 48 | ) -> None: 49 | headers = {"Authorization": f"Bearer {valid_jwt}"} 50 | 51 | database = crud_user_database.create( 52 | db=db, 53 | user_database_obj=UserDatabaseSchema( 54 | name="Test", 55 | user_id=valid_user_model.id, 56 | connection_string="postgresql://shuru:password@postgres:5432/mql_test" 57 | ), 58 | ) 59 | db.commit() 60 | 61 | response = client.get( 62 | f"api/v1/sql-data?db_id={database.id}&sql_query=DELETE FROM users;", 63 | headers=headers, 64 | ) 65 | 66 | assert response.status_code == 400 67 | assert response.json() == { 68 | "message": "Only DQL queries are allowed", 69 | "error": "Only DQL queries are allowed", 70 | } 71 | 72 | def test_query_executor_on_database_without_connection_string( 73 | client: TestClient, db: Session, valid_jwt: str, valid_user_model: UserModel 74 | ) -> None: 75 | headers = {"Authorization": f"Bearer {valid_jwt}"} 76 | database = crud_user_database.create( 77 | db=db, 78 | user_database_obj=UserDatabaseSchema( 79 | name="Test", 80 | user_id=valid_user_model.id, 81 | connection_string=None 82 | ), 83 | ) 84 | db.commit() 85 | response = client.get( 86 | f"api/v1/sql-data?db_id={database.id}&sql_query=SELECT name, email FROM users limit 1;", 87 | headers=headers, 88 | ) 89 | 90 | assert response.status_code == 400 91 | 92 | -------------------------------------------------------------------------------- /server/app/api/v1/query_executor.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, status 2 | from fastapi.responses import JSONResponse 3 | from sqlalchemy.orm import Session 4 | from app.api.v1.dependencies import get_current_user, get_db 5 | from app.crud.crud_user_database import crud_user_database 6 | from sqlalchemy import create_engine 7 | from fastapi.encoders import jsonable_encoder 8 | from sqlalchemy.sql import text 9 | from app.services.query_service import query_service 10 | from app.constants import DQL 11 | from app.utilities.fernet_manager import FernetManager 12 | import logging 13 | 14 | router = APIRouter() 15 | logger = logging.getLogger("mql") 16 | 17 | @router.get("/sql-data") 18 | async def query_executor( 19 | db_id: str, 20 | sql_query: str, 21 | db: Session = Depends(get_db), 22 | current_user: dict = Depends(get_current_user), 23 | ) -> JSONResponse: 24 | if not sql_query.lower().startswith(DQL["Statement_Type_Keyowrd"]): 25 | return JSONResponse( 26 | status_code=status.HTTP_400_BAD_REQUEST, 27 | content={ 28 | "message": "Only DQL queries are allowed", 29 | "error": "Only DQL queries are allowed", 30 | }, 31 | ) 32 | try: 33 | database_connection_string = crud_user_database.get_by_id(db, db_id).connection_string 34 | if database_connection_string is None: 35 | return JSONResponse( 36 | status_code=status.HTTP_400_BAD_REQUEST, 37 | content={ 38 | "message": "Query Executor not supoorted for this type of database", 39 | "error": "Query Executor only supported in connection string type databases", 40 | }, 41 | ) 42 | password = current_user.hashed_key 43 | fernet_manager = FernetManager(password) 44 | database_connection_string = fernet_manager.decrypt(database_connection_string) 45 | if database_connection_string: 46 | engine = create_engine(database_connection_string) 47 | with engine.connect() as connection: 48 | result = connection.execute(text(sql_query)) 49 | result = result.mappings().all() 50 | result_in_json_format = jsonable_encoder(result) 51 | result_in_2d_array = query_service.convert_to_2d_array(result_in_json_format) 52 | logger.info( 53 | "Query {} executed successfully for user {} and database {}".format( 54 | sql_query, current_user.id, db_id 55 | ) 56 | ) 57 | 58 | except Exception as e: 59 | logger.error( 60 | "Error while executing query {} for user {} and database {}. Error is {}".format( 61 | sql_query, current_user.id, db_id, e 62 | ) 63 | ) 64 | return JSONResponse( 65 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 66 | content={ 67 | "message": "Query execution failed", 68 | "error": str(e), 69 | } 70 | ) 71 | 72 | return JSONResponse( 73 | content={ 74 | "message": "Query executed successfully", 75 | "data": {"sql_query": sql_query, "query_result": result_in_2d_array}, 76 | }, 77 | status_code=status.HTTP_200_OK, 78 | ) 79 | -------------------------------------------------------------------------------- /client/app/(routes)/(public)/login/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import Header from "@/app/components/header"; 4 | import useLoginViewController from "@/app/viewControllers/loginViewController"; 5 | import appText from "../../../assets/strings"; 6 | import React from "react"; 7 | import { useState } from "react"; 8 | import Image from "next/image"; 9 | 10 | const Login:React.FC = () => { 11 | 12 | const { 13 | handleLogin, 14 | setEmail, 15 | setPassword, 16 | email, 17 | password, 18 | } = useLoginViewController(); 19 | 20 | const text = appText.login; 21 | 22 | const [showPassword, setShowPassword] = useState(false); 23 | 24 | return ( 25 | <> 26 |
27 |
28 |
29 |

{text.login}

30 |
31 |
32 | 38 | setEmail(e.target.value)} 46 | /> 47 |
48 |
49 | 55 |
56 | setPassword(e.target.value)} 64 | /> 65 | setShowPassword(!showPassword)}> 66 | eye-open 73 | 74 |
75 |
76 | 84 |
85 |
86 |
87 | 88 | ); 89 | }; 90 | 91 | export default Login; 92 | -------------------------------------------------------------------------------- /server/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = alembic 6 | 7 | # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s 8 | # Uncomment the line below if you want the files to be prepended with date and time 9 | # see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file 10 | # for all available tokens 11 | # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s 12 | 13 | # sys.path path, will be prepended to sys.path if present. 14 | # defaults to the current working directory. 15 | prepend_sys_path = . 16 | 17 | # timezone to use when rendering the date within the migration file 18 | # as well as the filename. 19 | # If specified, requires the python-dateutil library that can be 20 | # installed by adding `alembic[tz]` to the pip requirements 21 | # string value is passed to dateutil.tz.gettz() 22 | # leave blank for localtime 23 | # timezone = 24 | 25 | # max length of characters to apply to the 26 | # "slug" field 27 | # truncate_slug_length = 40 28 | 29 | # set to 'true' to run the environment during 30 | # the 'revision' command, regardless of autogenerate 31 | # revision_environment = false 32 | 33 | # set to 'true' to allow .pyc and .pyo files without 34 | # a source .py file to be detected as revisions in the 35 | # versions/ directory 36 | # sourceless = false 37 | 38 | # version location specification; This defaults 39 | # to alembic/versions. When using multiple version 40 | # directories, initial revisions must be specified with --version-path. 41 | # The path separator used here should be the separator specified by "version_path_separator" below. 42 | # version_locations = %(here)s/bar:%(here)s/bat:alembic/versions 43 | 44 | # version path separator; As mentioned above, this is the character used to split 45 | # version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. 46 | # If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. 47 | # Valid values for version_path_separator are: 48 | # 49 | # version_path_separator = : 50 | # version_path_separator = ; 51 | # version_path_separator = space 52 | version_path_separator = os # Use os.pathsep. Default configuration used for new projects. 53 | 54 | # set to 'true' to search source files recursively 55 | # in each "version_locations" directory 56 | # new in Alembic version 1.10 57 | # recursive_version_locations = false 58 | 59 | # the output encoding used when revision files 60 | # are written from script.py.mako 61 | # output_encoding = utf-8 62 | 63 | # sqlalchemy.url = 64 | 65 | 66 | [post_write_hooks] 67 | # post_write_hooks defines scripts or Python functions that are run 68 | # on newly generated revision scripts. See the documentation for further 69 | # detail and examples 70 | 71 | # format using "black" - use the console_scripts runner, against the "black" entrypoint 72 | # hooks = black 73 | # black.type = console_scripts 74 | # black.entrypoint = black 75 | # black.options = -l 79 REVISION_SCRIPT_FILENAME 76 | 77 | # Logging configuration 78 | [loggers] 79 | keys = root,sqlalchemy,alembic 80 | 81 | [handlers] 82 | keys = console 83 | 84 | [formatters] 85 | keys = generic 86 | 87 | [logger_root] 88 | level = WARN 89 | handlers = console 90 | qualname = 91 | 92 | [logger_sqlalchemy] 93 | level = WARN 94 | handlers = 95 | qualname = sqlalchemy.engine 96 | 97 | [logger_alembic] 98 | level = INFO 99 | handlers = 100 | qualname = alembic 101 | 102 | [handler_console] 103 | class = StreamHandler 104 | args = (sys.stderr,) 105 | level = NOTSET 106 | formatter = generic 107 | 108 | [formatter_generic] 109 | format = %(levelname)-5.5s [%(name)s] %(message)s 110 | datefmt = %H:%M:%S 111 | -------------------------------------------------------------------------------- /client/app/viewControllers/chatViewController.ts: -------------------------------------------------------------------------------- 1 | import React, { useState } from "react"; 2 | import { askQuery, getQuery, getQueries, executeQuery } from "@/app/lib/service"; 3 | import { toast } from "react-toastify"; 4 | import appText from "../assets/strings"; 5 | 6 | type Props = { 7 | dbId: string; 8 | }; 9 | 10 | 11 | const useChatViewController = ({ dbId }: Props) => { 12 | const [nlQuery, setNlQuery] = useState(""); 13 | const [showNlQuery, setShowNlQuery] = useState(null); 14 | const [sql, setSql] = useState(null); 15 | const [queries, setQueries] = useState([]); 16 | const [isFirst, setIsFirst] = useState(true); 17 | const [open, setOpen] = useState(false); 18 | const [queryResult, setQueryResult] = useState({column_names: [''],rows:[]}); 19 | const [hasQueryExecuted, setHasQueryExecuted] = useState(false); 20 | const [queryError, setQueryError] = useState(""); 21 | const [showError, setShowError] = useState(false); 22 | 23 | 24 | const getQueryHistory = async () => { 25 | try { 26 | const res = await getQueries(dbId); 27 | setQueries(res.data.data.queries); 28 | } catch (error) { 29 | toast.error(appText.toast.errGeneric); 30 | } 31 | }; 32 | 33 | const handleQuery = async (e: React.ChangeEvent) => { 34 | e.preventDefault(); 35 | setShowError(false); 36 | try { 37 | setIsFirst(false); 38 | setSql(null); 39 | setShowNlQuery(nlQuery); 40 | setHasQueryExecuted(false); 41 | const formData = new FormData(); 42 | formData.append("nl_query", nlQuery); 43 | formData.append("db_id", dbId); 44 | const res = await askQuery(formData); 45 | setSql(res.data.data.sql_query); 46 | setNlQuery(""); 47 | setShowError(false); 48 | } catch (error) { 49 | setShowError(true); 50 | toast.error(appText.toast.errGeneric); 51 | } 52 | }; 53 | 54 | const getQueryById = async (id: string) => { 55 | try { 56 | setIsFirst(false); 57 | setSql(null); 58 | setShowNlQuery(null); 59 | setHasQueryExecuted(false); 60 | const res = await getQuery({ id }); 61 | setSql(res.data.data.query.sql_query); 62 | setShowNlQuery(res.data.data.query.nl_query); 63 | setNlQuery(""); 64 | } catch (error) { 65 | toast.error(appText.toast.errGeneric); 66 | } 67 | }; 68 | 69 | const handleQueryResponse = async () => { 70 | setHasQueryExecuted(false); 71 | setQueryError(""); 72 | try { 73 | const payload = { 74 | db_id: dbId, 75 | sql_query: sql, 76 | }; 77 | const response = await executeQuery(payload); 78 | setQueryResult(response.data.data["query_result"]); 79 | } catch (error: any) { 80 | setQueryError(error.error); 81 | toast.error(appText.toast.errGeneric); 82 | } 83 | setHasQueryExecuted(true); 84 | } 85 | 86 | return { 87 | nlQuery, 88 | setNlQuery, 89 | showNlQuery, 90 | setShowNlQuery, 91 | sql, 92 | setSql, 93 | isFirst, 94 | setIsFirst, 95 | open, 96 | setOpen, 97 | getQueryHistory, 98 | handleQuery, 99 | getQueryById, 100 | queries, 101 | queryResult, 102 | setQueryResult, 103 | hasQueryExecuted, 104 | setHasQueryExecuted, 105 | handleQueryResponse, 106 | queryError, 107 | showError 108 | }; 109 | }; 110 | 111 | export default useChatViewController; -------------------------------------------------------------------------------- /client/app/components/databaseInfo.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { ChevronLeftIcon, ChevronRightIcon } from "@heroicons/react/20/solid"; 4 | import Link from "next/link"; 5 | import Accordion from "./accordion"; 6 | import useDatabaseInfoViewController from "../viewControllers/databaseInfoViewController"; 7 | import appText from "../assets/strings"; 8 | import React from "react"; 9 | 10 | type Database = { 11 | database: { 12 | database_id: string; 13 | database_name: string; 14 | database_tables: { 15 | table_id: string; 16 | table_name: string; 17 | table_columns: { 18 | column_id: string; 19 | column_name: string; 20 | column_type: string; 21 | }[]; 22 | }[]; 23 | }; 24 | }; 25 | 26 | const DatabaseInfo = ({ database }: Database):React.JSX.Element => { 27 | const { 28 | activeIndex, 29 | handleAccordionToggle, 30 | } = useDatabaseInfoViewController(); 31 | 32 | const text = appText.database; 33 | 34 | return ( 35 |
36 |
37 |
38 | 50 | 78 |
79 |
80 |
81 |

82 | {database.database_name} 83 |

84 | 88 |
89 |
90 |
91 | 92 |
93 | 94 |

95 | {text.tablesCount.replace("{tablesCount}", database.database_tables?.length.toString())} 96 |

97 |
98 |
99 |

{text.tableInfo}

100 | 104 |
105 | 106 | 111 |
112 | ); 113 | }; 114 | 115 | export default DatabaseInfo; 116 | -------------------------------------------------------------------------------- /server/app/clients/openai_client.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | from typing import List 3 | import openai 4 | import os 5 | import logging 6 | 7 | openai.api_key = os.getenv("OPENAI_API_KEY") 8 | 9 | logger = logging.getLogger("mql") 10 | 11 | 12 | class OpenAIClient: 13 | def get_embeddings(self, nodes: list) -> list: 14 | response = openai.embeddings.create( 15 | input=nodes, model="text-embedding-ada-002" 16 | ) 17 | embeddings = [v.embedding for v in response.data] 18 | return embeddings 19 | 20 | 21 | def get_chat_response( 22 | self, messages: List[dict], model: str = "gpt-4", temperature: float = 0.5 23 | ) -> dict: 24 | try: 25 | response = openai.chat.completions.create( 26 | model=model, 27 | temperature=temperature, 28 | messages=messages, 29 | ) 30 | chat_response = response 31 | except openai.error.Timeout as e: 32 | logger.error( 33 | "Timeout while getting chat response from openai. Error is {}".format(e) 34 | ) 35 | raise HTTPException( 36 | status_code=408, 37 | detail="Timeout while getting chat response from openai", 38 | ) 39 | except openai.error.APIError as e: 40 | logger.error( 41 | "APIError while getting chat response from openai. Error is {}".format( 42 | e 43 | ) 44 | ) 45 | raise HTTPException( 46 | status_code=500, 47 | detail="Internal Server Error", 48 | ) 49 | except openai.error.APIConnectionError as e: 50 | logger.error( 51 | "APIConnectionError while getting chat response from openai. Error is {}".format( 52 | e 53 | ) 54 | ) 55 | raise HTTPException( 56 | status_code=500, 57 | detail="Internal Server Error", 58 | ) 59 | except openai.error.InvalidRequestError as e: 60 | logger.error( 61 | "InvalidRequestError while getting chat response from openai. Error is {}".format( 62 | e 63 | ) 64 | ) 65 | raise HTTPException( 66 | status_code=500, 67 | detail="Internal Server Error", 68 | ) 69 | except openai.error.AuthenticationError as e: 70 | logger.error( 71 | "AuthenticationError while getting chat response from openai. Error is {}".format( 72 | e 73 | ) 74 | ) 75 | raise HTTPException( 76 | status_code=500, 77 | detail="Internal Server Error", 78 | ) 79 | except openai.error.PermissionError as e: 80 | logger.error( 81 | "PermissionError while getting chat response from openai. Error is {}".format( 82 | e 83 | ) 84 | ) 85 | raise HTTPException( 86 | status_code=500, 87 | detail="Internal Server Error", 88 | ) 89 | except openai.error.RateLimitError as e: 90 | logger.error( 91 | "RateLimitError while getting chat response from openai. Error is {}".format( 92 | e 93 | ) 94 | ) 95 | raise HTTPException( 96 | status_code=500, 97 | detail="Internal Server Error", 98 | ) 99 | except Exception as e: 100 | logger.error( 101 | "Error while getting chat response from openai. Error is {}".format(e) 102 | ) 103 | raise HTTPException( 104 | status_code=500, 105 | detail="Internal Server Error", 106 | ) 107 | result = {'response': chat_response} if chat_response else {} 108 | return result 109 | 110 | 111 | 112 | openai_client = OpenAIClient() 113 | -------------------------------------------------------------------------------- /server/app/tests/test_schemas/test_query_schema.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | from uuid import uuid4 4 | from app.schemas.query import Query as QuerySchema 5 | 6 | 7 | def test_query_schema_with_valid_data() -> None: 8 | nl_query = "What is the total sales for the month of January?" 9 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 10 | user_database_id = uuid4() 11 | id = uuid4() 12 | query = QuerySchema( 13 | id=id, nl_query=nl_query, sql_query=sql_query, user_database_id=user_database_id 14 | ) 15 | assert query.nl_query == nl_query 16 | assert query.sql_query == sql_query 17 | assert query.user_database_id == user_database_id 18 | assert query.id == id 19 | 20 | 21 | def test_query_schema_with_invalid_nl_query() -> None: 22 | with pytest.raises(ValidationError): 23 | nl_query = 1 24 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 25 | user_database_id = uuid4() 26 | QuerySchema( 27 | nl_query=nl_query, 28 | sql_query=sql_query, 29 | user_database_id=user_database_id, 30 | ) 31 | 32 | 33 | def test_query_schema_with_invalid_sql_query() -> None: 34 | with pytest.raises(ValidationError): 35 | nl_query = "What is the total sales for the month of January?" 36 | sql_query = 1 37 | user_database_id = uuid4() 38 | QuerySchema( 39 | nl_query=nl_query, 40 | sql_query=sql_query, 41 | user_database_id=user_database_id, 42 | ) 43 | 44 | 45 | def test_query_schema_with_invalid_user_database_id() -> None: 46 | with pytest.raises(ValidationError): 47 | nl_query = "What is the total sales for the month of January?" 48 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 49 | user_database_id = "1" 50 | QuerySchema( 51 | nl_query=nl_query, 52 | sql_query=sql_query, 53 | user_database_id=user_database_id, 54 | ) 55 | 56 | 57 | def test_query_schema_with_invalid_id() -> None: 58 | with pytest.raises(ValidationError): 59 | nl_query = "What is the total sales for the month of January?" 60 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 61 | user_database_id = uuid4() 62 | id = "1" 63 | QuerySchema( 64 | nl_query=nl_query, 65 | sql_query=sql_query, 66 | user_database_id=user_database_id, 67 | id=id, 68 | ) 69 | 70 | 71 | def test_query_schema_with_missing_nl_query() -> None: 72 | with pytest.raises(ValidationError): 73 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 74 | user_database_id = uuid4() 75 | QuerySchema( 76 | sql_query=sql_query, 77 | user_database_id=user_database_id, 78 | ) 79 | 80 | 81 | def test_query_schema_with_missing_user_database_id() -> None: 82 | with pytest.raises(ValidationError): 83 | nl_query = "What is the total sales for the month of January?" 84 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 85 | QuerySchema( 86 | nl_query=nl_query, 87 | sql_query=sql_query, 88 | ) 89 | 90 | 91 | def test_query_schema_with_missing_sql_query() -> None: 92 | nl_query = "What is the total sales for the month of January?" 93 | user_database_id = uuid4() 94 | query = QuerySchema( 95 | nl_query=nl_query, user_database_id=user_database_id 96 | ) 97 | assert query.nl_query == nl_query 98 | assert query.sql_query is None 99 | assert query.user_database_id == user_database_id 100 | assert query.id is None 101 | 102 | 103 | def test_query_schema_with_missing_id() -> None: 104 | nl_query = "What is the total sales for the month of January?" 105 | sql_query = "SELECT SUM(sales) FROM sales WHERE month = 'January'" 106 | user_database_id = uuid4() 107 | query = QuerySchema( 108 | nl_query=nl_query, sql_query=sql_query, user_database_id=user_database_id 109 | ) 110 | assert query.nl_query == nl_query 111 | assert query.sql_query == sql_query 112 | assert query.user_database_id == user_database_id 113 | assert query.id is None 114 | -------------------------------------------------------------------------------- /server/app/api/v1/query.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter, Depends, status, Form, HTTPException 2 | from fastapi.responses import JSONResponse 3 | from sqlalchemy.orm import Session 4 | from app.api.v1.dependencies import get_current_user, get_db 5 | from app.clients.openai_client import openai_client 6 | from app.services.openai_service import openai_service 7 | from app.services.embeddings_service import embeddings_service 8 | from app.crud.crud_query import crud_query 9 | from app.schemas.query import Query as QuerySchema 10 | from app.crud.crud_user_database import crud_user_database 11 | from sqlalchemy import create_engine 12 | from fastapi.encoders import jsonable_encoder 13 | from sqlalchemy.sql import text 14 | from sqlalchemy.exc import SQLAlchemyError,DBAPIError,StatementError,ProgrammingError 15 | from app.services.query_service import query_service 16 | import json 17 | from typing import Annotated 18 | import logging 19 | 20 | from app.utilities.fernet_manager import FernetManager 21 | 22 | router = APIRouter() 23 | logger = logging.getLogger("mql") 24 | 25 | 26 | @router.post("/queries") 27 | async def query( 28 | db_id: Annotated[str, Form()], 29 | nl_query: Annotated[str, Form()], 30 | db: Session = Depends(get_db), 31 | current_user: dict = Depends(get_current_user), 32 | ) -> JSONResponse: 33 | try: 34 | query_schema = QuerySchema( 35 | nl_query=nl_query, user_database_id=db_id 36 | ) 37 | query_record = crud_query.create(db, query_schema) 38 | query_embedding = openai_client.get_embeddings([nl_query])[0] 39 | relevant_table_text_nodes = embeddings_service.get_relevant_tables_for_query( 40 | query_embedding, db_id, db 41 | ) 42 | sql_query = openai_service.text_2_sql_query(nl_query, relevant_table_text_nodes) 43 | if sql_query[-1] != ";": 44 | sql_query += ";" 45 | crud_query.insert_sql_query_by_id( 46 | db, query_record.id, sql_query 47 | ) 48 | 49 | except HTTPException as e: 50 | raise HTTPException(status_code=e.status_code, detail=e.detail) 51 | except Exception as e: 52 | logger.error( 53 | "Error while processing query {} for user {} and database {}. Error is {}".format( 54 | nl_query, current_user.id, db_id, e 55 | ) 56 | ) 57 | raise HTTPException( 58 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 59 | detail="Internal server error", 60 | ) 61 | 62 | return JSONResponse( 63 | content={ 64 | "message": "Query processed successfully", 65 | "data": {"sql_query": sql_query, "nl_query": nl_query}, 66 | }, 67 | status_code=status.HTTP_200_OK, 68 | ) 69 | 70 | @router.get("/queries") 71 | async def get_queries( 72 | db_id: str, 73 | db: Session = Depends(get_db), 74 | current_user: dict = Depends(get_current_user), 75 | ) -> JSONResponse: 76 | try: 77 | query = crud_query.get_by_datatbase_id_where_sql_query_not_null( 78 | db, db_id 79 | ) 80 | except Exception as e: 81 | logger.error( 82 | "Error while fetching query for user {} and database {}. Error is {}".format( 83 | current_user.id, db_id, e 84 | ) 85 | ) 86 | raise HTTPException( 87 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 88 | detail="Internal server error", 89 | ) 90 | 91 | return JSONResponse( 92 | content={ 93 | "message": "Queries fetched successfully", 94 | "data": { 95 | "queries": [ 96 | query.as_dict() for query in query 97 | ], 98 | }, 99 | }, 100 | status_code=status.HTTP_200_OK, 101 | ) 102 | 103 | 104 | @router.get("/queries/{id}") 105 | async def get_query( 106 | id: str, 107 | db: Session = Depends(get_db), 108 | current_user: dict = Depends(get_current_user), 109 | ) -> JSONResponse: 110 | try: 111 | query = crud_query.get_by_id(db, id) 112 | except Exception as e: 113 | logger.error( 114 | "Error while fetching query {} for user {}. Error is {}".format( 115 | id, current_user.id, e 116 | ) 117 | ) 118 | raise HTTPException( 119 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 120 | detail="Internal server error", 121 | ) 122 | return JSONResponse( 123 | content={ 124 | "message": "Query fetched successfully", 125 | "data": {"query": query.as_dict()}, 126 | }, 127 | status_code=status.HTTP_200_OK, 128 | ) 129 | -------------------------------------------------------------------------------- /test_data/elearning_schema: -------------------------------------------------------------------------------- 1 | Table: account 2 | id | integer 3 | user_id | integer 4 | password | character varying 5 | role | character varying 6 | username | character varying 7 | 8 | 9 | Table: assignment_rating 10 | id | integer 11 | assignment | integer 12 | rating | integer 13 | student | integer 14 | timestamp | timestamp without time zone 15 | 16 | 17 | Table: book_rating 18 | id | integer 19 | book | integer 20 | rating | integer 21 | student | integer 22 | timestamp | timestamp without time zone 23 | 24 | 25 | Table: category 26 | id | integer 27 | parent | integer 28 | name | character varying 29 | 30 | 31 | Table: book_subject 32 | id | integer 33 | book | integer 34 | subject | integer 35 | 36 | 37 | Table: course_subject 38 | id | integer 39 | course | integer 40 | subject | integer 41 | 42 | 43 | Table: enrollment 44 | subject | integer 45 | student | integer 46 | id | integer 47 | status | integer 48 | lesson | integer 49 | 50 | 51 | Table: lecturer 52 | id | integer 53 | account | integer 54 | name | character varying 55 | email | character varying 56 | 57 | 58 | Table: faculty 59 | id | integer 60 | name | character 61 | faculty_describtion | text 62 | 63 | 64 | Table: friendship 65 | id | integer 66 | start | integer 67 | end | integer 68 | 69 | 70 | Table: group_discuss 71 | id | integer 72 | subject | integer 73 | type | integer 74 | group_name | character 75 | topic | character 76 | 77 | 78 | Table: lecture_note 79 | id | integer 80 | lesson | integer 81 | title | character 82 | note_url | text 83 | 84 | 85 | Table: lecturer_rating 86 | id | integer 87 | lecturer | integer 88 | student | integer 89 | rating | integer 90 | 91 | 92 | Table: multimedia 93 | id | integer 94 | type | integer 95 | url | character 96 | name | character 97 | author | character varying 98 | 99 | 100 | Table: note_rating 101 | id | integer 102 | rating | integer 103 | student | integer 104 | note | integer 105 | timestamp | timestamp without time zone 106 | 107 | 108 | Table: multimedia_rating 109 | id | integer 110 | rating | integer 111 | multimedia | integer 112 | student | integer 113 | timestamp | timestamp without time zone 114 | 115 | 116 | Table: multimedia_subject 117 | multimedia | integer 118 | subject | integer 119 | id | integer 120 | 121 | 122 | Table: note 123 | chapter | integer 124 | subject | integer 125 | student | integer 126 | id | integer 127 | type | integer 128 | content | text 129 | 130 | 131 | Table: student_preference 132 | id | integer 133 | student | integer 134 | style_preference | integer 135 | category_preference | integer 136 | 137 | 138 | Table: subject_requirement 139 | id | integer 140 | subject | integer 141 | pre-requires_subject | integer 142 | 143 | 144 | Table: student 145 | account | integer 146 | course | integer 147 | last_login | timestamp without time zone 148 | accumulated_online_time | timestamp without time zone 149 | id | integer 150 | email | character varying 151 | name | character varying 152 | dob | character varying 153 | gender | character varying 154 | nationality | character varying 155 | occupy | character varying 156 | graduated_university | character varying 157 | 158 | 159 | Table: subject 160 | id | integer 161 | category | integer 162 | name | character varying 163 | thumb | character 164 | pic | character 165 | description | character varying 166 | 167 | 168 | Table: subject_rating 169 | id | integer 170 | subject | integer 171 | student | integer 172 | rating | integer 173 | timestamp | timestamp without time zone 174 | commence | text 175 | 176 | 177 | Table: subject_tag 178 | id | integer 179 | student | integer 180 | subject | integer 181 | timestamp | timestamp without time zone 182 | tag | character 183 | 184 | 185 | Table: lesson 186 | id | integer 187 | year | integer 188 | lecturer | integer 189 | subject | integer 190 | book | integer 191 | semester | character 192 | 193 | 194 | Table: assignment 195 | due_date | date 196 | id | integer 197 | lesson | integer 198 | type | integer 199 | assignment_task | text 200 | 201 | 202 | Table: style_preference 203 | id | integer 204 | style_preference | character 205 | 206 | 207 | Table: book 208 | category | integer 209 | id | integer 210 | type | integer 211 | publish_year | integer 212 | name | character 213 | isbn | text 214 | author | character 215 | publisher | character 216 | 217 | 218 | Table: course 219 | faculty | integer 220 | id | integer 221 | category | integer 222 | name | character 223 | course_describtion | text 224 | 225 | 226 | Table: gorup_discuss_rating 227 | id | integer 228 | group_discuss | integer 229 | student | integer 230 | rating | integer 231 | timestamp | timestamp without time zone 232 | 233 | 234 | -------------------------------------------------------------------------------- /client/app/components/header.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Bars3Icon, XMarkIcon } from "@heroicons/react/24/outline"; 4 | import { Dialog } from "@headlessui/react"; 5 | 6 | import Link from "next/link"; 7 | import React from "react"; 8 | 9 | import useGenericViewController from "../viewControllers/genericViewController"; 10 | import appText from "../assets/strings"; 11 | 12 | type Props = { 13 | token: string | undefined; 14 | }; 15 | 16 | const Header = ({ token }: Props):React.JSX.Element => { 17 | const { 18 | headerNavigation, 19 | setMobileMenuOpen, 20 | mobileMenuOpen, 21 | pathname, 22 | } = useGenericViewController(); 23 | 24 | const text = appText.header; 25 | 26 | return ( 27 |
28 | 74 | 80 |
81 | 82 |
83 | 84 | MQL 85 | logo 86 | 87 | 95 |
96 |
97 |
98 |
99 | {headerNavigation.map((item) => ( 100 | 105 | {item.name} 106 | 107 | ))} 108 |
109 |
110 | 114 | {text.login} 115 | 116 |
117 |
118 |
119 | 120 |
121 |
122 | ); 123 | }; 124 | 125 | export default Header; 126 | --------------------------------------------------------------------------------