├── data-whisperer-backend ├── app │ ├── models │ │ ├── base.py │ │ └── __init__.py │ ├── schemas │ │ └── __init__.py │ ├── services │ │ ├── __init__.py │ │ └── file_upload.py │ ├── routes │ │ ├── __init__.py │ │ ├── main.py │ │ ├── upload_file.py │ │ └── run_command.py │ ├── Config.py │ └── __init__.py ├── sqlite_db │ └── .gitkeep ├── uploads │ └── .gitkeep ├── tests │ ├── __init__.py │ ├── Config.py │ ├── conftest.py │ ├── test_routes.py │ └── test_upload.py ├── .env.example ├── run.py ├── Dockerfile ├── README.md ├── requirements.txt └── setup │ ├── data │ ├── raw_customers.csv │ ├── raw_payments.csv │ └── raw_orders.csv │ └── create_and_seed_db.py ├── data-whisperer-frontend ├── src │ ├── react-app-env.d.ts │ ├── components │ │ ├── main-content-container │ │ │ ├── TableDataComponent │ │ │ │ ├── TableDataComponent.module.css │ │ │ │ └── TableDataComponent.tsx │ │ │ ├── TablesComponent │ │ │ │ ├── TablesComponent.module.css │ │ │ │ └── TablesComponent.tsx │ │ │ ├── MainContent.tsx │ │ │ └── UploadFileComponent │ │ │ │ └── UploadFileComponent.tsx │ │ ├── Footer.module.css │ │ ├── App.test.tsx │ │ ├── sidebar-chat-container │ │ │ ├── PromptComponent │ │ │ │ ├── PromptComponent.module.css │ │ │ │ └── PromptComponent.tsx │ │ │ └── Sidebar.tsx │ │ ├── Footer.tsx │ │ ├── Header.tsx │ │ └── App.tsx │ ├── index.css │ ├── setupTests.ts │ ├── styles │ │ └── styles.css │ ├── reportWebVitals.ts │ ├── index.tsx │ └── api │ │ └── index.ts ├── public │ ├── robots.txt │ ├── favicon.ico │ ├── manifest.json │ └── index.html ├── .gitignore ├── tsconfig.json ├── package.json └── README.md ├── images ├── whisp_10.PNG └── whisp_11.PNG ├── .gitignore ├── LICENSE └── README.md /data-whisperer-backend/app/models/base.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data-whisperer-backend/sqlite_db/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data-whisperer-backend/uploads/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data-whisperer-backend/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # tests/__init__.py 2 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/services/__init__.py: -------------------------------------------------------------------------------- 1 | from .file_upload import create_and_load_table 2 | -------------------------------------------------------------------------------- /data-whisperer-backend/.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY="your-open-ai-api-key" 2 | MODEL_NAME='gpt-3.5-turbo' -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/main-content-container/TableDataComponent/TableDataComponent.module.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/whisp_10.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cogitovirus/langchain-sql-agent-bootstrap/HEAD/images/whisp_10.PNG -------------------------------------------------------------------------------- /images/whisp_11.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cogitovirus/langchain-sql-agent-bootstrap/HEAD/images/whisp_11.PNG -------------------------------------------------------------------------------- /data-whisperer-frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /data-whisperer-frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cogitovirus/langchain-sql-agent-bootstrap/HEAD/data-whisperer-frontend/public/favicon.ico -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/Footer.module.css: -------------------------------------------------------------------------------- 1 | /* Footer.module.css */ 2 | .footer { 3 | background-color: #f5f5f5; /* Light gray background */ 4 | width: 100%; 5 | } 6 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/models/__init__.py: -------------------------------------------------------------------------------- 1 | # app/models/__init__.py 2 | 3 | from flask_sqlalchemy import SQLAlchemy 4 | from flask_marshmallow import Marshmallow 5 | 6 | db = SQLAlchemy() 7 | ma = Marshmallow() 8 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/routes/__init__.py: -------------------------------------------------------------------------------- 1 | # app/routes/__init__.py 2 | 3 | from flask import Blueprint 4 | 5 | main_blueprint = Blueprint('main', __name__) 6 | 7 | from . import main, run_command, upload_file 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # python-venv 2 | *myenv/ 3 | # env variables 4 | .env 5 | # pycache 6 | *__pycache__/ 7 | 8 | # don't commit the db 9 | data-whisperer-backend/sqlite_db/my_database.db 10 | data-whisperer-backend/sqlite_db/test_database.db 11 | 12 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/index.css: -------------------------------------------------------------------------------- 1 | /* Reset styles */ 2 | html, 3 | body { 4 | margin: 0; 5 | padding: 0; 6 | box-sizing: border-box; 7 | font-family: Arial, sans-serif; 8 | } 9 | 10 | /* Global styles */ 11 | body { 12 | background-color: #f4f4f4; 13 | } 14 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/main-content-container/TablesComponent/TablesComponent.module.css: -------------------------------------------------------------------------------- 1 | /* src/components/main-content-container/TablesComponent/TablesComponent.module.css */ 2 | 3 | 4 | 5 | @media screen and (max-width: 600px) { 6 | .tableContainer { 7 | width: 100%; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /data-whisperer-backend/run.py: -------------------------------------------------------------------------------- 1 | from app import create_app 2 | from app.Config import Config 3 | from flask_socketio import SocketIO 4 | 5 | 6 | if __name__ == '__main__': 7 | app = create_app(Config) 8 | socketio = SocketIO(app, cors_allowed_origins='http://localhost:3000') 9 | socketio.run(app, debug=True) 10 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/App.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render, screen } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | render(); 7 | const linkElement = screen.getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /data-whisperer-backend/tests/Config.py: -------------------------------------------------------------------------------- 1 | # tests/Config.py 2 | 3 | import os 4 | from dotenv import load_dotenv 5 | 6 | load_dotenv() 7 | basedir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 8 | 9 | class Config: 10 | SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.join(basedir, 'sqlite_db/test_database.db')}" 11 | SQLALCHEMY_TRACK_MODIFICATIONS = False 12 | UPLOAD_FOLDER = os.path.join(basedir, 'uploads') 13 | -------------------------------------------------------------------------------- /data-whisperer-frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | .env 4 | 5 | # dependencies 6 | /node_modules 7 | /.pnp 8 | .pnp.js 9 | 10 | # testing 11 | /coverage 12 | 13 | # production 14 | /build 15 | 16 | # misc 17 | .DS_Store 18 | .env.local 19 | .env.development.local 20 | .env.test.local 21 | .env.production.local 22 | 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | -------------------------------------------------------------------------------- /data-whisperer-backend/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # tests/conftest.py 2 | 3 | import pytest 4 | from app import create_app 5 | from app.models import db 6 | from tests.Config import Config 7 | 8 | 9 | @pytest.fixture 10 | def app(): 11 | app = create_app(Config) 12 | with app.app_context(): 13 | db.create_all() 14 | yield app 15 | db.drop_all() 16 | return app 17 | 18 | @pytest.fixture 19 | def client(app): 20 | return app.test_client() 21 | -------------------------------------------------------------------------------- /data-whisperer-backend/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official Python base image 2 | FROM python:3.9-slim 3 | 4 | # Set the working directory 5 | WORKDIR /app 6 | 7 | # Copy requirements.txt and install dependencies 8 | COPY requirements.txt . 9 | RUN pip install --no-cache-dir -r requirements.txt 10 | 11 | # Copy the rest of the application code 12 | COPY . . 13 | 14 | # Expose the port the app runs on 15 | EXPOSE 5000 16 | 17 | # Run the application 18 | CMD ["python", "run.py"] 19 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/sidebar-chat-container/PromptComponent/PromptComponent.module.css: -------------------------------------------------------------------------------- 1 | /* PromptComponent.module.css */ 2 | .Prompt { 3 | width: auto; 4 | height: 400px; 5 | overflow: auto; 6 | display: flex; 7 | flex-direction: column; 8 | gap: 16px; 9 | } 10 | 11 | .inputContainer { 12 | } 13 | 14 | .outputBox { 15 | flex-grow: 3; 16 | max-width: 100%; 17 | white-space: pre-wrap; 18 | word-wrap: break-word; 19 | } 20 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/styles/styles.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | margin: 0; 4 | padding: 0; 5 | } 6 | 7 | body { 8 | font-family: Arial, sans-serif; 9 | } 10 | 11 | .app { 12 | display: flex; 13 | flex-direction: column; 14 | justify-content: space-between; 15 | } 16 | 17 | .nav a { 18 | color: white; 19 | text-decoration: none; 20 | margin-left: 1rem; 21 | } 22 | 23 | @media (max-width: 768px) { 24 | .nav { 25 | display: none; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/Config.py: -------------------------------------------------------------------------------- 1 | # app/config.py 2 | 3 | import os 4 | from dotenv import load_dotenv 5 | 6 | load_dotenv() 7 | basedir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 8 | 9 | class Config: 10 | SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.join(basedir, 'sqlite_db/my_database.db')}" 11 | SQLALCHEMY_TRACK_MODIFICATIONS = False 12 | UPLOAD_FOLDER = f"sqlite:///{os.path.join(basedir, 'uploads')}" 13 | MODEL_NAME = os.environ.get('MODEL_NAME') 14 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals'; 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry); 7 | getFID(onPerfEntry); 8 | getFCP(onPerfEntry); 9 | getLCP(onPerfEntry); 10 | getTTFB(onPerfEntry); 11 | }); 12 | } 13 | }; 14 | 15 | export default reportWebVitals; 16 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/Footer.tsx: -------------------------------------------------------------------------------- 1 | // Footer.tsx 2 | import React from 'react'; 3 | import { Container, Box, Typography } from '@mui/material'; 4 | import './Footer.module.css'; 5 | 6 | const Footer: React.FC = () => ( 7 |
8 | 9 | 10 | 11 | 2023 cogitovirus.com 12 | 13 | 14 | 15 |
16 | ); 17 | 18 | export default Footer; 19 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/__init__.py: -------------------------------------------------------------------------------- 1 | # app/__init__.py 2 | 3 | from flask import Flask 4 | from flask_cors import CORS 5 | from .models import db, ma 6 | from .routes import main_blueprint 7 | 8 | def create_app(config_object): 9 | app = Flask(__name__) 10 | app.config.from_object(config_object) 11 | 12 | # TODO: CORS should be restricted to only the frontend domain on production 13 | # This will enable CORS for all routes 14 | CORS(app) 15 | 16 | # database init 17 | db.init_app(app) 18 | ma.init_app(app) 19 | 20 | app.register_blueprint(main_blueprint) 21 | 22 | return app 23 | -------------------------------------------------------------------------------- /data-whisperer-frontend/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './components/App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | 7 | const root = ReactDOM.createRoot( 8 | document.getElementById('root') as HTMLElement 9 | ); 10 | root.render( 11 | 12 | 13 | 14 | ); 15 | 16 | // If you want to start measuring performance in your app, pass a function 17 | // to log results (for example: reportWebVitals(console.log)) 18 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 19 | reportWebVitals(); 20 | -------------------------------------------------------------------------------- /data-whisperer-frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx" 22 | }, 23 | "include": [ 24 | "src" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/api/index.ts: -------------------------------------------------------------------------------- 1 | // src/api/index.ts 2 | 3 | const API_BASE_URL = process.env.REACT_APP_API_BASE_URL || 'http://localhost:5000'; 4 | 5 | export interface Customer { 6 | id: number; 7 | first_name: string; 8 | last_name: string; 9 | } 10 | 11 | export async function fetchTables(): Promise { 12 | const response = await fetch(`${API_BASE_URL}/api/v1/tables`); 13 | const data = await response.json(); 14 | return data; 15 | } 16 | 17 | export async function fetchTableData(tableName: string): Promise { 18 | const response = await fetch(`${API_BASE_URL}/api/v1/tables/${tableName}`); 19 | const data = await response.json(); 20 | return data; 21 | } 22 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/Header.tsx: -------------------------------------------------------------------------------- 1 | // Header.tsx 2 | import React from 'react'; 3 | import { AppBar, Toolbar, Typography, Button } from '@mui/material'; 4 | 5 | const Header: React.FC = () => ( 6 |
7 | 8 | 9 | 10 | LangChain SQL Agent Bootstrap v.0.1 11 | 12 | {/* 13 | 14 | */} 15 | 16 | 17 |
18 | ); 19 | 20 | export default Header; 21 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/sidebar-chat-container/Sidebar.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import PromptComponent from './PromptComponent/PromptComponent'; 3 | import Container from '@mui/material/Container'; 4 | import Typography from '@mui/material/Typography'; 5 | import Box from '@mui/material/Box'; 6 | 7 | const Sidebar = () => ( 8 | 16 | 17 | 18 | GPT Agent 19 | 20 | 21 | 22 | 23 | ); 24 | 25 | export default Sidebar; 26 | -------------------------------------------------------------------------------- /data-whisperer-backend/README.md: -------------------------------------------------------------------------------- 1 | # Data Whisperer Flask Backend 2 | 3 | ## Setup 4 | ### Virtual environment 5 | ```bash 6 | # create virtual environment 7 | python3 -m venv myenv 8 | # activate virtual environment 9 | source myenv/bin/activate 10 | # install dependencies 11 | pip install -r requirements.txt 12 | # deactivate virtual environment after you are done 13 | deactivate 14 | ``` 15 | 16 | ### Run db seed script 17 | ```bash 18 | # run from project-root/data-whisperer-backend/ 19 | python setup/create_and_seed_db.py 20 | ``` 21 | 22 | ## Running the server 23 | ```bash 24 | # run from project-root/data-whisperer-backend/ 25 | python run.py 26 | ``` 27 | 28 | ## Running tests 29 | ```bash 30 | # run from project-root/data-whisperer-backend/ 31 | pytest 32 | ``` 33 | 34 | ## Handy commands 35 | ```bash 36 | pip freeze > requirements.txt 37 | ``` -------------------------------------------------------------------------------- /data-whisperer-backend/tests/test_routes.py: -------------------------------------------------------------------------------- 1 | # tests/test_routes.py 2 | 3 | import json 4 | 5 | def test_get_tables(client): 6 | response = client.get('/api/v1/tables') 7 | assert response.status_code == 200 8 | data = json.loads(response.data) 9 | assert isinstance(data, list) 10 | assert len(data) > 0 11 | 12 | def test_get_table_data(client): 13 | # Use a table name that exists in your test database 14 | table_name = 'customers' 15 | response = client.get(f'/api/v1/tables/{table_name}') 16 | assert response.status_code == 200 17 | data = json.loads(response.data) 18 | assert isinstance(data, list) 19 | 20 | def test_get_table_data_not_found(client): 21 | # Use a table name that does not exist in your test database 22 | table_name = 'non_existent_table' 23 | response = client.get(f'/api/v1/tables/{table_name}') 24 | assert response.status_code == 404 -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/main-content-container/MainContent.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { TablesComponent } from './TablesComponent/TablesComponent'; 3 | import { UploadFileComponent } from './UploadFileComponent/UploadFileComponent'; 4 | import Container from '@mui/material/Container'; 5 | import Typography from '@mui/material/Typography'; 6 | import Grid from '@mui/material/Grid'; 7 | 8 | const MainContent = () => ( 9 | 10 | 11 | Table view 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | ); 23 | 24 | export default MainContent; 25 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/App.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import CssBaseline from '@mui/material/CssBaseline'; 3 | import { Container, Grid } from '@mui/material'; 4 | import Header from './Header'; 5 | import Sidebar from './sidebar-chat-container/Sidebar'; 6 | import MainContent from './main-content-container/MainContent'; 7 | import Footer from './Footer'; 8 | import '../styles/styles.css'; 9 | 10 | function App() { 11 | return ( 12 |
13 | 14 |
15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 |
25 | 26 |
27 | ); 28 | } 29 | 30 | export default App; 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2017, Jon Schlinkert. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /data-whisperer-backend/app/routes/main.py: -------------------------------------------------------------------------------- 1 | # app/routes/main.py 2 | 3 | from flask import jsonify 4 | from sqlalchemy import inspect, text 5 | from ..models import db 6 | from . import main_blueprint 7 | 8 | 9 | @main_blueprint.route('/api/v1/tables', methods=['GET']) 10 | def get_tables(): 11 | """ 12 | Get a list of all tables in the database 13 | """ 14 | inspector = inspect(db.engine) 15 | table_names = inspector.get_table_names() 16 | return jsonify(table_names) 17 | 18 | 19 | @main_blueprint.route('/api/v1/tables/', methods=['GET']) 20 | def get_table_data(table_name): 21 | # Validate if the table exists 22 | inspector = inspect(db.engine) 23 | if table_name not in inspector.get_table_names(): 24 | return jsonify({'error': f"Table '{table_name}' not found"}), 404 25 | 26 | # Fetch all the data from the table 27 | try: 28 | result = db.engine.execute(text(f"SELECT * FROM {table_name}")) 29 | column_names = result.keys() 30 | data = [dict(zip(column_names, row)) for row in result] 31 | 32 | return jsonify(data) 33 | except Exception as err: 34 | return jsonify({'error': str(err)}), 500 35 | -------------------------------------------------------------------------------- /data-whisperer-backend/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.8.4 2 | aiosignal==1.3.1 3 | async-timeout==4.0.2 4 | attrs==22.2.0 5 | autopep8==2.0.2 6 | bidict==0.22.1 7 | certifi==2022.12.7 8 | charset-normalizer==3.1.0 9 | click==8.1.3 10 | dataclasses-json==0.5.7 11 | exceptiongroup==1.1.1 12 | Flask==2.2.3 13 | Flask-Cors==3.0.10 14 | flask-marshmallow==0.14.0 15 | Flask-SocketIO==5.3.3 16 | Flask-SQLAlchemy==3.0.3 17 | frozenlist==1.3.3 18 | greenlet==2.0.2 19 | idna==3.4 20 | iniconfig==2.0.0 21 | itsdangerous==2.1.2 22 | Jinja2==3.1.2 23 | langchain==0.0.123 24 | MarkupSafe==2.1.2 25 | marshmallow==3.19.0 26 | marshmallow-enum==1.5.1 27 | marshmallow-sqlalchemy==0.29.0 28 | mock==5.0.1 29 | multidict==6.0.4 30 | mypy-extensions==1.0.0 31 | numpy==1.24.2 32 | openai==0.27.2 33 | packaging==23.0 34 | pandas==1.5.3 35 | pluggy==1.0.0 36 | pycodestyle==2.10.0 37 | pydantic==1.10.7 38 | pytest==7.2.2 39 | pytest-flask==1.2.0 40 | python-dateutil==2.8.2 41 | python-dotenv==1.0.0 42 | python-engineio==4.4.0 43 | python-socketio==5.8.0 44 | pytz==2023.2 45 | PyYAML==6.0 46 | requests==2.28.2 47 | six==1.16.0 48 | SQLAlchemy==1.4.47 49 | tenacity==8.2.2 50 | tomli==2.0.1 51 | tqdm==4.65.0 52 | typing-inspect==0.8.0 53 | typing_extensions==4.5.0 54 | urllib3==1.26.15 55 | Werkzeug==2.2.3 56 | yarl==1.8.2 57 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/routes/upload_file.py: -------------------------------------------------------------------------------- 1 | # app/routes/upload_file.py 2 | 3 | import os 4 | from werkzeug.utils import secure_filename 5 | from flask import request, jsonify, current_app 6 | 7 | from . import main_blueprint 8 | from ..services import create_and_load_table 9 | from ..models import db 10 | 11 | 12 | ALLOWED_EXTENSIONS = {'csv'} 13 | 14 | def allowed_file(filename): 15 | return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS 16 | 17 | 18 | @main_blueprint.route('/api/v1/upload', methods=['POST']) 19 | def upload_file(): 20 | if 'file' not in request.files: 21 | return jsonify({'error': 'No file part'}), 400 22 | file = request.files['file'] 23 | if file.filename == '': 24 | return jsonify({'error': 'No selected file'}), 400 25 | if file and allowed_file(file.filename): 26 | filename = secure_filename(file.filename) 27 | file.save(os.path.join(current_app.config['UPLOAD_FOLDER'], filename)) 28 | 29 | file_path = os.path.join(current_app.config['UPLOAD_FOLDER'], filename) 30 | # Split the filename into name and extension and use the name as the table_name 31 | table_name, _ = os.path.splitext(filename) 32 | 33 | create_and_load_table(file_path, db, table_name) 34 | return jsonify({'message': 'File uploaded and processed successfully'}), 200 35 | return jsonify({'error': 'Invalid request'}), 400 36 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/services/file_upload.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sqlalchemy import text 3 | 4 | def create_table(db, create_table_sql): 5 | try: 6 | db.engine.execute(text(create_table_sql)) 7 | except Exception as err: 8 | print(err) 9 | 10 | def insert_data(db, table_name, df): 11 | # Insert data from the DataFrame into the SQLite table 12 | df.to_sql(table_name, db.engine, if_exists='append', index=False) 13 | 14 | def create_and_load_table(file_path, db, table_name): 15 | # Read the file into a DataFrame and infer the schema 16 | df = pd.read_csv(file_path) 17 | schema = df.dtypes.to_dict() 18 | 19 | # Generate CREATE TABLE statement 20 | create_table_statement = f"CREATE TABLE IF NOT EXISTS {table_name} (" 21 | 22 | for column, dtype in schema.items(): 23 | if str(dtype) == 'object': 24 | sql_type = 'TEXT' 25 | elif str(dtype).startswith('int'): 26 | sql_type = 'INTEGER' 27 | elif str(dtype).startswith('float'): 28 | sql_type = 'REAL' 29 | else: 30 | sql_type = 'TEXT' 31 | 32 | create_table_statement += f"{column} {sql_type}, " 33 | 34 | create_table_statement = create_table_statement.rstrip(", ") + ");" 35 | 36 | # Create the table 37 | create_table(db, create_table_statement) 38 | 39 | # Insert data from the DataFrame into the SQLite table 40 | insert_data(db, table_name, df) 41 | -------------------------------------------------------------------------------- /data-whisperer-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "data-whisperer", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@emotion/react": "^11.10.6", 7 | "@emotion/styled": "^11.10.6", 8 | "@mui/icons-material": "^5.11.11", 9 | "@mui/material": "^5.11.15", 10 | "@testing-library/jest-dom": "^5.16.5", 11 | "@testing-library/react": "^13.4.0", 12 | "@testing-library/user-event": "^13.5.0", 13 | "@types/jest": "^27.5.2", 14 | "@types/node": "^16.18.19", 15 | "@types/react": "^18.0.29", 16 | "@types/react-dom": "^18.0.11", 17 | "axios": "^1.3.4", 18 | "react": "^18.2.0", 19 | "react-dom": "^18.2.0", 20 | "react-scripts": "5.0.1", 21 | "socket.io-client": "^4.6.1", 22 | "typescript": "^4.9.5", 23 | "web-vitals": "^2.1.4" 24 | }, 25 | "scripts": { 26 | "start": "react-scripts start", 27 | "build": "react-scripts build", 28 | "test": "react-scripts test", 29 | "eject": "react-scripts eject" 30 | }, 31 | "eslintConfig": { 32 | "extends": [ 33 | "react-app", 34 | "react-app/jest" 35 | ] 36 | }, 37 | "browserslist": { 38 | "production": [ 39 | ">0.2%", 40 | "not dead", 41 | "not op_mini all" 42 | ], 43 | "development": [ 44 | "last 1 chrome version", 45 | "last 1 firefox version", 46 | "last 1 safari version" 47 | ] 48 | }, 49 | "devDependencies": { 50 | "@types/axios": "^0.14.0" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/main-content-container/UploadFileComponent/UploadFileComponent.tsx: -------------------------------------------------------------------------------- 1 | // src/components/main-content-container/UploadFileComponent/UploadFileComponent.tsx 2 | 3 | import React from "react"; 4 | import { Box, Button, Card, CardContent, Typography } from "@mui/material"; 5 | import { styled } from "@mui/system"; 6 | 7 | const StyledInput = styled("input")({ 8 | display: "none", 9 | }); 10 | 11 | export const UploadFileComponent: React.FC = () => { 12 | const handleFileUpload = (event: React.ChangeEvent) => { 13 | const file = event.target.files && event.target.files[0]; 14 | if (file) { 15 | // Handle file upload logic here 16 | } 17 | }; 18 | 19 | return ( 20 | 21 | 22 | 28 | 29 | Upload a file 30 | 31 | 36 | 41 | 42 | 43 | 44 | ); 45 | }; 46 | -------------------------------------------------------------------------------- /data-whisperer-backend/tests/test_upload.py: -------------------------------------------------------------------------------- 1 | # tests/test_upload.py 2 | 3 | import os 4 | import io 5 | import json 6 | from app import db 7 | from flask import current_app 8 | from werkzeug.datastructures import FileStorage 9 | 10 | def test_upload_file(client, app): 11 | # Create an in-memory file with some content 12 | file_content = b"column1,column2\nvalue1,value2" 13 | file_storage = FileStorage(stream=io.BytesIO(file_content), filename="test.csv", content_type="text/csv") 14 | 15 | response = client.post('/api/v1/upload', content_type='multipart/form-data', data={'file': file_storage}) 16 | 17 | assert response.status_code == 200 18 | assert json.loads(response.data) == {'message': 'File uploaded and processed successfully'} 19 | 20 | # Check if the file was uploaded to the UPLOAD_FOLDER 21 | uploaded_file_path = os.path.join(app.config['UPLOAD_FOLDER'], "test.csv") 22 | assert os.path.exists(uploaded_file_path) 23 | 24 | # Clean up the uploaded file after testing 25 | os.remove(uploaded_file_path) 26 | 27 | # Check if the table was created in the database 28 | table_name = "test" 29 | with app.app_context(): 30 | result = db.engine.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'") 31 | assert result.fetchone()[0] == table_name 32 | 33 | # Check if the data was inserted into the table 34 | result = db.engine.execute(f"SELECT * FROM {table_name}") 35 | assert result.fetchone() == ('value1', 'value2') 36 | -------------------------------------------------------------------------------- /data-whisperer-backend/setup/data/raw_customers.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name 2 | 1,Michael,P. 3 | 2,Shawn,M. 4 | 3,Kathleen,P. 5 | 4,Jimmy,C. 6 | 5,Katherine,R. 7 | 6,Sarah,R. 8 | 7,,M. 9 | 8,Frank,R. 10 | 9,Jennifer,F. 11 | 10,Henry,W. 12 | 11,Fred,S. 13 | 12,Amy,D. 14 | 13,Kathleen,M. 15 | 14,Steve,F. 16 | 15,Teresa,H. 17 | 16,Amanda,H. 18 | 17,Kimberly,R. 19 | 18,Johnny,K. 20 | 19,Virginia,F. 21 | 20,Anna,A. 22 | 21,Willie,H. 23 | 22,Sean,H. 24 | 23,Mildred,A. 25 | 24,David,G. 26 | 25,Victor,H. 27 | 26,Aaron,R. 28 | 27,Benjamin,B. 29 | 28,Lisa,W. 30 | 29,Benjamin,K. 31 | 30,Christina,W. 32 | 31,Jane,G. 33 | 32,Thomas,O. 34 | 33,Katherine,M. 35 | 34,Jennifer,S. 36 | 35,Sara,T. 37 | 36,Harold,O. 38 | 37,Shirley,J. 39 | 38,Dennis,J. 40 | 39,Louise,W. 41 | 40,Maria,A. 42 | 41,Gloria,C. 43 | 42,Diana,S. 44 | 43,Kelly,N. 45 | 44,Jane,R. 46 | 45,Scott,B. 47 | 46,Norma,C. 48 | 47,Marie,P. 49 | 48,Lillian,C. 50 | 49,Judy,N. 51 | 50,Billy,L. 52 | 51,Howard,R. 53 | 52,Laura,F. 54 | 53,Anne,B. 55 | 54,Rose,M. 56 | 55,Nicholas,R. 57 | 56,Joshua,K. 58 | 57,Paul,W. 59 | 58,Kathryn,K. 60 | 59,Adam,A. 61 | 60,Norma,W. 62 | 61,Timothy,R. 63 | 62,Elizabeth,P. 64 | 63,Edward,G. 65 | 64,David,C. 66 | 65,Brenda,W. 67 | 66,Adam,W. 68 | 67,Michael,H. 69 | 68,Jesse,E. 70 | 69,Janet,P. 71 | 70,Helen,F. 72 | 71,Gerald,C. 73 | 72,Kathryn,O. 74 | 73,Alan,B. 75 | 74,Harry,A. 76 | 75,Andrea,H. 77 | 76,Barbara,W. 78 | 77,Anne,W. 79 | 78,Harry,H. 80 | 79,Jack,R. 81 | 80,Phillip,H. 82 | 81,Shirley,H. 83 | 82,Arthur,D. 84 | 83,Virginia,R. 85 | 84,Christina,R. 86 | 85,Theresa,M. 87 | 86,Jason,C. 88 | 87,Phillip,B. 89 | 88,Adam,T. 90 | 89,Margaret,J. 91 | 90,Paul,P. 92 | 91,Todd,W. 93 | 92,Willie,O. 94 | 93,Frances,R. 95 | 94,Gregory,H. 96 | 95,Lisa,P. 97 | 96,Jacqueline,A. 98 | 97,Shirley,D. 99 | 98,Nicole,M. 100 | 99,Mary,G. 101 | 100,Jean,M. 102 | -------------------------------------------------------------------------------- /data-whisperer-frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/main-content-container/TableDataComponent/TableDataComponent.tsx: -------------------------------------------------------------------------------- 1 | // TableDataComponent.tsx 2 | 3 | import React, { useEffect, useState } from "react"; 4 | import { fetchTableData } from "../../../api/index"; 5 | import { 6 | Table, 7 | TableBody, 8 | TableCell, 9 | TableContainer, 10 | TableHead, 11 | TableRow, 12 | Paper, 13 | Typography, 14 | } from "@mui/material"; 15 | import styles from "./TableDataComponent.module.css"; 16 | 17 | interface TableDataProps { 18 | tableName: string; 19 | } 20 | 21 | export const TableDataComponent: React.FC = ({ tableName }) => { 22 | const [tableData, setTableData] = useState([]); 23 | const [tableHeaders, setTableHeaders] = useState([]); 24 | 25 | useEffect(() => { 26 | fetchTableData(tableName).then((data) => { 27 | if (data.length > 0) { 28 | setTableHeaders(Object.keys(data[0])); 29 | } 30 | setTableData(data); 31 | }); 32 | }, [tableName]); 33 | 34 | return ( 35 |
36 | 37 | {tableName} 38 | 39 | 40 | 41 | 42 | 43 | {tableHeaders.map((header, index) => ( 44 | {header} 45 | ))} 46 | 47 | 48 | 49 | {tableData.map((row, rowIndex) => ( 50 | 51 | {tableHeaders.map((header, cellIndex) => ( 52 | {row[header]} 53 | ))} 54 | 55 | ))} 56 | 57 |
58 |
59 |
60 | ); 61 | }; 62 | -------------------------------------------------------------------------------- /data-whisperer-frontend/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Create React App 2 | 3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 4 | 5 | ## Available Scripts 6 | 7 | In the project directory, you can run: 8 | 9 | ### `npm start` 10 | 11 | Runs the app in the development mode.\ 12 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 13 | 14 | The page will reload if you make edits.\ 15 | You will also see any lint errors in the console. 16 | 17 | ### `npm test` 18 | 19 | Launches the test runner in the interactive watch mode.\ 20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 21 | 22 | ### `npm run build` 23 | 24 | Builds the app for production to the `build` folder.\ 25 | It correctly bundles React in production mode and optimizes the build for the best performance. 26 | 27 | The build is minified and the filenames include the hashes.\ 28 | Your app is ready to be deployed! 29 | 30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 31 | 32 | ### `npm run eject` 33 | 34 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 35 | 36 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 37 | 38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 39 | 40 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 41 | 42 | ## Learn More 43 | 44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 45 | 46 | To learn React, check out the [React documentation](https://reactjs.org/). 47 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/main-content-container/TablesComponent/TablesComponent.tsx: -------------------------------------------------------------------------------- 1 | // src/components/main-content-container/TablesComponent/TablesComponent.tsx 2 | 3 | import React, { useEffect, useState } from "react"; 4 | import Tabs from "@mui/material/Tabs"; 5 | import Tab from "@mui/material/Tab"; 6 | import Box from "@mui/material/Box"; 7 | import Typography from "@mui/material/Typography"; 8 | import { fetchTables } from "../../../api/index"; 9 | import { TableDataComponent } from "../TableDataComponent/TableDataComponent"; 10 | import Card from "@mui/material/Card"; 11 | import CardContent from "@mui/material/CardContent"; 12 | 13 | interface TabPanelProps { 14 | children: React.ReactNode; 15 | index: number; 16 | value: number; 17 | } 18 | 19 | function TabPanel(props: TabPanelProps) { 20 | const { children, value, index } = props; 21 | return ( 22 | 29 | ); 30 | } 31 | 32 | export const TablesComponent: React.FC = () => { 33 | const [tables, setTables] = useState([]); 34 | const [activeTab, setActiveTab] = useState(0); 35 | 36 | useEffect(() => { 37 | fetchTables().then((data) => setTables(data)); 38 | }, []); 39 | 40 | const handleTabChange = (event: React.SyntheticEvent, newValue: number) => { 41 | setActiveTab(newValue); 42 | }; 43 | 44 | return ( 45 | 46 | 56 | 57 | {tables.map((table, index) => ( 58 | 59 | ))} 60 | 61 | {tables.map((table, index) => ( 62 | 63 | 64 | 65 | ))} 66 | 67 | 68 | ); 69 | }; 70 | -------------------------------------------------------------------------------- /data-whisperer-backend/app/routes/run_command.py: -------------------------------------------------------------------------------- 1 | # app/routes/main.py 2 | 3 | import os 4 | import sys 5 | import re 6 | import io 7 | from contextlib import contextmanager, redirect_stdout 8 | import mock 9 | from langchain.agents import create_sql_agent 10 | from langchain.agents.agent_toolkits import SQLDatabaseToolkit 11 | from langchain.sql_database import SQLDatabase 12 | from langchain.llms.openai import OpenAI 13 | from flask_socketio import emit 14 | from flask import request, jsonify 15 | from flask import current_app 16 | 17 | 18 | from . import main_blueprint 19 | 20 | @contextmanager 21 | def st_capture(output_function): 22 | with io.StringIO() as stdout, redirect_stdout(stdout): 23 | old_write = stdout.write 24 | 25 | def new_write(string): 26 | ret = old_write(string) 27 | output_function(escape_ansi(stdout.getvalue())) 28 | return ret 29 | 30 | stdout.write = new_write 31 | yield 32 | 33 | def escape_ansi(line): 34 | return re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]').sub('', line) 35 | 36 | 37 | # Get the absolute path of the current file 38 | base_path = os.path.abspath(os.path.dirname(__file__)) 39 | 40 | @main_blueprint.route('/api/v1/run-command', methods=['POST']) 41 | def run_command(): 42 | # Construct the absolute path of the database file 43 | db_path = os.path.join(base_path, '..','..','sqlite_db', 'my_database.db') 44 | 45 | # Create the database connection 46 | db = SQLDatabase.from_uri(f"sqlite:///{db_path}") 47 | 48 | toolkit = SQLDatabaseToolkit(db=db) 49 | 50 | model_name = current_app.config['MODEL_NAME'] 51 | 52 | agent_executor = create_sql_agent( 53 | llm=OpenAI(temperature=0, model_name=model_name), 54 | toolkit=toolkit, 55 | verbose=True 56 | ) 57 | 58 | request_data = request.get_json() 59 | command = request_data.get("command", "") 60 | 61 | def emit_print(*args, **kwargs): 62 | message = " ".join(str(a) for a in args) 63 | 64 | # Use the escape_ansi function to remove ANSI escape codes before emitting the message 65 | escaped_message = escape_ansi(message) 66 | 67 | emit('output', {'message': escaped_message}, namespace='/', broadcast=True) 68 | sys.stdout.write(escaped_message + "\n") 69 | sys.stdout.flush() 70 | 71 | with mock.patch('builtins.print', emit_print): 72 | agent_executor.run(command) 73 | 74 | return jsonify({'running': command}) 75 | -------------------------------------------------------------------------------- /data-whisperer-frontend/src/components/sidebar-chat-container/PromptComponent/PromptComponent.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import { TextField, Button } from '@mui/material'; 3 | import SendIcon from '@mui/icons-material/Send'; 4 | import styles from './PromptComponent.module.css'; 5 | import axios from "axios"; 6 | import io from 'socket.io-client'; 7 | 8 | const PromptComponent: React.FC = () => { 9 | const [text, setText] = useState(""); 10 | const [generatedText, setGeneratedText] = useState(""); 11 | const [output, setOutput] = useState(""); // New state variable for the output 12 | 13 | const generateText = async () => { 14 | const API_BASE_URL = process.env.REACT_APP_API_BASE_URL || 'http://localhost:5000'; 15 | const API_URL = `${API_BASE_URL}/api/v1/run-command`; 16 | 17 | const data = { 18 | command: text, 19 | }; 20 | 21 | try { 22 | const response = await axios.post(API_URL, data); 23 | const generated = response.data.choices[0].message.content; 24 | setGeneratedText(generated); 25 | } catch (error) { 26 | console.error("Error generating text:", error); 27 | } 28 | }; 29 | 30 | // Socket.io integration 31 | useEffect(() => { 32 | const API_BASE_URL = process.env.REACT_APP_API_BASE_URL || 'http://localhost:5000'; 33 | const socket = io(API_BASE_URL); 34 | 35 | socket.on('output', (data) => { 36 | console.log(data.message); 37 | setOutput((prevOutput) => prevOutput + data.message + '\n'); // Update the output state 38 | }); 39 | 40 | return () => { 41 | socket.disconnect(); 42 | }; 43 | }, []); 44 | 45 | return ( 46 |
47 |
48 | setText(e.target.value)} 56 | /> 57 |
58 |
59 | 67 |
68 | {generatedText &&

Output: {generatedText}

} 69 | {/* Display the output */} 70 |
{output}
71 |
72 | ); 73 | }; 74 | 75 | export default PromptComponent; 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Langchain SQL Agent Bootstap 2 | This is a simple App for testing LLM to SQL commands on a sqlite database using [Langchain SQL Agent](https://python.langchain.com/en/latest/modules/agents/toolkits/examples/sql_database.html). The repo comes with a setup script that loads a sqlite database with some sample data. 3 | 4 | > This repo is intended to be a starter for Langchain and also a starter for programming with co-pilot GPT-4. The scope is small enough that you can even get by with copying/pasting whole clases to GPT-4 and asking for explanations or improvemnts to the code. 5 | >There's a single important TODO item that is not and will not be ever completed, which is to implement the file upload functionality in the UI. The backend is already setup to handle file uploads (csv). If you're interested you could probably also: 6 | > - clean the ouput of the agent after each execution 7 | > - add a auto-refresh or a refresh button to the tables 8 | 9 | Backend has several endpoints: 10 | - `/api/v1/tables` - returns a list of tables in the database 11 | - `/api/v1/tables/` - returns a list of columns in the table 12 | - `/api/v1/run-command` - takes a json payload with the command to execute and returns the results via a websocket 13 | - `/api/v1/upload` - takes a csv file and loads it into the database 14 | 15 | `run-command` is where all the fun is happening with the langchain sql agent. 16 | 17 | ![Example 1](images/whisp_10.PNG) 18 | ![Exampe 2](images/whisp_11.PNG) 19 | 20 | ## Running locally 21 | 22 | ### Pre-requisites 23 | Backend was setup with Python 3.10.10 24 | 25 | Frontend was setup with Node 18.15.0 26 | 27 | Experience may vary with other versions 28 | 29 | You have to have an open-api-api-key to use the langchain sql agent. 30 | 31 | Copy the `.env.example` file to `.env` and add your key to the `OPENAI_API_KEY` variable. You can also change the model which is used to make predictions. In the time of writing, `gpt-4` is in closed beta, so the backend might not work if you select that model. 32 | 33 | ### Backend 34 | Create virtual environment (only need to do this once) 35 | ```bash 36 | python3 -m venv data-whisperer-backend/myenv 37 | ``` 38 | activate virtual environment 39 | ```bash 40 | source data-whisperer-backend/myenv/bin/activate 41 | # install dependencies 42 | pip install -r data-whisperer-backend/requirements.txt 43 | ``` 44 | **seed the database** 45 | ```bash 46 | python data-whisperer-backend/setup/create_and_seed_db.py 47 | ``` 48 | finally, run the backend 49 | ```bash 50 | python data-whisperer-backend/run.py 51 | ``` 52 | 53 | ### Frontend 54 | ```bash 55 | cd data-whisperer-frontend/ 56 | npm install # only need to do this once 57 | npm start 58 | ``` 59 | 60 | ## Teardown 61 | ```bash 62 | # deactivate virtual environment after you are done 63 | deactivate 64 | ``` 65 | -------------------------------------------------------------------------------- /data-whisperer-backend/setup/data/raw_payments.csv: -------------------------------------------------------------------------------- 1 | id,order_id,payment_method,amount 2 | 1,1,credit_card,1000 3 | 2,2,credit_card,2000 4 | 3,3,coupon,100 5 | 4,4,coupon,2500 6 | 5,5,bank_transfer,1700 7 | 6,6,credit_card,600 8 | 7,7,credit_card,1600 9 | 8,8,credit_card,2300 10 | 9,9,gift_card,2300 11 | 10,9,bank_transfer,0 12 | 11,10,bank_transfer,2600 13 | 12,11,credit_card,2700 14 | 13,12,credit_card,100 15 | 14,13,credit_card,500 16 | 15,13,bank_transfer,1400 17 | 16,14,bank_transfer,300 18 | 17,15,coupon,2200 19 | 18,16,credit_card,1000 20 | 19,17,bank_transfer,200 21 | 20,18,credit_card,500 22 | 21,18,credit_card,800 23 | 22,19,gift_card,600 24 | 23,20,bank_transfer,1500 25 | 24,21,credit_card,1200 26 | 25,22,bank_transfer,800 27 | 26,23,gift_card,2300 28 | 27,24,coupon,2600 29 | 28,25,bank_transfer,2000 30 | 29,25,credit_card,2200 31 | 30,25,coupon,1600 32 | 31,26,credit_card,3000 33 | 32,27,credit_card,2300 34 | 33,28,bank_transfer,1900 35 | 34,29,bank_transfer,1200 36 | 35,30,credit_card,1300 37 | 36,31,credit_card,1200 38 | 37,32,credit_card,300 39 | 38,33,credit_card,2200 40 | 39,34,bank_transfer,1500 41 | 40,35,credit_card,2900 42 | 41,36,bank_transfer,900 43 | 42,37,credit_card,2300 44 | 43,38,credit_card,1500 45 | 44,39,bank_transfer,800 46 | 45,40,credit_card,1400 47 | 46,41,credit_card,1700 48 | 47,42,coupon,1700 49 | 48,43,gift_card,1800 50 | 49,44,gift_card,1100 51 | 50,45,bank_transfer,500 52 | 51,46,bank_transfer,800 53 | 52,47,credit_card,2200 54 | 53,48,bank_transfer,300 55 | 54,49,credit_card,600 56 | 55,49,credit_card,900 57 | 56,50,credit_card,2600 58 | 57,51,credit_card,2900 59 | 58,51,credit_card,100 60 | 59,52,bank_transfer,1500 61 | 60,53,credit_card,300 62 | 61,54,credit_card,1800 63 | 62,54,bank_transfer,1100 64 | 63,55,credit_card,2900 65 | 64,56,credit_card,400 66 | 65,57,bank_transfer,200 67 | 66,58,coupon,1800 68 | 67,58,gift_card,600 69 | 68,59,gift_card,2800 70 | 69,60,credit_card,400 71 | 70,61,bank_transfer,1600 72 | 71,62,gift_card,1400 73 | 72,63,credit_card,2900 74 | 73,64,bank_transfer,2600 75 | 74,65,credit_card,0 76 | 75,66,credit_card,2800 77 | 76,67,bank_transfer,400 78 | 77,67,credit_card,1900 79 | 78,68,credit_card,1600 80 | 79,69,credit_card,1900 81 | 80,70,credit_card,2600 82 | 81,71,credit_card,500 83 | 82,72,credit_card,2900 84 | 83,73,bank_transfer,300 85 | 84,74,credit_card,3000 86 | 85,75,credit_card,1900 87 | 86,76,coupon,200 88 | 87,77,credit_card,0 89 | 88,77,bank_transfer,1900 90 | 89,78,bank_transfer,2600 91 | 90,79,credit_card,1800 92 | 91,79,credit_card,900 93 | 92,80,gift_card,300 94 | 93,81,coupon,200 95 | 94,82,credit_card,800 96 | 95,83,credit_card,100 97 | 96,84,bank_transfer,2500 98 | 97,85,bank_transfer,1700 99 | 98,86,coupon,2300 100 | 99,87,gift_card,3000 101 | 100,87,credit_card,2600 102 | 101,88,credit_card,2900 103 | 102,89,bank_transfer,2200 104 | 103,90,bank_transfer,200 105 | 104,91,credit_card,1900 106 | 105,92,bank_transfer,1500 107 | 106,92,coupon,200 108 | 107,93,gift_card,2600 109 | 108,94,coupon,700 110 | 109,95,coupon,2400 111 | 110,96,gift_card,1700 112 | 111,97,bank_transfer,1400 113 | 112,98,bank_transfer,1000 114 | 113,99,credit_card,2400 115 | -------------------------------------------------------------------------------- /data-whisperer-backend/setup/data/raw_orders.csv: -------------------------------------------------------------------------------- 1 | id,user_id,order_date,status 2 | 1,1,2018-01-01,returned 3 | 2,3,2018-01-02,completed 4 | 3,94,2018-01-04,completed 5 | 4,50,2018-01-05,completed 6 | 5,64,2018-01-05,completed 7 | 6,54,2018-01-07,completed 8 | 7,88,2018-01-09,completed 9 | 8,2,2018-01-11,returned 10 | 9,53,2018-01-12,completed 11 | 10,7,2018-01-14,completed 12 | 11,99,2018-01-14,completed 13 | 12,59,2018-01-15,completed 14 | 13,84,2018-01-17,completed 15 | 14,40,2018-01-17,returned 16 | 15,25,2018-01-17,completed 17 | 16,39,2018-01-18,completed 18 | 17,71,2018-01-18,completed 19 | 18,64,2018-01-20,returned 20 | 19,54,2018-01-22,completed 21 | 20,20,2018-01-23,completed 22 | 21,71,2018-01-23,completed 23 | 22,86,2018-01-24,completed 24 | 23,22,2018-01-26,return_pending 25 | 24,3,2018-01-27,completed 26 | 25,51,2018-01-28,completed 27 | 26,32,2018-01-28,completed 28 | 27,94,2018-01-29,completed 29 | 28,8,2018-01-29,completed 30 | 29,57,2018-01-31,completed 31 | 30,69,2018-02-02,completed 32 | 31,16,2018-02-02,completed 33 | 32,28,2018-02-04,completed 34 | 33,42,2018-02-04,completed 35 | 34,38,2018-02-06,completed 36 | 35,80,2018-02-08,completed 37 | 36,85,2018-02-10,completed 38 | 37,1,2018-02-10,completed 39 | 38,51,2018-02-10,completed 40 | 39,26,2018-02-11,completed 41 | 40,33,2018-02-13,completed 42 | 41,99,2018-02-14,completed 43 | 42,92,2018-02-16,completed 44 | 43,31,2018-02-17,completed 45 | 44,66,2018-02-17,completed 46 | 45,22,2018-02-17,completed 47 | 46,6,2018-02-19,completed 48 | 47,50,2018-02-20,completed 49 | 48,27,2018-02-21,completed 50 | 49,35,2018-02-21,completed 51 | 50,51,2018-02-23,completed 52 | 51,71,2018-02-24,completed 53 | 52,54,2018-02-25,return_pending 54 | 53,34,2018-02-26,completed 55 | 54,54,2018-02-26,completed 56 | 55,18,2018-02-27,completed 57 | 56,79,2018-02-28,completed 58 | 57,93,2018-03-01,completed 59 | 58,22,2018-03-01,completed 60 | 59,30,2018-03-02,completed 61 | 60,12,2018-03-03,completed 62 | 61,63,2018-03-03,completed 63 | 62,57,2018-03-05,completed 64 | 63,70,2018-03-06,completed 65 | 64,13,2018-03-07,completed 66 | 65,26,2018-03-08,completed 67 | 66,36,2018-03-10,completed 68 | 67,79,2018-03-11,completed 69 | 68,53,2018-03-11,completed 70 | 69,3,2018-03-11,completed 71 | 70,8,2018-03-12,completed 72 | 71,42,2018-03-12,shipped 73 | 72,30,2018-03-14,shipped 74 | 73,19,2018-03-16,completed 75 | 74,9,2018-03-17,shipped 76 | 75,69,2018-03-18,completed 77 | 76,25,2018-03-20,completed 78 | 77,35,2018-03-21,shipped 79 | 78,90,2018-03-23,shipped 80 | 79,52,2018-03-23,shipped 81 | 80,11,2018-03-23,shipped 82 | 81,76,2018-03-23,shipped 83 | 82,46,2018-03-24,shipped 84 | 83,54,2018-03-24,shipped 85 | 84,70,2018-03-26,placed 86 | 85,47,2018-03-26,shipped 87 | 86,68,2018-03-26,placed 88 | 87,46,2018-03-27,placed 89 | 88,91,2018-03-27,shipped 90 | 89,21,2018-03-28,placed 91 | 90,66,2018-03-30,shipped 92 | 91,47,2018-03-31,placed 93 | 92,84,2018-04-02,placed 94 | 93,66,2018-04-03,placed 95 | 94,63,2018-04-03,placed 96 | 95,27,2018-04-04,placed 97 | 96,90,2018-04-06,placed 98 | 97,89,2018-04-07,placed 99 | 98,41,2018-04-07,placed 100 | 99,85,2018-04-09,placed 101 | -------------------------------------------------------------------------------- /data-whisperer-backend/setup/create_and_seed_db.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import csv 3 | from typing import Any 4 | 5 | def create_connection(db_file): 6 | conn = None 7 | try: 8 | conn = sqlite3.connect(db_file) 9 | print(f"SQLite version {sqlite3.version} - Connected to {db_file}") 10 | except Exception as err: 11 | print(err) 12 | 13 | return conn 14 | 15 | 16 | def create_table(conn, create_table_sql): 17 | try: 18 | cursor = conn.cursor() 19 | cursor.execute(create_table_sql) 20 | except Exception as err: 21 | print(err) 22 | 23 | 24 | def infer_data_type(value: str) -> Any: 25 | try: 26 | return int(value) 27 | except ValueError: 28 | try: 29 | return float(value) 30 | except ValueError: 31 | return value 32 | 33 | 34 | def insert_data(conn, sql_insert, data_path): 35 | data = [] 36 | 37 | with open(data_path, 'r', encoding='utf-8') as csvfile: 38 | csvreader = csv.DictReader(csvfile) 39 | 40 | for row in csvreader: 41 | values = tuple(infer_data_type(value) for value in row.values()) 42 | data.append(values) 43 | 44 | try: 45 | cursor = conn.cursor() 46 | cursor.executemany(sql_insert, data) 47 | conn.commit() 48 | print(f"{cursor.rowcount} rows inserted.") 49 | except Exception as err: 50 | print(err) 51 | 52 | def main(): 53 | # run this script from root/data-whisperer-backend 54 | # TODO: export this as a parameter 55 | database = "sqlite_db/my_database.db" 56 | 57 | sql_create_customers_table = """CREATE TABLE IF NOT EXISTS customers ( 58 | id INTEGER PRIMARY KEY AUTOINCREMENT, 59 | first_name TEXT NOT NULL, 60 | last_name TEXT NOT NULL 61 | );""" 62 | 63 | sql_create_orders_table = """CREATE TABLE IF NOT EXISTS orders ( 64 | id INTEGER PRIMARY KEY AUTOINCREMENT, 65 | user_id INTEGER NOT NULL, 66 | order_date TEXT NOT NULL, 67 | status TEXT NOT NULL, 68 | FOREIGN KEY (user_id) REFERENCES customers (id) 69 | );""" 70 | 71 | sql_create_payments_table = """CREATE TABLE IF NOT EXISTS payments ( 72 | id INTEGER PRIMARY KEY AUTOINCREMENT, 73 | order_id INTEGER NOT NULL, 74 | payment_method TEXT NOT NULL, 75 | amount REAL NOT NULL, 76 | FOREIGN KEY (order_id) REFERENCES orders (id) 77 | );""" 78 | 79 | sql_create_tables = [ 80 | sql_create_customers_table, 81 | sql_create_orders_table, 82 | sql_create_payments_table 83 | ] 84 | 85 | sql_insert_customerms = "INSERT INTO customers (id, first_name, last_name) VALUES (?, ?, ?)" 86 | sql_insert_orders = "INSERT INTO orders (id, user_id, order_date, status) VALUES (?, ?, ?, ?)" 87 | sql_insert_payments = "INSERT INTO payments (id, order_id, payment_method, amount) VALUES (?, ?, ?, ?)" 88 | 89 | customer_data_path = "setup/data/raw_customers.csv" 90 | order_data_path = "setup/data/raw_orders.csv" 91 | payment_data_path = "setup/data/raw_payments.csv" 92 | 93 | table_insert_data_touples = [ 94 | (sql_insert_customerms, customer_data_path), 95 | (sql_insert_orders, order_data_path), 96 | (sql_insert_payments, payment_data_path) 97 | ] 98 | 99 | # Create a connection to the SQLite database 100 | conn = create_connection(database) 101 | 102 | if conn is not None: 103 | 104 | for sql_create_table in sql_create_tables: 105 | create_table(conn, sql_create_table) 106 | 107 | for sql_insert, data_path in table_insert_data_touples: 108 | # Insert sample data 109 | insert_data(conn, sql_insert, data_path) 110 | else: 111 | print("Error! Cannot create the database connection.") 112 | 113 | 114 | if __name__ == '__main__': 115 | main() 116 | --------------------------------------------------------------------------------