├── data
└── bispap138.pdf
├── requirements.txt
├── screen-capture.gif
├── templates
└── index.html
├── static
├── css
│ └── main.css
└── js
│ └── app.js
├── README.md
├── .gitignore
└── app.py
/data/bispap138.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mickymultani/Streaming-LLM-Chat/HEAD/data/bispap138.pdf
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mickymultani/Streaming-LLM-Chat/HEAD/requirements.txt
--------------------------------------------------------------------------------
/screen-capture.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mickymultani/Streaming-LLM-Chat/HEAD/screen-capture.gif
--------------------------------------------------------------------------------
/templates/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | Local Chat Interface
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
tags
32 | chunk = chunk.replace(/\n/g, '
');
33 | botMessageContainer.innerHTML += chunk;
34 | }
35 | // Scroll the bot's message container into view
36 | botMessageContainer.scrollIntoView({ behavior: 'smooth' });
37 | } catch (error) {
38 | console.error('Error:', error);
39 | }
40 | }
41 |
42 | function appendMessage(text, className) {
43 | let messageContainer = createMessageContainer(className);
44 | messageContainer.innerHTML = text.replace(/\n/g, '
'); // Replace newlines with
45 | document.getElementById('chatArea').appendChild(messageContainer);
46 | messageContainer.scrollIntoView({ behavior: 'smooth' });
47 | }
48 |
49 | function createMessageContainer(className) {
50 | let div = document.createElement('div');
51 | div.classList.add('message', className);
52 | document.getElementById('chatArea').appendChild(div);
53 | return div;
54 | }
55 |
--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
1 | from flask import Flask, render_template, request, Response, stream_with_context
2 | from flask_cors import CORS # Import CORS
3 | from dotenv import load_dotenv
4 |
5 | import webbrowser
6 | from threading import Timer
7 | import os
8 | from llama_index import (
9 | ServiceContext,
10 | VectorStoreIndex,
11 | SimpleDirectoryReader,
12 | set_global_service_context,
13 | )
14 | from llama_index.llms import OpenAI
15 | from llama_index.memory import ChatMemoryBuffer
16 |
17 | app = Flask(__name__)
18 | CORS(app) # Enable CORS for all routes
19 |
20 | # Load environment variables
21 | load_dotenv()
22 |
23 | openai_api_key = os.getenv("OPENAI_API_KEY")
24 | if not openai_api_key:
25 | raise ValueError("The OPENAI_API_KEY environment variable is not set.")
26 |
27 |
28 | # Set up the service context for llama-index with the desired OpenAI model
29 | service_context = ServiceContext.from_defaults(
30 | llm=OpenAI(model="gpt-4", temperature=0)
31 | )
32 | set_global_service_context(service_context)
33 |
34 | # Load the data from the "data" directory
35 | data = SimpleDirectoryReader("data").load_data()
36 |
37 | # Create the index
38 | index = VectorStoreIndex.from_documents(data)
39 |
40 | # Configure the chat engine with a memory buffer
41 | memory = ChatMemoryBuffer.from_defaults(token_limit=20000)
42 | chat_engine = index.as_chat_engine(
43 | chat_mode="context",
44 | memory=memory,
45 | system_prompt=(
46 | "Act as an experienced risk and financial policy analyst"
47 | "You are now able to intelligently answer questions about the information you have been provided"
48 | ),
49 | )
50 |
51 | @app.route('/')
52 | def index():
53 | return render_template('index.html')
54 |
55 | @app.route('/chat', methods=['POST'])
56 | def chat():
57 | user_message = request.json.get('message')
58 | response = chat_engine.stream_chat(user_message)
59 | buffer = []
60 | buffer_size = 3
61 |
62 | def generate():
63 | for token in response.response_gen:
64 | buffer.append(token)
65 | if len(buffer) >= buffer_size:
66 | yield ''.join(buffer)
67 | buffer.clear()
68 | if buffer:
69 | yield ''.join(buffer)
70 |
71 | return Response(stream_with_context(generate()), content_type='text/plain')
72 |
73 | def open_browser():
74 | webbrowser.open_new('http://127.0.0.1:5000/')
75 |
76 | if __name__ == '__main__':
77 | if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
78 | Timer(1, open_browser).start()
79 | app.run(debug=True)
80 |
81 |
--------------------------------------------------------------------------------