├── Cli ├── __init__.py └── cli.py ├── localLlama ├── __init__.py └── server.py ├── litellm_uuid.txt ├── screenshots ├── promo1400.png ├── Screenshots.png ├── animate-use.gif ├── pixel-llama.png ├── Screenshots2.png ├── all screenshots.png └── llama-eatchrome-real.png ├── chrome-extension ├── images │ ├── 256.png │ ├── icon128.png │ ├── icon16.png │ ├── icon48.png │ └── llamalogo.png ├── manifest.json ├── popup.html ├── style.css └── javascript.js ├── .gitignore ├── setup.py └── Readme.md /Cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /localLlama/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /litellm_uuid.txt: -------------------------------------------------------------------------------- 1 | 8a5b31f9-9d54-4077-ad9c-626c671f0996 -------------------------------------------------------------------------------- /screenshots/promo1400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/promo1400.png -------------------------------------------------------------------------------- /screenshots/Screenshots.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/Screenshots.png -------------------------------------------------------------------------------- /screenshots/animate-use.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/animate-use.gif -------------------------------------------------------------------------------- /screenshots/pixel-llama.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/pixel-llama.png -------------------------------------------------------------------------------- /screenshots/Screenshots2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/Screenshots2.png -------------------------------------------------------------------------------- /chrome-extension/images/256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/chrome-extension/images/256.png -------------------------------------------------------------------------------- /screenshots/all screenshots.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/all screenshots.png -------------------------------------------------------------------------------- /chrome-extension/images/icon128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/chrome-extension/images/icon128.png -------------------------------------------------------------------------------- /chrome-extension/images/icon16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/chrome-extension/images/icon16.png -------------------------------------------------------------------------------- /chrome-extension/images/icon48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/chrome-extension/images/icon48.png -------------------------------------------------------------------------------- /chrome-extension/images/llamalogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/chrome-extension/images/llamalogo.png -------------------------------------------------------------------------------- /screenshots/llama-eatchrome-real.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdiamonddirt/local-llama-chrome-extension/HEAD/screenshots/llama-eatchrome-real.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /localLlama/__pycache__ 2 | build_publish.txt 3 | /tests 4 | .pypirc 5 | /build 6 | /dist 7 | litellm_uuid.txt 8 | local_llama.egg-info 9 | chrome-extension-v*.zip 10 | -------------------------------------------------------------------------------- /chrome-extension/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version": 3, 3 | "name": "Local LLama LLM AI Chat Query Tool", 4 | "version": "1.0.6", 5 | "description": "Query a local model from your browser.", 6 | "permissions": [ 7 | ], 8 | "action": { 9 | "default_popup": "popup.html", 10 | "default_icon": { 11 | "16": "images/icon16.png", 12 | "48": "images/icon48.png", 13 | "128": "images/icon128.png" 14 | } 15 | }, 16 | "icons": { 17 | "16": "images/icon16.png", 18 | "48": "images/icon48.png", 19 | "128": "images/icon128.png" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Cli/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from localLlama.server import app 3 | 4 | 5 | def start_server(ip='127.0.0.1', port=8000): 6 | print(f"API endpoint available at http://{ip}:{port}/") 7 | app.run(host=ip, port=port) 8 | 9 | 10 | def main(): 11 | parser = argparse.ArgumentParser( 12 | description="Start the server with optional IP and port arguments.") 13 | parser.add_argument("--ip", default="127.0.0.1", 14 | help="The IP address to bind to (default: 127.0.0.1)") 15 | parser.add_argument("--port", type=int, default=8000, 16 | help="The port to listen on (default: 8000)") 17 | args = parser.parse_args() 18 | 19 | start_server(ip=args.ip, port=args.port) 20 | 21 | 22 | if __name__ == "__main__": 23 | main() 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='local-llama', 5 | version='1.0.3', 6 | description='A flask server for a chrome extension that querys llm models', 7 | long_description='A flask server for a chrome extension that querys llm models for use with this chrome extention: https://chrome.google.com/webstore/detail/local-llama-llm-ai-chat-q/ekobbgdgkocdnnjoahoojakmoimfjlbm', 8 | url='https://github.com/mrdiamonddirt/local-llama-chrome-extension', 9 | author='Rowan Wood', 10 | author_email='mrdiamonddirt@gmail.com', 11 | packages=find_packages(), 12 | install_requires=[ 13 | 'Flask', 14 | 'flask-cors', 15 | 'rich', 16 | 'llama-cpp-python', 17 | 'typing-extensions', 18 | 'typing', 19 | ], 20 | entry_points={ 21 | 'console_scripts': [ 22 | 'local-llama = Cli.cli:main', 23 | ], 24 | }, 25 | ) 26 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Local LLama Chrome Extension 2 | ## What is this? 3 | 4 | This is a chrome extension and flask server that allows you to query the llama-cpp-python models while in the browser. It uses a local server to handle the queries and display the results in a popup. 5 | 6 | ## Why? 7 | 8 | To learn a little bit about chrome extensions and flask. And make a tool that i can use to query the models while using the browser. 9 | 10 | ## How does it work? 11 | 12 | The extension uses the chrome api to get the selected text and send it to the server. The server then queries the model and returns the results to the extension. The extension then displays the results in a popup. The conversations are stored in local storage. 13 | and can be cleared with the clear button in the popup. 14 | 15 | 16 | 17 | ## Showcase 18 | ![showcase](./screenshots/animate-use.gif) 19 | 20 | ## Prerequisites 21 | 22 | llama-cpp-python must be installed and some models must be downloaded. See [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) for more information. 23 | Models available for download from huggingface: 24 | - [TheBlokeAI]( 25 | https://huggingface.co/TheBloke/) 26 | i'v been using: 27 | - [TheBlokeAI/Llama-2-7B](https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF) 28 | for my testing but most of the gguf models should work. 29 | obviously the bigger the model the slower the query. and the more ram it will use. 30 | 31 | 32 | ## How to use it? 33 | 34 | ### easy mode (download from chrome store) 35 | 36 | 1. Download the extension from the chrome store 37 | 2. Pip install the server with `pip install local-llama` 38 | 3. Start the server with `local-llama` 39 | 4. Go to any page and click on the extension icon 40 | 5. query the model and press enter 41 | 6. The results will be displayed in the popup 42 | 43 | ### pro mode (download from github) 44 | 45 | 1. Clone this repo 46 | 2. Open Chrome and go to `chrome://extensions/` 47 | 3. Enable developer mode 48 | 4. Click on `Load unpacked` and select the folder where you cloned this repo 49 | 5. Go to any page and click on the extension icon 50 | 6. build the package with `python setup.py sdist bdist_wheel` 51 | 7. Install the package with `pip install .` 52 | 8. Start the server with `local-llama` 53 | 9. If this is the first time you are using the extension you will be prompted to enter the path for your default model 54 | 10. Type in the query and press enter 55 | 11. The results will be displayed in the popup 56 | 57 | ## TODO 58 | 59 | - [x] add a server to handle the queries 60 | - [x] add a popup to display the results 61 | - [x] store and retrieve conversations 62 | - [x] clear saved conversations 63 | - [x] add a settings page 64 | - [x] add a way to change the model easily 65 | - [x] turn the server.py into a proper python package, to make it easier to install and use if downloaded from the chrome store 66 | - [x] add a way to change the server address 67 | - [ ] hadle when an html is added to the codeblock 68 | - [ ] add a way to download models from huggingface 69 | - [ ] add a way to start the server from the extension -------------------------------------------------------------------------------- /chrome-extension/popup.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Local Model Query 5 | 6 | 7 | 8 | 9 |
10 |
11 |
12 | 13 |

Local Llama

14 |
15 | 16 |
17 | 18 | 19 | 20 | 21 |
22 |
23 |
24 |
25 | 26 |

Settings

27 |

Model Loaded:

28 |

29 |

Check the Server is Live:

30 |

Select Model 31 | 33 |

34 | 35 |

Server IP

36 | 37 |

Server Port

38 | 39 |
40 |
41 |
42 |
43 | 44 |

Help/Instructions

45 |

Thank you for choosing our Chrome extension. Below are step-by-step instructions to get started:

46 | 47 |
    48 |
  1. Install the accompanying Flask server using pip:
    2. 49 | 
    pip install local-llama
    50 | 51 |
  2. Run the server from the command line:
    3. 52 | 
    local-llama
    53 | 54 |
  3. Select your desired default model. If your folder contains multiple models, you can change them in the settings.
    4. 55 | 56 |
  4. Use the popup to query the server and harness its capabilities.
    5. 57 | 58 |
  5. Check the server's status with the "Check Server" button. Green indicates the server is up, while red indicates it's not.
    6. 59 |
60 | 61 |

For more detailed information, please visit our GitHub repository.

62 |
63 |
64 |
65 |
66 | 67 |

Creator: Rowan Wood

68 |

Local Llama is a project that aims to make it easier to query local models. It is currently in development.

69 |

For Issues, Latest Versions and to Support please visit the GitHub repository.

70 |
71 |
72 | 73 |
74 | 75 | 76 | -------------------------------------------------------------------------------- /localLlama/server.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from flask import Flask, request, jsonify 4 | from flask_cors import CORS 5 | import traceback 6 | 7 | from rich import print 8 | from llama_cpp import Llama 9 | from typing_extensions import TypedDict, Literal 10 | from typing import List 11 | 12 | Role = Literal["system", "user", "assistant"] 13 | 14 | 15 | class Message(TypedDict): 16 | role: Role 17 | content: str 18 | 19 | 20 | B_INST, E_INST = "[INST]", "[/INST]" 21 | B_SYS, E_SYS = "<>\n", "\n<>\n\n" 22 | DEFAULT_SYSTEM_PROMPT = """\ 23 | You are a helpful, respectful and honest assistant. Always answer as helpfully as possible. 24 | 25 | If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""" 26 | 27 | 28 | def make_prompt_llama2(llm, messages: List[Message]) -> List[int]: 29 | if messages[0]["role"] != "system": 30 | messages = [ 31 | { 32 | "role": "system", 33 | "content": DEFAULT_SYSTEM_PROMPT, 34 | } 35 | ] + messages 36 | 37 | messages = [ 38 | { 39 | "role": messages[1]["role"], 40 | "content": B_SYS + messages[0]["content"] + E_SYS + messages[1]["content"], 41 | } 42 | ] + messages[2:] 43 | 44 | # Print all messages 45 | print("All Messages:") 46 | for msg in messages: 47 | print(f"Role: {msg['role']}, Content: {msg['content']}") 48 | 49 | assert all([msg["role"] == "user" for msg in messages[::2]]) and all( 50 | [msg["role"] == "assistant" for msg in messages[1::2]] 51 | ), ( 52 | "model only supports 'system', 'user' and 'assistant' roles, " 53 | "starting with 'system', then 'user' and alternating (u/a/u/a/u...)" 54 | ) 55 | 56 | dialog_tokens = sum( 57 | [ 58 | llm.tokenize( 59 | bytes( 60 | f"{B_INST} {(prompt['content']).strip()} {E_INST} {(answer['content']).strip()} ", 61 | "utf-8", 62 | ), 63 | add_bos=True, 64 | ) 65 | + [llm.token_eos()] 66 | for prompt, answer in zip( 67 | messages[::2], 68 | messages[1::2], 69 | ) 70 | ], 71 | [], 72 | ) 73 | 74 | assert messages[-1]["role"] == "user", f"Last message must be from user, got {messages[-1]['role']}" 75 | 76 | dialog_tokens += llm.tokenize( 77 | bytes( 78 | f"{B_INST} {(messages[-1]['content']).strip()} {E_INST}", "utf-8"), 79 | add_bos=True, 80 | ) 81 | 82 | return dialog_tokens 83 | 84 | 85 | class LLMChatBot: 86 | def __init__(self, model_path): 87 | if not os.path.exists(model_path): 88 | print("Model not found at the specified path.") 89 | return 90 | 91 | self.llama = Llama(model_path=model_path, n_ctx=1024, n_gpu_layers=-1) 92 | print("Local LLM loaded successfully.") 93 | 94 | def get_response(self, user_message): 95 | print("User message:", user_message) 96 | print("Generating response...") 97 | messages: List[Message] = [ 98 | Message(role="user", content=user_message), 99 | ] 100 | 101 | tokens = make_prompt_llama2(self.llama, messages) 102 | 103 | completion = self.llama.generate(tokens=tokens, temp=0.01) 104 | 105 | response_text = "" 106 | 107 | for token in completion: 108 | if token == self.llama.token_eos(): 109 | break 110 | response_text += self.llama.detokenize([token]).decode("utf-8") 111 | print("Response:", response_text) 112 | return response_text 113 | 114 | 115 | # Flask app initialization 116 | app = Flask(__name__) 117 | 118 | # Enable CORS 119 | CORS(app) 120 | 121 | # File path to store the selected model information 122 | selected_model_path = os.path.join(os.path.expanduser( 123 | "~"), ".Llama_extension", "selected_model.json") 124 | 125 | # Function to save the selected model information to a JSON file 126 | def save_selected_model(model_path): 127 | try: 128 | os.makedirs(os.path.dirname(selected_model_path), exist_ok=True) 129 | with open(selected_model_path, "w") as file: 130 | json.dump({"modelPath": model_path}, file) 131 | except Exception as e: 132 | print(f'Error while saving selected model information: {str(e)}') 133 | 134 | # Function to load the selected model information from the JSON file 135 | def load_selected_model(): 136 | try: 137 | if os.path.exists(selected_model_path): 138 | with open(selected_model_path, "r") as file: 139 | data = json.load(file) 140 | return data.get("modelPath", None) 141 | except Exception as e: 142 | print(f'Error while loading selected model information: {str(e)}') 143 | return None 144 | 145 | 146 | # Check if a default model is already selected, otherwise prompt for selection 147 | if not load_selected_model(): 148 | # Prompt the user to select the default model and store it 149 | default_model_path = input( 150 | "Please enter the path to the default model: ").strip() 151 | save_selected_model(default_model_path) 152 | 153 | # Use the selected default model path 154 | model_path = load_selected_model() 155 | 156 | bot = LLMChatBot(model_path=model_path) 157 | 158 | 159 | @app.route('/set_selected_folder', methods=['POST', 'OPTIONS']) 160 | def set_selected_folder(): 161 | global selected_folder 162 | try: 163 | data = request.get_json() 164 | selected_folder = data.get('selectedFolder', '') 165 | print(f'Selected folder path: {selected_folder}') 166 | return jsonify({'status': 'Folder path set successfully'}) 167 | except Exception as e: 168 | print(f'Error: {str(e)}') 169 | return jsonify({'error': str(e)}), 500 170 | 171 | 172 | @app.route('/get_selected_folder', methods=['GET', 'OPTIONS']) 173 | def get_selected_folder(): 174 | global selected_folder 175 | try: 176 | return jsonify({'selectedFolder': selected_folder}) 177 | except Exception as e: 178 | print(f'Error: {str(e)}') 179 | return jsonify({'error': str(e)}), 500 180 | 181 | @app.route('/get_current_model', methods=['GET', 'OPTIONS']) 182 | def get_current_model(): 183 | try: 184 | return jsonify({'current_model': model_path}) 185 | except Exception as e: 186 | print(f'Error: {str(e)}') 187 | return jsonify({'error': str(e)}), 500 188 | 189 | @app.route('/load_model', methods=['POST']) 190 | def load_model(): 191 | try: 192 | data = request.get_json() 193 | print(f'Loading model: {data}') 194 | model_path_file = os.path.dirname(model_path) 195 | print(f'Loading model: {model_path_file}') 196 | print(f'Loading model: {data}') 197 | model_path_new = os.path.join(model_path_file, data.get('model', '')) 198 | bot = LLMChatBot(model_path=model_path_new) 199 | return jsonify({'status': 'Model loaded successfully'}) 200 | except Exception as e: 201 | print(f'Error: {str(e)}') 202 | return jsonify({'error': str(e)}), 500 203 | 204 | # Route to get the list of .gguf files in the model directory 205 | @app.route('/get_gguf_files', methods=['GET', 'OPTIONS']) 206 | def get_gguf_files(): 207 | try: 208 | gguf_files = [] 209 | if model_path: 210 | # remove the file name from the path 211 | model_path_file = os.path.dirname(model_path) 212 | for file in os.listdir(model_path_file): 213 | if file.endswith(".gguf"): 214 | gguf_files.append(file) 215 | return jsonify({'gguf_files': gguf_files}) 216 | except Exception as e: 217 | print(f'Error: {str(e)}') 218 | return jsonify({'error': str(e)}), 500 219 | 220 | @app.route('/query', methods=['POST']) 221 | def handle_query(): 222 | try: 223 | data = request.get_json() 224 | user_message = data.get('query', '') 225 | print(f'Received query: {user_message}') 226 | response = bot.get_response(user_message) 227 | print(f'Sending response: {response}') 228 | return jsonify({'response': response}) 229 | except Exception as e: 230 | # print(f'Error: {str(e)}') 231 | traceback.print_exc() 232 | return jsonify({'error': str(e)}), 500 233 | 234 | @app.route('/health', methods=['GET', 'OPTIONS']) 235 | def health_check(): 236 | try: 237 | return jsonify({'status': 'healthy'}) 238 | except Exception as e: 239 | print(f'Error: {str(e)}') 240 | return jsonify({'error': str(e)}), 500 241 | 242 | 243 | 244 | if __name__ == "__main__": 245 | print("API endpoint available.") 246 | app.run(host='127.0.0.1', port=8000) -------------------------------------------------------------------------------- /chrome-extension/style.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css2?family=Poppins:wght@100&display=swap"); 2 | 3 | :root { 4 | height: 600px; 5 | width: 300px; 6 | border: 0; 7 | margin: 0; 8 | padding: 0; 9 | /* background-color: blue; */ 10 | position: relative; 11 | overflow: hidden; 12 | font-family: "Poppins", sans-serif; 13 | color: white; 14 | } 15 | 16 | ::-webkit-scrollbar { 17 | width: 5px; 18 | height: 5px; 19 | } 20 | 21 | /* Track */ 22 | ::-webkit-scrollbar-track { 23 | margin-top: 20px; 24 | margin-bottom: 50px; 25 | box-shadow: inset 0 0 5px grey; 26 | border-radius: 10px; 27 | } 28 | 29 | /* Handle */ 30 | ::-webkit-scrollbar-thumb { 31 | background: rgb(4, 249, 4); 32 | border-radius: 10px; 33 | } 34 | 35 | body { 36 | height: 220px; 37 | width: 500px; 38 | overflow: hidden; 39 | margin-bottom: 5px; 40 | } 41 | 42 | h2 { 43 | font-size: 1.5rem; 44 | font-weight: 100; 45 | margin: 0px 0px 0px 5px; 46 | padding: 0; 47 | } 48 | 49 | .user { 50 | display: flex; 51 | width: fit-content; 52 | background-color: darkgreen; 53 | border: 1px solid grey; 54 | border-radius: 5px; 55 | margin: 5px; 56 | padding: 5px; 57 | flex: 1; 58 | } 59 | 60 | .bot { 61 | position: relative; 62 | background-color: darkblue; 63 | border: 1px solid grey; 64 | border-radius: 5px; 65 | margin: 5px; 66 | padding: 5px; 67 | } 68 | 69 | .error { 70 | background-color: rgb(249, 4, 4); 71 | border: 1px solid grey; 72 | border-radius: 5px; 73 | margin: 5px; 74 | padding: 5px; 75 | flex: 1; 76 | } 77 | 78 | .code { 79 | position: relative; 80 | background-color: #000000; /* Light gray background color */ 81 | padding: 10px; /* Padding for code blocks */ 82 | border: 1px solid #ccc; /* Border around code blocks */ 83 | border-radius: 4px; /* Rounded corners for code blocks */ 84 | font-family: monospace; /* Use a monospace font for code */ 85 | font-size: 14px; /* Adjust font size as needed */ 86 | color: #1eff00; /* Text color for code */ 87 | /* Add any other styling you need */ 88 | word-wrap: wrap; 89 | } 90 | 91 | .loading { 92 | background-color: rgb(70, 1, 182); 93 | border: 1px solid grey; 94 | border-radius: 5px; 95 | margin: 5px; 96 | padding: 5px; 97 | flex: 1; 98 | animation: hueshift 5s infinite; 99 | width: fit-content; 100 | } 101 | 102 | @keyframes hueshift { 103 | 0% { 104 | background-color: hsl(0, 100%, 50%); /* Red */ 105 | color: hsl(120, 100%, 50%); 106 | } 107 | 14.29% { 108 | background-color: hsl(30, 100%, 50%); /* Orange */ 109 | color: hsl(180, 100%, 50%); 110 | } 111 | 28.57% { 112 | background-color: hsl(60, 100%, 50%); /* Yellow */ 113 | color: hsl(240, 100%, 50%); 114 | } 115 | 42.86% { 116 | background-color: hsl(120, 100%, 50%); /* Green */ 117 | color: hsl(270, 100%, 50%); 118 | } 119 | 57.14% { 120 | background-color: hsl(180, 100%, 50%); /* Cyan */ 121 | color: hsl(300, 100%, 50%); 122 | } 123 | 71.43% { 124 | background-color: hsl(240, 100%, 50%); /* Blue */ 125 | color: hsl(0, 100%, 50%); 126 | } 127 | 85.71% { 128 | background-color: hsl(270, 100%, 50%); /* Indigo */ 129 | color: hsl(30, 100%, 50%); 130 | } 131 | 100% { 132 | background-color: hsl(300, 100%, 50%); /* Violet */ 133 | color: hsl(60, 100%, 50%); 134 | } 135 | } 136 | 137 | .repo-link { 138 | color: white; 139 | text-decoration: none; 140 | } 141 | 142 | .repo-link:hover { 143 | scale: 1.1; 144 | color:hsl(0, 100%, 50%) 145 | } 146 | 147 | pre { 148 | padding: 0; 149 | margin: 0; 150 | white-space: pre-wrap; 151 | overflow-wrap: break-word; 152 | word-wrap: break-word; 153 | } 154 | 155 | #copyCode { 156 | position: absolute; 157 | right: -5px; 158 | top: -5px; 159 | /* background-color: rgb(4, 249, 4); */ 160 | border: 1px solid grey; 161 | border-radius: 5px; 162 | cursor: crosshair; 163 | background-color: #000000; 164 | } 165 | 166 | #copyCode:hover { 167 | scale: 1.1; 168 | } 169 | 170 | #submitButton { 171 | background-color: rgb(4, 249, 4); 172 | border: 1px solid grey; 173 | border-radius: 5px; 174 | margin: 0px; 175 | padding: 5px; 176 | color: Black; 177 | font-size: 12px; 178 | font-weight: bold; 179 | cursor: pointer; 180 | outline: none; 181 | } 182 | 183 | #settingMenuBtn { 184 | background-color: rgb(4, 249, 4); 185 | border: 1px solid grey; 186 | border-radius: 5px; 187 | margin: 0px; 188 | padding: 5px; 189 | color: Black; 190 | font-size: 12px; 191 | font-weight: bold; 192 | cursor: pointer; 193 | outline: none; 194 | } 195 | 196 | #ClearConvo { 197 | position: absolute; 198 | right: 0; 199 | background-color: rgb(249, 114, 4); 200 | border: 1px solid grey; 201 | border-radius: 5px; 202 | margin: 5px; 203 | padding: 5px; 204 | color: Black; 205 | font-size: 12px; 206 | font-weight: bold; 207 | cursor: pointer; 208 | outline: none; 209 | } 210 | 211 | #queryInput { 212 | background-color: rgb(43, 43, 43); 213 | border: 1px solid grey; 214 | border-radius: 5px; 215 | margin: 5px; 216 | padding: 5px; 217 | color: white; 218 | font-size: 10px; 219 | font-weight: 100; 220 | outline: none; 221 | width: 90%; 222 | height: 20px; 223 | resize: none; 224 | overflow: hidden; 225 | overflow-wrap: break-word; 226 | word-wrap: break-word; 227 | white-space: pre-wrap; 228 | word-break: break-all; 229 | word-break: break-word; 230 | -ms-word-break: break-all; 231 | -ms-word-break: break-word; 232 | -webkit-hyphens: auto; 233 | -moz-hyphens: auto; 234 | hyphens: auto; 235 | text-align: left; 236 | line-height: 1.5; 237 | font-family: "Poppins", sans-serif; 238 | font-size: 10px; 239 | font-weight: 100; 240 | } 241 | 242 | #serverCheck { 243 | background-color: rgba(4, 241, 249, 0.795); 244 | border: 1px solid grey; 245 | border-radius: 5px; 246 | margin: 5px; 247 | padding: 5px; 248 | color: black; 249 | font-size: 12px; 250 | font-weight: 100; 251 | cursor: pointer; 252 | outline: none; 253 | } 254 | 255 | 256 | #result { 257 | background-color: rgb(43, 43, 43); 258 | height: 80vh; 259 | border: 1px solid black; 260 | border-radius: 5px; 261 | margin: 0px 5px 0px 5px; 262 | overflow-y: scroll; 263 | color: white; 264 | font-size: 10px; 265 | /* chrome scroll bar customisation */ 266 | -webkit-font-smoothing: antialiased; 267 | -moz-osx-font-smoothing: grayscale; 268 | } 269 | 270 | #chat-wrapper { 271 | position: fixed; 272 | top: 0; 273 | right: 0; 274 | height: 100vh; 275 | width: 300px; 276 | background-color: rgb(53, 53, 53); 277 | resize: both; /* Allow the window to be resized */ 278 | } 279 | 280 | #settings-wrapper { 281 | display: none; 282 | position: absolute; 283 | bottom: 0; 284 | background-color: rgb(53, 53, 53); 285 | height: fit-content; 286 | z-index: 1; 287 | margin: 5px; 288 | padding: 5px; 289 | } 290 | 291 | #controls-wrapper { 292 | display: flex; 293 | flex-direction: row; 294 | justify-content: space-around; 295 | align-items: flex-start; 296 | margin: 5px; 297 | padding: 5px; 298 | } 299 | 300 | #settingsClose { 301 | position: absolute; 302 | right: 0; 303 | top: 0; 304 | } 305 | 306 | #settings-content { 307 | font-size: 12px; 308 | width: 250px; 309 | height: 400px; 310 | color: white; 311 | } 312 | 313 | #creator-wrapper { 314 | display: none; 315 | position: absolute; 316 | bottom: 0; 317 | background-color: rgb(53, 53, 53); 318 | height: fit-content; 319 | z-index: 1; 320 | margin: 5px; 321 | padding: 5px; 322 | color: #1eff00; 323 | } 324 | 325 | 326 | #creator-content { 327 | font-size: 10px; 328 | } 329 | 330 | #creatorClose { 331 | position: absolute; 332 | right: 0; 333 | top: 0; 334 | } 335 | #creatorButton { 336 | background-color: yellow; 337 | border: 1px solid grey; 338 | border-radius: 5px; 339 | padding: 5px; 340 | color: black; 341 | font-size: 12px; 342 | font-weight: bold; 343 | cursor: pointer; 344 | outline: none; 345 | } 346 | 347 | #helpButton { 348 | background-color: rgb(4, 249, 4); 349 | border: 1px solid grey; 350 | border-radius: 5px; 351 | margin: 0px 5px 0px 5px; 352 | padding: 5px; 353 | color: black; 354 | font-size: 12px; 355 | font-weight: bold; 356 | cursor: pointer; 357 | outline: none; 358 | } 359 | 360 | #help-wrapper { 361 | display: none; 362 | position: absolute; 363 | bottom: 0; 364 | background-color: rgb(53, 53, 53); 365 | height: fit-content; 366 | z-index: 1; 367 | margin: 5px; 368 | padding: 5px; 369 | color: #1eff00; 370 | } 371 | 372 | #helpClose { 373 | position: absolute; 374 | right: 0; 375 | top: 0; 376 | } -------------------------------------------------------------------------------- /chrome-extension/javascript.js: -------------------------------------------------------------------------------- 1 | document.addEventListener('DOMContentLoaded', function () { 2 | const queryInput = document.getElementById('queryInput'); 3 | const submitButton = document.getElementById('submitButton'); 4 | const resultDiv = document.getElementById('result'); 5 | const serverCheckButton = document.getElementById('serverCheck'); 6 | const creatorButton = document.getElementById('creatorButton'); 7 | const creatorWindow = document.getElementById('creator-wrapper'); 8 | const creatorClose = document.getElementById('creatorClose'); 9 | const helpWrapper = document.getElementById('help-wrapper'); 10 | const helpButton = document.getElementById('helpButton'); 11 | const helpClose = document.getElementById('helpClose'); 12 | const clearConvo = document.getElementById('ClearConvo'); 13 | const settingsButton = document.getElementById('settingMenuBtn'); 14 | const settingsWindow = document.getElementById('settings-wrapper'); 15 | const settingsClose = document.getElementById('settingsClose'); 16 | const modelSelect = document.getElementById('modelSelect'); 17 | const changeModelButton = document.getElementById('changeModel'); 18 | const serverIP = document.getElementById('serverIP'); 19 | const serverPort = document.getElementById('serverPort'); 20 | // const serverStart = document.getElementById('serverStart'); 21 | 22 | var creatorContentOpen = false; 23 | var helpContentOpen = false; 24 | var code_response = null; 25 | 26 | var ip = '127.0.0.1'; 27 | var port = '8000'; 28 | var url = 'http://' + ip + ':' + port + '/'; 29 | 30 | function set_value_of_ip_and_port() { 31 | serverIP.value = ip; 32 | serverPort.value = port; 33 | } 34 | // get the current ip and port from the settings 35 | serverIP.addEventListener('change', function () { 36 | ip = serverIP.value; 37 | url = 'http://' + ip + ':' + port + '/'; 38 | console.log(url) 39 | }); 40 | 41 | serverPort.addEventListener('change', function () { 42 | port = serverPort.value; 43 | url = 'http://' + ip + ':' + port + '/'; 44 | console.log(url) 45 | }); 46 | 47 | 48 | // if the settings are open get the current model selected in modelSelect and send it to the server to be loaded 49 | changeModelButton.addEventListener('click', function () { 50 | var model = modelSelect.options[modelSelect.selectedIndex].value; 51 | console.log(model); 52 | //load the model url 53 | var req_url = 'http://' + ip + ':' + port + '/load_model'; 54 | fetch(req_url, { 55 | method: 'POST', 56 | headers: { 57 | 'Content-Type': 'application/json', 58 | 59 | }, 60 | mode: 'cors', 61 | body: JSON.stringify({ model: model }), 62 | }) 63 | .then(response => response.json()) 64 | .then(data => { 65 | console.log("Received data:", data); 66 | 67 | // Check if data.current_model is defined 68 | if (data.current_model) { 69 | // change the current model 70 | document.getElementById('modelLoaded').innerText = data.current_model; 71 | } else { 72 | console.log("No current_model data found."); 73 | } 74 | }) 75 | .catch(error => { 76 | console.error("Error:", error); 77 | }); 78 | }); 79 | 80 | function saveQuestionAndResponse(question, response) { 81 | const savedData = JSON.parse(localStorage.getItem('savedCoversation')) || []; 82 | savedData.push({ question, response }); 83 | localStorage.setItem('savedCoversation', JSON.stringify(savedData)); 84 | } 85 | 86 | // Function to recover and display saved questions and responses 87 | function displaySavedData() { 88 | const savedData = JSON.parse(localStorage.getItem('savedCoversation')) || []; 89 | for (const entry of savedData) { 90 | resultDiv.innerHTML += `
You said: ${entry.question}
`; 91 | resultDiv.innerHTML += `
Response: ${entry.response}
`; 92 | } 93 | } 94 | 95 | function clearSavedData() { 96 | localStorage.removeItem('savedCoversation'); 97 | resultDiv.innerHTML = ''; 98 | } 99 | 100 | // Call this function to display saved data when the window is opened 101 | displaySavedData(); 102 | 103 | resultDiv.addEventListener('click', function (event) { 104 | if (event.target.id === 'copyCode') { 105 | console.log('copying code'); 106 | // Code to copy the text when the copy button is clicked 107 | var div = event.target.parentNode; 108 | console.log(div); 109 | var code = div.innerText; 110 | // replace the last word copy with nothing 111 | code = code.slice(0, -4); 112 | console.log(code); 113 | navigator.clipboard.writeText(code); 114 | } 115 | }); 116 | 117 | queryInput.addEventListener('keyup', function (event) { 118 | if (event.key === 'Enter') { 119 | if (queryInput.value === '') { 120 | return; 121 | } else { 122 | submitButton.click(); 123 | queryInput.value = ''; 124 | } 125 | } 126 | }); 127 | 128 | clearConvo.addEventListener('click', function () { 129 | clearSavedData(); 130 | resultDiv.innerHTML = ''; 131 | }); 132 | 133 | creatorButton.addEventListener('click', function () { 134 | if (creatorContentOpen) { 135 | creatorContentOpen = false; 136 | creatorWindow.style.display = 'none'; 137 | } else { 138 | creatorContentOpen = true; 139 | creatorWindow.style.display = 'block'; 140 | } 141 | }); 142 | 143 | creatorClose.addEventListener('click', function () { 144 | creatorContentOpen = false; 145 | creatorWindow.style.display = 'none'; 146 | }); 147 | 148 | helpButton.addEventListener('click', function () { 149 | if (helpContentOpen) { 150 | helpContentOpen = false; 151 | helpWrapper.style.display = 'none'; 152 | } else { 153 | helpContentOpen = true; 154 | helpWrapper.style.display = 'block'; 155 | } 156 | }); 157 | 158 | helpClose.addEventListener('click', function () { 159 | helpContentOpen = false; 160 | helpWrapper.style.display = 'none'; 161 | }); 162 | 163 | settingsButton.addEventListener('click', function () { 164 | if (settingsWindow.style.display === 'block') { 165 | settingsWindow.style.display = 'none'; 166 | } else { 167 | settingsWindow.style.display = 'block'; 168 | set_value_of_ip_and_port(); 169 | get_current_model(); 170 | 171 | get_ggufs(); 172 | } 173 | }); 174 | 175 | function get_current_model() { 176 | // get the current model from the server 177 | var req_url = 'http://' + ip + ':' + port + '/get_current_model'; 178 | fetch(req_url, { 179 | method: 'GET', 180 | headers: { 181 | 'Content-Type': 'application/json', 182 | 'Access-Control-Allow-Origin': '*', 183 | }, 184 | mode: 'cors', 185 | }) 186 | .then(response => response.json()) 187 | .then(data => { 188 | console.log("Received data:", data); 189 | 190 | // Check if data.current_model is defined 191 | if (data.current_model) { 192 | // set the current model 193 | document.getElementById('modelLoaded').innerText = data.current_model; 194 | } else { 195 | console.log("No current_model data found."); 196 | } 197 | }) 198 | .catch(error => { 199 | console.error("Error:", error); 200 | }); 201 | } 202 | 203 | function get_ggufs() { 204 | // get models from the server 205 | var req_url = 'http://' + ip + ':' + port + '/get_gguf_files'; 206 | fetch(req_url, { 207 | method: 'GET', 208 | headers: { 209 | 'Content-Type': 'application/json', 210 | 'Access-Control-Allow-Origin': '*', 211 | }, 212 | mode: 'cors', 213 | }) 214 | .then(response => response.json()) 215 | .then(data => { 216 | console.log("Received data:", data); 217 | 218 | // Check if data.gguf_files is defined 219 | if (data.gguf_files && data.gguf_files.length) { 220 | // remove the current options 221 | var select = document.getElementById('modelSelect'); 222 | var length = select.options.length; 223 | for (i = length-1; i >= 0; i--) { 224 | select.options[i] = null; 225 | } 226 | 227 | // add the new options 228 | for (var i = 0; i < data.gguf_files.length; i++) { 229 | var opt = document.createElement('option'); 230 | opt.value = data.gguf_files[i]; 231 | opt.innerHTML = data.gguf_files[i]; 232 | select.appendChild(opt); 233 | } 234 | } else { 235 | console.log("No gguf_files data found."); 236 | } 237 | }) 238 | .catch(error => { 239 | console.error("Error:", error); 240 | }); 241 | } 242 | 243 | settingsClose.addEventListener('click', function () { 244 | settingsWindow.style.display = 'none'; 245 | }); 246 | 247 | 248 | serverCheckButton.addEventListener('click', function () { 249 | // guery the server to check if it is running 250 | // if you get a response then turn the button green 251 | // if you don't get a response then turn the button red 252 | var req_url = 'http://' + ip + ':' + port + '/health'; 253 | fetch(req_url, { 254 | method: 'GET', 255 | headers: { 256 | 'Content-Type': 'application/json', 257 | 'Access-Control-Allow-Origin': '*', 258 | }, 259 | mode: 'cors', 260 | }) 261 | .then(response => response.json()) 262 | .then(data => { 263 | serverCheckButton.style.backgroundColor = 'green'; 264 | }) 265 | .catch(error => { 266 | serverCheckButton.style.backgroundColor = 'red'; 267 | }); 268 | }) 269 | 270 | // serverStart.addEventListener('click', function () { 271 | // chrome.runtime.sendNativeMessage('com.local.llama', { command: 'run_python_script' }, function(response) { 272 | // // Handle the response from the Native Host application (if needed) 273 | // }); 274 | // }); 275 | 276 | 277 | 278 | submitButton.addEventListener('click', function () { 279 | const query = queryInput.value; 280 | your_prompt = queryInput.value 281 | resultDiv.innerHTML += `
You said: ${your_prompt}
`; 282 | 283 | resultDiv.innerHTML += `
"Llama is thinking..."
`; 284 | resultDiv.scrollTop = resultDiv.scrollHeight; 285 | 286 | var req_url = 'http://' + ip + ':' + port + '/query'; 287 | fetch(req_url, { 288 | method: 'POST', 289 | headers: { 290 | 'Content-Type': 'application/json', 291 | 'Access-Control-Allow-Origin': '*', 292 | }, 293 | body: JSON.stringify({ query: query }), 294 | mode: 'cors', 295 | }) 296 | .then(response => response.json()) 297 | .then(data => { 298 | // remove the loading text 299 | const loading = document.querySelector('.loading'); 300 | loading.parentNode.removeChild(loading); 301 | 302 | // resultDiv.textContent = `Response from the local model: ${data.response}`; 303 | data.response = data.response.replace(/(?:\r\n|\r|\n)/g, '
'); 304 | // if * or a number follows a period and a space before 305 | data.response = data.response.replace(/\. \*/g, '\n'); 306 | // also handle numbered lists 307 | data.response = data.response.replace(/\. \d/g, '\n'); 308 | // get the code between the 3 backticks 309 | code_response = data.response.match(/```(.*?)```/g); 310 | // on the the ''' if there is a
before the code remove it 311 | if (code_response) { 312 | // remove the first
if it exists 313 | data.response = data.response.replace(/```
/g, '```'); 314 | // replace ```python with ``` 315 | data.response = data.response.replace(/```python/g, '```'); 316 | data.response = data.response.replace(/```(.*?)```/g, '
$1
copy
'); 317 | } 318 | resultDiv.innerHTML += `
Response: ${data.response}
`; 319 | // Scroll to the bottom of the div 320 | resultDiv.scrollTop = resultDiv.scrollHeight; 321 | saveQuestionAndResponse(query, data.response); 322 | queryInput.value = ''; 323 | }) 324 | .catch(error => { 325 | // const loading = document.querySelector('.loading'); 326 | // loading.parentNode.removeChild(loading); 327 | 328 | resultDiv.innerHTML += `
${error}, is the server up?
`; 329 | // console.error(error); 330 | }); 331 | }); 332 | }); 333 | --------------------------------------------------------------------------------