├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── localstorage └── Placeholder.md ├── requirements.txt ├── run_api.py ├── solidgpt ├── definitions.py └── src │ ├── api │ ├── api.py │ ├── api_response.py │ ├── celery_config.py │ └── celery_tasks.py │ ├── configuration │ ├── Configuration.yaml │ └── configreader.py │ ├── constants.py │ ├── diy │ ├── chatgpt_diy_finetune │ │ ├── README.md │ │ ├── chatgpt_train.py │ │ └── dataset_checker.py │ ├── custom │ │ ├── customizedskilldefinition.py │ │ ├── customizeskillmanager.py │ │ └── customskillgenerator.py │ ├── llama2_diy_finetune │ │ ├── README.md │ │ ├── data_generator.py │ │ ├── dataloader.py │ │ ├── llama2modelsetting.py │ │ └── train.py │ └── requirements.txt │ ├── imports.py │ ├── manager │ ├── autogenmanager.py │ ├── blobmanager.py │ ├── embedding │ │ ├── embeddingmanager.py │ │ └── embeddingmodel.py │ ├── gptmanager.py │ ├── initializer.py │ ├── llamanager.py │ └── promptresource.py │ ├── orchestration │ └── orchestration.py │ ├── request │ └── basic_request.py │ ├── saveload │ └── saveload.py │ ├── tools │ ├── lowdefy │ │ ├── embedding │ │ │ ├── container_block_embedding.csv │ │ │ ├── display_block_embedding.csv │ │ │ └── input_block_embedding.csv │ │ ├── runner │ │ │ └── buildwebapprunner.py │ │ └── validator │ │ │ └── yaml_validator.py │ ├── notion │ │ └── notionactions.py │ ├── repo │ │ └── gpt_repository_loader.py │ └── templates │ │ ├── aws-python-flask-dynamodb-api │ │ ├── README.md │ │ ├── app.py │ │ ├── package.json │ │ ├── requirements.txt │ │ ├── serverless.template.yml │ │ └── serverless.yml │ │ ├── aws-python-http-api-with-dynamodb │ │ ├── .gitignore │ │ ├── README.md │ │ ├── package.json │ │ ├── serverless.yml │ │ └── todos │ │ │ ├── __init__.py │ │ │ ├── create.py │ │ │ ├── decimalencoder.py │ │ │ ├── delete.py │ │ │ ├── get.py │ │ │ ├── list.py │ │ │ └── update.py │ │ └── aws-python-telegram-bot │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── handler.py │ │ ├── package.json │ │ ├── requirements.txt │ │ └── serverless.yml │ ├── util │ └── util.py │ ├── workgraph │ ├── displayresult.py │ ├── graph.py │ ├── graph_helper.py │ └── workgraph.py │ ├── worknode │ └── worknode.py │ └── workskill │ ├── skillio.py │ ├── skills │ ├── code_chat.py │ ├── create_codeplan_v4.py │ ├── create_codesolution_v3.py │ ├── create_kanban.py │ ├── custom_skill.py │ ├── http_codeplan.py │ ├── http_codesolution.py │ ├── http_codesolution_v2.py │ ├── llama_analysis.py │ ├── llama_write_prd.py │ ├── notion_chat.py │ ├── notion_embed.py │ ├── repo_chat_v2.py │ ├── select_template.py │ └── vscode_embed_v2.py │ └── workskill.py ├── solidportal ├── .eslintrc.json ├── .vscodeignore ├── CHANGELOG.md ├── LICENSE ├── package-lock.json ├── package.json ├── server │ └── placeholder.md ├── solidgpticon.svg ├── solidgpticonpng.png ├── src │ ├── custom.d.ts │ ├── extension.ts │ ├── test │ │ ├── runTest.ts │ │ └── suite │ │ │ ├── extension.test.ts │ │ │ └── index.ts │ └── view │ │ ├── App.css │ │ ├── App.tsx │ │ ├── components │ │ ├── Avatar.tsx │ │ ├── ChatElement.tsx │ │ ├── FileList.css │ │ ├── FileList.tsx │ │ ├── Home.css │ │ ├── Home.tsx │ │ ├── Message.tsx │ │ ├── Setting.css │ │ ├── Setting.tsx │ │ └── stChat.css │ │ ├── index.html │ │ ├── index.tsx │ │ ├── static │ │ └── img │ │ │ ├── button.svg │ │ │ ├── leerob.png │ │ │ ├── solidgpt-1.svg │ │ │ ├── solidgpt.svg │ │ │ ├── solidgpticon.svg │ │ │ ├── solidgptlogo.svg │ │ │ └── useravatar.svg │ │ └── utils │ │ ├── ApiHelper.tsx │ │ ├── Constants.tsx │ │ ├── DataMappingHelper.tsx │ │ ├── config.tsx │ │ └── endPoints.tsx ├── tsconfig.json ├── webpack.base.js ├── webpack.dev.js └── webpack.prod.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | .idea/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # pdm 106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 107 | #pdm.lock 108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 109 | # in version control. 110 | # https://pdm.fming.dev/#use-with-ide 111 | .pdm.toml 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | #.idea/ 162 | 163 | # Mac 164 | .DS_Store 165 | solidgpt/src/.DS_Store 166 | 167 | # vs code 168 | .vscode/ 169 | 170 | # Portal 171 | node_modules 172 | 173 | # Local storage and workspace files 174 | localstorage/ 175 | !localstorage/workspace/ 176 | workspace/out/ 177 | solidgpt/src/tools/qdrant/embeddings/ 178 | solidgpt/src/tools/qdrant/embedding/ 179 | 180 | # DB 181 | celerydb.sqlite 182 | results.db -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use an official Python runtime as the parent image 2 | FROM python:3.11.5-bullseye 3 | 4 | # Set the working directory in the container to /app 5 | WORKDIR /app 6 | 7 | # Copy the current directory (where the Dockerfile is) into the container at /app 8 | COPY . /app 9 | 10 | # Install Python dependencies 11 | RUN pip install --no-cache-dir -r requirements.txt 12 | 13 | # Install Redis 14 | RUN apt-get update && apt-get install -y redis-server && apt-get clean 15 | 16 | # Expose ports 17 | EXPOSE 8000 6379 18 | 19 | RUN chmod -R 777 /app/localstorage 20 | 21 | # Start processes 22 | CMD export PYTHONPATH=$PYTHONPATH:$(git rev-parse --show-toplevel)/ && service redis-server start && celery -A solidgpt.src.api.celery_tasks worker --loglevel=info --detach && uvicorn solidgpt.src.api.api:app --proxy-headers --host 0.0.0.0 --port 8000 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🧱 SolidGPT 2 | 3 | # ❗️❗️We solemnly declare: 4 | 5 | **We have NOT issued any cryptocurrency.** 6 | 7 | **Any cryptocurrency claiming to be issued in our name is a SCAM. Please stay vigilant and avoid being deceived.** 8 | 9 | 10 | # 🚀 What's this 11 | SolidGPT is a AI searching assistant for developer that helps code and workspace semantic search 12 | 13 | 🔥🔥🔥 Try SolidGPT VSCode Extension from the [VSCode Marketplace](https://marketplace.visualstudio.com/items?itemName=aict.solidgpt) 14 | 15 | Appreciate Star 🌟 us on our [SolidGPT Github](https://github.com/AI-Citizen/SolidGPT) 16 | 17 | # Try SolidGPT VSCode Extension 18 | 1. Install SolidGPT VSCode Extension from the [VSCode Marketplace](https://marketplace.visualstudio.com/items?itemName=aict.solidgpt) 19 | 20 | # 🏁 Quick Start 21 | Highly recommend to try SolidGPT VSCode Extension from the [VSCode Marketplace](https://marketplace.visualstudio.com/items?itemName=aict.solidgpt). 22 | 23 | Or you can follow the steps below to build from source. 24 | ## 📦 Build From Source 25 | 1. Pull the latest version of the SolidGPT from the GitHub repository. 26 | 1. Pip install the requirements.txt file under the SolidGPT root directory. 27 | ```sh 28 | pip install -r requirements.txt 29 | ``` 30 | 1. Open terminal and run the following command to start the server. 31 | ```sh 32 | python run_api.py 33 | ``` 34 | 1. Open terminal and run the following command to start the webapp. 35 | ```sh 36 | cd solidportal 37 | npm install 38 | npm run dev 39 | ``` 40 | 41 | ## ❗️❗️ Onborading your Codebase and Notion 42 | 1. Click Settings button on the right bottom corner. 43 | image 44 | 45 | 46 | ### 1. Setup Codebase 47 | 1. Enter your OpenAI API key. [Get OpenAI API Key](https://openai.com/blog/openai-api) 48 | 1. Enter the full path of the folder you wish to onboard. **Suggest onboard lower than 100 files, maximum support 500 files onboarding**. 49 | image 50 | 51 | ### 2. Setup Notion (Optional) 52 | 1. Configure the Notion API by getting your Notion API secret from [Notion API](https://developers.notion.com/docs/create-a-notion-integration#getting-started) and entering it on the Settings page 53 | 1. Give your integration page permissions. [Details](https://developers.notion.com/docs/create-a-notion-integration#give-your-integration-page-permissions) 54 | 1. Get Notion Page ID and entering it on the Settings page 55 | image 56 | image 57 | 58 | ### 3. Pick chat resources and start chat 59 | 60 | 61 | ## 🔥 Usecase 62 | - Talk to your codebase, save time hunting for the place to start a change or the right method to call. 63 | - Ask any questions about your codebase, get the answer in seconds. 64 | - Semantic search and summary in Notion, knows your project from documents and track the project sprint board and tickets. 65 | - Get questions answered from your codebase and Notion, no more context switching. 66 | 67 | ## 📖 Known Issue 68 | 1. [Intel Chip Mac]: permission denied 69 | - Please run \`cd ~/.vscode/extensions\` and \`chmod -R 777 aict.solidgpt*\` in your terminal to allow the app to run. 70 | 71 | ## 📣 Feedback!! 72 | If you have any questions or feedback about our project, please don't hesitate to reach out to us. We greatly appreciate your suggestions! 73 | - Email: aict@ai-citi.com 74 | - GitHub Issues: For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/AI-Citizen/SolidGPT/issues). 75 | We will respond to all questions within 2-3 business days. 76 | 77 | ## Data Safety 78 | - SolidGPT will not collect any of users' data. 79 | - SolidGPT using OpenAI series model API, using SolidGPT indicates that you have read, understood, and agreed to adhere to all terms of use associated with the OpenAI GPT series model API. 80 | -------------------------------------------------------------------------------- /localstorage/Placeholder.md: -------------------------------------------------------------------------------- 1 | # Local Storage Folder 2 | Default folder for local storage. This folder is used to store files that are not synced to the cloud. This folder is not synced to the cloud. -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.8.5 2 | aiosignal==1.3.1 3 | async-timeout==4.0.3 4 | attrs==23.1.0 5 | certifi==2023.7.22 6 | charset-normalizer==3.2.0 7 | frozenlist==1.4.0 8 | idna==3.4 9 | multidict==6.0.4 10 | openai==0.27.8 11 | PyYAML==6.0.1 12 | requests==2.31.0 13 | tqdm==4.66.1 14 | urllib3==1.26.16 15 | yarl==1.9.2 16 | iniconfig==2.0.0 17 | packaging==23.1 18 | pluggy==1.3.0 19 | pytest==7.4.0 20 | numpy==1.24.0 21 | pandas==2.1.1 22 | notional==0.8.0 23 | notion2md==2.8.3 24 | azure-storage-blob==12.18.1 25 | langchain==0.0.292 26 | langsmith==0.0.38 27 | qdrant_client==1.5.4 28 | fastapi==0.103.1 29 | python-multipart==0.0.6 30 | uvicorn==0.23.2 31 | celery~=5.3.4 32 | redis==4.6.0 33 | rq==1.15.1 34 | eventlet==0.33.3 35 | tiktoken==0.5.1 36 | text-generation==0.6.1 37 | pyautogen==0.1.13 38 | -------------------------------------------------------------------------------- /run_api.py: -------------------------------------------------------------------------------- 1 | import uvicorn 2 | import threading 3 | import solidgpt.src.api.api 4 | import celery.fixups 5 | import kombu.transport.sqlalchemy 6 | import celery.fixups.django 7 | import celery.app.amqp 8 | import celery.backends 9 | import celery.backends.redis 10 | 11 | from celery import Celery 12 | 13 | # PyInstaller friendly imports --start-- 14 | import celery.app.amqp 15 | import celery.app.log 16 | import celery.worker.autoscale 17 | import celery.worker.components 18 | import celery.bin 19 | import celery.utils 20 | import celery.utils.dispatch 21 | import celery.contrib.testing 22 | import celery.utils.static 23 | import celery.concurrency.prefork 24 | import celery.app.events 25 | import celery.events.state 26 | import celery.app.control 27 | import celery.backends.redis 28 | import celery.backends 29 | import celery.backends.database 30 | import celery.worker 31 | import celery.worker.consumer 32 | import celery.app 33 | import celery.loaders 34 | import celery.security 35 | import celery.fixups 36 | import celery.concurrency 37 | import celery.concurrency.thread 38 | import celery.events 39 | import celery.contrib 40 | import celery.apps 41 | import celery 42 | import celery.fixups 43 | import celery.fixups.django 44 | import celery.apps.worker 45 | import celery.worker.strategy 46 | import kombu.transport.redis 47 | import sqlalchemy.sql.default_comparator 48 | import sqlalchemy.ext.baked 49 | import subprocess 50 | import platform 51 | 52 | 53 | if __name__ == "__main__": 54 | # Define the Celery command as a list of strings 55 | celery_command = [] 56 | if platform.system() == 'Windows': 57 | celery_command = [ 58 | 'celery', 59 | '-A', 60 | 'solidgpt.src.api.celery_tasks', 61 | 'worker', 62 | '--loglevel=info', 63 | '-P', 64 | 'eventlet' 65 | ] 66 | else: 67 | celery_command = [ 68 | 'celery', 69 | '-A', 70 | 'solidgpt.src.api.celery_tasks', 71 | 'worker', 72 | '--loglevel=info' 73 | ] 74 | 75 | # Start the Celery worker process in the background 76 | #celery_process = subprocess.Popen(celery_command) 77 | # Run your UVicorn server 78 | uvicorn.run("solidgpt.src.api.api:app", port=8000) -------------------------------------------------------------------------------- /solidgpt/definitions.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import logging 4 | 5 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # This is your Project Root 6 | SRC_DIR = os.path.join(ROOT_DIR, "src") 7 | TEST_DIR = os.path.join(ROOT_DIR, "test") 8 | LOCAL_STORAGE_DIR = os.path.join(ROOT_DIR, "../localstorage") 9 | REPO_STORAGE_DIR = os.path.join(LOCAL_STORAGE_DIR, "repo") 10 | LOCAL_STORAGE_WORKSPACE_DIR = os.path.join(LOCAL_STORAGE_DIR, "workspace") 11 | LOCAL_STORAGE_OUTPUT_DIR = os.path.join(LOCAL_STORAGE_WORKSPACE_DIR, "out0325") # You can change this to your own output dir 12 | TEST_SKILL_WORKSPACE = os.path.join(TEST_DIR, "workskill", "skills", "workspace") 13 | EMBEDDING_BLOB_CONTAINER = "embedding1002" 14 | logging.basicConfig(level=logging.INFO) 15 | SUPPORT_EXTENSION = [".py", ".java", ".js", ".ts", "tsx", "css", "jsx", "go", ".cs", ".cpp", ".rb", ".rs", ".swift", ".m", ".h", ".kt", ".php", ".scala", ".dart", ".R", ".clj", ".ex", ".hs", ".jl", ".lua", ".ml", ".pl", ".sql", ".vb"] -------------------------------------------------------------------------------- /solidgpt/src/api/api_response.py: -------------------------------------------------------------------------------- 1 | def response_upload(message="", status="", progress="", error=""): 2 | if progress == "": 3 | progress = {} 4 | return { 5 | "message": message, 6 | "status": status, 7 | "progress": progress, 8 | "error": error, 9 | } 10 | 11 | 12 | def response_serverless(message="", status="", error=""): 13 | return { 14 | "message": message, 15 | "status": status, 16 | "error": error, 17 | } 18 | 19 | 20 | def response_graph(graph="", message="", status="", progress="", error="", result="", extra_payload=None): 21 | if progress == "": 22 | progress = {} 23 | return { 24 | "graph": graph, 25 | "message": message, 26 | "status": status, 27 | "progress": progress, 28 | "error": error, 29 | "result": result, 30 | "payload": extra_payload, 31 | } -------------------------------------------------------------------------------- /solidgpt/src/api/celery_config.py: -------------------------------------------------------------------------------- 1 | # celery -A solidgpt.src.api.celery_tasks worker --loglevel=info -P eventlet 2 | # celery_config.py 3 | 4 | BROKER_URL = 'redis://localhost:6379/0' # Using Redis as the broker 5 | CELERY_RESULT_BACKEND = 'redis://localhost:6379/1' 6 | -------------------------------------------------------------------------------- /solidgpt/src/configuration/Configuration.yaml: -------------------------------------------------------------------------------- 1 | # config.yaml 2 | openai_api_key: 3 | # gpt-3.5-turbo-16k/gpt-4//... 4 | openai_model: gpt-3.5-turbo-16k 5 | # notion 6 | notion_api_key : 7 | notion_page_id : 8 | 9 | azure_blob_connection_string: 10 | 11 | #Huggingface API 12 | HF_API_LLAMA2_BASE : 13 | HF_API_KEY : 14 | -------------------------------------------------------------------------------- /solidgpt/src/configuration/configreader.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import os 3 | 4 | from solidgpt.definitions import ROOT_DIR 5 | 6 | class ConfigReader: 7 | # gpt-4-1106-preview, gpt-3.5-turbo-16k 8 | config_map = { "openai_model": "gpt-3.5-turbo-16k" } 9 | def __init__(self): 10 | # self.file_path = os.path.join(ROOT_DIR, "src", "configuration", "Configuration.yaml") 11 | # config_path = "configuration.yaml" # This should match the expected path in your code. 12 | pass 13 | 14 | def read(self): 15 | # with open(self.file_path, 'r') as file: 16 | # data = yaml.safe_load(file) 17 | # return data 18 | pass 19 | 20 | def get_property(self, key): 21 | return self.config_map.get(key, "") 22 | 23 | def set_default_openai_model(self, model:str): 24 | self.__set_property("openai_model", model) 25 | 26 | def __set_property(self, key, value): 27 | self.config_map[key] = value 28 | -------------------------------------------------------------------------------- /solidgpt/src/constants.py: -------------------------------------------------------------------------------- 1 | """Skill Names""" 2 | SKILL_NAME_WRITE_PRODUCT_REQUIREMENTS_DOCUMENTATION = "Write Product Requirement Documentation" 3 | SKILL_NAME_WRITE_HLD = "Write High Level Design" 4 | SKILL_NAME_CREATE_KANBAN_BOARD = "Create Kanban Board" 5 | SKILL_NAME_CUSTOM_SKILL = "Custom Skill" 6 | SKILL_NAME_WRITE_YAML = "Write lowdefy YAML" 7 | SKILL_NAME_RUN_APP = "Host and run web app" 8 | SKILL_NAME_LOAD_REPO = "Load repo" 9 | SKILL_NAME_SUMMARY_PROJECT = "Summary project" 10 | SKILL_NAME_QUERY_CODE = "Query code" 11 | SKILL_NAME_SUMMARY_FILE = "Summary file" 12 | SKILL_NAME_ANALYSIS_PRODUCT = "Analysis Product" 13 | SKILL_NAME_PROVIDE_TECH_SOLUTION = "Provide Tech Solution" 14 | SKILL_NAME_REPO_CHAT = "Repo Chat" 15 | SKILL_NAME_AUTOGEN_ANALYSIS = "AutoGen analysis" 16 | SKILL_NAME_VSCODE_EMBED = "VSCODE embed" 17 | SKILL_NAME_VSCODE_EMBED2 = "VSCODE embed 2" 18 | SKILL_NAME_CREATE_CODE_PLAN = "Create Code Plan" 19 | SKILL_NAME_CREATE_CODE_SOLUTION = "Create Code Solution" 20 | SKILL_NAME_REPO_CHAT_V2 = "Repo Chat 2" 21 | SKILL_NAME_SELECT_TEMPLATE = "Select Template" 22 | SKILL_NAME_HTTP_CODE_PLAN = "HTTP Code Plan" 23 | SKILL_NAME_HTTP_SOLUTION = "HTTP Code Solution" 24 | SKILL_NAME_HTTP_SOLUTION_V2 = "HTTP Code Solution V2" 25 | SKILL_NAME_NOTION_EMBED = "Notion Embed" 26 | SKILL_NAME_CODE_CHAT = "Code chat" 27 | SKILL_NAME_NOTION_CHAT = "Notion chat" 28 | -------------------------------------------------------------------------------- /solidgpt/src/diy/chatgpt_diy_finetune/README.md: -------------------------------------------------------------------------------- 1 | # GPT Finetune (Beta Version) 2 | 3 | ## About the project 4 | The implementation the lastest of the gpt-3.5 fine tuning. This code allows you to fine-tune a GPT-3.5 model for a custom chatbot application using OpenAI's API. The fine-tuned model can then be used to generate responses based on input prompts. Follow the steps below to use the code for fine-tuning and checking your dataset compatibility. 5 | 6 | ## Prerequisites 7 | 8 | Before you begin, make sure you have the following: 9 | 10 | 1. An OpenAI API key. 11 | 2. Python 3.x installed on your machine. 12 | 13 | ## Setup 14 | 15 | 1. Clone or download this repository to your local machine. 16 | 17 | 2. Install the required Python packages using the following command: 18 | ```sh 19 | pip install -r requirements.txt 20 | ``` 21 | 22 | 23 | ## Fine-Tuning Your Model 24 | 25 | 1. Prepare your training dataset in JSONL format. Each line should contain a JSON object with a `"prompt"` key representing the input prompt and a `"completion"` key containing the expected completion. 26 | 27 | 2. Open the `chatgpt_train.py` file and locate the `training_file_path` variable. Set its value to the path of your training dataset JSONL file. 28 | 29 | 3. Run the script using the following command: 30 | ```sh 31 | python chatgpt_train.py 32 | ``` 33 | 34 | 4. You will be prompted to enter the path to your training dataset JSONL file. Provide the correct path and press Enter. 35 | 36 | 5. The script will start the fine-tuning process on your specified GPT model. The status will be displayed as the fine-tuning progresses. 37 | 38 | 6. Once the fine-tuning is completed, the script will exit, and your fine-tuned model will be ready for use. 39 | 40 | ## Checking Dataset Compatibility 41 | 42 | To ensure your training dataset is compatible with the fine-tuning process, you can use the `dataset_checker.py` script. This script checks if your dataset meets the required format for fine-tuning. 43 | 44 | 1. Open the `dataset_checker.py` file and locate the `training_file_path` variable. Set its value to the path of your training dataset JSONL file. 45 | 46 | 2. Run the script using the following command: 47 | ```sh 48 | python dataset_checker.py 49 | ``` 50 | 51 | 52 | 3. The script will check your dataset and provide feedback on whether it's compatible with the fine-tuning process. 53 | 54 | ## Notes 55 | 56 | - This code uses asyncio to handle asynchronous operations. The `aiohttp` package is required for asynchronous HTTP requests. 57 | 58 | - Ensure you have your OpenAI API key handy, as it will be used to authenticate API requests. 59 | 60 | - Fine-tuning may take some time to complete depending on your dataset size and the model you choose. 61 | 62 | - Remember to respect OpenAI's usage policies and guidelines when fine-tuning and deploying models. 63 | 64 | Feel free to customize the script according to your needs and explore further functionalities provided by the OpenAI API. 65 | 66 | -------------------------------------------------------------------------------- /solidgpt/src/diy/chatgpt_diy_finetune/chatgpt_train.py: -------------------------------------------------------------------------------- 1 | import openai 2 | import logging 3 | import asyncio 4 | 5 | class GPTFinetune: 6 | def __init__(self, model_name, suffix_name, training_file_path): 7 | self.training_file_id = None 8 | self.model_name = model_name 9 | self.suffix_name = suffix_name 10 | self.job_id = None 11 | self.training_file_path = training_file_path 12 | openai.api_key = ConfigReader().get_property("openai_api_key") 13 | 14 | async def start_fine_tuning(self): 15 | await self.__upload_training_file() 16 | model_details = openai.FineTuningJob.create( 17 | training_file=self.training_file_id, 18 | model=self.model_name, 19 | suffix=self.suffix_name 20 | ) 21 | self.job_id = model_details["id"] 22 | logging.info("Fine-tuning job started: %s", self.job_id) 23 | 24 | async def __upload_training_file(self): 25 | training_response = await openai.File.create( 26 | file=open(self.training_file_path, "rb"), purpose="fine-tune" 27 | ) 28 | self.training_file_id = training_response["id"] 29 | logging.info("Training file ID is ready: %s", self.training_file_id) 30 | 31 | async def get_fine_tuning_status(self): 32 | response = await openai.FineTuningJob.retrieve(self.job_id) 33 | return response["status"] 34 | 35 | # Sample 36 | if __name__ == "__main__": 37 | training_file_path = input("Enter the path to train.jsonl: ") 38 | model_name = "gpt-3.5-turbo" 39 | suffix_name = "Quantchat" 40 | finetune_instance = GPTFinetune(model_name, suffix_name, training_file_path) 41 | loop = asyncio.get_event_loop() 42 | loop.run_until_complete(finetune_instance.start_fine_tuning()) 43 | 44 | async def wait_for_finetuning_complete(): 45 | while True: 46 | status = await finetune_instance.get_fine_tuning_status() 47 | logging.info("Fine-tuning status: %s", status) 48 | if status == "succeeded" or status == "failed": 49 | break 50 | await asyncio.sleep(60) 51 | 52 | loop.run_until_complete(wait_for_finetuning_complete()) 53 | -------------------------------------------------------------------------------- /solidgpt/src/diy/chatgpt_diy_finetune/dataset_checker.py: -------------------------------------------------------------------------------- 1 | # We start by importing the required packages 2 | # The scripts adopt from the https://platform.openai.com/docs/guides/fine-tuning 3 | # The script will check if the data a good to use chatgpt to train. User need 4 | # to update the `data_path` to the training files. The output is the format 5 | # error or format approved. Also provide an estimation of the token usa byto 6 | # the training files. The output is the format error or format approved. Also 7 | # provide an estimation of the token usage. 8 | import json 9 | import os 10 | import tiktoken 11 | import numpy as np 12 | from collections import defaultdict 13 | 14 | # Next, we specify the data path and open the JSONL file 15 | 16 | data_path = "" 17 | 18 | # Load dataset 19 | with open(data_path) as f: 20 | dataset = [json.loads(line) for line in f] 21 | 22 | # We can inspect the data quickly by checking the number of examples and the first item 23 | 24 | # Initial dataset stats 25 | print("Num examples:", len(dataset)) 26 | print("First example:") 27 | for message in dataset[0]["messages"]: 28 | print(message) 29 | 30 | # Now that we have a sense of the data, we need to go through all the different examples and check to make sure the formatting is correct and matches the Chat completions message structure 31 | 32 | # Format error checks 33 | format_errors = defaultdict(int) 34 | 35 | for ex in dataset: 36 | if not isinstance(ex, dict): 37 | format_errors["data_type"] += 1 38 | continue 39 | 40 | messages = ex.get("messages", None) 41 | if not messages: 42 | format_errors["missing_messages_list"] += 1 43 | continue 44 | 45 | for message in messages: 46 | if "role" not in message or "content" not in message: 47 | format_errors["message_missing_key"] += 1 48 | 49 | if any(k not in ("role", "content", "name") for k in message): 50 | format_errors["message_unrecognized_key"] += 1 51 | 52 | if message.get("role", None) not in ("system", "user", "assistant"): 53 | format_errors["unrecognized_role"] += 1 54 | 55 | content = message.get("content", None) 56 | if not content or not isinstance(content, str): 57 | format_errors["missing_content"] += 1 58 | 59 | if not any(message.get("role", None) == "assistant" for message in messages): 60 | format_errors["example_missing_assistant_message"] += 1 61 | 62 | if format_errors: 63 | print("Found errors:") 64 | for k, v in format_errors.items(): 65 | print(f"{k}: {v}") 66 | else: 67 | print("No errors found") 68 | 69 | # Beyond the structure of the message, we also need to ensure that the length does not exceed the 4096 token limit. 70 | 71 | # Token counting functions 72 | encoding = tiktoken.get_encoding("cl100k_base") 73 | 74 | # not exact! 75 | # simplified from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb 76 | def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1): 77 | num_tokens = 0 78 | for message in messages: 79 | num_tokens += tokens_per_message 80 | for key, value in message.items(): 81 | num_tokens += len(encoding.encode(value)) 82 | if key == "name": 83 | num_tokens += tokens_per_name 84 | num_tokens += 3 85 | return num_tokens 86 | 87 | def num_assistant_tokens_from_messages(messages): 88 | num_tokens = 0 89 | for message in messages: 90 | if message["role"] == "assistant": 91 | num_tokens += len(encoding.encode(message["content"])) 92 | return num_tokens 93 | 94 | def print_distribution(values, name): 95 | print(f"\n#### Distribution of {name}:") 96 | print(f"min / max: {min(values)}, {max(values)}") 97 | print(f"mean / median: {np.mean(values)}, {np.median(values)}") 98 | print(f"p5 / p95: {np.quantile(values, 0.1)}, {np.quantile(values, 0.9)}") 99 | 100 | # Last, we can look at the results of the different formatting operations before proceeding with creating a fine-tuning job: 101 | 102 | # Warnings and tokens counts 103 | n_missing_system = 0 104 | n_missing_user = 0 105 | n_messages = [] 106 | convo_lens = [] 107 | assistant_message_lens = [] 108 | 109 | for ex in dataset: 110 | messages = ex["messages"] 111 | if not any(message["role"] == "system" for message in messages): 112 | n_missing_system += 1 113 | if not any(message["role"] == "user" for message in messages): 114 | n_missing_user += 1 115 | n_messages.append(len(messages)) 116 | convo_lens.append(num_tokens_from_messages(messages)) 117 | assistant_message_lens.append(num_assistant_tokens_from_messages(messages)) 118 | 119 | print("Num examples missing system message:", n_missing_system) 120 | print("Num examples missing user message:", n_missing_user) 121 | print_distribution(n_messages, "num_messages_per_example") 122 | print_distribution(convo_lens, "num_total_tokens_per_example") 123 | print_distribution(assistant_message_lens, "num_assistant_tokens_per_example") 124 | n_too_long = sum(l > 4096 for l in convo_lens) 125 | print(f"\n{n_too_long} examples may be over the 4096 token limit, they will be truncated during fine-tuning") 126 | 127 | # Pricing and default n_epochs estimate 128 | MAX_TOKENS_PER_EXAMPLE = 4096 129 | 130 | MIN_TARGET_EXAMPLES = 100 131 | MAX_TARGET_EXAMPLES = 25000 132 | TARGET_EPOCHS = 3 133 | MIN_EPOCHS = 1 134 | MAX_EPOCHS = 25 135 | 136 | n_epochs = TARGET_EPOCHS 137 | n_train_examples = len(dataset) 138 | if n_train_examples * TARGET_EPOCHS < MIN_TARGET_EXAMPLES: 139 | n_epochs = min(MAX_EPOCHS, MIN_TARGET_EXAMPLES // n_train_examples) 140 | elif n_train_examples * TARGET_EPOCHS > MAX_TARGET_EXAMPLES: 141 | n_epochs = max(MIN_EPOCHS, MAX_TARGET_EXAMPLES // n_train_examples) 142 | 143 | n_billing_tokens_in_dataset = sum(min(MAX_TOKENS_PER_EXAMPLE, length) for length in convo_lens) 144 | print(f"Dataset has ~{n_billing_tokens_in_dataset} tokens that will be charged for during training") 145 | print(f"By default, you'll train for {n_epochs} epochs on this dataset") 146 | print(f"By default, you'll be charged for ~{n_epochs * n_billing_tokens_in_dataset} tokens") 147 | print("See pricing page to estimate total costs") 148 | -------------------------------------------------------------------------------- /solidgpt/src/diy/custom/customizedskilldefinition.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from solidgpt.src.workskill.skillio import SkillIOParamCategory 4 | 5 | 6 | @dataclass 7 | class CustomizedSkillDefinition: 8 | def __init__(self, skill_name, basic_description, instruction, 9 | qa_example, principles, embedding_background_data_list, 10 | input_method : SkillIOParamCategory, output_method : SkillIOParamCategory, model_name = None): 11 | self.skill_name : str = skill_name 12 | self.basic_description : str = basic_description 13 | self.instruction : str = instruction 14 | self.qa_example : str = qa_example 15 | self.principles : str = principles 16 | self.embedding_background_data_list : str = embedding_background_data_list 17 | self.model_name : str = model_name 18 | self.input_method : SkillIOParamCategory = input_method 19 | self.output_method : SkillIOParamCategory = output_method 20 | 21 | def toDict(self): 22 | return { 23 | "skill_name": self.skill_name, 24 | "basic_description": self.basic_description, 25 | "instruction": self.instruction, 26 | "qa_example": self.qa_example, 27 | "principles": self.principles, 28 | "embedding_background_data_list": self.embedding_background_data_list, 29 | "model_name": self.model_name, 30 | "input_method": self.input_method, 31 | "output_method": self.output_method 32 | } -------------------------------------------------------------------------------- /solidgpt/src/diy/custom/customizeskillmanager.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from solidgpt.definitions import * 4 | from solidgpt.src.diy.custom.customizedskilldefinition import CustomizedSkillDefinition 5 | from solidgpt.src.util.util import load_from_json 6 | from solidgpt.src.workskill.skills.custom_skill import CustomSkill 7 | 8 | class CustomizeSkillManager: 9 | _instance = None 10 | 11 | def __new__(cls, *args, **kwargs): 12 | if cls._instance is None: 13 | cls._instance = super().__new__(cls) 14 | # You can initialize the instance attributes here 15 | return cls._instance 16 | 17 | def __init__(self, custom_skill_definition_folder_path=os.path.join(LOCAL_STORAGE_DIR, 'customizedskilldefinition')): 18 | self.customized_skills_map: dict[str, CustomSkill] = {} 19 | self.custom_skill_definition_folder_path = custom_skill_definition_folder_path 20 | self.__load_customized_skills() 21 | 22 | def get_customzied_skill(self, skill_name: str)-> CustomSkill: 23 | skill = self.customized_skills_map.get(skill_name) 24 | if skill is None: 25 | logging.error(f"Error, Customized skill {skill_name} is not found.") 26 | return skill 27 | 28 | def __load_customized_skills(self): 29 | # load all of the customized skills josn files 30 | skill_definitions = self.__load_customzied_skill_from_folder() 31 | for skill_definition in skill_definitions: 32 | skill = self.__load_customized_skill(skill_definition) 33 | self.customized_skills_map[skill_definition.skill_name] = skill 34 | return 35 | 36 | def __load_customized_skill(self, skill_definition: CustomizedSkillDefinition)-> CustomSkill: 37 | # load all of the customized skills josn files 38 | return CustomSkill(skill_definition) 39 | 40 | def __load_customzied_skill_from_folder(self): 41 | # Get a list of all files in the folder 42 | file_list = os.listdir(self.custom_skill_definition_folder_path) 43 | 44 | # Filter JSON files from the list 45 | json_files = [file for file in file_list if file.endswith('.json')] 46 | logging.info(f"Found {json_files} json files in {self.custom_skill_definition_folder_path}") 47 | 48 | # Load JSON data from each JSON file 49 | customized_skills_definition: list(CustomizedSkillDefinition)= [] 50 | for json_file in json_files: 51 | customized_skills_definition.append(CustomizedSkillDefinition(**load_from_json(os.path.join(self.custom_skill_definition_folder_path, json_file)))) 52 | return customized_skills_definition -------------------------------------------------------------------------------- /solidgpt/src/diy/custom/customskillgenerator.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import re 4 | import logging 5 | import os 6 | from solidgpt.definitions import ROOT_DIR 7 | from solidgpt.src.diy.custom.customizedskilldefinition import CustomizedSkillDefinition 8 | from solidgpt.src.manager.gptmanager import GPTManager 9 | from solidgpt.src.manager.promptresource import CUSTOM_GENERATE_LIST_SKILLS_OUTPUT_FORMAT, CUSTOM_GENERATE_PRINCIPLES, \ 10 | build_gpt_prompt, get_custom_skills_assumption_role_prompt 11 | from solidgpt.src.util.util import save_to_json 12 | 13 | class CustomSkillGenerator: 14 | def __init__(self): 15 | self.gpt_manager = GPTManager._instance 16 | self.cache = {} 17 | 18 | def generate_custom_skill(self, business:str): 19 | logging.info("Generating custom skill...") 20 | skills = self.__list_essential_skill_list(business) 21 | for skill in skills: 22 | self.cache["skill_short_description"] = skill 23 | self.__get_custom_skill_detail() 24 | self.__generate_principles() 25 | custom_skill_definition = self.__format_custom_skill() 26 | logging.info(custom_skill_definition) 27 | return 28 | 29 | def __list_essential_skill_list(self, business:str): 30 | role_prompt = get_custom_skills_assumption_role_prompt(business) 31 | prompt = build_gpt_prompt(role_assumption=role_prompt, output_format=CUSTOM_GENERATE_LIST_SKILLS_OUTPUT_FORMAT) 32 | skill_list : str = self.gpt_manager.create_and_chat_with_model( 33 | prompt=prompt, 34 | gpt_model_label="list_essential_skill_list", 35 | input_message="Always use && to separate each skill ", 36 | temperature=0 37 | ) 38 | skill_list_tmp = skill_list.split("&&") 39 | if len(skill_list_tmp) < 2: 40 | lines = skill_list.split('\n') 41 | skill_list_tmp = [line for line in lines if re.match(r'^\d+\.', line)] 42 | skill_list_final = [item for item in skill_list_tmp if not item.isdigit() and item != ''] 43 | return skill_list_final 44 | 45 | def __get_custom_skill_detail(self): 46 | skill_short_description = self.cache["skill_short_description"] 47 | logging.info(f"""Explore skill {skill_short_description}""") 48 | detail = self.gpt_manager.create_and_chat_with_model( 49 | prompt=f"""I want to create the {skill_short_description} AI agent, 50 | Can you list more detail about the {skill_short_description}? 51 | can you give me an input and output format of agent? 52 | And also give an instruction of how to do/implement {skill_short_description} step by step""", 53 | gpt_model_label="get_custom_skills_detail", 54 | input_message=skill_short_description 55 | ) 56 | qa_example = self.gpt_manager.create_and_chat_with_model( 57 | prompt= f"""Your idea for skill {skill_short_description} is: {detail}. Directly response no extra words """, 58 | gpt_model_label="get_custom_skills_example", 59 | input_message="""Can you give a example Input and output base on your idea.""" 60 | ) 61 | self.cache["detail"] = detail 62 | self.cache["qa_example"] = qa_example 63 | 64 | def __generate_principles(self): 65 | if self.cache.get("detail") is None or self.cache.get("skill_short_description") is None: 66 | logging.error("Don't have enough information to generate principles") 67 | prompt = build_gpt_prompt(f'''Assuem you are the expert with {self.cache["skill_short_description"]}''', 68 | CUSTOM_GENERATE_PRINCIPLES) 69 | self.cache['principles'] = self.gpt_manager.create_and_chat_with_model( 70 | prompt=prompt, 71 | gpt_model_label="generate principles", 72 | input_message=f'''Task description: {self.cache["skill_short_description"]}\n\n 73 | Task instruction{self.cache["detail"]}''' 74 | ) 75 | 76 | def __format_custom_skill(self): 77 | if self.cache.get("qa_example") is None or self.cache.get("detail") is None: 78 | logging.error("Please run list_essential_skill_list or get_custom_skill_detail first") 79 | 80 | skill_name = self.gpt_manager.create_and_chat_with_model( 81 | prompt=f"""Base on the skill short description, give me short clear Camel Case no space name . 82 | For example - Quantitative Analyst, Programming, Write PRD etc.""", 83 | gpt_model_label="format_custom_skill", 84 | input_message=f'''Describtion: {self.cache["detail"]}''' 85 | ) 86 | definition = CustomizedSkillDefinition( 87 | skill_name = skill_name, 88 | basic_description = self.cache["skill_short_description"], 89 | instruction= self.cache["detail"], 90 | qa_example = self.cache["qa_example"], 91 | principles = self.cache['principles'], 92 | embedding_background_data_list= "", 93 | input_method= "SkillIOParamCategory.PlainText", 94 | output_method= "SkillIOParamCategory.PlainText" 95 | ) 96 | save_to_json(definition.toDict(), os.path.join(ROOT_DIR, "localstorage", "customizedskilldefinition", f"{skill_name}.json")) 97 | # Clean cache 98 | self.cache = {} 99 | return 100 | 101 | # Sample code 102 | # GPTManager() 103 | # c = CustomSkillGenerator() 104 | # c.generate_custom_skill("Product Manager") -------------------------------------------------------------------------------- /solidgpt/src/diy/llama2_diy_finetune/README.md: -------------------------------------------------------------------------------- 1 | # LLama2 Fine-Tuning for Lowdefy YAML Files (Test Version) 2 | 3 | This code utilizes the LLama2 model to fine-tune Lowdefy YAML files, enabling the generation of bug-free Lowdefy files using a large language model. Follow the steps below to understand and use the provided code for fine-tuning and generating Lowdefy YAML files. 4 | 5 | ## Prerequisites 6 | 7 | Before you begin, ensure you have the following: 8 | 9 | 1. An environment with the required dependencies installed. You can install them using the command: 10 | ```sh 11 | pip install -r requirements.txt 12 | ``` 13 | 14 | ## Code Overview 15 | 16 | The provided code comprises several components to fine-tune the LLama2 model and generate Lowdefy YAML files: 17 | 18 | 1. `train.py`: Fine-tunes the LLama2 model using the specified training and validation datasets. Run the script using the command: 19 | 20 | ```sh 21 | python train.py 22 | ``` 23 | 24 | 2. `llamamanager.py`: Manages the LLama2 model, including loading the model, generating YAML files, and saving model parameters. 25 | 26 | 3. `dataloader.py`: Loads and preprocesses the training and validation datasets. 27 | 28 | 4. `llama2modelsetting.py`: Defines the LLama2 model settings, including model type and configuration. 29 | 30 | 5. `requirements.txt`: Lists the required Python packages. Install them using `pip install -r requirements.txt`. 31 | 32 | ## Usage Example 33 | 34 | 1. Prepare your training and validation datasets in JSONL format, containing prompts and expected completions for fine-tuning. 35 | 36 | 2. Run the `train.py` script to fine-tune the LLama2 model with your datasets and specified settings. The fine-tuned model parameters will be saved. 37 | 38 | 3. Modify the `prompt` variable in the `if __name__ == "__main__":` section of the script to define the prompt for generating a Lowdefy YAML file. 39 | 40 | 4. Optionally, specify the path to a template YAML file using the `user_template_path` variable. 41 | 42 | 5. The script will generate a Lowdefy YAML file based on the provided prompt and template (if specified). 43 | 44 | ## Notes 45 | 46 | - Obtain permissions for using the LLama2 model from the official LLama2 site: [LLama2 Official Site](https://ai.meta.com/llama/). 47 | 48 | - Make sure to provide paths to your training and validation datasets in the `LLamaModel` class. 49 | 50 | - Fine-tuning may take some time depending on your dataset size and settings. 51 | 52 | - Please follow ethical considerations and usage guidelines for fine-tuning and deploying large language models. 53 | 54 | - Refer to the official LLama2 documentation and resources for further information and support. 55 | 56 | 57 | -------------------------------------------------------------------------------- /solidgpt/src/diy/llama2_diy_finetune/data_generator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import tiktoken 4 | from solidgpt.src.manager.gptmanager import GPTModel # Import GPTModel from the correct location 5 | from solidgpt.src.configuration.configreader import ConfigReader 6 | 7 | # Set your OpenAI API key here 8 | openai.api_key = ConfigReader().get_property("openai_api_key") 9 | 10 | 11 | MAX_TOKENS_LIMIT = 16000 12 | 13 | class GenerateDataset: 14 | def __init__(self, model_name): 15 | self.model_name = model_name 16 | prompt = "Explain the content of a Lowdefy YAML file." # Define the prompt 17 | self.gpt_model = GPTModel(prompt, model_name) 18 | 19 | def _num_tokens_from_string(self, string: str, encoding_name: str) -> int: 20 | encoding = tiktoken.encoding_for_model(encoding_name) 21 | num_tokens = len(encoding.encode(string)) 22 | return num_tokens 23 | 24 | def get_files_with_filter(self, root_path: str, regex_filter: str) -> list: 25 | filtered_files = [] 26 | for dirpath, dirnames, filenames in os.walk(root_path): 27 | for filename in filenames: 28 | if filename.lower().endswith(regex_filter.lower()): 29 | filtered_files.append(os.path.join(dirpath, filename)) 30 | return filtered_files 31 | 32 | def generate_dataset(self, files_list): 33 | dataset = [] 34 | for file_path in files_list: 35 | with open(file_path, "r") as f: 36 | yaml_content = f.read() 37 | 38 | yaml_prompt = f"Explain the following lowdefy YAML file's content:\n{yaml_content}" 39 | try: 40 | yaml_prompt_tokens = self._num_tokens_from_string(yaml_prompt, self.model_name) 41 | if yaml_prompt_tokens <= MAX_TOKENS_LIMIT: 42 | yaml_explanation = self.gpt_model.chat_with_model(yaml_prompt) 43 | dataset.append({"prompt": yaml_explanation, "completion": yaml_content}) 44 | else: 45 | print(f"Skipping {file_path} as it exceeds the token limit.") 46 | except openai.error.OpenAIError as e: 47 | print(f"Error generating YAML explanation: {e}") 48 | continue 49 | return dataset 50 | 51 | def main(): 52 | model_name = "gpt-3.5-turbo-16k" # Replace with your desired model name 53 | root_folder = input("Enter the root folder path containing sub-folders with YAML files: ") 54 | 55 | dataset_generator = GenerateDataset(model_name) 56 | 57 | yaml_files = dataset_generator.get_files_with_filter(root_folder, ".yaml") 58 | dataset = dataset_generator.generate_dataset(yaml_files) 59 | 60 | output_jsonl_file = os.path.join(root_folder, 'localstorage', 'train.jsonl') 61 | with open(output_jsonl_file, "w") as f: 62 | for item in dataset: 63 | json.dump(item, f) 64 | f.write("\n") 65 | 66 | print(f"Dataset saved to {output_jsonl_file}") 67 | 68 | if __name__ == "__main__": 69 | main() 70 | 71 | -------------------------------------------------------------------------------- /solidgpt/src/diy/llama2_diy_finetune/dataloader.py: -------------------------------------------------------------------------------- 1 | from datasets import load_dataset 2 | 3 | class ModelDataLoader: 4 | def __init__(self, train_dataset_path, valid_dataset_path, system_message=None): 5 | self.train_dataset_path = train_dataset_path 6 | self.valid_dataset_path = valid_dataset_path 7 | self.system_message = system_message 8 | 9 | def load_and_preprocess_datasets(self): 10 | train_dataset = load_dataset('json', data_files=self.train_dataset_path, split="train") 11 | valid_dataset = load_dataset('json', data_files=self.valid_dataset_path, split="train") 12 | 13 | # Define a default system message if not provided 14 | if self.system_message is None: 15 | self.system_message = "The system message is: `Given a function description, you will generate the Lowdefy YAML file to configure a website. The generated YAML file should follow the general Lowdefy indentation format.`. Feel free to re-run this cell if you want a better result." 16 | 17 | train_dataset_mapped = train_dataset.map(lambda examples: {'text': [f'[INST] <>\n{self.system_message.strip()}\n<>\n\n' + prompt + ' [/INST] ' + response for prompt, response in zip(examples['prompt'], examples['completion'])]}, batched=True) 18 | valid_dataset_mapped = valid_dataset.map(lambda examples: {'text': [f'[INST] <>\n{self.system_message.strip()}\n<>\n\n' + prompt + ' [/INST] ' + response for prompt, response in zip(examples['prompt'], examples['completion'])]}, batched=True) 19 | 20 | return train_dataset_mapped, valid_dataset_mapped 21 | 22 | -------------------------------------------------------------------------------- /solidgpt/src/diy/llama2_diy_finetune/llama2modelsetting.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from transformers import ( 3 | AutoModelForCausalLM, 4 | AutoTokenizer, 5 | BitsAndBytesConfig, 6 | ) 7 | 8 | class LlamaBasicModelFactory: 9 | def __init__(self, model_name, lora_r, lora_alpha, lora_dropout, use_4bit, bnb_4bit_compute_dtype, 10 | bnb_4bit_quant_type, use_nested_quant, device_map): 11 | self.model_name = model_name 12 | self.lora_r = lora_r 13 | self.lora_alpha = lora_alpha 14 | self.lora_dropout = lora_dropout 15 | self.use_4bit = use_4bit 16 | self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype 17 | self.bnb_4bit_quant_type = bnb_4bit_quant_type 18 | self.use_nested_quant = use_nested_quant 19 | self.device_map = device_map 20 | 21 | def create_model(self): 22 | compute_dtype = getattr(torch, self.bnb_4bit_compute_dtype) 23 | bnb_config = BitsAndBytesConfig( 24 | load_in_4bit=self.use_4bit, 25 | bnb_4bit_quant_type=self.bnb_4bit_quant_type, 26 | bnb_4bit_compute_dtype=compute_dtype, 27 | bnb_4bit_use_double_quant=self.use_nested_quant, 28 | ) 29 | model = AutoModelForCausalLM.from_pretrained( 30 | self.model_name, 31 | quantization_config=bnb_config, 32 | device_map=self.device_map 33 | ) 34 | model.config.use_cache = False 35 | model.config.pretraining_tp = 1 36 | tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True) 37 | tokenizer.pad_token = tokenizer.eos_token 38 | tokenizer.padding_side = "left" 39 | return model, tokenizer 40 | -------------------------------------------------------------------------------- /solidgpt/src/diy/llama2_diy_finetune/train.py: -------------------------------------------------------------------------------- 1 | from transformers import TrainingArguments, pipeline, AutoModelForCausalLM, AutoTokenizer 2 | import torch 3 | from llamamanager import LlamaManager 4 | from trl import SFTTrainer 5 | from peft import LoraConfig, PeftModel 6 | from dataloader import ModelDataLoader 7 | from llama2modelsetting import LlamaBasicModelFactory 8 | 9 | class Llama2TrainerParam: 10 | def __init__(self, model, tokenizer, train_dataset, valid_dataset, peft_config, max_seq_length, training_arguments, packing, model_name): 11 | self.model = model 12 | self.tokenizer = tokenizer 13 | self.train_dataset = train_dataset 14 | self.valid_dataset = valid_dataset 15 | self.peft_config = peft_config 16 | self.max_seq_length = max_seq_length 17 | self.training_arguments = training_arguments 18 | self.packing = packing 19 | self.model_name = model_name 20 | 21 | class LLamaModel: 22 | def __init__(self, model_type="togethercomputer/LLaMA-2-7B-32K-Instruct", 23 | model_name="llama-2-7b-lowdefy_Instruct", 24 | result_dir="./result", 25 | model_path='./llama2-7b-lowdefy_generator_saved', 26 | train_dataset_path="./dataset/train.jsonl", 27 | valid_dataset_path="./dataset/test.jsonl"): 28 | self.model_type = model_type 29 | self.model_name_ = model_name 30 | self.result_dir = result_dir 31 | self.model_path = model_path 32 | self.train_dataset_path = train_dataset_path 33 | self.valid_dataset_path = valid_dataset_path 34 | self.train_param = None 35 | 36 | def set_train_config(self, train_param: Llama2TrainerParam): 37 | self.train_param = train_param 38 | 39 | def __set_default_config(self): 40 | train, valid = self.__load_train_dataset() 41 | model_settings = LlamaBasicModelFactory( 42 | model_name=self.model_type, 43 | lora_r=512, 44 | lora_alpha=64, 45 | lora_dropout=0.1, 46 | use_4bit=True, 47 | bnb_4bit_compute_dtype="float16", 48 | bnb_4bit_quant_type="nf4", 49 | use_nested_quant=False, 50 | device_map={"": 0} 51 | ) 52 | model, tokenizer = model_settings.create_model() 53 | peft_config = LoraConfig( 54 | lora_alpha=64, 55 | lora_dropout=0.1, 56 | r=512, 57 | bias="none", 58 | task_type="CAUSAL_LM", 59 | ) 60 | training_arguments = TrainingArguments( 61 | output_dir=self.result_dir, 62 | num_train_epochs=1, 63 | per_device_train_batch_size=8, 64 | gradient_accumulation_steps=1, 65 | optim="paged_adamw_32bit", 66 | save_steps=25, 67 | logging_steps=5, 68 | learning_rate=1e-3, 69 | weight_decay=0.001, 70 | fp16=False, 71 | bf16=False, 72 | max_grad_norm=0.3, 73 | max_steps=-1, 74 | warmup_ratio=0.03, 75 | group_by_length=True, 76 | lr_scheduler_type="constant", 77 | report_to="all", 78 | evaluation_strategy="steps", 79 | eval_steps=5 80 | ) 81 | return Llama2TrainerParam( 82 | model=model, 83 | tokenizer=tokenizer, 84 | train_dataset=train, 85 | valid_dataset=valid, 86 | peft_config=peft_config, 87 | max_seq_length=None, # Set your desired max sequence length 88 | training_arguments=training_arguments, 89 | packing=False, 90 | model_name=self.model_name_ 91 | ) 92 | 93 | def __load_train_dataset(self): 94 | data_loader = ModelDataLoader(self.train_dataset_path, self.valid_dataset_path) 95 | train_dataset_mapped, valid_dataset_mapped = data_loader.load_and_preprocess_datasets() 96 | return train_dataset_mapped, valid_dataset_mapped 97 | 98 | def train_model(self): 99 | if self.train_param is None: 100 | self.train_param = self.__set_default_config() 101 | 102 | trainer = SFTTrainer( 103 | model=self.train_param.model, 104 | train_dataset=self.train_param.train_dataset, 105 | eval_dataset=self.train_param.valid_dataset, 106 | peft_config=self.train_param.peft_config, 107 | dataset_text_field="text", 108 | max_seq_length=self.train_param.max_seq_length, 109 | tokenizer=self.train_param.tokenizer, 110 | args=self.train_param.training_arguments, 111 | packing=self.train_param.packing, 112 | ) 113 | 114 | trainer.train() 115 | trainer.model.save_pretrained(self.model_name_) 116 | self.__save_model() 117 | 118 | def __save_model(self): 119 | base_model = AutoModelForCausalLM.from_pretrained(self.model_type, low_cpu_mem_usage=True, return_dict=True, torch_dtype=torch.float16, device_map={"":0}) 120 | model = PeftModel.from_pretrained(base_model, self.model_name_) 121 | model = model.merge_and_unload() 122 | tokenizer = AutoTokenizer.from_pretrained(self.model_type, trust_remote_code=True) 123 | tokenizer.pad_token = tokenizer.eos_token 124 | tokenizer.padding_side = "left" 125 | model.save_pretrained(self.model_path) 126 | tokenizer.save_pretrained(self.model_path) 127 | 128 | def predict(self, prompt, template_path=None): 129 | if self.train_param is None: 130 | self.train_param = self.__set_default_config() 131 | 132 | user_prompt = self.template + "\n" + prompt 133 | llama_manager = LlamaManager(model_path=self.model_path, template_path=template_path) 134 | generated_yaml = llama_manager.generate_yaml(user_prompt) 135 | return generated_yaml 136 | 137 | # Usage example 138 | if __name__ == "__main__": 139 | llama_model = LLamaModel() 140 | llama_model.train_model() 141 | 142 | prompt = "help me create lowdefy file for personal website" 143 | user_template_path = "./dataset/template.yaml" 144 | generated_yaml = llama_model.predict(prompt, template_path=user_template_path) 145 | print("Generated YAML:\n", generated_yaml) 146 | -------------------------------------------------------------------------------- /solidgpt/src/diy/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==0.21.0 2 | aiohttp==3.8.5 3 | aiosignal==1.3.1 4 | async-timeout==4.0.3 5 | attrs==23.1.0 6 | bitsandbytes==0.40.2 7 | certifi==2023.7.22 8 | charset-normalizer==3.2.0 9 | cmake==3.27.2 10 | datasets==2.14.4 11 | dill==0.3.7 12 | filelock==3.12.2 13 | frozenlist==1.4.0 14 | fsspec==2023.6.0 15 | huggingface-hub==0.16.4 16 | idna==3.4 17 | Jinja2==3.1.2 18 | lit==16.0.6 19 | MarkupSafe==2.1.3 20 | mpmath==1.3.0 21 | multidict==6.0.4 22 | multiprocess==0.70.15 23 | networkx==3.1 24 | numpy==1.25.2 25 | nvidia-cublas-cu11==11.10.3.66 26 | nvidia-cuda-cupti-cu11==11.7.101 27 | nvidia-cuda-nvrtc-cu11==11.7.99 28 | nvidia-cuda-runtime-cu11==11.7.99 29 | nvidia-cudnn-cu11==8.5.0.96 30 | nvidia-cufft-cu11==10.9.0.58 31 | nvidia-curand-cu11==10.2.10.91 32 | nvidia-cusolver-cu11==11.4.0.1 33 | nvidia-cusparse-cu11==11.7.4.91 34 | nvidia-nccl-cu11==2.14.3 35 | nvidia-nvtx-cu11==11.7.91 36 | openai==0.27.8 37 | packaging==23.1 38 | pandas==2.0.3 39 | peft==0.4.0 40 | protobuf==4.24.1 41 | psutil==5.9.5 42 | pyarrow==12.0.1 43 | python-dateutil==2.8.2 44 | pytz==2023.3 45 | PyYAML==6.0.1 46 | regex==2023.8.8 47 | requests==2.31.0 48 | safetensors==0.3.2 49 | scipy==1.11.2 50 | sentencepiece==0.1.99 51 | sympy==1.12 52 | tokenizers==0.13.3 53 | torch==2.0.1 54 | tqdm==4.66.1 55 | transformers==4.31.0 56 | triton==2.0.0 57 | trl==0.4.7 58 | typing_extensions==4.7.1 59 | tzdata==2023.3 60 | urllib3==2.0.4 61 | xxhash==3.3.0 62 | yarl==1.9.2 63 | xformers 64 | -------------------------------------------------------------------------------- /solidgpt/src/imports.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.workskill.skills.create_kanban import CreateKanBan 2 | -------------------------------------------------------------------------------- /solidgpt/src/manager/blobmanager.py: -------------------------------------------------------------------------------- 1 | from azure.storage.blob import BlobServiceClient, ContainerClient, BlobProperties 2 | from solidgpt.src.configuration.configreader import ConfigReader 3 | 4 | 5 | class AzureBlobManager: 6 | def __init__(self, connection_string): 7 | self.blob_service_client = BlobServiceClient.from_connection_string(connection_string) 8 | 9 | def upload_blob(self, container_name, blob_name, data, overwrite=True): 10 | blob_client = self.blob_service_client.get_blob_client(container=container_name, blob=blob_name) 11 | blob_client.upload_blob(data, overwrite=overwrite) 12 | 13 | def download_blob(self, container_name, blob_name): 14 | blob_client = self.blob_service_client.get_blob_client(container=container_name, blob=blob_name) 15 | return blob_client.download_blob().readall() 16 | 17 | def list_blobs(self, container_name : str): 18 | container_client = self.blob_service_client.get_container_client(container=container_name) 19 | return [blob.name for blob in container_client.list_blobs()] 20 | 21 | def delete_blob(self, container_name, blob_name): 22 | blob_client = self.blob_service_client.get_blob_client(container=container_name, blob=blob_name) 23 | blob_client.delete_blob() 24 | 25 | def clear_container(self, container_name): 26 | container_client: ContainerClient = self.blob_service_client.get_container_client(container_name) 27 | container_client.delete_blobs(*container_client.list_blobs()) 28 | 29 | 30 | # Usage 31 | if __name__ == "__main__": 32 | CONNECTION_STRING = ConfigReader().get_property("azure_blob_connection_string") 33 | manager = AzureBlobManager(CONNECTION_STRING) 34 | 35 | # Sample Usage 36 | manager.upload_blob("repos", "sample.txt", "This is a sample text.", overwrite=True) 37 | manager.upload_blob("repos", "sample1.txt", "This is another sample text.", overwrite=True) 38 | print(manager.list_blobs("repos")) 39 | sample_text = manager.download_blob("repos", "sample.txt").decode("utf-8") 40 | # manager.clear_container("repos") 41 | print(sample_text) 42 | -------------------------------------------------------------------------------- /solidgpt/src/manager/embedding/embeddingmanager.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | from solidgpt.src.manager.embedding.embeddingmodel import LOCAL_EMBEDDING_STORAGE_DIVIDED_RESOURCE_DIR, LOCAL_EMBEDDING_STORAGE_EMBEDDED_RESOURCE_DIR, LOCAL_EMBEDDING_STORAGE_ORIGINAL_RESOURCE_DIR, EmbeddingModel, EmbeddingModelParameter 4 | 5 | 6 | DEFAULT_EMBEDDING_MODEL_LABEL = 'DefaultEmbeddingModel' 7 | DEFAULT_EMBEDDING_RESOURCE_NAME = 'Default' 8 | 9 | class EmbeddingManager: 10 | 11 | _instance = None 12 | 13 | def __new__(cls): 14 | if cls._instance is None: 15 | cls._instance = super(EmbeddingManager, cls).__new__(cls) 16 | # You can initialize the instance attributes here 17 | return cls._instance 18 | 19 | def __init__(self): 20 | self.embed_models_container : dict(str, EmbeddingModel) = {} 21 | 22 | def add_default_embed_model(self): 23 | default_param = EmbeddingModelParameter(resource_name=DEFAULT_EMBEDDING_RESOURCE_NAME, 24 | original_resources_folder_path=LOCAL_EMBEDDING_STORAGE_ORIGINAL_RESOURCE_DIR, 25 | divided_resources_folder_path=LOCAL_EMBEDDING_STORAGE_DIVIDED_RESOURCE_DIR, 26 | embedded_resources_folder_path=LOCAL_EMBEDDING_STORAGE_EMBEDDED_RESOURCE_DIR, 27 | has_embedded=False) 28 | self.add_embed_model(DEFAULT_EMBEDDING_MODEL_LABEL, default_param) 29 | 30 | def add_embed_model(self, label : str, param : EmbeddingModelParameter, do_embedding = True): 31 | self.embed_models_container[label] = EmbeddingModel(param) 32 | if do_embedding: 33 | self.embed_models_container[label].embed_resources() 34 | 35 | def query_from_embed_model(self, query_message : str, model_label = DEFAULT_EMBEDDING_MODEL_LABEL, response_num = 12): 36 | if self.embed_models_container.get(model_label) is None: 37 | logging.error(f"Embedding model {model_label} not found.") 38 | return 39 | return self.embed_models_container[model_label].query_most_match_result_from_resource(query_message, response_num) -------------------------------------------------------------------------------- /solidgpt/src/manager/gptmanager.py: -------------------------------------------------------------------------------- 1 | import openai 2 | from solidgpt.src.configuration.configreader import ConfigReader 3 | 4 | class GPTManager: 5 | 6 | _instance = None 7 | 8 | def __new__(cls): 9 | if cls._instance is None: 10 | cls._instance = super(GPTManager, cls).__new__(cls) 11 | # You can initialize the instance attributes here 12 | return cls._instance 13 | 14 | def __init__(self, if_show_reply = False): 15 | # read api key from config file 16 | global_openai_key = ConfigReader().get_property("openai_api_key") 17 | if global_openai_key is not None and global_openai_key != "": 18 | openai.api_key = global_openai_key 19 | self.__default_model = ConfigReader().get_property("openai_model") 20 | self.gpt_models_container = {} 21 | self.if_show_reply = if_show_reply 22 | 23 | def create_model(self, prompt, gpt_model_label, temperature=1.0, model = None): 24 | if model is None: 25 | model = self.__default_model 26 | gpt_model = GPTModel(prompt, self.__default_model, self.if_show_reply, temperature) 27 | self.gpt_models_container[gpt_model_label] = gpt_model 28 | return gpt_model 29 | 30 | def create_and_chat_with_model(self, prompt, gpt_model_label, input_message, temperature=0.1, model=None, is_stream = False): 31 | gpt_model = self.create_model(prompt, gpt_model_label, temperature, model) 32 | return gpt_model.chat_with_model(input_message, is_stream=is_stream) 33 | 34 | def get_gpt_model(self, gpt_model_label): 35 | return self.completions_container[gpt_model_label] 36 | 37 | def remove_gpt_model(self, gpt_model_label): 38 | self.completions_container.pop(gpt_model_label) 39 | 40 | class GPTModel: 41 | def __init__(self, prompt, model, if_show_reply = True, temperature = 0.1): 42 | self.prompt = prompt 43 | self.model = model 44 | self.messages = [{"role": "system", "content": self.prompt}] 45 | self.last_reply = None 46 | self.if_show_reply = if_show_reply 47 | self.temperature = temperature 48 | 49 | def chat_with_model(self, input_message, is_stream=False): 50 | self.messages.append({"role": "user", "content": input_message}) 51 | if not is_stream: 52 | self._run_model() 53 | else: 54 | return self._run_model_stream() 55 | return self.last_reply 56 | 57 | def _run_model(self): 58 | chat = openai.ChatCompletion.create( 59 | model=self.model, 60 | messages=self.messages, 61 | temperature=self.temperature, 62 | ) 63 | reply = chat.choices[0].message.content 64 | if self.if_show_reply: 65 | print(f"ChatGPT: {reply}") 66 | self.messages.append({"role": "assistant", "content": reply}) 67 | self.last_reply = reply 68 | 69 | def _run_model_stream(self): 70 | stream = openai.ChatCompletion.create( 71 | model=self.model, 72 | messages=self.messages, 73 | temperature=self.temperature, 74 | stream=True, 75 | ) 76 | return stream 77 | 78 | 79 | def add_background(self, background_message): 80 | self.messages.append({"role": "assistant", "content": background_message}) -------------------------------------------------------------------------------- /solidgpt/src/manager/initializer.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.diy.custom.customizeskillmanager import CustomizeSkillManager 2 | from solidgpt.src.manager.gptmanager import GPTManager 3 | 4 | 5 | class Initializer: 6 | 7 | _instance = None 8 | 9 | def __new__(cls): 10 | if cls._instance is None: 11 | cls._instance = super(Initializer, cls).__new__(cls) 12 | # You can initialize the instance attributes here 13 | return cls._instance 14 | 15 | def __init__(self): 16 | self._initialize() 17 | 18 | def _initialize(self): 19 | self.gpt_manager = GPTManager() 20 | # self.customize_skill_manager = CustomizeSkillManager() 21 | -------------------------------------------------------------------------------- /solidgpt/src/manager/llamanager.py: -------------------------------------------------------------------------------- 1 | import os 2 | from solidgpt.src.configuration.configreader import ConfigReader 3 | from text_generation import Client 4 | from solidgpt.src.manager.promptresource import llama_v2_prompt 5 | 6 | class LLAManager: 7 | _instance = None 8 | 9 | def __new__(cls): 10 | if cls._instance is None: 11 | cls._instance = super(LLAManager, cls).__new__(cls) 12 | return cls._instance 13 | 14 | def __init__(self, if_show_reply=False): 15 | self.llama2_base_url = ConfigReader().get_property("HF_API_LLAMA2_BASE") 16 | self.llama2_api_key = ConfigReader().get_property("HF_API_KEY") 17 | self.llama_models_container = {} 18 | self.if_show_reply = if_show_reply 19 | 20 | def create_model(self, prompt, llama_api, llama_model_label, temperature=1, model=None): 21 | if model is None: 22 | model = self.llama2_base_url # Use LLAMA2 base URL as the model 23 | llama_model = LLamaModel(prompt, self.llama2_api_key, self.llama2_base_url, self.if_show_reply, temperature) 24 | self.llama_models_container[llama_model_label] = llama_model 25 | return llama_model 26 | 27 | def create_and_chat_with_model(self, prompt, llama_model_label, input_message, temperature=0.1, model=None): 28 | llama_model = self.create_model(prompt, llama_model_label, temperature, model) 29 | return llama_model.chat_with_model(input_message) 30 | 31 | def get_llama_model(self, llama_model_label): 32 | return self.llama_models_container.get(llama_model_label) 33 | 34 | def remove_llama_model(self, llama_model_label): 35 | self.llama_models_container.pop(llama_model_label, None) 36 | 37 | class LLamaModel: 38 | def __init__(self, prompt, api, model, if_show_reply=True, temperature=0.1): 39 | self.prompt = prompt 40 | self.api = api 41 | self.model = model 42 | self.messages = [{"role": "system", "content": self.prompt}] 43 | self.last_reply = None 44 | self.if_show_reply = if_show_reply 45 | self.temperature = temperature 46 | 47 | def chat_with_model(self, input_message): 48 | self.messages.append({"role": "user", "content": input_message}) 49 | self._run_model() 50 | return self.last_reply 51 | 52 | def _run_model(self): 53 | client = Client(self.model, headers={"Authorization": f"Bearer {self.api}"}, timeout=120) 54 | chat = client.generate( 55 | llama_v2_prompt(self.messages), # Convert messages to LLAMA2 prompt 56 | temperature=self.temperature, 57 | max_new_tokens=1000 58 | ) 59 | reply = chat.generated_text 60 | if self.if_show_reply: 61 | print(f"LLAMA2: {reply}") 62 | self.messages.append({"role": "assistant", "content": reply}) 63 | self.last_reply = reply 64 | 65 | def add_background(self, background_message): 66 | self.messages.append({"role": "assistant", "content": background_message}) 67 | 68 | -------------------------------------------------------------------------------- /solidgpt/src/orchestration/orchestration.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from solidgpt.src.workgraph.graph import build_onboarding_graph 3 | from solidgpt.src.workgraph.graph_helper import GraphStatus, GraphType 4 | from solidgpt.src.manager.initializer import Initializer 5 | from solidgpt.src.workgraph.workgraph import * 6 | 7 | 8 | class GraphInfo: 9 | __graph: WorkGraph = None 10 | __graph_name: str = "" 11 | graph_id: str 12 | graph_type: GraphType 13 | graph_status: GraphStatus 14 | 15 | def __init__(self, graph_name): 16 | self.__graph = WorkGraph() 17 | self.__graph_name = graph_name 18 | 19 | def __init__(self, init_graph: WorkGraph, init_graph_id: str, graph_type: GraphType = None, graph_status: GraphStatus = None): 20 | self.__graph = init_graph 21 | self.graph_type = graph_type 22 | self.graph_status = graph_status 23 | self.graph_id = init_graph_id 24 | 25 | def get_graph(self): 26 | return self.__graph 27 | 28 | def get_name(self): 29 | return self.__graph_name 30 | 31 | 32 | class Orchestration: 33 | _instance = None 34 | __graphs: list[GraphInfo] = [] 35 | 36 | # Using a dict to monitor the status of each graph, always update the status of the graph in the dict 37 | # When the graph is completed and expired (e.g. 1 hour), remove the graph from the dict 38 | graph_monitor: dict[str, GraphInfo] = {} 39 | 40 | def __new__(cls): 41 | if cls._instance is None: 42 | cls._instance = super(Orchestration, cls).__new__(cls) 43 | # You can initialize the instance attributes here 44 | return cls._instance 45 | 46 | def __init__(self): 47 | Initializer() 48 | self.__graphs = [] 49 | 50 | def add_graph(self, json_file_path: str, graph_name: str): 51 | if not os.path.isfile(json_file_path): 52 | print("Cannot add graph, the specified json file path does not contain a file.", file=sys.stderr) 53 | return 54 | if not json_file_path.endswith(".json"): 55 | print("Cannot add graph, the specified file is not a json file.", file=sys.stderr) 56 | return 57 | 58 | temp_graph = GraphInfo(graph_name) 59 | temp_graph.get_graph().load_data(json_file_path) 60 | # try: 61 | # temp_graph.get_graph().load_data(json_file_path) 62 | # except: 63 | # print("Cannot add graph, loading json file into graph failed.", file=sys.stderr) 64 | # return 65 | 66 | self.__graphs.append(temp_graph) 67 | return 68 | 69 | def add_graph(self, init_graph: WorkGraph, init_graph_id: str, graph_type: GraphType): 70 | graph_info = GraphInfo(init_graph, init_graph_id, graph_type=graph_type, graph_status=GraphStatus.NotStarted) 71 | self.graph_monitor[init_graph_id] = graph_info 72 | return init_graph_id 73 | 74 | def get_graph_status(self, graph_id: str) -> GraphStatus: 75 | if graph_id not in self.graph_monitor: 76 | return GraphStatus.NotStarted 77 | return self.graph_monitor.get(graph_id).graph_status 78 | 79 | def run_graph_with_id(self, graph_id: str): 80 | if graph_id not in self.graph_monitor: 81 | logging.error("Cannot run graph, invalid graph id.") 82 | return 83 | self.graph_monitor[graph_id].graph_status = GraphStatus.Running 84 | graph = self.graph_monitor[graph_id].get_graph() 85 | graph.init_node_dependencies() 86 | graph.execute() 87 | self.graph_monitor[graph_id].graph_status = GraphStatus.Completed 88 | return 89 | 90 | def run_graph_with_index(self, graph_index: int): 91 | if 0 < graph_index < len(self.__graphs): 92 | print("Cannot run graph, index out of range.", file=sys.stderr) 93 | return 94 | self.__graphs[graph_index].get_graph().execute() 95 | 96 | def run_graph_with_name(self, graph_name: str): 97 | idx = self.get_graph_index(graph_name) 98 | if idx < 0: 99 | print_error_message("Cannot run graph, invalid graph name.") 100 | return 101 | return self.run_graph_with_index(idx) 102 | 103 | def get_graph_index(self, graph_name: str): 104 | for graph in self.__graphs: 105 | if graph.get_name() == graph_name: 106 | return self.__graphs.index(graph) 107 | return -1 108 | 109 | def remove_graph(self, graph_index: int): 110 | if 0 <= graph_index < len(self.__graphs): 111 | print("Cannot remove graph, index out of range.", file=sys.stderr) 112 | return 113 | self.__graphs.pop(graph_index) 114 | 115 | def remove_all_graphs(self): 116 | self.__graphs.clear() 117 | 118 | def show_graphs(self): 119 | graph_str = "Graphs: \n" 120 | idx = 0 121 | for graph in self.__graphs: 122 | graph_str += str(idx) + ": " + graph.get_name() + "\n" 123 | idx += 1 124 | print(graph_str) 125 | 126 | 127 | # Running sample 128 | if __name__ == "__main__": 129 | orchestration = Orchestration() 130 | # onborading API call will trigger the following code 131 | graph = build_onboarding_graph(os.path.join(TEST_SKILL_WORKSPACE, "in", "repo"), True) 132 | graph_id = orchestration.add_graph(graph, GraphType.OnboardingGraph) 133 | orchestration.run_graph_with_id(graph_id) 134 | -------------------------------------------------------------------------------- /solidgpt/src/request/basic_request.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import logging 3 | from enum import Enum 4 | 5 | class RequestMethod(Enum): 6 | POST = 1 7 | GET = 2 8 | PATCH = 3 9 | DELETE = 3 10 | 11 | class BasicRequest: 12 | def __init__(self, url : str, method : RequestMethod, headers : str, params : str, data : dict): 13 | self.url = url 14 | self.method = method 15 | self.headers = headers 16 | self.params = params 17 | self.data = data 18 | self.response = None 19 | 20 | def __str__(self): 21 | return f"basic_request(url={self.url}, method={self.method}, headers={self.headers}, params={self.params}, data={self.data})" 22 | 23 | def __repr__(self): 24 | return self.__str__() 25 | 26 | def call(self): 27 | if self.method == RequestMethod.GET: 28 | self.response = requests.get(self.url, headers=self.headers, params=self.params) 29 | elif self.method == RequestMethod.POST: 30 | self.response = requests.post(self.url, headers=self.headers, json=self.data) 31 | else: 32 | raise Exception("Invalid method") 33 | logging.info(f"Response: {self.response}") 34 | return self.response -------------------------------------------------------------------------------- /solidgpt/src/saveload/saveload.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | from solidgpt.src.diy.custom.customizeskillmanager import CustomizeSkillManager 3 | from solidgpt.src.worknode.worknode import * 4 | from solidgpt.src.imports import * 5 | from solidgpt.src.constants import * 6 | 7 | 8 | SKILL_NAME_TO_CONSTRUCTOR: dict[str, Type[WorkSkill]] = { 9 | SKILL_NAME_CREATE_KANBAN_BOARD: CreateKanBan, 10 | } 11 | 12 | 13 | def generate_save_data_from_nodes(nodes: list[WorkNode], generate_debug_info: bool = False): 14 | save_data = [] 15 | for node in nodes: 16 | node_data = { 17 | "node_id": node.node_id, 18 | "manual_review_result": node.manual_review_result 19 | } 20 | 21 | if generate_debug_info: 22 | node_data["next_node_ids"] = list(node.next_node_ids) 23 | node_data["output_id_dependencies"] = list(node.output_id_dependencies) 24 | 25 | skill = node.skill 26 | node_data["skill"] = skill.name 27 | node_data["inputs"] = [] 28 | node_data["outputs"] = [] 29 | 30 | for i in skill.inputs: 31 | temp_input = { 32 | # "param_type": str(i.param_type), 33 | "param_path": i.param_path, 34 | "loading_method": str(i.loading_method), 35 | "load_from_output_id": i.load_from_output_id, 36 | } 37 | if generate_debug_info: 38 | temp_input["param_name"] = i.param_name 39 | temp_input["param_category"] = str(i.param_category) 40 | temp_input["optional"] = str(i.optional) 41 | 42 | node_data["inputs"].append(temp_input) 43 | 44 | for o in skill.outputs: 45 | temp_output = { 46 | "id": o.id, 47 | } 48 | 49 | if generate_debug_info: 50 | temp_output["param_category"] = str(o.param_category) 51 | # temp_output["param_type"] = str(o.param_type) 52 | temp_output["id"] = o.id 53 | 54 | node_data["outputs"].append(temp_output) 55 | 56 | save_data.append(node_data) 57 | return save_data 58 | 59 | 60 | def load_save_data_to_nodes(loaded_data): 61 | nodes: list[WorkNode] = [] 62 | for node_data in loaded_data: 63 | skill_name = node_data["skill"] 64 | inputs_data = node_data["inputs"] 65 | outputs_data = node_data["outputs"] 66 | skill_constructor = SKILL_NAME_TO_CONSTRUCTOR.get(skill_name, None) 67 | if skill_constructor is not None: 68 | skill: WorkSkill = skill_constructor() 69 | else: 70 | skill = CustomizeSkillManager._instance.get_customzied_skill(skill_name) 71 | if skill is not None: 72 | skill.init_config(inputs_data, outputs_data) 73 | node = WorkNode(node_data["node_id"], skill, node_data["manual_review_result"]) 74 | nodes.append(node) 75 | return nodes 76 | 77 | -------------------------------------------------------------------------------- /solidgpt/src/tools/lowdefy/runner/buildwebapprunner.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | class WebAppRunner: 5 | 6 | def __init__(self, app_name : str, project_folder : str): 7 | self.app_name = app_name 8 | self.project_folder = project_folder 9 | 10 | def build_run_webapp(self): 11 | # Command to run 12 | command = ["pnpx", "lowdefy@rc", "dev"] 13 | if os.name == 'nt': 14 | command = ["cmd.exe", "/c", "pnpx", "lowdefy@rc", "dev"] 15 | # Change the current working directory to the absolute path 16 | subprocess.run(command, cwd=self.project_folder) 17 | -------------------------------------------------------------------------------- /solidgpt/src/tools/notion/notionactions.py: -------------------------------------------------------------------------------- 1 | # import time 2 | # 3 | # from solidgpt.definitions import ROOT_DIR 4 | # from solidgpt.src.configuration.configreader import ConfigReader 5 | # import os 6 | # import notional 7 | # from notional import blocks 8 | # from notion2md.exporter.block import MarkdownExporter 9 | # 10 | # NOTION_API_KEY = ConfigReader().get_property("notion_api_key") 11 | # PAGE_ID = ConfigReader().get_property("notion_page_id") 12 | # 13 | # class NotionActions: 14 | # 15 | # def __init__(self): 16 | # self.auth_token = NOTION_API_KEY 17 | # os.environ["NOTION_TOKEN"] = NOTION_API_KEY 18 | # self.notion = notional.connect(auth=self.auth_token) 19 | # self.page = self.notion.pages.retrieve(PAGE_ID) 20 | # 21 | # def process_markdown_and_upload(self, md_file_path: str): 22 | # 23 | # # clear current notion page 24 | # for child_block in self.notion.blocks.children.list(self.page): 25 | # self.notion.blocks.delete(child_block.id) 26 | # 27 | # # Initialize time variables 28 | # start_time = time.time() 29 | # timeout = 20 # 20 seconds 30 | # 31 | # # Check if all blocks are deleted, with a timeout 32 | # while True: 33 | # elapsed_time = time.time() - start_time 34 | # if elapsed_time > timeout: 35 | # print("Timeout reached. Exiting loop. Not all blocks have been cleared.") 36 | # break 37 | # 38 | # remaining_blocks = self.notion.blocks.children.list(self.page) 39 | # for block in remaining_blocks: 40 | # time.sleep(1) 41 | # continue 42 | # 43 | # print("All blocks deleted. Exiting loop.") 44 | # break 45 | # 46 | # with open(md_file_path, "r", encoding="utf-8") as mdFile: 47 | # lastLine = "" 48 | # table = None 49 | # content = None 50 | # for line in mdFile: 51 | # line = line.strip() 52 | # if line.startswith("|"): 53 | # if not lastLine.startswith("|"): 54 | # headers = line.split("|") 55 | # headers = list(filter(None, headers)) 56 | # table = blocks.Table[blocks.TableRow[headers]] 57 | # else: 58 | # row = line.split("|") 59 | # row = list( 60 | # map(lambda item: item.replace('-', ''), row)) 61 | # row = list(filter(None, row)) 62 | # if len(row) > 0: 63 | # table.append(blocks.TableRow[row]) 64 | # elif line.startswith("###"): 65 | # content = blocks.Heading3[line.replace("###", "", 1)] 66 | # elif line.startswith("##"): 67 | # content = blocks.Heading2[line.replace("##", "", 1)] 68 | # elif line.startswith("#"): 69 | # content = blocks.Heading1[line.replace("#", "", 1)] 70 | # elif line.startswith("-"): 71 | # content = blocks.BulletedListItem[line.replace("-", "", 1)] 72 | # else: 73 | # content = blocks.Paragraph[line] 74 | # 75 | # if table is not None and not line.startswith("|"): 76 | # self.notion.blocks.children.append(self.page, table) 77 | # table = None 78 | # if content is not None: 79 | # self.notion.blocks.children.append(self.page, content) 80 | # content = None 81 | # 82 | # lastLine = line 83 | # 84 | # if table is not None: 85 | # self.notion.blocks.children.append(self.page, table) 86 | # if content is not None: 87 | # self.notion.blocks.children.append(self.page, content) 88 | # 89 | # def sync_from_notion(self, path: str = os.path.join(ROOT_DIR), doc_name: str = "PRDDocument"): 90 | # # MarkdownExporter will make markdown file on your output path 91 | # MarkdownExporter(block_id=PAGE_ID, output_path=path, download=True, unzipped=True, output_filename= doc_name).export() 92 | # 93 | # 94 | # # # Test will remove later 95 | # # actions = NotionActions() 96 | # # #actions.process_markdown_and_upload(os.path.join(ROOT_DIR, "PRDDocument.md")) 97 | # # actions.sync_from_notion() 98 | -------------------------------------------------------------------------------- /solidgpt/src/tools/repo/gpt_repository_loader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import fnmatch 4 | 5 | from solidgpt.definitions import ROOT_DIR 6 | 7 | class GitToTextConverter: 8 | def __init__(self, repo_path, ignore_file_path=None, preamble_file=None, output_file_path='output.txt'): 9 | self.repo_path = repo_path 10 | self.ignore_file_path = ignore_file_path 11 | self.preamble_file = preamble_file 12 | self.output_file_path = output_file_path 13 | 14 | def get_ignore_list(self): 15 | ignore_list = [] 16 | if not self.ignore_file_path: 17 | return ignore_list 18 | with open(self.ignore_file_path, 'r') as ignore_file: 19 | for line in ignore_file: 20 | if sys.platform == "win32": 21 | line = line.replace("/", "\\") 22 | ignore_list.append(line.strip()) 23 | # Add default ignore patterns 24 | ignore_list.append('.DS_Store') 25 | return ignore_list 26 | 27 | def should_ignore(self, file_path, ignore_list): 28 | for pattern in ignore_list: 29 | if fnmatch.fnmatch(file_path, pattern): 30 | return True 31 | return False 32 | 33 | def process_repository(self): 34 | with open(self.output_file_path, 'w') as output_file: 35 | if self.preamble_file: 36 | with open(self.preamble_file, 'r') as pf: 37 | preamble_text = pf.read() 38 | output_file.write(f"{preamble_text}\n") 39 | else: 40 | output_file.write("The following text is a Git repository with code. The structure of the text are sections that begin with **-****-****-****-**, followed by a single line containing the file path and file name, followed by a variable amount of lines containing the file contents. The text representing the Git repository ends when the symbols --END-- are encounted. Any further text beyond --END-- are meant to be interpreted as instructions using the aforementioned Git repository as context.\n") 41 | 42 | ignore_list = self.get_ignore_list() 43 | for root, _, files in os.walk(self.repo_path): 44 | for file in files: 45 | file_path = os.path.join(root, file) 46 | relative_file_path = os.path.relpath(file_path, self.repo_path) 47 | 48 | if not self.should_ignore(relative_file_path, ignore_list): 49 | with open(file_path, 'r', errors='ignore') as file: 50 | contents = file.read() 51 | output_file.write("**-**" * 4 + "\n") 52 | output_file.write(f"{relative_file_path}\n") 53 | output_file.write(f"{contents}\n") 54 | 55 | def convert(self): 56 | self.process_repository() 57 | with open(self.output_file_path, 'a') as output_file: 58 | output_file.write("--END--") 59 | print(f"Repository contents written to {self.output_file_path}.") 60 | 61 | if __name__ == "__main__": 62 | converter = GitToTextConverter(os.path.join(ROOT_DIR, '..', '..', 'textbase', 'textbase'), output_file_path='output.txt') 63 | converter.convert() 64 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-flask-dynamodb-api/README.md: -------------------------------------------------------------------------------- 1 | 13 | 14 | # Serverless Framework Python Flask API service backed by DynamoDB on AWS 15 | 16 | This template demonstrates how to develop and deploy a simple Python Flask API service, backed by DynamoDB, running on AWS Lambda using the traditional Serverless Framework. 17 | 18 | 19 | ## Anatomy of the template 20 | 21 | This template configures a single function, `api`, which is responsible for handling all incoming requests thanks to configured `httpApi` events. To learn more about `httpApi` event configuration options, please refer to [httpApi event docs](https://www.serverless.com/framework/docs/providers/aws/events/http-api/). As the events are configured in a way to accept all incoming requests, `Flask` framework is responsible for routing and handling requests internally. The implementation takes advantage of `serverless-wsgi`, which allows you to wrap WSGI applications such as Flask apps. To learn more about `serverless-wsgi`, please refer to corresponding [GitHub repository](https://github.com/logandk/serverless-wsgi). The template also relies on `serverless-python-requirements` plugin for packaging dependencies from `requirements.txt` file. For more details about `serverless-python-requirements` configuration, please refer to corresponding [GitHub repository](https://github.com/UnitedIncome/serverless-python-requirements). 22 | 23 | Additionally, the template also handles provisioning of a DynamoDB database that is used for storing data about users. The Flask application exposes two endpoints, `POST /users` and `GET /user/{userId}`, which allow to create and retrieve users. 24 | 25 | ## Usage 26 | 27 | ### Prerequisites 28 | 29 | In order to package your dependencies locally with `serverless-python-requirements`, you need to have `Python3.9` installed locally. You can create and activate a dedicated virtual environment with the following command: 30 | 31 | ```bash 32 | python3.9 -m venv ./venv 33 | source ./venv/bin/activate 34 | ``` 35 | 36 | Alternatively, you can also use `dockerizePip` configuration from `serverless-python-requirements`. For details on that, please refer to corresponding [GitHub repository](https://github.com/UnitedIncome/serverless-python-requirements). 37 | 38 | ### Deployment 39 | 40 | This example is made to work with the Serverless Framework dashboard, which includes advanced features such as CI/CD, monitoring, metrics, etc. 41 | 42 | In order to deploy with dashboard, you need to first login with: 43 | 44 | ``` 45 | serverless login 46 | ``` 47 | 48 | install dependencies with: 49 | 50 | ``` 51 | npm install 52 | ``` 53 | 54 | and then perform deployment with: 55 | 56 | ``` 57 | serverless deploy 58 | ``` 59 | 60 | After running deploy, you should see output similar to: 61 | 62 | ```bash 63 | Deploying aws-python-flask-dynamodb-api-project to stage dev (us-east-1) 64 | 65 | ✔ Service deployed to stack aws-python-flask-dynamodb-api-project-dev (182s) 66 | 67 | endpoint: ANY - https://xxxxxxxx.execute-api.us-east-1.amazonaws.com 68 | functions: 69 | api: aws-python-flask-dynamodb-api-project-dev-api (1.5 MB) 70 | ``` 71 | 72 | _Note_: In current form, after deployment, your API is public and can be invoked by anyone. For production deployments, you might want to configure an authorizer. For details on how to do that, refer to [httpApi event docs](https://www.serverless.com/framework/docs/providers/aws/events/http-api/). 73 | 74 | ### Invocation 75 | 76 | After successful deployment, you can create a new user by calling the corresponding endpoint: 77 | 78 | ```bash 79 | curl --request POST 'https://xxxxxx.execute-api.us-east-1.amazonaws.com/users' --header 'Content-Type: application/json' --data-raw '{"name": "John", "userId": "someUserId"}' 80 | ``` 81 | 82 | Which should result in the following response: 83 | 84 | ```bash 85 | {"userId":"someUserId","name":"John"} 86 | ``` 87 | 88 | You can later retrieve the user by `userId` by calling the following endpoint: 89 | 90 | ```bash 91 | curl https://xxxxxxx.execute-api.us-east-1.amazonaws.com/users/someUserId 92 | ``` 93 | 94 | Which should result in the following response: 95 | 96 | ```bash 97 | {"userId":"someUserId","name":"John"} 98 | ``` 99 | 100 | If you try to retrieve user that does not exist, you should receive the following response: 101 | 102 | ```bash 103 | {"error":"Could not find user with provided \"userId\""} 104 | ``` 105 | 106 | ### Local development 107 | 108 | Thanks to capabilities of `serverless-wsgi`, it is also possible to run your application locally, however, in order to do that, you will need to first install `werkzeug`, `boto3` dependencies, as well as all other dependencies listed in `requirements.txt`. It is recommended to use a dedicated virtual environment for that purpose. You can install all needed dependencies with the following commands: 109 | 110 | ```bash 111 | pip install werkzeug boto3 112 | pip install -r requirements.txt 113 | ``` 114 | 115 | Additionally, you will need to emulate DynamoDB locally, which can be done by using `serverless-dynamodb-local` plugin. In order to do that, execute the following commands: 116 | 117 | ```bash 118 | serverless plugin install -n serverless-dynamodb-local 119 | serverless dynamodb install 120 | ``` 121 | 122 | It will add the plugin to `devDependencies` in `package.json` file as well as to `plugins` section in `serverless.yml`. Additionally, it will also install DynamoDB locally. 123 | 124 | You should also add the following config to `custom` section in `serverless.yml`: 125 | 126 | 127 | ```yml 128 | custom: 129 | (...) 130 | dynamodb: 131 | start: 132 | migrate: true 133 | stages: 134 | - dev 135 | ``` 136 | 137 | Additionally, we need to reconfigure DynamoDB Client to connect to our local instance of DynamoDB. We can take advantage of `IS_OFFLINE` environment variable set by `serverless-wsgi` plugin and replace: 138 | 139 | 140 | ```python 141 | dynamodb_client = boto3.client('dynamodb') 142 | ``` 143 | 144 | with 145 | 146 | ```python 147 | dynamodb_client = boto3.client('dynamodb') 148 | 149 | if os.environ.get('IS_OFFLINE'): 150 | dynamodb_client = boto3.client('dynamodb', region_name='localhost', endpoint_url='http://localhost:8000') 151 | ``` 152 | 153 | Now you can start DynamoDB local with the following command: 154 | 155 | ```bash 156 | serverless dynamodb start 157 | ``` 158 | 159 | At this point, you can run your application locally with the following command: 160 | 161 | ```bash 162 | serverless wsgi serve 163 | ``` 164 | 165 | For additional local development capabilities of `serverless-wsgi` and `serverless-dynamodb-local` plugins, please refer to corresponding GitHub repositories: 166 | - https://github.com/logandk/serverless-wsgi 167 | - https://github.com/99x/serverless-dynamodb-local 168 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-flask-dynamodb-api/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | from flask import Flask, jsonify, make_response, request 5 | 6 | app = Flask(__name__) 7 | 8 | 9 | dynamodb_client = boto3.client('dynamodb') 10 | 11 | if os.environ.get('IS_OFFLINE'): 12 | dynamodb_client = boto3.client( 13 | 'dynamodb', region_name='localhost', endpoint_url='http://localhost:8000' 14 | ) 15 | 16 | 17 | USERS_TABLE = os.environ['USERS_TABLE'] 18 | 19 | 20 | @app.route('/users/') 21 | def get_user(user_id): 22 | result = dynamodb_client.get_item( 23 | TableName=USERS_TABLE, Key={'userId': {'S': user_id}} 24 | ) 25 | item = result.get('Item') 26 | if not item: 27 | return jsonify({'error': 'Could not find user with provided "userId"'}), 404 28 | 29 | return jsonify( 30 | {'userId': item.get('userId').get('S'), 'name': item.get('name').get('S')} 31 | ) 32 | 33 | 34 | @app.route('/users', methods=['POST']) 35 | def create_user(): 36 | user_id = request.json.get('userId') 37 | name = request.json.get('name') 38 | if not user_id or not name: 39 | return jsonify({'error': 'Please provide both "userId" and "name"'}), 400 40 | 41 | dynamodb_client.put_item( 42 | TableName=USERS_TABLE, Item={'userId': {'S': user_id}, 'name': {'S': name}} 43 | ) 44 | 45 | return jsonify({'userId': user_id, 'name': name}) 46 | 47 | 48 | @app.errorhandler(404) 49 | def resource_not_found(e): 50 | return make_response(jsonify(error='Not found!'), 404) 51 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-flask-dynamodb-api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-python-flask-dynamodb-api", 3 | "version": "1.0.0", 4 | "description": "Example of a Python Flask API service backed by DynamoDB with traditional Serverless Framework", 5 | "author": "", 6 | "devDependencies": { 7 | "serverless-python-requirements": "^6.0.0", 8 | "serverless-wsgi": "^3.0.1" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-flask-dynamodb-api/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==1.1.4 2 | Werkzeug==1.0.1 3 | markupsafe==2.0.1 4 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-flask-dynamodb-api/serverless.template.yml: -------------------------------------------------------------------------------- 1 | name: aws-python-flask-dynamodb-api 2 | org: serverlessinc 3 | description: Deploys a Python Flask API service, backed by DynamoDB, with traditional Serverless Framework 4 | keywords: aws, serverless, faas, lambda, python, flask, dynamodb 5 | repo: https://github.com/serverless/examples/aws-python-flask-dynamodb-api 6 | license: MIT 7 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-flask-dynamodb-api/serverless.yml: -------------------------------------------------------------------------------- 1 | service: aws-python-flask-dynamodb-api 2 | 3 | frameworkVersion: '3' 4 | 5 | custom: 6 | tableName: 'users-table-${sls:stage}' 7 | wsgi: 8 | app: app.app 9 | 10 | provider: 11 | name: aws 12 | runtime: python3.9 13 | iam: 14 | role: 15 | statements: 16 | - Effect: Allow 17 | Action: 18 | - dynamodb:Query 19 | - dynamodb:Scan 20 | - dynamodb:GetItem 21 | - dynamodb:PutItem 22 | - dynamodb:UpdateItem 23 | - dynamodb:DeleteItem 24 | Resource: 25 | - Fn::GetAtt: [ UsersTable, Arn ] 26 | environment: 27 | USERS_TABLE: ${self:custom.tableName} 28 | 29 | functions: 30 | api: 31 | handler: wsgi_handler.handler 32 | events: 33 | - httpApi: '*' 34 | 35 | plugins: 36 | - serverless-wsgi 37 | - serverless-python-requirements 38 | 39 | resources: 40 | Resources: 41 | UsersTable: 42 | Type: AWS::DynamoDB::Table 43 | Properties: 44 | AttributeDefinitions: 45 | - AttributeName: userId 46 | AttributeType: S 47 | KeySchema: 48 | - AttributeName: userId 49 | KeyType: HASH 50 | ProvisionedThroughput: 51 | ReadCapacityUnits: 1 52 | WriteCapacityUnits: 1 53 | TableName: ${self:custom.tableName} 54 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/.gitignore: -------------------------------------------------------------------------------- 1 | .serverless 2 | *.pyc 3 | *.pyo 4 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/README.md: -------------------------------------------------------------------------------- 1 | 12 | # Serverless HTTP API 13 | 14 | This example demonstrates how to setup an HTTP API allowing you to create, list, get, update and delete Todos. DynamoDB is used to store the data. This is just an example and of course you could use any data storage as a backend. 15 | 16 | ## Structure 17 | 18 | This service has a separate directory for all the todo operations. For each operation exactly one file exists e.g. `todos/delete.py`. In each of these files there is exactly one function defined. 19 | 20 | The idea behind the `todos` directory is that in case you want to create a service containing multiple resources e.g. users, notes, comments you could do so in the same service. While this is certainly possible you might consider creating a separate service for each resource. It depends on the use-case and your preference. 21 | 22 | ## Use-cases 23 | 24 | - API for a Web Application 25 | - API for a Mobile Application 26 | 27 | ## Setup 28 | 29 | ```bash 30 | npm install -g serverless 31 | ``` 32 | 33 | ## Deploy 34 | 35 | In order to deploy the endpoint simply run 36 | 37 | ```bash 38 | serverless deploy 39 | ``` 40 | 41 | The expected result should be similar to: 42 | 43 | ```bash 44 | Serverless: Packaging service… 45 | Serverless: Uploading CloudFormation file to S3… 46 | Serverless: Uploading service .zip file to S3… 47 | Serverless: Updating Stack… 48 | Serverless: Checking Stack update progress… 49 | Serverless: Stack update finished… 50 | 51 | Service Information 52 | service: serverless-http-api-dynamodb 53 | stage: dev 54 | region: us-east-1 55 | api keys: 56 | None 57 | endpoints: 58 | POST - https://45wf34z5yf.execute-api.us-east-1.amazonaws.com/todos 59 | GET - https://45wf34z5yf.execute-api.us-east-1.amazonaws.com/todos 60 | GET - https://45wf34z5yf.execute-api.us-east-1.amazonaws.com/todos/{id} 61 | PUT - https://45wf34z5yf.execute-api.us-east-1.amazonaws.com/todos/{id} 62 | DELETE - https://45wf34z5yf.execute-api.us-east-1.amazonaws.com/todos/{id} 63 | functions: 64 | update: serverless-http-api-dynamodb-dev-update 65 | get: serverless-http-api-dynamodb-dev-get 66 | list: serverless-http-api-dynamodb-dev-list 67 | create: serverless-http-api-dynamodb-dev-create 68 | delete: serverless-http-api-dynamodb-dev-delete 69 | ``` 70 | 71 | ## Usage 72 | 73 | You can create, retrieve, update, or delete todos with the following commands: 74 | 75 | ### Create a Todo 76 | 77 | ```bash 78 | curl -X POST https://XXXXXXX.execute-api.us-east-1.amazonaws.com/todos --data '{ "text": "Learn Serverless" }' -H "Content-Type: application/json" 79 | ``` 80 | 81 | No output 82 | 83 | ### List all Todos 84 | 85 | ```bash 86 | curl https://XXXXXXX.execute-api.us-east-1.amazonaws.com/todos 87 | ``` 88 | 89 | Example output: 90 | ```bash 91 | [{"text":"Deploy my first service","id":"ac90feaa11e6-9ede-afdfa051af86","checked":true,"updatedAt":1479139961304},{"text":"Learn Serverless","id":"206793aa11e6-9ede-afdfa051af86","createdAt":1479139943241,"checked":false,"updatedAt":1479139943241}]% 92 | ``` 93 | 94 | ### Get one Todo 95 | 96 | ```bash 97 | # Replace the part with a real id from your todos table 98 | curl https://XXXXXXX.execute-api.us-east-1.amazonaws.com/todos/ 99 | ``` 100 | 101 | Example Result: 102 | ```bash 103 | {"text":"Learn Serverless","id":"ee6490d0-aa11e6-9ede-afdfa051af86","createdAt":1479138570824,"checked":false,"updatedAt":1479138570824}% 104 | ``` 105 | 106 | ### Update a Todo 107 | 108 | ```bash 109 | # Replace the part with a real id from your todos table 110 | curl -X PUT https://XXXXXXX.execute-api.us-east-1.amazonaws.com/todos/ --data '{ "text": "Learn Serverless", "checked": true }' -H "Content-Type: application/json" 111 | ``` 112 | 113 | Example Result: 114 | ```bash 115 | {"text":"Learn Serverless","id":"ee6490d0-aa11e6-9ede-afdfa051af86","createdAt":1479138570824,"checked":true,"updatedAt":1479138570824}% 116 | ``` 117 | 118 | ### Delete a Todo 119 | 120 | ```bash 121 | # Replace the part with a real id from your todos table 122 | curl -X DELETE https://XXXXXXX.execute-api.us-east-1.amazonaws.com/todos/ 123 | ``` 124 | 125 | No output 126 | 127 | ## Scaling 128 | 129 | ### AWS Lambda 130 | 131 | By default, AWS Lambda limits the total concurrent executions across all functions within a given region to 1000. The default limit is a safety limit that protects you from costs due to potential runaway or recursive functions during initial development and testing. To increase this limit above the default, follow the steps in [To request a limit increase for concurrent executions](http://docs.aws.amazon.com/lambda/latest/dg/concurrent-executions.html#increase-concurrent-executions-limit). 132 | 133 | ### DynamoDB 134 | 135 | When you create a table, you specify how much provisioned throughput capacity you want to reserve for reads and writes. DynamoDB will reserve the necessary resources to meet your throughput needs while ensuring consistent, low-latency performance. You can change the provisioned throughput and increasing or decreasing capacity as needed. 136 | 137 | This is can be done via settings in the `serverless.yml`. 138 | 139 | ```yaml 140 | ProvisionedThroughput: 141 | ReadCapacityUnits: 1 142 | WriteCapacityUnits: 1 143 | ``` 144 | 145 | In case you expect a lot of traffic fluctuation we recommend to checkout this guide on how to auto scale DynamoDB [https://aws.amazon.com/blogs/aws/auto-scale-dynamodb-with-dynamic-dynamodb/](https://aws.amazon.com/blogs/aws/auto-scale-dynamodb-with-dynamic-dynamodb/) 146 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-http-with-dynamodb", 3 | "version": "1.0.0", 4 | "description": "Serverless HTTP API", 5 | "author": "", 6 | "license": "MIT" 7 | } 8 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/serverless.yml: -------------------------------------------------------------------------------- 1 | service: serverless-http-api-dynamodb 2 | frameworkVersion: "3" 3 | 4 | provider: 5 | name: aws 6 | runtime: python3.9 7 | environment: 8 | DYNAMODB_TABLE: ${self:service}-${sls:stage} 9 | httpApi: 10 | cors: true 11 | iam: 12 | role: 13 | statements: 14 | - Effect: Allow 15 | Action: 16 | - dynamodb:Query 17 | - dynamodb:Scan 18 | - dynamodb:GetItem 19 | - dynamodb:PutItem 20 | - dynamodb:UpdateItem 21 | - dynamodb:DeleteItem 22 | Resource: "arn:aws:dynamodb:${aws:region}:*:table/${self:provider.environment.DYNAMODB_TABLE}" 23 | 24 | functions: 25 | create: 26 | handler: todos/create.create 27 | events: 28 | - httpApi: 29 | path: /todos 30 | method: post 31 | 32 | list: 33 | handler: todos/list.list 34 | events: 35 | - httpApi: 36 | path: /todos 37 | method: get 38 | 39 | get: 40 | handler: todos/get.get 41 | events: 42 | - httpApi: 43 | path: /todos/{id} 44 | method: get 45 | 46 | update: 47 | handler: todos/update.update 48 | events: 49 | - httpApi: 50 | path: /todos/{id} 51 | method: put 52 | 53 | delete: 54 | handler: todos/delete.delete 55 | events: 56 | - httpApi: 57 | path: /todos/{id} 58 | method: delete 59 | 60 | resources: 61 | Resources: 62 | TodosDynamoDbTable: 63 | Type: 'AWS::DynamoDB::Table' 64 | DeletionPolicy: Retain 65 | Properties: 66 | AttributeDefinitions: 67 | - 68 | AttributeName: id 69 | AttributeType: S 70 | KeySchema: 71 | - 72 | AttributeName: id 73 | KeyType: HASH 74 | BillingMode: PAY_PER_REQUEST 75 | TableName: ${self:provider.environment.DYNAMODB_TABLE} 76 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Citizen/SolidGPT/f39ee7da341bbcda6cea926eece7b087307588e6/solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/__init__.py -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/create.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import time 5 | import uuid 6 | 7 | import boto3 8 | dynamodb = boto3.resource('dynamodb') 9 | 10 | 11 | def create(event, context): 12 | data = json.loads(event['body']) 13 | if 'text' not in data: 14 | logging.error("Validation Failed") 15 | raise Exception("Couldn't create the todo item.") 16 | 17 | timestamp = str(time.time()) 18 | 19 | table = dynamodb.Table(os.environ['DYNAMODB_TABLE']) 20 | 21 | item = { 22 | 'id': str(uuid.uuid1()), 23 | 'text': data['text'], 24 | 'checked': False, 25 | 'createdAt': timestamp, 26 | 'updatedAt': timestamp, 27 | } 28 | 29 | # write the todo to the database 30 | table.put_item(Item=item) 31 | 32 | # create a response 33 | response = { 34 | "statusCode": 200, 35 | "body": json.dumps(item) 36 | } 37 | 38 | return response 39 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/decimalencoder.py: -------------------------------------------------------------------------------- 1 | import decimal 2 | import json 3 | 4 | 5 | # This is a workaround for: http://bugs.python.org/issue16535 6 | class DecimalEncoder(json.JSONEncoder): 7 | def default(self, obj): 8 | if isinstance(obj, decimal.Decimal): 9 | return int(obj) 10 | return super(DecimalEncoder, self).default(obj) 11 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/delete.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | dynamodb = boto3.resource('dynamodb') 5 | 6 | 7 | def delete(event, context): 8 | table = dynamodb.Table(os.environ['DYNAMODB_TABLE']) 9 | 10 | # delete the todo from the database 11 | table.delete_item( 12 | Key={ 13 | 'id': event['pathParameters']['id'] 14 | } 15 | ) 16 | 17 | # create a response 18 | response = { 19 | "statusCode": 200 20 | } 21 | 22 | return response 23 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/get.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | from todos import decimalencoder 5 | import boto3 6 | dynamodb = boto3.resource('dynamodb') 7 | 8 | 9 | def get(event, context): 10 | table = dynamodb.Table(os.environ['DYNAMODB_TABLE']) 11 | 12 | # fetch todo from the database 13 | result = table.get_item( 14 | Key={ 15 | 'id': event['pathParameters']['id'] 16 | } 17 | ) 18 | 19 | # create a response 20 | response = { 21 | "statusCode": 200, 22 | "body": json.dumps(result['Item'], 23 | cls=decimalencoder.DecimalEncoder) 24 | } 25 | 26 | return response 27 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/list.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from todos import decimalencoder 5 | import boto3 6 | dynamodb = boto3.resource('dynamodb') 7 | 8 | 9 | def list(event, context): 10 | table = dynamodb.Table(os.environ['DYNAMODB_TABLE']) 11 | 12 | # fetch all todos from the database 13 | result = table.scan() 14 | 15 | # create a response 16 | response = { 17 | "statusCode": 200, 18 | "body": json.dumps(result['Items'], cls=decimalencoder.DecimalEncoder) 19 | } 20 | 21 | return response 22 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-http-api-with-dynamodb/todos/update.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import logging 4 | import os 5 | 6 | from todos import decimalencoder 7 | import boto3 8 | dynamodb = boto3.resource('dynamodb') 9 | 10 | 11 | def update(event, context): 12 | data = json.loads(event['body']) 13 | if 'text' not in data or 'checked' not in data: 14 | logging.error("Validation Failed") 15 | raise Exception("Couldn't update the todo item.") 16 | return 17 | 18 | timestamp = int(time.time() * 1000) 19 | 20 | table = dynamodb.Table(os.environ['DYNAMODB_TABLE']) 21 | 22 | # update the todo in the database 23 | result = table.update_item( 24 | Key={ 25 | 'id': event['pathParameters']['id'] 26 | }, 27 | ExpressionAttributeNames={ 28 | '#todo_text': 'text', 29 | }, 30 | ExpressionAttributeValues={ 31 | ':text': data['text'], 32 | ':checked': data['checked'], 33 | ':updatedAt': timestamp, 34 | }, 35 | UpdateExpression='SET #todo_text = :text, ' 36 | 'checked = :checked, ' 37 | 'updatedAt = :updatedAt', 38 | ReturnValues='ALL_NEW', 39 | ) 40 | 41 | # create a response 42 | response = { 43 | "statusCode": 200, 44 | "body": json.dumps(result['Attributes'], 45 | cls=decimalencoder.DecimalEncoder) 46 | } 47 | 48 | return response 49 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | hackernews/db.sqlite3 104 | 105 | # Logs 106 | logs 107 | *.log 108 | npm-debug.log* 109 | yarn-debug.log* 110 | yarn-error.log* 111 | 112 | # Runtime data 113 | pids 114 | *.pid 115 | *.seed 116 | *.pid.lock 117 | 118 | # Directory for instrumented libs generated by jscoverage/JSCover 119 | lib-cov 120 | 121 | # Coverage directory used by tools like istanbul 122 | coverage 123 | 124 | # nyc test coverage 125 | .nyc_output 126 | 127 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 128 | .grunt 129 | 130 | # Bower dependency directory (https://bower.io/) 131 | bower_components 132 | 133 | # node-waf configuration 134 | .lock-wscript 135 | 136 | # Compiled binary addons (https://nodejs.org/api/addons.html) 137 | build/Release 138 | 139 | # Dependency directories 140 | node_modules/ 141 | jspm_packages/ 142 | 143 | # Typescript v1 declaration files 144 | typings/ 145 | 146 | # Optional npm cache directory 147 | .npm 148 | 149 | # Optional eslint cache 150 | .eslintcache 151 | 152 | # Optional REPL history 153 | .node_repl_history 154 | 155 | # Output of 'npm pack' 156 | *.tgz 157 | 158 | # Yarn Integrity file 159 | .yarn-integrity 160 | 161 | # dotenv environment variables file 162 | .env 163 | 164 | # Token file 165 | serverless.env.yml 166 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jonatas Baldin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/README.md: -------------------------------------------------------------------------------- 1 | 13 | # Serverless Telegram Bot 14 | This example demonstrates how to setup an echo Telegram Bot using the Serverless Framework ⚡🤖 15 | 16 | ## Usage 17 | 18 | ### What do I need? 19 | - A AWS key configured locally, see [here](https://serverless.com/framework/docs/providers/aws/guide/credentials/). 20 | - NodeJS. I tested with v8.9.0. 21 | - A Telegram account. 22 | 23 | ### Installing 24 | ``` 25 | # Install the Serverless Framework 26 | $ npm install serverless -g 27 | 28 | # Install the necessary plugins 29 | $ npm install 30 | 31 | # Get a bot from Telegram, sending this message to @BotFather 32 | $ /newbot 33 | 34 | # Put the token received into a file called serverless.env.yml, like this 35 | $ cat serverless.env.yml 36 | TELEGRAM_TOKEN: 37 | 38 | # Deploy it! 39 | $ serverless deploy 40 | 41 | # With the URL returned in the output, configure the Webhook 42 | $ curl -X POST https://.amazonaws.com/dev/set_webhook 43 | ``` 44 | 45 | Now, just start a conversation with the bot :) 46 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import telegram 3 | import os 4 | import logging 5 | 6 | 7 | # Logging is cool! 8 | logger = logging.getLogger() 9 | if logger.handlers: 10 | for handler in logger.handlers: 11 | logger.removeHandler(handler) 12 | logging.basicConfig(level=logging.INFO) 13 | 14 | OK_RESPONSE = { 15 | 'statusCode': 200, 16 | 'headers': {'Content-Type': 'application/json'}, 17 | 'body': json.dumps('ok') 18 | } 19 | ERROR_RESPONSE = { 20 | 'statusCode': 400, 21 | 'body': json.dumps('Oops, something went wrong!') 22 | } 23 | 24 | 25 | def configure_telegram(): 26 | """ 27 | Configures the bot with a Telegram Token. 28 | 29 | Returns a bot instance. 30 | """ 31 | 32 | TELEGRAM_TOKEN = os.environ.get('TELEGRAM_TOKEN') 33 | if not TELEGRAM_TOKEN: 34 | logger.error('The TELEGRAM_TOKEN must be set') 35 | raise NotImplementedError 36 | 37 | return telegram.Bot(TELEGRAM_TOKEN) 38 | 39 | 40 | def webhook(event, context): 41 | """ 42 | Runs the Telegram webhook. 43 | """ 44 | 45 | bot = configure_telegram() 46 | logger.info('Event: {}'.format(event)) 47 | 48 | if event.get('requestContext', {}).get('http', {}).get('method') == 'POST' and event.get('body'): 49 | logger.info('Message received') 50 | update = telegram.Update.de_json(json.loads(event.get('body')), bot) 51 | chat_id = update.message.chat.id 52 | text = update.message.text 53 | 54 | if text == '/start': 55 | text = """Hello, human! I am an echo bot, built with Python and the Serverless Framework. 56 | You can take a look at my source code here: https://github.com/jonatasbaldin/serverless-telegram-bot. 57 | If you have any issues, please drop a tweet to my creator: https://twitter.com/jonatsbaldin. Happy botting!""" 58 | 59 | bot.sendMessage(chat_id=chat_id, text=text) 60 | logger.info('Message sent') 61 | 62 | return OK_RESPONSE 63 | 64 | return ERROR_RESPONSE 65 | 66 | 67 | def set_webhook(event, context): 68 | """ 69 | Sets the Telegram bot webhook. 70 | """ 71 | 72 | logger.info('Event: {}'.format(event)) 73 | bot = configure_telegram() 74 | url = 'https://{}/{}/'.format( 75 | event.get('headers').get('host'), 76 | event.get('requestContext').get('stage'), 77 | ) 78 | webhook = bot.set_webhook(url) 79 | 80 | if webhook: 81 | return OK_RESPONSE 82 | 83 | return ERROR_RESPONSE 84 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-telegram-bot", 3 | "description": "This example demonstrates how to setup an echo Telegram Bot using the Serverless Framework ⚡🤖 ", 4 | "version": "0.1.0", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/jonatasbaldin/serverless-telegram-bot.git" 8 | }, 9 | "keywords": [ 10 | "python", 11 | "aws", 12 | "lambda", 13 | "serverless" 14 | ], 15 | "author": "jonatasbaldin", 16 | "license": "MIT", 17 | "dependencies": { 18 | "serverless-python-requirements": "^5.4.0" 19 | }, 20 | "bugs": { 21 | "url": "https://github.com/jonatasbaldin/serverless-telegram-bot/issues" 22 | }, 23 | "homepage": "https://github.com/jonatasbaldin/serverless-telegram-bot#readme" 24 | } 25 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/requirements.txt: -------------------------------------------------------------------------------- 1 | python-telegram-bot==13.11 2 | -------------------------------------------------------------------------------- /solidgpt/src/tools/templates/aws-python-telegram-bot/serverless.yml: -------------------------------------------------------------------------------- 1 | service: serverless-telegram-bot 2 | 3 | frameworkVersion: "3" 4 | 5 | provider: 6 | name: aws 7 | runtime: python3.9 8 | profile: ckl 9 | environment: 10 | TELEGRAM_TOKEN: ${file(./serverless.env.yml):TELEGRAM_TOKEN, ''} 11 | 12 | functions: 13 | webhook: 14 | handler: handler.webhook 15 | events: 16 | - httpApi: 17 | path: / 18 | method: POST 19 | 20 | set_webhook: 21 | handler: handler.set_webhook 22 | events: 23 | - httpApi: 24 | path: /set_webhook 25 | method: POST 26 | 27 | plugins: 28 | - serverless-python-requirements 29 | -------------------------------------------------------------------------------- /solidgpt/src/util/util.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import openai 5 | import uuid 6 | from qdrant_client import QdrantClient 7 | from langchain.embeddings import OpenAIEmbeddings 8 | from qdrant_client.http.models import PointStruct, Distance, VectorParams 9 | from solidgpt.definitions import * 10 | 11 | 12 | def save_to_json(data, filename="data.json"): 13 | create_directories_if_not_exist(filename) 14 | # Save data to a JSON file 15 | with open(filename, "w") as json_file: 16 | print(json.dump(data, json_file, indent=4)) 17 | 18 | 19 | def load_from_json(filename="data.json"): 20 | # Load data from a JSON file 21 | with open(filename, "r") as json_file: 22 | loaded_data = json.load(json_file) 23 | return loaded_data 24 | 25 | 26 | def save_to_md(filename, content: str, path = "") -> str: 27 | create_directories_if_not_exist(filename) 28 | path = os.path.join(ROOT_DIR, path) 29 | full_path = os.path.join(path, filename) 30 | with open(full_path, "w") as md_file: 31 | md_file.write(content) 32 | logging.info(f"Information saved to {full_path}") 33 | return full_path 34 | 35 | 36 | def save_to_md2(filename, content: str) -> str: 37 | create_directories_if_not_exist(filename) 38 | full_path = filename 39 | full_path = add_extension_if_not_exist(full_path, ".md") 40 | with open(full_path, "w") as md_file: 41 | md_file.write(content) 42 | md_file.flush() 43 | logging.info(f"Information saved to {full_path}") 44 | return full_path 45 | 46 | 47 | def save_to_yaml(filename, content: str) -> str: 48 | create_directories_if_not_exist(filename) 49 | full_path = filename 50 | full_path = add_extension_if_not_exist(full_path, ".yaml") 51 | with open(full_path, "w", encoding='utf-8') as md_file: 52 | md_file.write(content) 53 | md_file.flush() 54 | logging.info(f"Information saved to {full_path}") 55 | return full_path 56 | 57 | 58 | def save_to_text(filename, content): 59 | create_directories_if_not_exist(filename) 60 | full_path = filename 61 | full_path = add_extension_if_not_exist(full_path, ".txt") 62 | with open(full_path, "w", encoding='utf-8') as txt_file: 63 | txt_file.write(content) 64 | logging.info(f"Information saved to {full_path}") 65 | return full_path 66 | 67 | 68 | def create_directories_if_not_exist(filepath: str): 69 | dir_name = os.path.dirname(filepath) 70 | if not os.path.exists(dir_name): 71 | os.makedirs(dir_name) 72 | return 73 | 74 | 75 | def load_from_text(filename, path = "", extension = ".md") -> str: 76 | full_path = os.path.join(path, filename) 77 | full_path = add_extension_if_not_exist(full_path, extension) 78 | with open(full_path, "r", encoding='utf-8') as md_file: 79 | content = md_file.read() 80 | logging.info(f"Information loaded from {full_path}") 81 | return content 82 | 83 | def add_extension_if_not_exist(input_string, extension): 84 | if not input_string.endswith(extension): 85 | return input_string + extension 86 | else: 87 | return input_string 88 | 89 | 90 | def same_string(s1: str, s2: str, case_sensitive: bool = False): 91 | if case_sensitive: 92 | return s1 == s2 93 | return s1.lower() == s2.lower() 94 | 95 | 96 | def print_error_message(message): 97 | print(f"Error: {message}", file=sys.stderr) 98 | 99 | 100 | def delete_directory_contents(directory): 101 | for root, dirs, files in os.walk(directory, topdown=False): 102 | for file in files: 103 | file_path = os.path.join(root, file) 104 | try: 105 | os.remove(file_path) 106 | print(f"Deleted file: {file_path}") 107 | except Exception as e: 108 | print(f"Error deleting file {file_path}: {str(e)}") 109 | for dir_name in dirs: 110 | dir_path = os.path.join(root, dir_name) 111 | try: 112 | os.rmdir(dir_path) 113 | print(f"Deleted directory: {dir_path}") 114 | except Exception as e: 115 | print(f"Error deleting directory {dir_path}: {str(e)}") 116 | 117 | 118 | def embed_templates(): 119 | qdrant_path = os.path.join(SRC_DIR, "tools", "qdrant", "embedding") 120 | client = QdrantClient(path=qdrant_path) 121 | template_lists = list(filter(lambda x: os.path.isdir(os.path.join(SRC_DIR, "tools", "templates", x)), 122 | os.listdir(os.path.join(SRC_DIR, "tools", "templates")))) 123 | 124 | def get_uuid(): 125 | return str(uuid.uuid4().hex) 126 | 127 | def __embed_summary(summary, path): 128 | payload_dict = {"path": path, "summary": summary} 129 | embeddings_model = OpenAIEmbeddings(openai_api_key=openai.api_key) 130 | embedded_query = embeddings_model.embed_query(summary) 131 | try: 132 | client.upsert( 133 | collection_name="templates", 134 | points=[ 135 | PointStruct(id=get_uuid(), vector=embedded_query, payload=payload_dict) 136 | ] 137 | ) 138 | except ValueError: 139 | client.recreate_collection( 140 | collection_name="templates", 141 | vectors_config=VectorParams(size=len(embedded_query), distance=Distance.COSINE), 142 | ) 143 | client.upsert( 144 | collection_name="templates", 145 | points=[ 146 | PointStruct(id=get_uuid(), vector=embedded_query, payload=payload_dict) 147 | ] 148 | ) 149 | return 150 | 151 | for cur_dir in template_lists: 152 | cur_path = os.path.join(SRC_DIR, "tools", "templates", cur_dir) 153 | readme = os.path.join(cur_path, "README.md") 154 | with open(readme) as f: 155 | content = f.read() 156 | __embed_summary(content, cur_path) 157 | return 158 | 159 | 160 | if __name__ == "__main__": 161 | # embed_templates() 162 | pass 163 | -------------------------------------------------------------------------------- /solidgpt/src/workgraph/displayresult.py: -------------------------------------------------------------------------------- 1 | class DisplayResult: 2 | result: str = "" 3 | custom_result = None 4 | 5 | def __init__(self): 6 | self.result = "No result" 7 | 8 | def set_result(self, s): 9 | self.result = s 10 | 11 | def get_result(self): 12 | return self.result 13 | 14 | def set_custom_result(self, data): 15 | self.custom_result = data 16 | 17 | def get_custom_result(self): 18 | return self.custom_result 19 | -------------------------------------------------------------------------------- /solidgpt/src/workgraph/graph_helper.py: -------------------------------------------------------------------------------- 1 | 2 | from enum import Enum 3 | import uuid 4 | 5 | 6 | class GraphType(Enum): 7 | OnboardingGraph = 1 8 | WritePRDGraph = 2 9 | ProvideTechSolutionGraph = 3 10 | RepoChat = 4 11 | 12 | class GraphStatus(Enum): 13 | NotStarted = 0 14 | Running = 1 15 | Completed = 2 16 | Failed = 3 -------------------------------------------------------------------------------- /solidgpt/src/worknode/worknode.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.workgraph.displayresult import DisplayResult 2 | from solidgpt.src.workskill.workskill import * 3 | from solidgpt.src.util.util import * 4 | 5 | 6 | class WorkNode: 7 | node_id: str = 0 8 | skill: WorkSkill = None 9 | next_node_ids: set[str] = [] 10 | output_id_dependencies: set[int] = [] 11 | manual_review_result: bool = False 12 | graph_cache: dict = {} 13 | display_result: DisplayResult = None 14 | 15 | def __init__(self, node_id: str, work_skill: WorkSkill, manual_review_result: bool = False, graph_cache : dict = {}): 16 | # Initialization 17 | self.node_id = node_id 18 | self.graph_cache = graph_cache 19 | work_skill.graph_cache = self.graph_cache 20 | self.skill = work_skill 21 | self.next_node_ids = set([]) 22 | self.output_id_dependencies = set([]) 23 | self.manual_review_result = manual_review_result 24 | self.display_result = DisplayResult() 25 | return 26 | 27 | def can_execute(self): 28 | if len(self.output_id_dependencies) == 0: 29 | return True 30 | print("still need to wait other nodes to finish first for node " + str(self.node_id)) 31 | return False 32 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skillio.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class SkillIOParamCategory(int, Enum): 5 | BusinessRequirementsDocument = 1 6 | ProductRequirementsDocument = 2 7 | HighLevelDesignDocument = 3 8 | SourceCode = 4 9 | PlainText = 5 10 | KanbanBoard = 6 11 | YAML = 7 12 | CODE_PLAIN_TEXT = 8 13 | CODE_PLAN = 9 14 | 15 | class SkillInputLoadingMethod(Enum): 16 | LOAD_FROM_OUTPUT_ID = 1 17 | LOAD_FROM_STRING = 2 18 | LOAD_FROM_CACHE_STRING = 3 19 | 20 | 21 | STRING_TO_SKILL_INPUT_LOADING_METHOD_DICT: dict[str, SkillInputLoadingMethod] = { 22 | str(SkillInputLoadingMethod.LOAD_FROM_OUTPUT_ID): SkillInputLoadingMethod.LOAD_FROM_OUTPUT_ID, 23 | str(SkillInputLoadingMethod.LOAD_FROM_STRING): SkillInputLoadingMethod.LOAD_FROM_STRING, 24 | str(SkillInputLoadingMethod.LOAD_FROM_CACHE_STRING): SkillInputLoadingMethod.LOAD_FROM_CACHE_STRING, 25 | } 26 | 27 | 28 | def string_to_skill_input_loading_method(s: str): 29 | return STRING_TO_SKILL_INPUT_LOADING_METHOD_DICT.get(s, SkillInputLoadingMethod.LOAD_FROM_STRING) 30 | 31 | 32 | class SkillOutput: 33 | config = None 34 | param_name: str = "" 35 | param_category: SkillIOParamCategory = SkillIOParamCategory.BusinessRequirementsDocument 36 | param_path: str = "" 37 | id: int = -1 38 | to_display: bool = False 39 | 40 | def __init__(self, 41 | param_name: str, 42 | param_category: SkillIOParamCategory, 43 | ): 44 | # Initialization 45 | self.param_name = param_name 46 | self.param_category = param_category 47 | self.param_path = "" 48 | self.id = -1 49 | self.to_display = False 50 | 51 | def apply_config(self, config): 52 | if config is None: 53 | return 54 | self.config = config 55 | self.id = config["id"] 56 | self.to_display = config.get("to_display", False) 57 | 58 | 59 | class SkillInput: 60 | config: dict = None 61 | param_name: str = "" 62 | param_category: SkillIOParamCategory = SkillIOParamCategory.BusinessRequirementsDocument 63 | param_path: str = "" 64 | optional: bool = False 65 | loading_method: SkillInputLoadingMethod = SkillInputLoadingMethod.LOAD_FROM_STRING 66 | load_from_output_id: int = -1 67 | skill_output: SkillOutput = None 68 | content: str = "" 69 | 70 | def __init__(self, 71 | param_name: str, 72 | param_category: SkillIOParamCategory, 73 | optional: bool = False, 74 | ): 75 | # Initialization 76 | self.param_name = param_name 77 | self.param_category = param_category 78 | self.optional = optional 79 | self.loading_method = SkillInputLoadingMethod.LOAD_FROM_STRING 80 | self.load_from_output_id = -1 81 | return 82 | 83 | def apply_config(self, config): 84 | if config is None: 85 | return 86 | self.config = config 87 | self.param_path = config["param_path"] 88 | self.loading_method = string_to_skill_input_loading_method(config["loading_method"]) 89 | if "load_from_output_id" in config: 90 | self.load_from_output_id = config["load_from_output_id"] 91 | else: 92 | self.load_from_output_id = -1 93 | if self.loading_method == SkillInputLoadingMethod.LOAD_FROM_CACHE_STRING: 94 | self.content = config["content"] 95 | 96 | def get_input_path(self): 97 | if self.loading_method == SkillInputLoadingMethod.LOAD_FROM_STRING: 98 | return self.param_path 99 | elif self.loading_method == SkillInputLoadingMethod.LOAD_FROM_OUTPUT_ID: 100 | return self.skill_output.param_path 101 | return "" 102 | 103 | class SkillInputConfig: 104 | def __init__(self, param_path: str, loading_method: SkillInputLoadingMethod, load_from_output_id: int, content: str = None): 105 | self.param_path = param_path 106 | self.loading_method = loading_method 107 | self.load_from_output_id = load_from_output_id 108 | self.content = content 109 | 110 | def to_dict(self): 111 | return { 112 | "param_path": self.param_path, 113 | "loading_method": str(self.loading_method), 114 | "load_from_output_id": self.load_from_output_id, 115 | "content": self.content 116 | } 117 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/create_kanban.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.manager.gptmanager import GPTManager 2 | from solidgpt.src.manager.promptresource import PE_KANBAN_OUTPUT_TEMPLATE, PE_ROE_ASSUMPTION, build_gpt_prompt 3 | from solidgpt.src.util.util import * 4 | from solidgpt.src.workskill.workskill import * 5 | 6 | 7 | class CreateKanBan(WorkSkill): 8 | 9 | def __init__(self): 10 | super().__init__() 11 | self.gpt_manager = GPTManager._instance 12 | self.name = SKILL_NAME_CREATE_KANBAN_BOARD 13 | self.skill_input = SkillInput( 14 | "HLD Doc", 15 | SkillIOParamCategory.HighLevelDesignDocument, 16 | ) 17 | self.add_input(self.skill_input) 18 | self.skill_output = SkillOutput( 19 | "Create Kanban Board", 20 | SkillIOParamCategory.KanbanBoard, 21 | ) 22 | self.add_output(self.skill_output) 23 | self.input_hld : str = None 24 | 25 | def _read_input(self): 26 | input_path = self.get_input_path(self.skill_input) 27 | self.input_hld = load_from_text(input_path) 28 | 29 | def execution_impl(self): 30 | print("Printing Kanban result here...") 31 | kanban = self.__run_create_kanban_model() 32 | save_to_md2(self.skill_output.param_path, kanban) 33 | return 34 | 35 | def __run_create_kanban_model(self,): 36 | logging.info("Running create kanban model...") 37 | prompt = build_gpt_prompt(PE_ROE_ASSUMPTION, PE_KANBAN_OUTPUT_TEMPLATE) 38 | return self.gpt_manager.create_and_chat_with_model( 39 | prompt=prompt, 40 | gpt_model_label="create_kanban", 41 | input_message=self.input_hld 42 | ) 43 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/custom_skill.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from solidgpt.src.diy.custom.customizedskilldefinition import CustomizedSkillDefinition 3 | from solidgpt.src.manager.embedding.embeddingmanager import EmbeddingManager 4 | from solidgpt.src.manager.gptmanager import GPTManager, GPTModel 5 | from solidgpt.src.manager.promptresource import build_custom_skill_gpt_prompt 6 | from solidgpt.src.util.util import load_from_text, save_to_md2 7 | from solidgpt.src.workskill.workskill import * 8 | 9 | 10 | class CustomSkill(WorkSkill): 11 | 12 | __skill_io_str_map = { 13 | "SkillIOParamCategory.PlainText": SkillIOParamCategory.PlainText , 14 | "SkillIOParamCategory.ProductRequirementsDocument": SkillIOParamCategory.ProductRequirementsDocument, 15 | "SkillIOParamCategory.BusinessRequirementsDocument": SkillIOParamCategory.BusinessRequirementsDocument, 16 | "SkillIOParamCategory.HighLevelDesignDocument": SkillIOParamCategory.HighLevelDesignDocument, 17 | } 18 | 19 | def __init__(self, definition : CustomizedSkillDefinition): 20 | super().__init__() 21 | self.name = definition.skill_name 22 | self.definition = definition 23 | self.skill_input = SkillInput( 24 | "Custom Skill Input", 25 | self.__skill_io_str_map[definition.input_method], 26 | ) 27 | self.add_input(self.skill_input) 28 | self.skill_output = SkillOutput( 29 | "Custom Skill Output", 30 | self.__skill_io_str_map[definition.output_method], 31 | ) 32 | self.add_output(self.skill_output) 33 | self.gpt_manager = GPTManager() 34 | self.role_assumption = f'''Assuem you are the expert with {self.definition.skill_name}''' 35 | #Embedding resources setup 36 | self.embedding_manager = EmbeddingManager._instance 37 | self.embedding_model_label_list = [item for item in definition.embedding_background_data_list.split(",") if item != ""] 38 | self.model = self.__generate_custom_model() 39 | self.prompt : str 40 | 41 | # Generate the model for the skill based on the definition 42 | def __generate_custom_model(self) -> GPTModel: 43 | # Generate principle 44 | self.prompt = build_custom_skill_gpt_prompt(self.role_assumption, self.definition.instruction, self.definition.principles, self.definition.qa_example) 45 | # Generate final prompt 46 | return self.gpt_manager.create_model( 47 | model=self.definition.model_name, 48 | prompt=self.prompt, 49 | gpt_model_label="generate custom model", 50 | ) 51 | 52 | def __improve_output_content(self, content: str) -> str: 53 | prompt = f"""{self.role_assumption} Enhance the content 54 | I provide by adding more details base on your expert knowledge about {self.definition.basic_description}""" 55 | return self.gpt_manager.create_and_chat_with_model( 56 | model=self.definition.model_name, 57 | prompt=prompt, 58 | input_message=content, 59 | gpt_model_label="impove output content", 60 | ) 61 | 62 | def _read_input(self): 63 | self.input = load_from_text(self.get_input_path(self.skill_input)) 64 | if self.embedding_model_label_list is not None and len(self.embedding_model_label_list) > 0: 65 | embedding_info = self.__get_embedding_resource_message(f"""{self.prompt}\n{self.input}""") 66 | self.input = f"""{self.input}\n When responding, 67 | please consider the available background information and tailor your answer accordingly. 68 | Here is background information: {embedding_info}""" 69 | logging.info(f"""Skill: {self.definition.skill_name} Input: {self.input}""") 70 | 71 | 72 | def execution_impl(self): 73 | draft = self.model.chat_with_model(self.input) 74 | logging.info(f"""Skill: {self.definition.skill_name} Output: {draft}""") 75 | model_output = self.__improve_output_content(draft) 76 | if self.definition.output_method == SkillIOParamCategory.PlainText: 77 | logging.info(model_output) 78 | else: 79 | save_to_md2(self.skill_output.param_path, model_output) 80 | 81 | def __get_embedding_resource_message(self, message: str) -> list: 82 | res = [] 83 | try: 84 | for model_label in self.embedding_model_label_list: 85 | res.extend(self.embedding_manager.query_from_embed_model(message, model_label)) 86 | return res 87 | except: 88 | logging.error("Failed to query from embedding model") 89 | 90 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/http_codesolution.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | from solidgpt.src.manager.gptmanager import GPTManager 4 | from solidgpt.src.manager.promptresource import * 5 | from solidgpt.src.util.util import * 6 | from solidgpt.src.workskill.workskill import * 7 | 8 | 9 | # To create a code plan for all target files, we should output List 10 | class HTTPCodeSolution(WorkSkill): 11 | 12 | def __init__(self): 13 | super().__init__() 14 | self.gpt_manager = GPTManager._instance 15 | self.name = SKILL_NAME_HTTP_SOLUTION 16 | self.input_user_requirement = SkillInput( 17 | "User requirement", 18 | SkillIOParamCategory.PlainText, 19 | ) 20 | 21 | self.add_input(self.input_user_requirement) 22 | 23 | self.output_path = SkillOutput( 24 | "HTTP Solution Path", 25 | SkillIOParamCategory.PlainText, 26 | ) 27 | self.add_output(self.output_path) 28 | self.input_content = None 29 | # self.collection_name = "" 30 | self.schema_dict = {} 31 | self.need_modify = False 32 | self.session_id = None 33 | 34 | def _read_input(self): 35 | self.input_content = self.input_user_requirement.content 36 | self.session_id = self.get_uuid() 37 | 38 | def execution_impl(self): 39 | logging.info("Start to Generate HTTP Code...") 40 | self.remove_dir() 41 | self.copy_templates() 42 | self.create_schema() 43 | # self.modify_dir_name() 44 | if self.need_modify: 45 | self.modify_todo_create() 46 | self.modify_todo_update() 47 | dst_path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture") 48 | self._save_to_result_cache(self.output_path, dst_path) 49 | 50 | def copy_templates(self): 51 | src = os.path.join(SRC_DIR, "tools", "templates", "aws-python-http-api-with-dynamodb") 52 | dst = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture") 53 | shutil.copytree(src, dst) 54 | 55 | def create_schema(self): 56 | schema = self.gpt_manager.create_and_chat_with_model( 57 | prompt=SCHEMA_DESIGN_PRINCIPLE, 58 | gpt_model_label="HTTP Collection", 59 | input_message=f"{self.input_content}. Only output the schema, do not output anything else including introductions and explanations.", 60 | ) 61 | # print(schema) 62 | for line in schema.split("\n"): 63 | if ":" not in line: 64 | continue 65 | tokens = line.split(":") 66 | key = tokens[0].strip() 67 | val = tokens[1].strip() 68 | if key == "##Collection##": 69 | continue 70 | else: 71 | self.schema_dict[key] = val 72 | if set(self.schema_dict.keys()) != {"'createdAt'", "'updatedAt'", "'id'"}: 73 | self.need_modify = True 74 | 75 | 76 | def modify_yml(self): 77 | # filelist = os.listdir(os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", self.collection_name)) 78 | # path1 = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", "serverless.yml") 79 | # 80 | # def change_todos(dest_path): 81 | # with open(dest_path, 'r') as file: 82 | # # Reading the content of the file 83 | # # using the read() function and storing 84 | # # them in a new variable 85 | # data = file.read() 86 | # 87 | # # Searching and replacing the text 88 | # # using the replace() function 89 | # data = data.replace("todos", self.collection_name) 90 | # 91 | # # Opening our text file in write only 92 | # # mode to write the replaced content 93 | # with open(dest_path, 'w') as file: 94 | # # Writing the replaced data in our 95 | # # text file 96 | # file.write(data) 97 | # 98 | # change_todos(path1) 99 | # for file2 in filelist: 100 | # path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", self.collection_name, file2) 101 | # change_todos(path) 102 | pass 103 | 104 | def modify_dir_name(self): 105 | # src_path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", "todos") 106 | # dest_path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", self.collection_name) 107 | # shutil.move(src_path, dest_path) 108 | pass 109 | 110 | def modify_todo_create(self): 111 | search_text = """'id': str(uuid.uuid1()), 112 | 'text': data['text'], 113 | 'checked': False, 114 | 'createdAt': timestamp, 115 | 'updatedAt': timestamp,""" 116 | replace_list = [] 117 | for k, v in self.schema_dict.items(): 118 | replace_list.append(f"{k}: {v}") 119 | replace_text = "\n".join(replace_list) 120 | dest_path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", "todos", "create.py") 121 | with open(dest_path, 'r') as file: 122 | 123 | data = file.read() 124 | data = data.replace(search_text, replace_text) 125 | with open(dest_path, 'w') as file: 126 | file.write(data) 127 | 128 | def modify_todo_update(self): 129 | replace1 = [] 130 | replace2 = [] 131 | replace3 = [] 132 | replace4 = [] 133 | for k, v in self.schema_dict.items(): 134 | if v != "timestamp," and v != "str(uuid.uuid1()),": 135 | replace1.append(f"{k} not in data") 136 | replace2.append(f"'#{k[1:-1]}': {k},") 137 | replace3.append(f"':{k[1:-1]}': {v}") 138 | replace4.append(f"'#{k[1:-1]} = :{k[1:-1]}, '") 139 | # print(replace3) 140 | replace_text1 = " or ".join(replace1) 141 | replace_text2 = "\n".join(replace2) 142 | replace_text3 = "\n".join(replace3) 143 | replace4.append("'updatedAt = :updatedAt'") 144 | replace_text4 = "\n".join(replace4) 145 | dest_path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture", "todos", "update.py") 146 | with open(dest_path, 'r') as file: 147 | # Reading the content of the file 148 | # using the read() function and storing 149 | # them in a new variable 150 | data = file.read() 151 | 152 | # Searching and replacing the text 153 | # using the replace() function 154 | search_text1 = "'text' not in data or 'checked' not in data" 155 | data = data.replace(search_text1, replace_text1) 156 | search_text2 = "'#todo_text': 'text'," 157 | data = data.replace(search_text2, replace_text2) 158 | search_text3 = """':text': data['text'], 159 | ':checked': data['checked'],""" 160 | data = data.replace(search_text3, replace_text3) 161 | search_text4 = """'SET #todo_text = :text, ' 162 | 'checked = :checked, ' 163 | 'updatedAt = :updatedAt'""" 164 | # Opening our text file in write only 165 | data = data.replace(search_text4, f"'SET {replace_text4[1:]}") 166 | # mode to write the replaced content 167 | with open(dest_path, 'w') as file: 168 | file.write(data) 169 | 170 | def remove_dir(self): 171 | if os.path.exists(os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture")): 172 | shutil.rmtree(os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture")) 173 | print(f'path exist: {os.path.exists(os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture"))}') 174 | return 175 | 176 | @staticmethod 177 | def get_uuid(): 178 | return str(uuid.uuid4().hex) 179 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/llama_analysis.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.manager.llamanager import LLAManager 2 | from solidgpt.src.manager.promptresource import PRODUCT_MANAGER_5H2W_OUTPUT_TEMPLATE, PRODUCT_MANAGER_ANALYSIS_ROLE_ASSUMPTION, PRODUCT_MANAGER_BRAINSTORM_OUTPUT_TEMPLATE, PRODUCT_MANAGER_BRAINSTORM_ROLE_ASSUMPTION, PRODUCT_MANAGER_PRD_OUTPUT_TEMPLATE, PRODUCT_MANAGER_PRD_ROLE_ASSUMPTION, build_gpt_prompt 3 | from solidgpt.src.util.util import * 4 | from solidgpt.src.workskill.workskill import * 5 | 6 | class ProductAnalysisLlama(WorkSkill): 7 | def __init__(self): 8 | super().__init__() 9 | self.llm_manager = LLAManager._instance 10 | self.name = SKILL_NAME_ANALYSIS_PRODUCT 11 | self.repo_summary = SkillInput( 12 | "Product Analysis Repo Summary", 13 | SkillIOParamCategory.PlainText, 14 | ) 15 | self.additional_info = SkillInput( 16 | "Product Analysis Additional Info", 17 | SkillIOParamCategory.PlainText, 18 | ) 19 | self.requirements = SkillInput( 20 | "Product Analysis Requirements", 21 | SkillIOParamCategory.PlainText, 22 | ) 23 | self.add_input(self.repo_summary) 24 | self.add_input(self.additional_info) 25 | self.add_input(self.requirements) 26 | self.output_md = SkillOutput( 27 | "Requirments Analysis Markdown", 28 | SkillIOParamCategory.PlainText, 29 | ) 30 | self.add_output(self.output_md) 31 | self.additional_info_content = None 32 | self.repo_summary_content = None 33 | self.requirements_content = None 34 | 35 | def _read_input(self): 36 | if self.additional_info is None: 37 | self.additional_info_content = self.additional_info.content 38 | if self.repo_summary_content is None: 39 | self.repo_summary_content = self.__get_input_content(self.repo_summary) 40 | if self.requirements_content is None: 41 | self.requirements_content = self.requirements.content 42 | 43 | def __get_input_content(self, skill_input : SkillInput): 44 | return load_from_text(self.get_input_path(skill_input), extension=".txt") 45 | 46 | def execution_impl(self): 47 | print("Generate product analysis here...") 48 | product_analysis = self._run_product_analysis_model() 49 | save_to_md2(self.output_md.param_path, product_analysis) 50 | self._save_to_result_cache(self.output_md, product_analysis) 51 | return 52 | 53 | def _run_product_analysis_model(self): 54 | logging.info("Running product analysis model...") 55 | prompt = build_gpt_prompt(PRODUCT_MANAGER_ANALYSIS_ROLE_ASSUMPTION, PRODUCT_MANAGER_5H2W_OUTPUT_TEMPLATE) 56 | model = self.llm_manager.create_model( 57 | prompt=prompt, 58 | llama_model_label="product_brainstorm", 59 | temperature=0.01, 60 | ) 61 | analysis = model.chat_with_model(self.__get_model_input()) 62 | logging.info("Product analysis report: %s", analysis) 63 | return analysis 64 | 65 | def __get_model_input(self): 66 | return f'''Requirements: {self.requirements_content} \n Product Instruction: {self.repo_summary_content} \n Product additional background information: {self.additional_info_content}''' 67 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/llama_write_prd.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.manager.llamanager import LLAManager 2 | from solidgpt.src.manager.promptresource import PRODUCT_MANAGER_BRAINSTORM_OUTPUT_TEMPLATE, PRODUCT_MANAGER_BRAINSTORM_ROLE_ASSUMPTION, PRODUCT_MANAGER_PRD_OUTPUT_TEMPLATE, PRODUCT_MANAGER_PRD_ROLE_ASSUMPTION, build_gpt_prompt 3 | from solidgpt.src.util.util import * 4 | from solidgpt.src.workskill.workskill import * 5 | 6 | class WritePRDLlama(WorkSkill): 7 | def __init__(self): 8 | super().__init__() 9 | self.llm_manager = LLAManager._instance 10 | self.name = SKILL_NAME_WRITE_PRODUCT_REQUIREMENTS_DOCUMENTATION 11 | self.input_product_key_info = SkillInput( 12 | "Design Doc", 13 | SkillIOParamCategory.PlainText, 14 | ) 15 | self.add_input(self.input_product_key_info) 16 | self.output_md = SkillOutput( 17 | "Write prd Model PRD Result", 18 | SkillIOParamCategory.ProductRequirementsDocument, 19 | ) 20 | self.add_output(self.output_md) 21 | self.input_content = None 22 | 23 | def _read_input(self): 24 | input_path = self.get_input_path(self.input_product_key_info) 25 | 26 | # if input is not a path, infer it as a string content 27 | try: 28 | self.input_content = load_from_text(input_path, extension=".md") 29 | except Exception as e: 30 | self.input_content = self.input_product_key_info.content 31 | 32 | def execution_impl(self): 33 | print("Printing PRD result here...") 34 | brain_storm_product_info = self._run_product_brainstorm_model() 35 | prd = self.__run_write_prd_model(brain_storm_product_info) 36 | self._save_to_result_cache(self.output_md, prd) 37 | save_to_md2(self.output_md.param_path, prd) 38 | return 39 | 40 | def __run_write_prd_model(self, brain_storm_product_info): 41 | logging.info("Running write prd model...") 42 | prompt = build_gpt_prompt(PRODUCT_MANAGER_PRD_ROLE_ASSUMPTION, PRODUCT_MANAGER_PRD_OUTPUT_TEMPLATE) 43 | return self.llm_manager.create_and_chat_with_model( 44 | prompt=prompt, 45 | llama_model_label="write_prd", 46 | input_message=brain_storm_product_info 47 | ) 48 | 49 | def _run_product_brainstorm_model(self): 50 | logging.info("Running product brainstorm model...") 51 | prompt = build_gpt_prompt(PRODUCT_MANAGER_BRAINSTORM_ROLE_ASSUMPTION, PRODUCT_MANAGER_BRAINSTORM_OUTPUT_TEMPLATE) 52 | model = self.llm_manager.create_model( 53 | prompt=prompt, 54 | llama_model_label="product_brainstorm", 55 | temperature=0.01, 56 | ) 57 | brainstorm = model.chat_with_model(self.input_content) 58 | logging.info("Brainstorm result: %s", brainstorm) 59 | return brainstorm 60 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/repo_chat_v2.py: -------------------------------------------------------------------------------- 1 | from solidgpt.src.manager.gptmanager import GPTManager 2 | from solidgpt.src.manager.promptresource import PRODUCT_MANAGER_5H2W_OUTPUT_TEMPLATE, PRODUCT_MANAGER_ANALYSIS_ROLE_ASSUMPTION, PRODUCT_MANAGER_BRAINSTORM_OUTPUT_TEMPLATE, PRODUCT_MANAGER_BRAINSTORM_ROLE_ASSUMPTION, PRODUCT_MANAGER_PRD_OUTPUT_TEMPLATE, PRODUCT_MANAGER_PRD_ROLE_ASSUMPTION, SDE_CHAT_ADVISOR_ASSUMPTION, SDE_TECH_SOLUTION_ASSUMPTION, build_gpt_prompt 3 | from solidgpt.src.util.util import * 4 | from solidgpt.src.workskill.workskill import * 5 | 6 | HistoryContextSeperator = '*------*' 7 | HistoryUserInuptLabel = 'UserInput:' 8 | HistorySystemOutputLabel = 'SystemOutput:' 9 | 10 | class HistoryContext(): 11 | system_output: str 12 | user_input: str 13 | 14 | def __init__(self, system_output, user_input): 15 | self.system_output = system_output 16 | self.user_input = user_input 17 | 18 | 19 | def __str__(self): 20 | return f"User Input: {self.user_input}, System Output: {self.system_output}" 21 | 22 | class RepoChatV2(WorkSkill): 23 | Memory_Length = 3 24 | 25 | def __init__(self): 26 | super().__init__() 27 | self.gpt_manager = GPTManager._instance 28 | self.name = SKILL_NAME_REPO_CHAT 29 | self.code_plan = SkillInput( 30 | "Code Plan", 31 | SkillIOParamCategory.PlainText, 32 | ) 33 | self.message = SkillInput( 34 | "User Message", 35 | SkillIOParamCategory.PlainText, 36 | ) 37 | self.history_context = SkillInput( 38 | "History Context", 39 | SkillIOParamCategory.PlainText, 40 | ) 41 | self.relatived_code_file = SkillInput( 42 | "Relatived Code Files", 43 | SkillIOParamCategory.PlainText, 44 | ) 45 | self.add_input(self.code_plan) 46 | self.add_input(self.message) 47 | self.add_input(self.history_context) 48 | self.add_input(self.relatived_code_file) 49 | self.output_md = SkillOutput( 50 | "Chat Context", 51 | SkillIOParamCategory.PlainText, 52 | ) 53 | self.add_output(self.output_md) 54 | self.code_plan_content = None 55 | self.history_contexts_content = [] 56 | self.message_content = None 57 | self.relatived_code_file_content = None 58 | 59 | def _read_input(self): 60 | # Get from cache or read from file 61 | self.code_plan_content = self.__get_input_content(self.code_plan) 62 | self.relatived_code_file_content = self.__get_input_content(self.relatived_code_file) 63 | self.message_content = self.message.content 64 | self.history_contexts_content = self.__get_history_context() 65 | 66 | def __get_input_content(self, skill_input : SkillInput): 67 | return load_from_text(self.get_input_path(skill_input), extension=".txt") 68 | 69 | def __get_history_context(self): 70 | json_data = load_from_text(self.get_input_path(self.history_context),extension=".json") 71 | print(json_data) 72 | # Load JSON data 73 | data = json.loads(json_data) 74 | 75 | # Extract HistoryContent list 76 | history_content = data["HistoryContent"] 77 | history_contexts_content = [] 78 | 79 | # Create a list of HistoryContext objects 80 | for item in history_content: 81 | system_output = item["SystemOutput"] 82 | user_input = item["UserInput"] 83 | history_context = HistoryContext(system_output, user_input) 84 | history_contexts_content.append(history_context) 85 | return history_contexts_content 86 | 87 | def execution_impl(self): 88 | system_output = self.__run_chat_with_repo_model() 89 | # Save system_output into the history context 90 | current_context = HistoryContext(system_output, self.message_content) 91 | self.history_contexts_content.append(current_context) 92 | # Convert the list of HistoryContext objects to a list of dictionaries 93 | history_list = [{"UserInput": hc.user_input, "SystemOutput": hc.system_output} for hc in self.history_contexts_content] 94 | 95 | # Create a dictionary with the HistoryContent key 96 | data = {"HistoryContent": history_list} 97 | 98 | save_to_json(data, self.history_context.param_path ) 99 | # Show the result 100 | self._save_to_result_cache(self.output_md, str(self.__get_last_response())) 101 | return 102 | 103 | def __run_chat_with_repo_model(self): 104 | logging.info("Running repo chat model...") 105 | model = self.gpt_manager.create_model( 106 | prompt=f"""{SDE_CHAT_ADVISOR_ASSUMPTION}""", 107 | gpt_model_label="repo_chat", 108 | temperature=0.01, 109 | model="gpt4", 110 | ) 111 | solution = model.chat_with_model(self.__get_model_input()) 112 | return solution 113 | 114 | def __get_model_input(self): 115 | return f'''QUESTION: {self.message_content} \n 116 | CODE PLAN: {self.code_plan_content} \n 117 | Relatived Code Files: {self.relatived_code_file_content}\n 118 | Chat History Context: { self.history_contexts_content[-self.Memory_Length:] if len(self.history_contexts_content) > self.Memory_Length else self.history_contexts_content}\n''' 119 | 120 | def __get_display_format(self): 121 | display_content = '' 122 | for context in self.history_contexts_content[::-1]: 123 | display_content += '**You:** \n' 124 | display_content += '\n' 125 | display_content += f"{context.user_input} \n" 126 | display_content += '\n' 127 | display_content += '**SolidGPT:** \n' 128 | display_content += '\n' 129 | display_content += f"{context.system_output} \n" 130 | display_content += '\n' 131 | display_content += "-------------------------------------\n" 132 | return display_content 133 | 134 | def __get_last_response(self): 135 | return self.history_contexts_content[::-1][0].system_output -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/select_template.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | from solidgpt.src.manager.gptmanager import GPTManager 3 | from solidgpt.src.util.util import * 4 | from solidgpt.src.workskill.workskill import * 5 | from solidgpt.src.manager.promptresource import CODE_SUMMARY_V2 6 | from qdrant_client import QdrantClient 7 | from langchain.embeddings import OpenAIEmbeddings 8 | import openai 9 | 10 | # To create a code plan for all target files, we should output List 11 | MAX_ONBOARDING_LENGTH = 500 12 | 13 | 14 | class SelectTemplate(WorkSkill): 15 | def __init__(self): 16 | super().__init__() 17 | self.gpt_manager = GPTManager._instance 18 | self.name = SKILL_NAME_SELECT_TEMPLATE 19 | self.session_id_input = SkillInput( 20 | "Session ID", 21 | SkillIOParamCategory.PlainText, 22 | ) 23 | self.input_user_requirement = SkillInput( 24 | "User requirement", 25 | SkillIOParamCategory.PlainText, 26 | ) 27 | self.add_input(self.session_id_input) 28 | self.add_input(self.input_user_requirement) 29 | 30 | self.skill_output = SkillOutput( 31 | "result message", 32 | SkillIOParamCategory.PlainText, 33 | ) 34 | self.add_output(self.skill_output) 35 | self.input_content = None 36 | self.session_id = None 37 | self.dest_path = "" 38 | self.qdrant_path = "" 39 | self.file_list: list = [] 40 | 41 | 42 | def _read_input(self): 43 | self.input_content = self.input_user_requirement.content 44 | self.session_id = self.session_id_input.content 45 | self.template_client = QdrantClient(path=os.path.join(SRC_DIR, "tools", "qdrant", "embedding")) 46 | self.client = QdrantClient(path=os.path.join(LOCAL_STORAGE_WORKSPACE_DIR, "qdrant", self.session_id)) 47 | 48 | def execution_impl(self): 49 | logging.info("Start to Create Code Plan...") 50 | top_template = self.__find_top_code()[0] 51 | self.template_client.close() 52 | # Check which part of code we actually need to be included into the code plan 53 | src_path = top_template.payload["path"] 54 | dest_path = os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.session_id, "architecture") 55 | self.copy_all_files(src_path, dest_path) 56 | self.file_list = self.list_all_dirs(dest_path) 57 | self.onboard_template() 58 | return 59 | 60 | def __find_top_code(self): 61 | embeddings_model = OpenAIEmbeddings(openai_api_key=openai.api_key) 62 | embedding_query = embeddings_model.embed_query(self.input_user_requirement.content) 63 | search = self.template_client.search( 64 | collection_name="templates", 65 | query_vector=embedding_query, 66 | limit=1 67 | ) 68 | return search 69 | 70 | def __summary_file(self, filename): 71 | py_file = load_from_text(filename, extension="") 72 | real_name = os.path.basename(filename) 73 | if py_file is None: 74 | logging.warn("No python file found") 75 | return 76 | python_summary = self.gpt_manager.create_and_chat_with_model( 77 | prompt=CODE_SUMMARY_V2, 78 | gpt_model_label="summary_python", 79 | input_message=py_file, 80 | model="gpt-3.5-turbo-16k", 81 | ) 82 | python_summary = python_summary.replace("\n", " ") 83 | root, ext = os.path.splitext(real_name) 84 | # save_to_text(os.path.join(self.skill_output1.param_path, f"{root}_{ext}"), python_summary) 85 | self.__embed_summary(f"{root}{ext}", python_summary, py_file) 86 | return 87 | 88 | def __embed_summary(self, filename, summary, code): 89 | payload_dict = {"code": code, "summary": summary, "filename": filename} 90 | embeddings_model = OpenAIEmbeddings(openai_api_key=openai.api_key) 91 | embedded_query = embeddings_model.embed_query(summary) 92 | logging.info(f"Onboarding ID:{self.session_id}\nSave this id to retrieve embedded data later.") 93 | try: 94 | self.client.upsert( 95 | collection_name=self.session_id, 96 | points=[ 97 | PointStruct(id=self.get_uuid(), vector=embedded_query, payload=payload_dict) 98 | ] 99 | ) 100 | except ValueError: 101 | self.client.recreate_collection( 102 | collection_name=self.session_id, 103 | vectors_config=VectorParams(size=len(embedded_query), distance=Distance.COSINE), 104 | ) 105 | self.client.upsert( 106 | collection_name=self.session_id, 107 | points=[ 108 | PointStruct(id=self.get_uuid(), vector=embedded_query, payload=payload_dict) 109 | ] 110 | ) 111 | return 112 | 113 | def onboard_template(self): 114 | file_count = len(self.file_list) 115 | current_file_idx = 0 116 | if file_count > MAX_ONBOARDING_LENGTH: 117 | outbound_message = f"## 👈 Failed to onboard, too many files, change your workspace, limit files count to {MAX_ONBOARDING_LENGTH}" 118 | logging.warn(outbound_message) 119 | self._save_to_result_cache(self.skill_output, outbound_message) 120 | else: 121 | for file in self.file_list: 122 | current_file_idx += 1 123 | if self.callback_func: 124 | self.callback_func(current_file_idx, file_count) 125 | _, ext = os.path.splitext(file) 126 | if ext in SUPPORT_EXTENSION: 127 | self.__summary_file(file) 128 | self._save_to_result_cache(self.skill_output, f"## 👈 Onboard Finished! Please use the left dropdown to continue") 129 | self.client.close() 130 | pass 131 | 132 | @staticmethod 133 | def get_uuid(): 134 | return str(uuid.uuid4().hex) 135 | 136 | @staticmethod 137 | def copy_all_files(src, dest): 138 | ret = [] 139 | visited = set() 140 | src_len = len(src) 141 | stack = [src] 142 | while stack: 143 | cur_path = stack.pop(0) 144 | visited.add(cur_path) 145 | if os.path.isfile(cur_path): 146 | rest = cur_path[src_len:] 147 | shutil.copy(cur_path, dest + rest) 148 | continue 149 | if os.path.isdir(cur_path): 150 | rest = cur_path[src_len:] 151 | if not os.path.exists(dest + rest): 152 | os.mkdir(dest + rest) 153 | neighbors = os.listdir(cur_path) 154 | for neighbor in neighbors: 155 | if neighbor not in visited: 156 | neighbor_path = os.path.join(cur_path, neighbor) 157 | stack.append(neighbor_path) 158 | visited.add(neighbor_path) 159 | return ret 160 | 161 | @staticmethod 162 | def list_all_dirs(path): 163 | ret = [] 164 | visited = set() 165 | stack = [path] 166 | while stack: 167 | cur_path = stack.pop() 168 | visited.add(cur_path) 169 | if os.path.isfile(cur_path): 170 | ret.append(cur_path) 171 | if len(ret) > MAX_ONBOARDING_LENGTH: 172 | return ret 173 | continue 174 | neighbors = os.listdir(cur_path) 175 | for neighbor in neighbors: 176 | if neighbor not in visited: 177 | neighbor_path = os.path.join(cur_path, neighbor) 178 | stack.append(neighbor_path) 179 | visited.add(neighbor_path) 180 | return ret 181 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/skills/vscode_embed_v2.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import shutil 4 | import openai 5 | import uuid 6 | from solidgpt.src.constants import SKILL_NAME_VSCODE_EMBED2 7 | from solidgpt.src.configuration.configreader import ConfigReader 8 | from solidgpt.src.manager.gptmanager import GPTManager 9 | from solidgpt.src.manager.blobmanager import AzureBlobManager 10 | from solidgpt.src.manager.promptresource import CODE_SUMMARY_V2, SUMMARY_CODE_SUMMARY_PYTHON, SUMMARY_PROJECT 11 | from solidgpt.src.util.util import load_from_text, save_to_text 12 | from solidgpt.src.workskill.skillio import SkillIOParamCategory, SkillInput, SkillOutput 13 | from solidgpt.src.workskill.workskill import WorkSkill 14 | from solidgpt.definitions import LOCAL_STORAGE_WORKSPACE_DIR, SUPPORT_EXTENSION, LOCAL_STORAGE_OUTPUT_DIR 15 | 16 | from langchain.document_loaders import TextLoader 17 | from langchain.embeddings import OpenAIEmbeddings 18 | from langchain.vectorstores import Qdrant 19 | from qdrant_client import QdrantClient 20 | from qdrant_client.http.models import PointStruct, Distance, VectorParams 21 | 22 | 23 | MAX_ONBOARDING_LENGTH = 500 24 | class VscodeEmbedV2(WorkSkill): 25 | onboarding_id: str = None 26 | 27 | def __init__(self): 28 | super().__init__() 29 | self.name = SKILL_NAME_VSCODE_EMBED2 30 | self.gpt_manager = GPTManager._instance 31 | self.onboarding_id_input = SkillInput( 32 | "Onboarding ID", 33 | SkillIOParamCategory.PlainText, 34 | ) 35 | self.skill_input = SkillInput( 36 | "base path", 37 | SkillIOParamCategory.PlainText, 38 | ) 39 | self.add_input(self.onboarding_id_input) 40 | self.add_input(self.skill_input) 41 | self.skill_output = SkillOutput( 42 | "result message", 43 | SkillIOParamCategory.PlainText, 44 | ) 45 | self.summary_output = SkillOutput( 46 | "Project Summary", 47 | SkillIOParamCategory.PlainText, 48 | ) 49 | self.add_output(self.skill_output) 50 | self.add_output(self.summary_output) 51 | self.base_path: str = "" 52 | self.file_list: list = [] 53 | self.summary_list: list = [] 54 | self.qdrant_path = "" 55 | 56 | def _read_input(self): 57 | self.base_path = self.get_input_path(self.skill_input) 58 | self.file_list = self.list_all_dirs(self.base_path) 59 | # onboard id is an uuid for vscode user 60 | self.onboarding_id = self.onboarding_id_input.content 61 | self.qdrant_path = os.path.join(LOCAL_STORAGE_WORKSPACE_DIR, "qdrant", self.onboarding_id) 62 | self.client = QdrantClient(path=self.qdrant_path) 63 | #self._set_graph_cached_content("qdrant_client", self.client) 64 | 65 | def execution_impl(self): 66 | logging.info("Start to summary code...") 67 | current_file_idx = 0 68 | file_count = len(self.file_list) 69 | try: 70 | if file_count > MAX_ONBOARDING_LENGTH: 71 | outbound_message = f"The number of code files exceeds {MAX_ONBOARDING_LENGTH}, adjust your workspace path to a smaller scope in the Settings page." 72 | logging.warn(outbound_message) 73 | self._save_to_result_cache(self.skill_output, outbound_message) 74 | else: 75 | self._save_custom_result_to_result_cache(self.file_list) 76 | for file in self.file_list: 77 | current_file_idx += 1 78 | if self.callback_func: 79 | self.callback_func(current_file_idx, file_count) 80 | _, ext = os.path.splitext(file) 81 | if ext in SUPPORT_EXTENSION: 82 | self.__summary_file(file) 83 | self.__summary_project() 84 | self._save_to_result_cache(self.skill_output, f"Indexing Successful. You can now start chatting with Codebase. Be sure to enable **Chat with Codebase** in the Settings. 👉") 85 | except Exception as e: 86 | logging.error(f"Error: {e}") 87 | self.client.close() 88 | raise e 89 | self.client.close() 90 | return 91 | 92 | def __summary_file(self, filename): 93 | py_file = load_from_text(filename, extension="") 94 | real_name = os.path.basename(filename) 95 | if py_file is None: 96 | logging.warn("No python file found") 97 | return 98 | python_summary = self.gpt_manager.create_and_chat_with_model( 99 | prompt=CODE_SUMMARY_V2, 100 | gpt_model_label="summary_python", 101 | input_message=py_file 102 | ) 103 | python_summary = python_summary.replace("\n", " ") 104 | self.summary_list.append(python_summary) 105 | root, ext = os.path.splitext(real_name) 106 | # save_to_text(os.path.join(self.skill_output1.param_path, f"{root}_{ext}"), python_summary) 107 | self.__embed_summary(f"{root}{ext}", python_summary, py_file) 108 | return 109 | 110 | def __summary_project(self): 111 | all_summary = "\n".join(self.summary_list) 112 | summary = self.gpt_manager.create_and_chat_with_model( 113 | prompt=SUMMARY_PROJECT, 114 | gpt_model_label="summary_readme", 115 | input_message=all_summary, 116 | ) 117 | save_to_text(os.path.join(LOCAL_STORAGE_OUTPUT_DIR, self.onboarding_id, "ProjectSummary"), summary) 118 | self._save_to_result_cache(self.summary_output, summary) 119 | return 120 | 121 | def __embed_summary(self, filename, summary, code): 122 | payload_dict = {"code": code, "summary": summary, "filename": filename} 123 | embeddings_model = OpenAIEmbeddings(openai_api_key=openai.api_key) 124 | embedded_query = embeddings_model.embed_query(summary) 125 | logging.info(f"Onboarding ID:{self.onboarding_id}\nSave this id to retrieve embedded data later.") 126 | try: 127 | self.client.upsert( 128 | collection_name=self.onboarding_id, 129 | points=[ 130 | PointStruct(id=self.get_uuid(), vector=embedded_query, payload=payload_dict) 131 | ] 132 | ) 133 | except ValueError: 134 | self.client.recreate_collection( 135 | collection_name=self.onboarding_id, 136 | vectors_config=VectorParams(size=len(embedded_query), distance=Distance.COSINE), 137 | ) 138 | self.client.upsert( 139 | collection_name=self.onboarding_id, 140 | points=[ 141 | PointStruct(id=self.get_uuid(), vector=embedded_query, payload=payload_dict) 142 | ] 143 | ) 144 | return 145 | 146 | @staticmethod 147 | def get_uuid(): 148 | return str(uuid.uuid4().hex) 149 | 150 | @staticmethod 151 | def list_all_dirs(path): 152 | ret = [] 153 | visited = set() 154 | stack = [path] 155 | while stack: 156 | cur_path = stack.pop() 157 | visited.add(cur_path) 158 | if os.path.isfile(cur_path): 159 | ret.append(cur_path) 160 | if len(ret) > MAX_ONBOARDING_LENGTH: 161 | return ret 162 | continue 163 | neighbors = os.listdir(cur_path) 164 | for neighbor in neighbors: 165 | if neighbor not in visited: 166 | neighbor_path = os.path.join(cur_path, neighbor) 167 | stack.append(neighbor_path) 168 | visited.add(neighbor_path) 169 | return ret 170 | -------------------------------------------------------------------------------- /solidgpt/src/workskill/workskill.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from solidgpt.src.workgraph.displayresult import DisplayResult 4 | from solidgpt.src.workskill.skillio import * 5 | from solidgpt.src.constants import * 6 | 7 | 8 | class WorkSkill: 9 | name: str = "" 10 | inputs: list[SkillInput] = [] 11 | outputs: list[SkillOutput] = [] 12 | action: str = "" 13 | # Setup by node 14 | graph_cache: dict = {} 15 | display_result: DisplayResult = None 16 | callback_func = None 17 | graph = None 18 | 19 | def __init__(self): 20 | self.name = "" 21 | self.inputs = [] 22 | self.outputs = [] 23 | self.action = "" 24 | self.display_result = DisplayResult() 25 | self.callback_func = None 26 | self.graph = None 27 | return 28 | 29 | def add_input(self, skill_input: SkillInput): 30 | self.inputs.append(skill_input) 31 | 32 | def add_output(self, skill_output: SkillOutput): 33 | self.outputs.append(skill_output) 34 | 35 | def init_config(self, input_config, output_config): 36 | if len(input_config) != len(self.inputs): 37 | logging.error("Skill %s: Input config is not correct, expected number of input: %d, actual number of input: %d." 38 | % (self.name, len(self.inputs), len(input_config))) 39 | return 40 | if len(output_config) != len(self.outputs): 41 | logging.error("Skill %s: Output config is not correct, expected number of output: %d, actual number of output: %d." 42 | % (self.name, len(self.outputs), len(output_config))) 43 | return 44 | for i in range(len(input_config)): 45 | self.inputs[i].apply_config(input_config[i]) 46 | for o in range(len(output_config)): 47 | self.outputs[o].apply_config(output_config[o]) 48 | return 49 | 50 | def execute(self): 51 | self.begin_execution() 52 | self.execution_impl() 53 | self.finish_execution() 54 | 55 | def execution_impl(self): 56 | pass 57 | 58 | def _save_to_result_cache(self, skill_output: SkillOutput, content: str): 59 | if skill_output is not None and skill_output.to_display and content is not None: 60 | self.display_result.set_result(content) 61 | 62 | def _save_custom_result_to_result_cache(self, data): 63 | if data is not None: 64 | self.display_result.set_custom_result(data) 65 | 66 | def begin_execution(self): 67 | print("Node begins " + str(self.name) + " task...") 68 | self._read_input() 69 | pass 70 | 71 | def finish_execution(self): 72 | print("Node finishes " + str(self.name) + " task...") 73 | return 74 | 75 | def _read_input(self): 76 | pass 77 | 78 | def get_input_path(self, skill_input: SkillInput): 79 | return skill_input.get_input_path() 80 | 81 | def _get_graph_cached_content(self, label): 82 | if label not in self.graph_cache: 83 | return None 84 | return self.graph_cache[label] 85 | 86 | def _set_graph_cached_content(self, label, content): 87 | self.graph_cache[label] = content 88 | return 89 | -------------------------------------------------------------------------------- /solidportal/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "parserOptions": { 5 | "ecmaVersion": 6, 6 | "sourceType": "module" 7 | }, 8 | "plugins": [ 9 | "@typescript-eslint" 10 | ], 11 | "rules": { 12 | "@typescript-eslint/naming-convention": [ 13 | "warn", 14 | { 15 | "selector": "import", 16 | "format": [ "camelCase", "PascalCase" ] 17 | } 18 | ], 19 | "@typescript-eslint/semi": "warn", 20 | "curly": "warn", 21 | "eqeqeq": "warn", 22 | "no-throw-literal": "warn", 23 | "semi": "off" 24 | }, 25 | "ignorePatterns": [ 26 | "out", 27 | "dist", 28 | "**/*.d.ts" 29 | ] 30 | } -------------------------------------------------------------------------------- /solidportal/.vscodeignore: -------------------------------------------------------------------------------- 1 | .vscode/** 2 | .vscode-test/** 3 | src/** 4 | .gitignore 5 | .yarnrc 6 | vsc-extension-quickstart.md 7 | **/tsconfig.json 8 | **/.eslintrc.json 9 | **/*.map 10 | **/*.ts 11 | -------------------------------------------------------------------------------- /solidportal/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to the "vscodepluginsolidgpt" extension will be documented in this file. 4 | 5 | Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how to structure this file. 6 | 7 | ## [Unreleased] 8 | 9 | - Initial release -------------------------------------------------------------------------------- /solidportal/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "solidgpt", 3 | "scripts": { 4 | "compile": "tsc -p ./", 5 | "watch": "tsc -watch -p ./", 6 | "pretest": "npm run compile && npm run lint", 7 | "lint": "eslint src --ext ts", 8 | "test": "node ./out/test/runTest.js", 9 | "dev": "webpack-dev-server -c webpack.dev.js", 10 | "build": "webpack -c webpack.prod.js", 11 | "package": "vsce package", 12 | "buildall": "npm run compile && npm run build && npm run package" 13 | }, 14 | "devDependencies": { 15 | "@babel/core": "^7.23.3", 16 | "@babel/preset-react": "^7.23.3", 17 | "@babel/preset-typescript": "^7.23.3", 18 | "@types/mocha": "^10.0.3", 19 | "@types/node": "18.x", 20 | "@types/react": "^18.2.38", 21 | "@types/react-dom": "^18.2.17", 22 | "@types/vscode": "^1.64.0", 23 | "@typescript-eslint/eslint-plugin": "^6.9.0", 24 | "@typescript-eslint/parser": "^6.9.0", 25 | "@vscode/test-electron": "^2.3.6", 26 | "babel-loader": "^9.1.3", 27 | "css-loader": "^6.8.1", 28 | "eslint": "^8.52.0", 29 | "file-loader": "^6.2.0", 30 | "glob": "^10.3.10", 31 | "html-webpack-plugin": "^5.5.3", 32 | "mocha": "^10.2.0", 33 | "style-loader": "^3.3.3", 34 | "svg-url-loader": "^8.0.0", 35 | "svgo": "^3.0.4", 36 | "svgo-loader": "^4.0.0", 37 | "typescript": "^5.2.2", 38 | "webpack": "^5.89.0", 39 | "webpack-cli": "^5.1.4", 40 | "webpack-dev-server": "^4.15.1", 41 | "webpack-merge": "^5.10.0" 42 | }, 43 | "dependencies": { 44 | "antd": "^5.11.3", 45 | "axios": "^1.6.2", 46 | "react": "^18.2.0", 47 | "react-dom": "^18.2.0", 48 | "react-markdown": "^9.0.1", 49 | "react-router-dom": "^6.20.0", 50 | "rehype-highlight": "^7.0.0", 51 | "rehype-katex": "^7.0.0", 52 | "rehype-raw": "^7.0.0", 53 | "remark-gfm": "^4.0.0", 54 | "remark-math": "^6.0.0" 55 | }, 56 | "files": [ 57 | "server" 58 | ] 59 | } 60 | -------------------------------------------------------------------------------- /solidportal/server/ placeholder.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Citizen/SolidGPT/f39ee7da341bbcda6cea926eece7b087307588e6/solidportal/server/ placeholder.md -------------------------------------------------------------------------------- /solidportal/solidgpticonpng.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Citizen/SolidGPT/f39ee7da341bbcda6cea926eece7b087307588e6/solidportal/solidgpticonpng.png -------------------------------------------------------------------------------- /solidportal/src/custom.d.ts: -------------------------------------------------------------------------------- 1 | declare module "*.svg" { 2 | const content: React.FunctionComponent>; 3 | export default content; 4 | } -------------------------------------------------------------------------------- /solidportal/src/test/runTest.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | 3 | import { runTests } from '@vscode/test-electron'; 4 | 5 | async function main() { 6 | try { 7 | // The folder containing the Extension Manifest package.json 8 | // Passed to `--extensionDevelopmentPath` 9 | const extensionDevelopmentPath = path.resolve(__dirname, '../../'); 10 | 11 | // The path to test runner 12 | // Passed to --extensionTestsPath 13 | const extensionTestsPath = path.resolve(__dirname, './suite/index'); 14 | 15 | // Download VS Code, unzip it and run the integration test 16 | await runTests({ extensionDevelopmentPath, extensionTestsPath }); 17 | } catch (err) { 18 | console.error('Failed to run tests', err); 19 | process.exit(1); 20 | } 21 | } 22 | 23 | main(); 24 | -------------------------------------------------------------------------------- /solidportal/src/test/suite/extension.test.ts: -------------------------------------------------------------------------------- 1 | import * as assert from 'assert'; 2 | 3 | // You can import and use all API from the 'vscode' module 4 | // as well as import your extension to test it 5 | import * as vscode from 'vscode'; 6 | // import * as myExtension from '../../extension'; 7 | 8 | suite('Extension Test Suite', () => { 9 | vscode.window.showInformationMessage('Start all tests.'); 10 | 11 | test('Sample test', () => { 12 | assert.strictEqual(-1, [1, 2, 3].indexOf(5)); 13 | assert.strictEqual(-1, [1, 2, 3].indexOf(0)); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /solidportal/src/test/suite/index.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | import Mocha from 'mocha'; 3 | import { glob } from 'glob'; 4 | 5 | export async function run(): Promise { 6 | // Create the mocha test 7 | const mocha = new Mocha({ 8 | ui: 'tdd', 9 | color: true 10 | }); 11 | 12 | const testsRoot = path.resolve(__dirname, '..'); 13 | const files = await glob('**/**.test.js', { cwd: testsRoot }); 14 | 15 | // Add files to the test suite 16 | files.forEach(f => mocha.addFile(path.resolve(testsRoot, f))); 17 | 18 | try { 19 | return new Promise((c, e) => { 20 | // Run the mocha test 21 | mocha.run(failures => { 22 | if (failures > 0) { 23 | e(new Error(`${failures} tests failed.`)); 24 | } else { 25 | c(); 26 | } 27 | }); 28 | }); 29 | } catch (err) { 30 | console.error(err); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /solidportal/src/view/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | @media (prefers-reduced-motion: no-preference) { 11 | .App-logo { 12 | animation: App-logo-spin infinite 20s linear; 13 | } 14 | } 15 | 16 | .App-header { 17 | background-color: #cccccc33; 18 | min-height: 100vh; 19 | display: flex; 20 | flex-direction: column; 21 | align-items: center; 22 | justify-content: center; 23 | font-size: calc(10px + 2vmin); 24 | color: white; 25 | } 26 | 27 | .App-link { 28 | color: #61dafb; 29 | } 30 | 31 | @keyframes App-logo-spin { 32 | from { 33 | transform: rotate(0deg); 34 | } 35 | to { 36 | transform: rotate(360deg); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /solidportal/src/view/App.tsx: -------------------------------------------------------------------------------- 1 | import './App.css'; 2 | import { BrowserRouter as Router, Routes, Route } from 'react-router-dom'; 3 | import Home from "./components/Home"; 4 | import React from 'react'; 5 | import {ConfigProvider} from 'antd'; 6 | 7 | const App: React.FC = () => ( 8 | 40 |
41 | 42 | ); 43 | 44 | export default App; 45 | -------------------------------------------------------------------------------- /solidportal/src/view/components/Avatar.tsx: -------------------------------------------------------------------------------- 1 | import { ReactElement } from "react"; 2 | import "./stChat.css"; 3 | import React from 'react'; 4 | 5 | function Avatar(props: {src: string}): ReactElement { 6 | return ( 7 |
8 | profile 9 |
10 | ); 11 | } 12 | 13 | export default Avatar; -------------------------------------------------------------------------------- /solidportal/src/view/components/ChatElement.tsx: -------------------------------------------------------------------------------- 1 | import { ReactNode } from "react"; 2 | import React from 'react'; 3 | 4 | function ChatElement(props: React.PropsWithChildren<{ 5 | isUser: boolean, 6 | avatar: boolean, 7 | children: ReactNode 8 | }>) { 9 | const { isUser, avatar, children } = props; 10 | let classList: string[] = ['chat']; 11 | 12 | if (isUser){ 13 | classList.push('user'); 14 | } 15 | 16 | 17 | if (!avatar){ 18 | classList.push('no-avatar'); 19 | } 20 | 21 | 22 | return ( 23 |
24 | {children} 25 |
26 | ); 27 | } 28 | 29 | export default ChatElement; 30 | -------------------------------------------------------------------------------- /solidportal/src/view/components/FileList.css: -------------------------------------------------------------------------------- 1 | .file-list-container { 2 | display: flex; 3 | flex-direction: column; 4 | } 5 | 6 | .file-row { 7 | flex: 1; 8 | overflow-x: auto; 9 | white-space: nowrap; 10 | } 11 | 12 | .file-scroll { 13 | display: flex; 14 | } 15 | 16 | .file-item { 17 | flex: 0 0 auto; 18 | margin: 5px; 19 | padding: 5px 10px; 20 | border-radius: 20px; 21 | background-color: #454545; 22 | color: #cccccc; 23 | position: relative; 24 | border: #007acc 0.3px solid; 25 | } 26 | 27 | .remove-button { 28 | position: absolute; 29 | top: -5px; 30 | right: -5px; 31 | background-color: transparent; 32 | border: none; 33 | cursor: pointer; 34 | color: red; 35 | font-weight: bold; 36 | } 37 | -------------------------------------------------------------------------------- /solidportal/src/view/components/FileList.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import './FileList.css'; 3 | import { DeleteOutlined } from '@ant-design/icons'; 4 | 5 | const FileList = ({files, setFiles}) => { 6 | const removeFile = (index) => { 7 | const updatedFiles = [...files]; 8 | updatedFiles.splice(index, 1); 9 | setFiles(updatedFiles); 10 | }; 11 | 12 | return ( 13 |
14 |
15 |
16 | {files.map((file, index) => ( 17 |
18 | {file} 19 | 20 |
21 | ))} 22 |
23 |
24 |
25 | ); 26 | }; 27 | 28 | export default FileList; 29 | -------------------------------------------------------------------------------- /solidportal/src/view/components/Home.css: -------------------------------------------------------------------------------- 1 | .ant-btn-primary { 2 | background-color: #007acc; 3 | border-color: #007acc; 4 | } 5 | 6 | .home-style { 7 | align-items: start; 8 | display: flex; 9 | flex-direction: column; 10 | height: calc(100vh); 11 | position: relative; 12 | background-color: #252526; 13 | } 14 | 15 | .home-style .nav { 16 | align-items: start; 17 | background-color: rgb(0, 0, 0); 18 | box-shadow: 0px 1px 2px #0000000d; 19 | display: flex; 20 | flex: 0 0 auto; 21 | flex-direction: column; 22 | position: relative; 23 | width: 100%; 24 | } 25 | 26 | .home-style .div-mx-auto { 27 | align-items: flex-start; 28 | display: flex; 29 | flex: 0 0 auto; 30 | flex-direction: column; 31 | position: relative; 32 | width: 100%; 33 | } 34 | 35 | .home-style .div-flex { 36 | align-items: flex-start; 37 | display: flex; 38 | height: 64px; 39 | justify-content: space-between; 40 | position: relative; 41 | width: 100%; 42 | } 43 | 44 | .home-style .div { 45 | padding-left: 20px; 46 | align-items: flex-start; 47 | align-self: stretch; 48 | display: inline-flex; 49 | flex: 0 0 auto; 50 | position: relative; 51 | } 52 | 53 | .home-style .img { 54 | align-self: stretch; 55 | flex: 0 0 auto; 56 | position: relative; 57 | cursor: pointer; 58 | } 59 | 60 | .home-style .text-wrapper { 61 | -webkit-box-orient: vertical; 62 | -webkit-line-clamp: 3; 63 | color: rgba(17, 24, 39, 1); 64 | display: -webkit-box; 65 | font-family: "Roboto", Helvetica; 66 | font-size: 14px; 67 | font-weight: 500; 68 | letter-spacing: 0; 69 | line-height: 20px; 70 | margin-top: -2px; 71 | overflow: hidden; 72 | position: relative; 73 | text-overflow: ellipsis; 74 | white-space: nowrap; 75 | width: fit-content; 76 | } 77 | 78 | .home-style .div-hidden-wrapper { 79 | align-items: flex-start; 80 | align-self: stretch; 81 | display: inline-flex; 82 | flex: 0 0 auto; 83 | flex-direction: column; 84 | padding: 0px 0px 0px 24px; 85 | position: relative; 86 | } 87 | 88 | .home-style .div-hidden { 89 | align-items: center; 90 | display: inline-flex; 91 | height: 64px; 92 | padding: 16px 20px 16px 12px; 93 | position: relative; 94 | } 95 | 96 | .home-style .div-relative { 97 | align-items: flex-start; 98 | display: inline-flex; 99 | flex: 0 0 auto; 100 | flex-direction: column; 101 | position: relative; 102 | } 103 | 104 | .home-style .div-hd-ec { 105 | align-items: flex-start; 106 | align-self: stretch; 107 | display: flex; 108 | flex: 0 0 auto; 109 | flex-direction: column; 110 | position: relative; 111 | width: 100%; 112 | } 113 | 114 | .home-style .button { 115 | align-items: flex-start; 116 | background-color: rgba(255, 255, 255, 1); 117 | border-radius: 9999px; 118 | display: inline-flex; 119 | flex: 0 0 auto; 120 | position: relative; 121 | } 122 | 123 | .home-style .leerob { 124 | background-image: url(../static/img/leerob.png); 125 | background-position: 50% 50%; 126 | background-size: cover; 127 | border-radius: 9999px; 128 | height: 32px; 129 | max-width: 32px; 130 | position: relative; 131 | width: 32px; 132 | cursor: pointer; 133 | } 134 | 135 | .home-style .div-p { 136 | align-items: flex-start; 137 | display: flex; 138 | flex-direction: column; 139 | height: 100%; 140 | max-width: 100%; 141 | position: relative; 142 | width: 100%; 143 | } 144 | 145 | .home-style .main { 146 | align-items: flex-start; 147 | align-self: stretch; 148 | display: flex; 149 | flex-direction: column; 150 | position: relative; 151 | width: 100%; 152 | height: 100%; 153 | } 154 | 155 | 156 | .home-style .div-text-primary { 157 | align-items: center; 158 | border-radius: 4px; 159 | display: flex; 160 | flex-direction: column; 161 | overflow: hidden; 162 | position: relative; 163 | width: 100%; 164 | } 165 | 166 | .home-style .div-flex-margin-wrapper { 167 | align-items: flex-start; 168 | border-radius: 4px; 169 | display: flex; 170 | flex-direction: column; 171 | height: 82vh; 172 | overflow: hidden; 173 | position: relative; 174 | width: 100%; 175 | } 176 | 177 | 178 | 179 | .home-style .div-mt { 180 | align-items: end; 181 | background-color: rgba(0, 0, 0, 0); 182 | display: flex; 183 | width: 90%; 184 | position: fixed; 185 | bottom: 25px; 186 | border: none; 187 | } 188 | 189 | .home-style .form { 190 | align-items: flex-start; 191 | align-self: center; 192 | display: flex; 193 | flex: 1; 194 | flex-direction: column; 195 | position: relative; 196 | width: 65%; 197 | } 198 | 199 | .home-style .file-list { 200 | width: 100%; 201 | overflow-x: auto; 202 | white-space: nowrap; 203 | } 204 | 205 | .home-style .input { 206 | align-self: stretch; 207 | height: 40px; 208 | position: relative; 209 | width: 100%; 210 | } 211 | 212 | .home-style .list-container { 213 | margin-block-start: 0em; 214 | padding-inline-start: 0px; 215 | width: 100%; 216 | overflow-y: auto; /* Enable vertical scrolling */ 217 | } 218 | 219 | .home-style .list-item { 220 | padding: 10px; 221 | list-style-type: none !important;; /* Remove the default list-style-type */ 222 | 223 | } 224 | .home-style .list-item::after { 225 | content: none !important; 226 | } 227 | 228 | .home-style .floating-button-container { 229 | position: fixed; 230 | bottom: 80px; 231 | right: 50px; 232 | } 233 | 234 | .home-style .setting { 235 | position: relative; 236 | } 237 | 238 | .ant-select-dropdown { 239 | max-height: 100px; /* Set the maximum height for the dropdown */ 240 | overflow-y: auto; /* Enable vertical scrollbar when content exceeds the height */ 241 | } 242 | 243 | .config-container { 244 | display: flex; 245 | flex-direction: column; 246 | gap: 20px; /* adjust this value as needed */ 247 | } 248 | 249 | -------------------------------------------------------------------------------- /solidportal/src/view/components/Message.tsx: -------------------------------------------------------------------------------- 1 | import ReactMarkdown from "react-markdown"; 2 | import remarkMath from "remark-math"; 3 | import rehypeKatex from "rehype-katex"; 4 | import rehypeRaw from "rehype-raw"; 5 | import remarkGfm from "remark-gfm"; 6 | import rehypeHighlight from "rehype-highlight"; 7 | 8 | import 'katex/dist/katex.min.css'; 9 | import 'highlight.js/styles/monokai-sublime.css'; 10 | import React, { ReactElement } from "react"; 11 | import "./stChat.css"; 12 | 13 | function Message(props: React.PropsWithChildren<{ is_table: boolean, message: string, allow_html: boolean }>): ReactElement { 14 | // Init React Markdown plugins 15 | const remarkPlugins = [ 16 | remarkMath, 17 | remarkGfm 18 | ]; 19 | const rehypePlugins = [ 20 | rehypeKatex, 21 | ...(props.allow_html ? [rehypeRaw] : []) 22 | ]; 23 | 24 | let classList = ["msg"]; 25 | 26 | if (props.is_table){ 27 | classList.push("msg-table"); 28 | } 29 | 30 | 31 | return ( 32 |
33 | 37 | {props.message} 38 | 39 |
40 | ); 41 | } 42 | 43 | export default Message; 44 | -------------------------------------------------------------------------------- /solidportal/src/view/components/Setting.css: -------------------------------------------------------------------------------- 1 | .ant-btn-primary { 2 | background-color: "#007acc"; 3 | border-color: "#007acc"; 4 | } 5 | 6 | .settings-container { 7 | position: fixed; 8 | bottom: 130px; 9 | right: 50px; 10 | z-index: 999; 11 | } 12 | 13 | .settings-content { 14 | position: absolute; 15 | width: 260px; 16 | bottom: 46px; 17 | right: 0; 18 | padding: 20px; 19 | background: #454545; 20 | border-radius: 5px; 21 | } 22 | 23 | .settings-content-input { 24 | color: #cccccc; 25 | } 26 | 27 | .settings-paragraph { 28 | word-wrap: break-word; 29 | color: #cccccc; 30 | } 31 | 32 | .settings-addon-before-container { 33 | display: inline-block; 34 | width: 85px; 35 | color: #cccccc; 36 | font-size: smaller; 37 | } 38 | 39 | .settings-checkbox { 40 | color: #cccccc; 41 | } 42 | 43 | 44 | @media screen and (max-width: 350px) { 45 | .settings-content { 46 | width: 25vh; 47 | } 48 | } 49 | 50 | @media screen and (max-width: 250px) { 51 | .settings-content { 52 | width: 15vh; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /solidportal/src/view/components/Setting.tsx: -------------------------------------------------------------------------------- 1 | import React, {useEffect, useState} from 'react'; 2 | import {Button, Checkbox, Input} from 'antd'; 3 | import {SettingOutlined} from '@ant-design/icons'; 4 | import './Setting.css'; 5 | import Constants from "../utils/Constants"; 6 | import DataMappingHelper from "../utils/DataMappingHelper"; 7 | 8 | const Settings = ({handleCodebaseSync, onBoardProgress, handleNotionSync, visible, setVisible, isSendDisabled}) => { 9 | 10 | const [APIToken, setAPIToken] = useState(localStorage.getItem(Constants.settingConstants.APIToken)); 11 | const [Path, setPath] = useState(localStorage.getItem(Constants.settingConstants.Path)); 12 | const [NotionToken, setNotionToken] = useState(localStorage.getItem(Constants.settingConstants.NotionToken)); 13 | const [NotionPageId, setNotionPageId] = useState(localStorage.getItem(Constants.settingConstants.NotionPageId)); 14 | const [AWSCred, setAWSCred] = useState(localStorage.getItem(Constants.settingConstants.AWSCred)); 15 | const [fileNames, setFileNames] = useState([]); 16 | const [targetFilePath, setTargetFilePath] = useState([""]); 17 | const [codebaseChecked, setCodebaseChecked] = useState(true); 18 | const [notionChecked, setNotionChecked] = useState(false); 19 | 20 | useEffect(() => { 21 | if (fileNames.length > 0) { 22 | if(localStorage.getItem(Constants.settingConstants.Path) === null){ 23 | setPath(DataMappingHelper.extractLastFolder(fileNames[0])) 24 | } 25 | setTargetFilePath(fileNames); 26 | } else { 27 | setTargetFilePath(['']); 28 | } 29 | console.log(fileNames[0]); 30 | console.log(targetFilePath); 31 | }, [fileNames]); 32 | 33 | window.addEventListener('message', event => { 34 | const dataReceived = event.data; 35 | if (dataReceived.action === 'fileOpen' || dataReceived.action === 'fileClose') { 36 | // Check if fileNames data is present 37 | setFileNames(dataReceived.fileNames || []); 38 | } 39 | 40 | }); 41 | 42 | 43 | const toggleSettings = () => { 44 | setVisible(!visible); 45 | }; 46 | 47 | const isChecked = (key) => { 48 | return localStorage.getItem(key) === "true" 49 | }; 50 | 51 | return ( 52 |
53 | 92 | 93 |

Last Indexing Time: {localStorage.getItem(Constants.settingConstants.CodebaseSyncTime)} 94 |
{onBoardProgress}

95 | 99 | 100 | { 104 | console.log(e.target.checked.toString()); 105 | setNotionChecked(e.target.checked); 106 | localStorage.setItem(Constants.settingConstants.NotionChecked, e.target.checked.toString()); 107 | }}>Chat 108 | with Notion 109 | { 113 | console.log(e.target.checked.toString()); 114 | setCodebaseChecked(e.target.checked); 115 | localStorage.setItem(Constants.settingConstants.CodebaseChecked, e.target.checked.toString()); 116 | }}>Chat 117 | with Codebase 118 |
119 | )} 120 | 121 | ); 122 | }; 123 | 124 | export default Settings; 125 | -------------------------------------------------------------------------------- /solidportal/src/view/components/stChat.css: -------------------------------------------------------------------------------- 1 | #root { 2 | margin: 0 auto; 3 | } 4 | 5 | body { 6 | margin: 0; 7 | } 8 | 9 | .chat { 10 | display: flex; 11 | font-family: var(--font), 'Segoe UI', 'Roboto', 'sans-serif'; 12 | flex-direction: row; 13 | height: auto; 14 | margin: 0; 15 | width: 100%; 16 | } 17 | 18 | .chat.user { 19 | flex-direction: row-reverse; 20 | } 21 | 22 | .chat .avatar { 23 | align-items: center; 24 | border: 1px solid transparent; 25 | border-radius: 50%; 26 | color: var(--text-color); 27 | display: flex; 28 | /* display: inline-block; */ 29 | height: 2rem; 30 | justify-content: center; 31 | margin: 3px; 32 | overflow: hidden; 33 | width: 2rem; 34 | } 35 | 36 | .chat .avatar img { 37 | height: 100%; 38 | } 39 | 40 | .chat .msg { 41 | display: inline-block; 42 | background: #3c3c3c; 43 | border: 1px solid transparent; 44 | border-radius: 5px; 45 | padding: 10px; 46 | line-height: 1.2; 47 | margin: 0 5px; 48 | max-width: 70%; 49 | min-height: 1.5rem; 50 | white-space: pre-line; 51 | word-wrap: break-word; 52 | overflow-wrap: break-word; 53 | color:#cccccc; 54 | } 55 | 56 | .chat .msg::after { 57 | content: ""; 58 | top: 0; 59 | position: absolute; 60 | border: 0.75em solid transparent; 61 | border-top-color: var(--secondary-bg-color); 62 | display: block; 63 | left: 3.1rem; 64 | content: none !important; 65 | } 66 | 67 | .chat.user .msg::after { 68 | right: 3.1rem; 69 | left: auto; 70 | } 71 | 72 | .chat .msg p { 73 | margin-block: 0; 74 | } 75 | 76 | .chat .msg a:hover { 77 | color: var(--primary-color); 78 | } 79 | 80 | .chat .msg.msg-table { 81 | white-space: normal; 82 | color: rgb(255, 255, 255) 83 | } 84 | 85 | .chat .msg:has(> table) { 86 | white-space: normal; 87 | } 88 | 89 | .chat .msg table { 90 | border-collapse: collapse; 91 | margin: 10px; 92 | } 93 | 94 | .chat .msg td, 95 | .chat .msg th { 96 | border: 1px solid var(--text-color); 97 | padding: 3px; 98 | } 99 | 100 | /* Styles for No Avatar */ 101 | .chat.no-avatar .avatar { 102 | height: 0; 103 | width: 0; 104 | } 105 | 106 | .chat.no-avatar .msg { 107 | max-width: 85%; 108 | } 109 | 110 | .chat.no-avatar .msg::after { 111 | left: 0.2rem; 112 | } 113 | 114 | .chat.no-avatar.user .msg::after { 115 | right: 0.2rem; 116 | left: auto; 117 | } 118 | 119 | @media screen and (max-width: 260px) { 120 | /* Hide the item but don't collapse it */ 121 | .chat .avatar { 122 | visibility: hidden; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /solidportal/src/view/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | solidgpt 8 | 9 | 10 |
11 | 12 | 13 | -------------------------------------------------------------------------------- /solidportal/src/view/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { createRoot } from 'react-dom/client'; 3 | import App from './App'; 4 | 5 | const root = document.getElementById('root'); 6 | if (root) { 7 | createRoot(root).render(); 8 | } 9 | 10 | -------------------------------------------------------------------------------- /solidportal/src/view/static/img/button.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /solidportal/src/view/static/img/leerob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Citizen/SolidGPT/f39ee7da341bbcda6cea926eece7b087307588e6/solidportal/src/view/static/img/leerob.png -------------------------------------------------------------------------------- /solidportal/src/view/static/img/useravatar.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /solidportal/src/view/utils/ApiHelper.tsx: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import config from "./config"; 3 | 4 | 5 | export class ApiHelper { 6 | static async postRequest(endPoint: any, requestBody: any) { 7 | try { 8 | return await axios.post(this.joinAndNormalizeUrl(config.ApiBaseUrl, endPoint), requestBody, { 9 | headers: config.CustomHeaders, 10 | }); 11 | } catch (error) { 12 | throw error; 13 | } 14 | } 15 | 16 | static async getRequest(endPoint) { 17 | try { 18 | return await axios.get(this.joinAndNormalizeUrl(config.ApiBaseUrl, endPoint), { 19 | headers: config.CustomHeaders, 20 | }); 21 | } catch (error) { 22 | throw error; 23 | } 24 | } 25 | 26 | static joinAndNormalizeUrl(base: any, ...parts: any[]) { 27 | return [base, ...parts] 28 | .map(part => part.trim().replace(/(^\/+|\/+$)/g, '')) 29 | .filter(part => part.length > 0) 30 | .join('/'); 31 | } 32 | 33 | static generateUUID = (): string => { 34 | return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => { 35 | const r = (Math.random() * 16) | 0; 36 | const v = c === 'x' ? r : (r & 0x3) | 0x8; 37 | return v.toString(16); 38 | }); 39 | }; 40 | } 41 | -------------------------------------------------------------------------------- /solidportal/src/view/utils/Constants.tsx: -------------------------------------------------------------------------------- 1 | class Constants { 2 | static constants = { 3 | waitHint: "Wait, solidgpt is generating!", 4 | error: "An error occurred. Please ensure you've correctly index to Notion/Codebase and configured the settings properly in **Settings Paage** . 👉. Please check https://github.com/AI-Citizen/SolidGPT for more details.", 5 | onBoardError:"OnBoard error, please make sure you have correct absolute path and open AI API key.", 6 | waitHintStringify: "{\"isUser\":false,\"message\":\"Wait, solidgpt is generating!\"}", 7 | errorHint: "Please open setting and click Index Notion/Index Codebase to onboard Notion/Codebase." 8 | }; 9 | static settingConstants = { 10 | APIToken: 'API Token', 11 | Path:'Path', 12 | NotionToken: 'Notion Workspace Token', 13 | NotionPageId: 'Notion Page Id', 14 | AWSCred: 'AWS Cred', 15 | NotionSyncTime: 'Notion Sync Time', 16 | CodebaseSyncTime: 'Codebase Sync Time', 17 | NotionChecked: 'Notion Checked', 18 | CodebaseChecked: 'Codebase Checked', 19 | }; 20 | static onBoardProject = { 21 | GraphId: "Graph Id", 22 | CodeBaseFileList: "CodeBase File List", 23 | NotionFileList: "Notion File List" 24 | }; 25 | static maxID: number = 12; 26 | } 27 | 28 | export default Constants; 29 | -------------------------------------------------------------------------------- /solidportal/src/view/utils/DataMappingHelper.tsx: -------------------------------------------------------------------------------- 1 | import Constants from "./Constants"; 2 | 3 | export type CommandOption = { value: string }; 4 | class DataMappingHelper { 5 | //outputs are formatted selected notionItems and codebaseItems. 6 | static mapData(selectedFile: string[], notionList: [string, string][], codebaseList: string[]): { notionItems: string[], codebaseItems: string[] } { 7 | const notionItems: string[] = []; 8 | const codebaseItems: string[] = []; 9 | 10 | selectedFile.forEach(item => { 11 | const notionItem = notionList.find(notion => notion[1] === this.extractFileName(item)); 12 | const codebaseItem = codebaseList.find(code => code.includes(this.extractFileName(item))); 13 | 14 | if (notionItem) { 15 | notionItems.push(`notion/${notionItem[0]}`); 16 | } else if (codebaseItem) { 17 | codebaseItems.push(`codebase/${codebaseItem}`); 18 | } 19 | }); 20 | 21 | return { notionItems, codebaseItems }; 22 | } 23 | 24 | static removeBasePath(paths: string[]): string[] { 25 | let pathList = paths.map(path => path.replace(/\\\\/g, '\\').replace(/\\/g, '/')); 26 | const basePath = pathList.reduce((acc, curr) => { 27 | if (!acc) return curr; 28 | const basePathParts = acc.split("/"); 29 | const currPathParts = curr.split("/"); 30 | for (let i = 0; i < basePathParts.length; i++) { 31 | if (basePathParts[i] !== currPathParts[i]) { 32 | return basePathParts.slice(0, i).join("/") + "/"; 33 | } 34 | } 35 | return acc; 36 | }, ""); 37 | return pathList.map(path => path.replace(new RegExp('^' + basePath), '')); 38 | } 39 | 40 | static getInputHintValue(searchText: string, optionsCommand: string[], type: string, topN): CommandOption[] { 41 | const foundCommands: CommandOption[] = []; 42 | const searchCommands: string[] = []; 43 | const prefix = type === Constants.onBoardProject.CodeBaseFileList ? "Codebase: " : "Notion: "; 44 | optionsCommand.forEach(option => { 45 | searchCommands.push(prefix + option ); 46 | }); 47 | searchCommands.forEach(option => { 48 | if (option.toLowerCase().replaceAll(" ", "").includes(searchText.toLowerCase().replaceAll(" ", ""))) { 49 | foundCommands.push({ value: option }); 50 | } 51 | if (foundCommands.length >= topN) { 52 | return; 53 | } 54 | }); 55 | // this.findClosestFiles(searchText,searchCommands,topN).forEach( result => { 56 | 57 | // foundCommands.push({ value: result }); 58 | // }); 59 | return foundCommands; 60 | } 61 | 62 | static extractFileName(fullPath) { 63 | const parts = this.extractFileNameWithColon(fullPath).replace(/\\\\/g, '\\').replace(/\\/g, '/').split('/'); 64 | return parts[parts.length - 1]; 65 | } 66 | 67 | static extractFileNameWithColon(fullPath) { 68 | const index = fullPath.indexOf(':'); 69 | if (index !== -1) { 70 | return fullPath.slice(index + 1).trim(); 71 | } else { 72 | return fullPath.trim(); 73 | } 74 | } 75 | 76 | static getCurrentTimeString(): string { 77 | const currentDate = new Date(); 78 | 79 | const year = currentDate.getFullYear(); 80 | const month = String(currentDate.getMonth() + 1).padStart(2, '0'); 81 | const day = String(currentDate.getDate()).padStart(2, '0'); 82 | const hours = String(currentDate.getHours()).padStart(2, '0'); 83 | const minutes = String(currentDate.getMinutes()).padStart(2, '0'); 84 | const seconds = String(currentDate.getSeconds()).padStart(2, '0'); 85 | 86 | return `${year}/${month}/${day} ${hours}:${minutes}:${seconds}`; 87 | } 88 | 89 | static extractLastFolder(filePath) { 90 | const parts = filePath.replace(/\\\\/g, '\\').replace(/\\/g, '/').split('/'); 91 | return parts.slice(0, -1).join('/'); 92 | } 93 | 94 | static findClosestFiles(inputString, fileNamesList, topN) { 95 | const charsInFileNames = new Set(); 96 | fileNamesList.forEach(fileName => { 97 | fileName.split('').forEach(char => charsInFileNames.add(char.toLowerCase())); 98 | }); 99 | 100 | let inputCharsInFileNames = true; 101 | inputString.split('').forEach(char => { 102 | if (!charsInFileNames.has(char.toLowerCase())) { 103 | inputCharsInFileNames = false; 104 | } 105 | }); 106 | 107 | if (!inputCharsInFileNames) { 108 | return []; 109 | } 110 | 111 | const similarityScores = fileNamesList.map(fileName => ({ 112 | fileName, 113 | similarity: this.similarText(inputString.toLowerCase(), fileName.toLowerCase()) 114 | })); 115 | 116 | similarityScores.sort((a, b) => b.similarity - a.similarity); 117 | 118 | return similarityScores.slice(0, topN).map(score => score.fileName); 119 | } 120 | 121 | static similarText(first, second) { 122 | let pos1 = 0, 123 | pos2 = 0, 124 | max = 0, 125 | firstLength = first.length, 126 | secondLength = second.length, 127 | p, q, l, sum; 128 | 129 | for (p = 0; p < firstLength; p++) { 130 | for (q = 0; q < secondLength; q++) { 131 | for (l = 0; 132 | (p + l < firstLength) && (q + l < secondLength) && (first.charAt(p + l) === second.charAt(q + l)); l++); 133 | if (l > max) { 134 | max = l; 135 | pos1 = p; 136 | pos2 = q; 137 | } 138 | } 139 | } 140 | 141 | sum = max; 142 | 143 | if (sum) { 144 | if (pos1 && pos2) { 145 | sum += this.similarText(first.substr(0, pos2), second.substr(0, pos2)); 146 | } 147 | 148 | if ((pos1 + max < firstLength) && (pos2 + max < secondLength)) { 149 | sum += this.similarText(first.substr(pos1 + max, firstLength - pos1 - max), second.substr(pos2 + max, secondLength - pos2 - max)); 150 | } 151 | } 152 | 153 | return sum; 154 | } 155 | } 156 | 157 | export default DataMappingHelper; 158 | -------------------------------------------------------------------------------- /solidportal/src/view/utils/config.tsx: -------------------------------------------------------------------------------- 1 | const config = { 2 | ApiBaseUrl: "http://127.0.0.1:8000", 3 | CustomHeaders: { 4 | 'Content-Type': 'application/json', // Set the appropriate content type 5 | }, 6 | Avatar: require("../static/img/solidgpt-1.svg"), 7 | UserAvatar: require("../static/img/useravatar.svg"), 8 | OpenaiKey: "", 9 | FontColor:'grey', 10 | ChatBoxColor: 'black', 11 | Font:"Arial" 12 | }; 13 | 14 | export default config; 15 | -------------------------------------------------------------------------------- /solidportal/src/view/utils/endPoints.tsx: -------------------------------------------------------------------------------- 1 | const endPoints = { 2 | OnBoardProject: '/onboardrepo/v4', 3 | StatusGraph: '/status/graph', 4 | Notionembed: '/notionembed', 5 | CodeChatAPI: '/codechatapi', 6 | NotionChatApI: '/notionchatapi', 7 | CleanHistory: '/clean/chat', 8 | InitCheck: '/test/getmsg', 9 | }; 10 | 11 | export default endPoints; 12 | -------------------------------------------------------------------------------- /solidportal/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES2022", 5 | "outDir": "out", 6 | "lib": [ 7 | "ES2022", 8 | "DOM", 9 | "DOM.Iterable" 10 | ], 11 | "sourceMap": true, 12 | "rootDir": "src", 13 | "strict": true, /* enable all strict type-checking options */ 14 | /* Additional Checks */ 15 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 16 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 17 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 18 | "jsx": "react-jsx", // react18 用 react-jsx，18之前的版本用 react 19 | "esModuleInterop": true, 20 | "noImplicitAny": false 21 | }, 22 | "exclude": ["src/test"] 23 | } 24 | -------------------------------------------------------------------------------- /solidportal/webpack.base.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const HtmlWebpackPlugin = require('html-webpack-plugin'); 3 | 4 | module.exports = { 5 | entry: path.join(__dirname, './src/view/index.tsx'), // 入口文件 6 | output: { 7 | filename: 'static/js/[name].js', 8 | path: path.join(__dirname, './dist'), 9 | clean: true, 10 | publicPath: '/', 11 | }, 12 | module: { 13 | rules: [ 14 | { 15 | test: /.(ts|tsx)$/, // 匹配.ts, tsx文件 16 | use: { 17 | loader: 'babel-loader', 18 | options: { 19 | presets: [ 20 | '@babel/preset-react', 21 | '@babel/preset-typescript' 22 | ] 23 | } 24 | } 25 | }, 26 | 27 | { 28 | test: /\.css$/, 29 | use: ['style-loader', 'css-loader'], 30 | }, 31 | 32 | { 33 | test: /\.svg$/, 34 | use: ['svg-url-loader'], 35 | }, 36 | 37 | ] 38 | }, 39 | resolve: { 40 | extensions: ['.js', '.tsx', '.ts'], 41 | }, 42 | plugins: [ 43 | new HtmlWebpackPlugin({ 44 | template: path.resolve(__dirname, './src/view/index.html'), 45 | inject: true, // 自动注入静态资源 46 | }), 47 | ], 48 | }; 49 | -------------------------------------------------------------------------------- /solidportal/webpack.dev.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const { merge } = require('webpack-merge'); 3 | const baseConfig = require('./webpack.base.js'); 4 | 5 | module.exports = merge(baseConfig, { 6 | mode: 'development', 7 | devtool: 'source-map', 8 | devServer: { 9 | port: 3000, 10 | compress: false, 11 | hot: true, 12 | historyApiFallback: true, 13 | static: { 14 | directory: path.join(__dirname, './src/view/public'), 15 | } 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /solidportal/webpack.prod.js: -------------------------------------------------------------------------------- 1 | const { merge } = require('webpack-merge'); 2 | const baseConfig = require('./webpack.base.js'); 3 | module.exports = merge(baseConfig, { 4 | mode: 'production', 5 | }); 6 | --------------------------------------------------------------------------------