├── filegpt ├── __init__.py ├── openai_wrapper.py ├── personalities.py └── FileGPT.py ├── .gitignore ├── docs ├── improvements.md └── publish.md ├── create.sh ├── setup.py ├── publish.py └── README.md /filegpt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | *.egg-info 3 | *.pyc 4 | build 5 | dist -------------------------------------------------------------------------------- /docs/improvements.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bachittle/FileGPT/HEAD/docs/improvements.md -------------------------------------------------------------------------------- /filegpt/openai_wrapper.py: -------------------------------------------------------------------------------- 1 | import openai 2 | import os 3 | from typing import List 4 | 5 | def get_openai_response(model_name:str, messages:List[dict]): 6 | openai.api_key = os.getenv("OPENAI_API_KEY") 7 | 8 | response = openai.ChatCompletion.create( 9 | model=model_name, 10 | messages=messages, 11 | stream=True 12 | ) 13 | return response 14 | -------------------------------------------------------------------------------- /create.sh: -------------------------------------------------------------------------------- 1 | # changelog of autocompletion commands used 2 | 3 | # filegpt -f filegpt/*.py setup.py README.md -m gpt-4 >> README.md # update readme based on the new code: it is now a package 4 | # filegpt -f filegpt/FileGPT.py >> filegpt/FileGPT.py # update filegpt to get personality list 5 | # filegpt -f filegpt/*.py setup.py README.md -m gpt-4 >> README.md # update readme to introduce personalities 6 | # filegpt -f README.md setup.py docs/publish.md -m gpt-4 >> docs/publish.md # document steps on how to publish to pypi 7 | filegpt -f docs/publish.md setup.py publish.py -m gpt-4 -p py-coder >> publish.py # script to publish to pypi automatically -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="filegpt", 5 | version="0.1.8", 6 | description="A tool for autocompleting text files using OpenAI's GPT models.", 7 | long_description= open("README.md", "r").read(), 8 | long_description_content_type="text/markdown", 9 | url="https://github.com/bachittle/FileGPT", 10 | author="Bailey Chittle", 11 | author_email="bachittle@gmail.com", 12 | classifiers=[ 13 | "Programming Language :: Python :: 3", 14 | "License :: OSI Approved :: MIT License", 15 | "Operating System :: OS Independent", 16 | ], 17 | packages=["filegpt"], 18 | install_requires=[ 19 | "openai", 20 | "tiktoken" 21 | ], 22 | entry_points={ 23 | "console_scripts": [ 24 | "filegpt = filegpt.FileGPT:main" 25 | ] 26 | }, 27 | ) -------------------------------------------------------------------------------- /filegpt/personalities.py: -------------------------------------------------------------------------------- 1 | PERSONALITIES = [ 2 | "py-coder", 3 | "writer", 4 | ] 5 | 6 | MAP_PERSONALITY_TO_OPENAI = { 7 | "py-coder": {"role": "system", "content": """ 8 | You only give your responses with python code. 9 | When you need to tell the user something, use code comments. 10 | 11 | Example: 12 | User: What is your name? 13 | Bot: # My name is GPT-3 14 | User: Write some code that performs the fibonacci sequence 15 | Bot: 16 | # Sure! Here you go: 17 | def fibonacci(n): 18 | ... 19 | 20 | Do not use markdown or html in your responses. 21 | Do not wrap your response in backticks (```python) 22 | """}, 23 | "writer": {"role": "system", "content": """ 24 | You give your responses in markdown. 25 | You can use markdown to format your responses. 26 | You write technical documentation, blog posts, and other text-based content. 27 | """}, 28 | } 29 | 30 | def get_personality(personality: str) -> str: 31 | if personality in PERSONALITIES: 32 | return personality 33 | else: 34 | return None 35 | 36 | def get_openai_personality(personality: str) -> str: 37 | if personality in MAP_PERSONALITY_TO_OPENAI: 38 | return MAP_PERSONALITY_TO_OPENAI[personality] 39 | else: 40 | return None -------------------------------------------------------------------------------- /publish.py: -------------------------------------------------------------------------------- 1 | import os 2 | import getpass 3 | 4 | TOOLS = ["twine", "setuptools", "wheel"] 5 | 6 | def install_tools(): 7 | os.system(f"pip install {' '.join(TOOLS)}") 8 | 9 | def build_package(): 10 | os.system("rm -r dist") # Remove old build 11 | os.system("python setup.py sdist bdist_wheel") # Build new build 12 | 13 | def check_package(): 14 | os.system("twine check dist/*") 15 | 16 | def upload_package(username, password): 17 | os.environ['TWINE_USERNAME'] = username 18 | os.environ['TWINE_PASSWORD'] = password 19 | os.system("twine upload dist/* --skip-existing") 20 | del os.environ['TWINE_USERNAME'] 21 | del os.environ['TWINE_PASSWORD'] 22 | 23 | def main(): 24 | install_tools() 25 | build_package() 26 | check_package() 27 | 28 | # Check for environment variables 29 | username = os.environ.get('TWINE_USERNAME') 30 | password = os.environ.get('TWINE_PASSWORD') 31 | 32 | if username is None or password is None: 33 | # If not found in environment variables, ask for user input 34 | username = input("Enter your PyPI username: ") 35 | password = getpass.getpass("Enter your PyPI password: ") 36 | 37 | upload_package(username, password) 38 | 39 | if __name__ == "__main__": 40 | main() -------------------------------------------------------------------------------- /docs/publish.md: -------------------------------------------------------------------------------- 1 | ## Publishing the FileGPT project to PyPI 2 | 3 | To publish FileGPT to the Python Package Index (PyPI), follow these steps: 4 | 5 | ### Prerequisites 6 | 7 | 1. Make sure you have a PyPI account. If you don't have one, [register here](https://pypi.org/account/register/). 8 | 2. Install the necessary tools for publishing packages: `twine`, `setuptools` and `wheel`. You can do this by running the following command: 9 | 10 | ``` 11 | pip install twine setuptools wheel 12 | ``` 13 | 14 | ### Steps 15 | 16 | 1. **Update the package version:** Before publishing a new version of the package, you should update the `version` parameter in the `setup` function within `setup.py`. For example: 17 | 18 | ```python 19 | setup( 20 | ... 21 | version="0.1.6", # Increment this version number 22 | ... 23 | ) 24 | ``` 25 | 26 | Make sure to use [Semantic Versioning](https://semver.org/) when updating the package version. 27 | 28 | 2. **Build the package:** Navigate to the project directory where the `setup.py` file is located, and execute the following command to build the package: 29 | 30 | ``` 31 | python setup.py sdist 32 | ``` 33 | 34 | This will generate a `dist` directory containing the `.tar.gz` and `.whl` distribution files. 35 | 36 | 3. **Check the package with Twine:** Before uploading the package to PyPI, it's a good idea to check it for errors using Twine: 37 | 38 | ``` 39 | twine check dist/* 40 | ``` 41 | 42 | 4. **Upload the package to PyPI:** Use Twine to upload the package: 43 | 44 | ``` 45 | twine upload dist/* 46 | ``` 47 | 48 | You will be prompted for your PyPI username and password (or API token). 49 | 50 | 5. **Verify the publication:** Once the upload is complete, visit the FileGPT package page on [PyPI](https://pypi.org/project/filegpt/) to verify that the new version has been published. 51 | 52 | That's it! Your updated FileGPT package is now available for installation through PyPI. Users can install or update it using the following command: 53 | 54 | ``` 55 | pip install --upgrade filegpt 56 | ``` -------------------------------------------------------------------------------- /filegpt/FileGPT.py: -------------------------------------------------------------------------------- 1 | # this is the code for FileGPT.py 2 | #!/usr/bin/env python 3 | 4 | import sys 5 | import os 6 | import logging 7 | import tempfile 8 | 9 | from filegpt.openai_wrapper import get_openai_response 10 | from filegpt.personalities import PERSONALITIES, get_personality, get_openai_personality 11 | 12 | # from openai_wrapper import get_openai_response 13 | # from personalities import PERSONALITIES, get_personality, get_openai_personality 14 | 15 | import tiktoken 16 | from typing import List 17 | 18 | PERSONALITY = None 19 | 20 | MODELS = [ 21 | "gpt-3.5-turbo", 22 | "gpt-4" 23 | ] 24 | 25 | TOKEN_LIMIT = { 26 | "gpt-3.5-turbo": 4096, 27 | "gpt-4": 8196 28 | } 29 | 30 | def setup_logging(): 31 | log_dir = tempfile.gettempdir() 32 | log_file = os.path.join(log_dir, "filegpt.log") 33 | logging.basicConfig(filename=log_file, level=logging.INFO, format="%(asctime)s [%(levelname)s]: %(message)s") 34 | logging.info("Starting FileGPT") 35 | 36 | def read_input(files: List[str] = None) -> str: 37 | if files: 38 | logging.info("Reading input from files: %s", files) 39 | contents = [] 40 | for file in files: 41 | with open(file, "r") as f: 42 | contents.append(f.read()) 43 | content = "\n".join(contents) 44 | else: 45 | logging.info("Reading input from stdin") 46 | if os.isatty(sys.stdin.fileno()): 47 | print("Current personality:", get_personality(PERSONALITY)) 48 | print("Reading from stdin (press CTRL+D for linux/mac or Enter+CTRL+Z+Enter for windows to stop)...") 49 | content = sys.stdin.read() 50 | return content 51 | 52 | 53 | def select_model(model_name:str) -> str: 54 | use_default = True 55 | for name in MODELS: 56 | if model_name == name: 57 | use_default = False 58 | break 59 | 60 | if use_default: 61 | logging.info("Using default model: %s", MODELS[0]) 62 | model_name = MODELS[0] 63 | else: 64 | logging.info("Using specified model: %s", model_name) 65 | 66 | return model_name 67 | 68 | def write_output(response): 69 | logging.info("Writing output") 70 | response_str = "" 71 | 72 | for chunk in response: 73 | chunk_msg = chunk['choices'][0]['delta'] 74 | if 'content' in chunk_msg: 75 | sys.stdout.write(chunk_msg['content']) 76 | response_str += chunk_msg['content'] 77 | sys.stdout.flush() 78 | 79 | return response_str 80 | 81 | def process_text(model_name: str, input_files: List[str] = None) -> str: 82 | setup_logging() 83 | input_content = read_input(input_files) 84 | model_name = select_model(model_name) 85 | 86 | enc = tiktoken.encoding_for_model(model_name) 87 | 88 | logging.info("Input has %d tokens", len(enc.encode(input_content))) 89 | 90 | personality = get_openai_personality(PERSONALITY) 91 | messages = [{"role": "user", "content": input_content}] 92 | if personality is not None: 93 | messages.append(personality) 94 | response = get_openai_response(model_name, messages) 95 | resp_str = write_output(response) 96 | logging.info("FileGPT finished, response has %d tokens", len(enc.encode(resp_str))) 97 | 98 | return resp_str 99 | 100 | def main(): 101 | import argparse 102 | parser = argparse.ArgumentParser(description="FileGPT - A simple tool that autocompletes text files.") 103 | parser.add_argument("-f", "--file", help="Specify one or more text files as input.", type=str, nargs="+") 104 | parser.add_argument("-m", "--model", help="Specify the model to use for autocompletion.", type=str) 105 | parser.add_argument("-p", "--personality", help="Specify the personality to use for autocompletion.", type=str) 106 | 107 | parser.add_argument("personalities", help="Perform operations on available personalities.", type=str, nargs="?", choices=["p-ls"]) 108 | args = parser.parse_args() 109 | 110 | if args.personalities: 111 | # handle personalities argument 112 | if args.personalities == "p-ls": 113 | # list available personalities 114 | print(f"Available personalities: {', '.join(PERSONALITIES)}") 115 | return 116 | 117 | global PERSONALITY 118 | PERSONALITY = get_personality(args.personality) 119 | process_text(args.model, args.file) 120 | 121 | 122 | if __name__ == "__main__": 123 | main() 124 | 125 | # (run FileGPT on itself and ask it the question below, using `filegpt -f FileGPT.py`) 126 | # How can this code be improved? -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FileGPT 2 | 3 | FileGPT is a simple command-line tool that uses OpenAI's GPT-3.5-turbo or GPT-4 to autocomplete text files. It reads text input from a file or standard input (stdin) and outputs the completed text to standard output (stdout). FileGPT is useful for generating content, adding context or explanations, and autocompleting code, among other use cases. 4 | 5 | ## Prerequisites 6 | 7 | To use FileGPT, you need: 8 | 9 | 1. An OpenAI API key (sign up for one [here](https://platform.openai.com)) 10 | 2. Python 3.6 or later 11 | 12 | ## Installation 13 | 14 | Install FileGPT directly from PyPI: 15 | 16 | ``` 17 | pip install filegpt 18 | ``` 19 | 20 | ## Setup 21 | 22 | Set the OpenAI API key as an environment variable: 23 | 24 | ``` 25 | export OPENAI_API_KEY= 26 | ``` 27 | 28 | (for Windows, use `set OPENAI_API_KEY=`) 29 | 30 | ## Usage 31 | 32 | To use FileGPT, run the following command: 33 | 34 | ``` 35 | filegpt 36 | ``` 37 | 38 | You can provide input to FileGPT in two ways: 39 | 40 | 1. By passing one or more input text files as arguments, using the `-f` or `--file` flag: 41 | 42 | ``` 43 | filegpt -f input1.txt input2.txt 44 | ``` 45 | 46 | 2. By providing input through standard input (stdin): 47 | 48 | ``` 49 | echo "Hello, this is FileGPT!" | filegpt 50 | ``` 51 | 52 | To specify the model to use for autocompletion, use the `-m` or `--model` flag: 53 | 54 | ``` 55 | filegpt -f input.txt -m gpt-3.5-turbo # old model 56 | filegpt -f input.txt -m gpt-4 # new model 57 | ``` 58 | 59 | If no model is specified, FileGPT will use the default model (gpt-3.5-turbo). 60 | 61 | ## Logging 62 | 63 | FileGPT generates log messages for major operations and saves them to a local log file named `filegpt.log`. You can check this file for any issues or status updates during execution. 64 | 65 | ## Example 66 | 67 | Let's create an example input file for FileGPT. Write the following content to a file named `input.txt`: 68 | 69 | ``` 70 | Once upon a time, in a small village near a dense forest, 71 | ``` 72 | 73 | Now, we can use FileGPT to autocomplete the story: 74 | 75 | ``` 76 | filegpt -f input.txt 77 | ``` 78 | 79 | Sample output: 80 | 81 | ``` 82 | Once upon a time, in a small village near a dense forest, there lived a young boy named Jack. Jack was an adventurous and curious child who loved to explore the woods with his trusty dog, Toby. 83 | 84 | One sunny morning, Jack and Toby ventured deep into the forest, far from the village. As they wandered through the beautiful landscape, they discovered a hidden cave amidst the trees. Filled with curiosity, Jack decided to explore the cave with Toby by his side. 85 | 86 | Inside the cave, they found an ancient treasure chest, overflowing with gold and precious gems. But guarding the treasure was a fierce dragon, awakened by the sound of their footsteps. Jack had heard stories of dragons and knew that they were dangerous creatures, so he needed to think of a plan to escape the cave without alerting the dragon. 87 | 88 | As Jack carefully considered his options, he remembered a folk tale his father had once told him about a dragon's weakness: their inability to resist the sound of a beautiful melody. Jack took a deep breath and began to sing the most enchanting song he could muster. As the sweet music filled the cave, the dragon's fiery eyes softened, and it listened intently, captivated by the melody. 89 | 90 | Seizing the opportunity, Jack and Toby made their way past the enchanted dragon, out of the cave, and back into the sunlight. They returned to their village, treasure in hand, and shared their incredible adventure with the villagers. From that day on, Jack was known as a hero in the small village, and his tale of bravery and wit was passed down through generations. And as for Toby, he was never far from Jack's side, the two forging a lifelong bond the likes of which the village had never seen. 91 | ``` 92 | 93 | add a new section to this readme detailing the new use of personalities 94 | 95 | ## Personalities 96 | 97 | FileGPT now includes personalities that allow you to adjust the style and format of the output. Currently, there are two available personalities: 98 | 99 | 1. Py Coder: Generates code-based responses, usually with comments for user interaction. Currently only for python, will experiment with other langs later. 100 | 2. Writer: Generates text-based content in Markdown format, suitable for documentation, blog posts, and other textual use cases. 101 | 102 | To specify a personality when using FileGPT, use the `-p` or `--personality` flag: 103 | 104 | ``` 105 | filegpt -f input.txt -p py-coder 106 | filegpt -f input.txt -p writer 107 | ``` 108 | 109 | You can also list the available personalities by running the following command: 110 | 111 | ``` 112 | filegpt p-ls 113 | ``` 114 | 115 | ### Example with Personalities 116 | 117 | Let's create an example input file for FileGPT using the 'py-coder' personality. Write the following content to a file named `code_input.txt`: 118 | 119 | ``` 120 | # Calculate the factorial of a number 121 | def factorial(n): 122 | if n == 0: 123 | return 1 124 | ``` 125 | 126 | Now, we can use FileGPT to autocomplete the code: 127 | 128 | ``` 129 | filegpt -f code_input.txt -p py-coder 130 | ``` 131 | 132 | Sample output: 133 | 134 | ``` 135 | # Calculate the factorial of a number 136 | def factorial(n): 137 | if n == 0: 138 | return 1 139 | else: 140 | return n * factorial(n - 1) 141 | 142 | # Now, let's test the factorial function 143 | n = 5 144 | result = factorial(n) 145 | print(f"The factorial of {n} is {result}") 146 | ``` 147 | 148 | When using the 'writer' personality, FileGPT will produce text-based content with Markdown formatting. This can be useful for generating documentation or writing blog posts. --------------------------------------------------------------------------------