├── LICENSE ├── MANIFEST.in ├── README.md ├── dist ├── llm_plugin_generator-0.5-py3-none-any.whl ├── llm_plugin_generator-0.5.tar.gz ├── llm_plugin_generator-0.6-py3-none-any.whl └── llm_plugin_generator-0.6.tar.gz ├── llm_plugin_generator.egg-info ├── PKG-INFO ├── SOURCES.txt ├── dependency_links.txt ├── entry_points.txt ├── requires.txt └── top_level.txt ├── llm_plugin_generator ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-312.pyc │ └── __init__.cpython-313.pyc ├── few_shot_prompt_llm_plugin_all.xml ├── few_shot_prompt_llm_plugin_all_with_tests.xml ├── few_shot_prompt_llm_plugin_model.xml ├── few_shot_prompt_llm_plugin_utility.xml └── few_shot_prompt_llm_plugin_utility_with_tests.xml └── pyproject.toml /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.xml 2 | include *.py 3 | global-include *.py 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # llm-plugin-generator 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/llm-plugin-generator.svg)](https://pypi.org/project/llm-plugin-generator/) 4 | [![Changelog](https://img.shields.io/github/v/release/irthomasthomas/llm-plugin-generator?include_prereleases&label=changelog)](https://github.com/irthomasthomas/llm-plugin-generator/releases) 5 | [![Tests](https://github.com/irthomasthomas/llm-plugin-generator/workflows/Test/badge.svg)](https://github.com/irthomasthomas/llm-plugin-generator/actions?query=workflow%3ATest) 6 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/irthomasthomas/llm-plugin-generator/blob/main/LICENSE) 7 | 8 | LLM plugin to generate plugins for LLM 9 | 10 | ## Installation 11 | 12 | Install this plugin in the same environment as LLM: 13 | 14 | ```bash 15 | llm install llm-plugin-generator 16 | ``` 17 | 18 | ## Usage 19 | 20 | To generate a new LLM plugin, use the `generate-plugin` command: 21 | 22 | ```bash 23 | llm generate-plugin "Description of your plugin" 24 | ``` 25 | 26 | Options: 27 | 28 | - `PROMPT`: Description of your plugin (optional) 29 | - `INPUT_FILES`: Path(s) to input README or prompt file(s) (optional, multiple allowed) 30 | - `--output-dir`: Directory to save generated plugin files (default: current directory) 31 | - `--type`: Type of plugin to generate (default, model, or utility) 32 | - `--model`, `-m`: Model to use for generation 33 | 34 | ## --type 35 | --type model will use a few-shot prompt focused on llm model plugins. 36 | --type utility focuses on utilities. 37 | leaving off --type will use a default prompt that combines all off them. I suggest picking one of the focused options which should be faster. 38 | 39 | Examples: 40 | 41 | 1. Basic usage: 42 | ```bash 43 | llm generate-plugin "Create a plugin that translates text to emoji" --output-dir ./my-new-plugin --type utility --model gpt-4 44 | ``` 45 | 46 | 2. Using a prompt and input files - Generating plugin from a README.md 47 | ``` 48 | llm generate-plugin "Few-shot Prompt Generator. Call it llm-few-shot-generator" \ 49 | 'files/README.md' --output-dir plugins/Utilities/few-shot-generator \ 50 | --type utility -m claude-3.5-sonnet 51 | ``` 52 | 53 | 3. Using websites or remote files: 54 | ``` 55 | llm generate-plugin "Write an llm-cerebras plugin from these docs: $(curl -s https://raw.githubusercontent.com/irthomasthomas/llm-cerebras/refs/heads/main/.artefacts/cerebras-api-notes.txt)" \ 56 | --output-dir llm-cerebras --type model -m sonnet-3.5 57 | ``` 58 | 59 | This will generate a new LLM plugin based on the provided description and/or input files. The files will be saved in the specified output directory. 60 | 61 | ## Features 62 | # New: Requests are now logged to the llm db. 63 | - Generates fully functional LLM plugins based on descriptions or input files 64 | - Supports different plugin types: default, model, and utility 65 | - Uses few-shot learning with predefined examples for better generation 66 | - Allows specifying custom output directory 67 | - Compatible with various LLM models 68 | - Generates main plugin file, README.md, and pyproject.toml 69 | - Extracts plugin name from generated pyproject.toml for consistent naming 70 | 71 | ## Development 72 | 73 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 74 | 75 | ```bash 76 | cd llm-plugin-generator 77 | python -m venv venv 78 | source venv/bin/activate 79 | ``` 80 | 81 | Now install the dependencies and test dependencies: 82 | 83 | ```bash 84 | pip install -e '.[test]' 85 | ``` 86 | 87 | To run the tests: 88 | 89 | ```bash 90 | pytest 91 | ``` 92 | 93 | ## Contributing 94 | 95 | Contributions to llm-plugin-generator are welcome! Please refer to the [GitHub repository](https://github.com/irthomasthomas/llm-plugin-generator) for more information on how to contribute. 96 | 97 | ## License 98 | 99 | This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. 100 | -------------------------------------------------------------------------------- /dist/llm_plugin_generator-0.5-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irthomasthomas/llm-plugin-generator/4a9a3ad97bcb7aac0da5e4ae7c113fb31903d48f/dist/llm_plugin_generator-0.5-py3-none-any.whl -------------------------------------------------------------------------------- /dist/llm_plugin_generator-0.5.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irthomasthomas/llm-plugin-generator/4a9a3ad97bcb7aac0da5e4ae7c113fb31903d48f/dist/llm_plugin_generator-0.5.tar.gz -------------------------------------------------------------------------------- /dist/llm_plugin_generator-0.6-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irthomasthomas/llm-plugin-generator/4a9a3ad97bcb7aac0da5e4ae7c113fb31903d48f/dist/llm_plugin_generator-0.6-py3-none-any.whl -------------------------------------------------------------------------------- /dist/llm_plugin_generator-0.6.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irthomasthomas/llm-plugin-generator/4a9a3ad97bcb7aac0da5e4ae7c113fb31903d48f/dist/llm_plugin_generator-0.6.tar.gz -------------------------------------------------------------------------------- /llm_plugin_generator.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: llm-plugin-generator 3 | Version: 0.6 4 | Summary: LLM plugin to generate few-shot prompts for plugin creation 5 | Author: llm-plugin-generator 6 | License: Apache-2.0 7 | Project-URL: Homepage, https://github.com/irthomasthomas/llm-plugin-generator 8 | Project-URL: Changelog, https://github.com/irthomasthomas/llm-plugin-generator/releases 9 | Project-URL: Issues, https://github.com/irthomasthomas/llm-plugin-generator/issues 10 | Classifier: License :: OSI Approved :: Apache Software License 11 | Description-Content-Type: text/markdown 12 | License-File: LICENSE 13 | Requires-Dist: llm 14 | Requires-Dist: click 15 | Requires-Dist: gitpython 16 | Provides-Extra: test 17 | Requires-Dist: pytest; extra == "test" 18 | 19 | # llm-plugin-generator 20 | 21 | [![PyPI](https://img.shields.io/pypi/v/llm-plugin-generator.svg)](https://pypi.org/project/llm-plugin-generator/) 22 | [![Changelog](https://img.shields.io/github/v/release/irthomasthomas/llm-plugin-generator?include_prereleases&label=changelog)](https://github.com/irthomasthomas/llm-plugin-generator/releases) 23 | [![Tests](https://github.com/irthomasthomas/llm-plugin-generator/workflows/Test/badge.svg)](https://github.com/irthomasthomas/llm-plugin-generator/actions?query=workflow%3ATest) 24 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/irthomasthomas/llm-plugin-generator/blob/main/LICENSE) 25 | 26 | LLM plugin to generate plugins for LLM 27 | 28 | ## Installation 29 | 30 | Install this plugin in the same environment as LLM: 31 | 32 | ```bash 33 | llm install llm-plugin-generator 34 | ``` 35 | 36 | ## Usage 37 | 38 | To generate a new LLM plugin, use the `generate-plugin` command: 39 | 40 | ```bash 41 | llm generate-plugin "Description of your plugin" 42 | ``` 43 | 44 | Options: 45 | 46 | - `PROMPT`: Description of your plugin (optional) 47 | - `INPUT_FILES`: Path(s) to input README or prompt file(s) (optional, multiple allowed) 48 | - `--output-dir`: Directory to save generated plugin files (default: current directory) 49 | - `--type`: Type of plugin to generate (default, model, or utility) 50 | - `--model`, `-m`: Model to use for generation 51 | 52 | ## --type 53 | --type model will use a few-shot prompt focused on llm model plugins. 54 | --type utility focuses on utilities. 55 | leaving off --type will use a default prompt that combines all off them. I suggest picking one of the focused options which should be faster. 56 | 57 | Examples: 58 | 59 | 1. Basic usage: 60 | ```bash 61 | llm generate-plugin "Create a plugin that translates text to emoji" --output-dir ./my-new-plugin --type utility --model gpt-4 62 | ``` 63 | 64 | 2. Using a prompt and input files - Generating plugin from a README.md 65 | ``` 66 | llm generate-plugin "Few-shot Prompt Generator. Call it llm-few-shot-generator" \ 67 | 'files/README.md' --output-dir plugins/Utilities/few-shot-generator \ 68 | --type utility -m claude-3.5-sonnet 69 | ``` 70 | 71 | 3. Using websites or remote files: 72 | ``` 73 | llm generate-plugin "Write an llm-cerebras plugin from these docs: $(curl -s https://raw.githubusercontent.com/irthomasthomas/llm-cerebras/refs/heads/main/.artefacts/cerebras-api-notes.txt)" \ 74 | --output-dir llm-cerebras --type model -m sonnet-3.5 75 | ``` 76 | 77 | This will generate a new LLM plugin based on the provided description and/or input files. The files will be saved in the specified output directory. 78 | 79 | ## Features 80 | # New: Requests are now logged to the llm db. 81 | - Generates fully functional LLM plugins based on descriptions or input files 82 | - Supports different plugin types: default, model, and utility 83 | - Uses few-shot learning with predefined examples for better generation 84 | - Allows specifying custom output directory 85 | - Compatible with various LLM models 86 | - Generates main plugin file, README.md, and pyproject.toml 87 | - Extracts plugin name from generated pyproject.toml for consistent naming 88 | 89 | ## Development 90 | 91 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 92 | 93 | ```bash 94 | cd llm-plugin-generator 95 | python -m venv venv 96 | source venv/bin/activate 97 | ``` 98 | 99 | Now install the dependencies and test dependencies: 100 | 101 | ```bash 102 | pip install -e '.[test]' 103 | ``` 104 | 105 | To run the tests: 106 | 107 | ```bash 108 | pytest 109 | ``` 110 | 111 | ## Contributing 112 | 113 | Contributions to llm-plugin-generator are welcome! Please refer to the [GitHub repository](https://github.com/irthomasthomas/llm-plugin-generator) for more information on how to contribute. 114 | 115 | ## License 116 | 117 | This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. 118 | -------------------------------------------------------------------------------- /llm_plugin_generator.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | LICENSE 2 | MANIFEST.in 3 | README.md 4 | pyproject.toml 5 | llm_plugin_generator/__init__.py 6 | llm_plugin_generator/few_shot_prompt_llm_plugin_all.xml 7 | llm_plugin_generator/few_shot_prompt_llm_plugin_all_with_tests.xml 8 | llm_plugin_generator/few_shot_prompt_llm_plugin_model.xml 9 | llm_plugin_generator/few_shot_prompt_llm_plugin_utility.xml 10 | llm_plugin_generator/few_shot_prompt_llm_plugin_utility_with_tests.xml 11 | llm_plugin_generator.egg-info/PKG-INFO 12 | llm_plugin_generator.egg-info/SOURCES.txt 13 | llm_plugin_generator.egg-info/dependency_links.txt 14 | llm_plugin_generator.egg-info/entry_points.txt 15 | llm_plugin_generator.egg-info/requires.txt 16 | llm_plugin_generator.egg-info/top_level.txt -------------------------------------------------------------------------------- /llm_plugin_generator.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /llm_plugin_generator.egg-info/entry_points.txt: -------------------------------------------------------------------------------- 1 | [llm] 2 | plugin-generator = llm_plugin_generator 3 | -------------------------------------------------------------------------------- /llm_plugin_generator.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | llm 2 | click 3 | gitpython 4 | 5 | [test] 6 | pytest 7 | -------------------------------------------------------------------------------- /llm_plugin_generator.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | llm_plugin_generator 2 | -------------------------------------------------------------------------------- /llm_plugin_generator/__init__.py: -------------------------------------------------------------------------------- 1 | import llm 2 | import click 3 | import os 4 | import pathlib 5 | import sqlite_utils 6 | import toml 7 | from importlib import resources 8 | 9 | # Update these lines to use just the filename 10 | DEFAULT_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_all.xml" 11 | MODEL_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_model.xml" 12 | UTILITY_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_utility.xml" 13 | 14 | def user_dir(): 15 | llm_user_path = os.environ.get("LLM_USER_PATH") 16 | if llm_user_path: 17 | path = pathlib.Path(llm_user_path) 18 | else: 19 | path = pathlib.Path(click.get_app_dir("io.datasette.llm")) 20 | path.mkdir(exist_ok=True, parents=True) 21 | return path 22 | 23 | def logs_db_path(): 24 | return user_dir() / "logs.db" 25 | 26 | def read_few_shot_prompt(file_name): 27 | with resources.open_text("llm_plugin_generator", file_name) as file: 28 | return file.read() 29 | 30 | def write_main_python_file(content, output_dir, filename): 31 | main_file = output_dir / filename 32 | with main_file.open("w") as f: 33 | f.write(content) 34 | click.echo(f"Main Python file written to {main_file}") 35 | 36 | def write_readme(content, output_dir): 37 | readme_file = output_dir / "README.md" 38 | with readme_file.open("w") as f: 39 | f.write(content) 40 | click.echo(f"README file written to {readme_file}") 41 | 42 | def write_pyproject_toml(content, output_dir): 43 | pyproject_file = output_dir / "pyproject.toml" 44 | with pyproject_file.open("w") as f: 45 | f.write(content) 46 | click.echo(f"pyproject.toml file written to {pyproject_file}") 47 | 48 | def extract_plugin_name(pyproject_content): 49 | try: 50 | pyproject_dict = toml.loads(pyproject_content) 51 | name = pyproject_dict['project']['name'] 52 | # Convert kebab-case to snake_case 53 | return name.replace('-', '_') 54 | except: 55 | # If parsing fails, return a default name 56 | return "plugin" 57 | 58 | @llm.hookimpl 59 | def register_commands(cli): 60 | @cli.command() 61 | @click.argument("prompt", required=False) 62 | @click.argument("input_files", nargs=-1, type=click.Path(exists=True)) 63 | @click.option("--output-dir", type=click.Path(), default=".", help="Directory to save generated plugin files") 64 | @click.option("--type", default="default", type=click.Choice(["default", "model", "utility"]), help="Type of plugin to generate") 65 | @click.option("--model", "-m", help="Model to use") 66 | def generate_plugin(prompt, input_files, output_dir, type, model): 67 | """Generate a new LLM plugin based on examples and a prompt or README file(s).""" 68 | # Select the appropriate few-shot prompt file based on the type 69 | if type == "model": 70 | few_shot_file = MODEL_FEW_SHOT_PROMPT_FILE 71 | elif type == "utility": 72 | few_shot_file = UTILITY_FEW_SHOT_PROMPT_FILE 73 | else: 74 | few_shot_file = DEFAULT_FEW_SHOT_PROMPT_FILE 75 | 76 | few_shot_prompt = read_few_shot_prompt(few_shot_file) 77 | 78 | input_content = "" 79 | for input_file in input_files: 80 | with open(input_file, "r") as f: 81 | input_content += f"""Content from {input_file}: 82 | {f.read()} 83 | 84 | """ 85 | if prompt: 86 | input_content += f"""Additional prompt: 87 | {prompt} 88 | """ 89 | 90 | if not input_content: 91 | input_content = click.prompt("Enter your plugin description or requirements") 92 | 93 | llm_model = llm.get_model(model) 94 | db = sqlite_utils.Database("") 95 | full_prompt = f"""Generate a new LLM plugin based on the following few-shot examples and the given input: 96 | Few-shot examples: 97 | {few_shot_prompt} 98 | 99 | Input: 100 | {input_content} 101 | 102 | Generate the plugin code, including the main plugin file, README.md, and pyproject.toml. 103 | Ensure the generated plugin follows best practices and is fully functional. 104 | Provide the content for each file separately, enclosed in XML tags like , , and .""" 105 | 106 | db = sqlite_utils.Database(logs_db_path()) 107 | response = llm_model.prompt(full_prompt) 108 | response.log_to_db(db) 109 | generated_plugin = response.text() 110 | 111 | output_path = pathlib.Path(output_dir) 112 | output_path.mkdir(parents=True, exist_ok=True) 113 | 114 | plugin_py_content = extract_content(generated_plugin, "plugin_py") 115 | readme_content = extract_content(generated_plugin, "readme_md") 116 | pyproject_content = extract_content(generated_plugin, "pyproject_toml") 117 | 118 | # Extract the plugin name from pyproject.toml 119 | plugin_name = extract_plugin_name(pyproject_content) 120 | 121 | # Use the extracted name for the main Python file 122 | write_main_python_file(plugin_py_content, output_path, f"{plugin_name}.py") 123 | write_readme(readme_content, output_path) 124 | write_pyproject_toml(pyproject_content, output_path) 125 | 126 | click.echo("Plugin generation completed.") 127 | 128 | def extract_content(text, tag): 129 | start_tag = f"<{tag}>" 130 | end_tag = f"" 131 | start = text.find(start_tag) + len(start_tag) 132 | end = text.find(end_tag) 133 | return text[start:end].strip() 134 | 135 | @llm.hookimpl 136 | def register_models(register): 137 | pass # No custom models to register for this plugin 138 | 139 | @llm.hookimpl 140 | def register_prompts(register): 141 | pass # No custom prompts to register for this plugin 142 | -------------------------------------------------------------------------------- /llm_plugin_generator/__pycache__/__init__.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irthomasthomas/llm-plugin-generator/4a9a3ad97bcb7aac0da5e4ae7c113fb31903d48f/llm_plugin_generator/__pycache__/__init__.cpython-312.pyc -------------------------------------------------------------------------------- /llm_plugin_generator/__pycache__/__init__.cpython-313.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/irthomasthomas/llm-plugin-generator/4a9a3ad97bcb7aac0da5e4ae7c113fb31903d48f/llm_plugin_generator/__pycache__/__init__.cpython-313.pyc -------------------------------------------------------------------------------- /llm_plugin_generator/few_shot_prompt_llm_plugin_model.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | from anthropic import Anthropic, AsyncAnthropic 6 | import llm 7 | from pydantic import Field, field_validator, model_validator 8 | from typing import Optional, List 9 | 10 | 11 | @llm.hookimpl 12 | def register_models(register): 13 | # https://docs.anthropic.com/claude/docs/models-overview 14 | register( 15 | ClaudeMessages("claude-3-opus-20240229"), 16 | AsyncClaudeMessages("claude-3-opus-20240229"), 17 | ), 18 | register( 19 | ClaudeMessages("claude-3-opus-latest"), 20 | AsyncClaudeMessages("claude-3-opus-latest"), 21 | aliases=("claude-3-opus",), 22 | ) 23 | register( 24 | ClaudeMessages("claude-3-sonnet-20240229"), 25 | AsyncClaudeMessages("claude-3-sonnet-20240229"), 26 | aliases=("claude-3-sonnet",), 27 | ) 28 | register( 29 | ClaudeMessages("claude-3-haiku-20240307"), 30 | AsyncClaudeMessages("claude-3-haiku-20240307"), 31 | aliases=("claude-3-haiku",), 32 | ) 33 | # 3.5 models 34 | register( 35 | ClaudeMessagesLong("claude-3-5-sonnet-20240620", supports_pdf=True), 36 | AsyncClaudeMessagesLong("claude-3-5-sonnet-20240620", supports_pdf=True), 37 | ) 38 | register( 39 | ClaudeMessagesLong("claude-3-5-sonnet-20241022", supports_pdf=True), 40 | AsyncClaudeMessagesLong("claude-3-5-sonnet-20241022", supports_pdf=True), 41 | ) 42 | register( 43 | ClaudeMessagesLong("claude-3-5-sonnet-latest", supports_pdf=True), 44 | AsyncClaudeMessagesLong("claude-3-5-sonnet-latest", supports_pdf=True), 45 | aliases=("claude-3.5-sonnet", "claude-3.5-sonnet-latest"), 46 | ) 47 | register( 48 | ClaudeMessagesLong("claude-3-5-haiku-latest", supports_images=False), 49 | AsyncClaudeMessagesLong("claude-3-5-haiku-latest", supports_images=False), 50 | aliases=("claude-3.5-haiku",), 51 | ) 52 | 53 | 54 | class ClaudeOptions(llm.Options): 55 | max_tokens: Optional[int] = Field( 56 | description="The maximum number of tokens to generate before stopping", 57 | default=4_096, 58 | ) 59 | 60 | temperature: Optional[float] = Field( 61 | description="Amount of randomness injected into the response. Defaults to 1.0. Ranges from 0.0 to 1.0. Use temperature closer to 0.0 for analytical / multiple choice, and closer to 1.0 for creative and generative tasks. Note that even with temperature of 0.0, the results will not be fully deterministic.", 62 | default=1.0, 63 | ) 64 | 65 | top_p: Optional[float] = Field( 66 | description="Use nucleus sampling. In nucleus sampling, we compute the cumulative distribution over all the options for each subsequent token in decreasing probability order and cut it off once it reaches a particular probability specified by top_p. You should either alter temperature or top_p, but not both. Recommended for advanced use cases only. You usually only need to use temperature.", 67 | default=None, 68 | ) 69 | 70 | top_k: Optional[int] = Field( 71 | description="Only sample from the top K options for each subsequent token. Used to remove 'long tail' low probability responses. Recommended for advanced use cases only. You usually only need to use temperature.", 72 | default=None, 73 | ) 74 | 75 | user_id: Optional[str] = Field( 76 | description="An external identifier for the user who is associated with the request", 77 | default=None, 78 | ) 79 | 80 | @field_validator("max_tokens") 81 | @classmethod 82 | def validate_max_tokens(cls, max_tokens): 83 | real_max = cls.model_fields["max_tokens"].default 84 | if not (0 < max_tokens <= real_max): 85 | raise ValueError("max_tokens must be in range 1-{}".format(real_max)) 86 | return max_tokens 87 | 88 | @field_validator("temperature") 89 | @classmethod 90 | def validate_temperature(cls, temperature): 91 | if not (0.0 <= temperature <= 1.0): 92 | raise ValueError("temperature must be in range 0.0-1.0") 93 | return temperature 94 | 95 | @field_validator("top_p") 96 | @classmethod 97 | def validate_top_p(cls, top_p): 98 | if top_p is not None and not (0.0 <= top_p <= 1.0): 99 | raise ValueError("top_p must be in range 0.0-1.0") 100 | return top_p 101 | 102 | @field_validator("top_k") 103 | @classmethod 104 | def validate_top_k(cls, top_k): 105 | if top_k is not None and top_k <= 0: 106 | raise ValueError("top_k must be a positive integer") 107 | return top_k 108 | 109 | @model_validator(mode="after") 110 | def validate_temperature_top_p(self): 111 | if self.temperature != 1.0 and self.top_p is not None: 112 | raise ValueError("Only one of temperature and top_p can be set") 113 | return self 114 | 115 | 116 | long_field = Field( 117 | description="The maximum number of tokens to generate before stopping", 118 | default=4_096 * 2, 119 | ) 120 | 121 | 122 | class _Shared: 123 | needs_key = "claude" 124 | key_env_var = "ANTHROPIC_API_KEY" 125 | can_stream = True 126 | 127 | class Options(ClaudeOptions): ... 128 | 129 | def __init__( 130 | self, 131 | model_id, 132 | claude_model_id=None, 133 | extra_headers=None, 134 | supports_images=True, 135 | supports_pdf=False, 136 | ): 137 | self.model_id = model_id 138 | self.claude_model_id = claude_model_id or model_id 139 | self.extra_headers = extra_headers or {} 140 | if supports_pdf: 141 | self.extra_headers["anthropic-beta"] = "pdfs-2024-09-25" 142 | self.attachment_types = set() 143 | if supports_images: 144 | self.attachment_types.update( 145 | { 146 | "image/png", 147 | "image/jpeg", 148 | "image/webp", 149 | "image/gif", 150 | } 151 | ) 152 | if supports_pdf: 153 | self.attachment_types.add("application/pdf") 154 | 155 | def build_messages(self, prompt, conversation) -> List[dict]: 156 | messages = [] 157 | if conversation: 158 | for response in conversation.responses: 159 | if response.attachments: 160 | content = [ 161 | { 162 | "type": ( 163 | "document" 164 | if attachment.resolve_type() == "application/pdf" 165 | else "image" 166 | ), 167 | "source": { 168 | "data": attachment.base64_content(), 169 | "media_type": attachment.resolve_type(), 170 | "type": "base64", 171 | }, 172 | } 173 | for attachment in response.attachments 174 | ] 175 | content.append({"type": "text", "text": response.prompt.prompt}) 176 | else: 177 | content = response.prompt.prompt 178 | messages.extend( 179 | [ 180 | { 181 | "role": "user", 182 | "content": content, 183 | }, 184 | {"role": "assistant", "content": response.text()}, 185 | ] 186 | ) 187 | if prompt.attachments: 188 | content = [ 189 | { 190 | "type": ( 191 | "document" 192 | if attachment.resolve_type() == "application/pdf" 193 | else "image" 194 | ), 195 | "source": { 196 | "data": attachment.base64_content(), 197 | "media_type": attachment.resolve_type(), 198 | "type": "base64", 199 | }, 200 | } 201 | for attachment in prompt.attachments 202 | ] 203 | content.append({"type": "text", "text": prompt.prompt}) 204 | messages.append( 205 | { 206 | "role": "user", 207 | "content": content, 208 | } 209 | ) 210 | else: 211 | messages.append({"role": "user", "content": prompt.prompt}) 212 | return messages 213 | 214 | def build_kwargs(self, prompt, conversation): 215 | kwargs = { 216 | "model": self.claude_model_id, 217 | "messages": self.build_messages(prompt, conversation), 218 | "max_tokens": prompt.options.max_tokens, 219 | } 220 | if prompt.options.user_id: 221 | kwargs["metadata"] = {"user_id": prompt.options.user_id} 222 | 223 | if prompt.options.top_p: 224 | kwargs["top_p"] = prompt.options.top_p 225 | else: 226 | kwargs["temperature"] = prompt.options.temperature 227 | 228 | if prompt.options.top_k: 229 | kwargs["top_k"] = prompt.options.top_k 230 | 231 | if prompt.system: 232 | kwargs["system"] = prompt.system 233 | 234 | if self.extra_headers: 235 | kwargs["extra_headers"] = self.extra_headers 236 | return kwargs 237 | 238 | def set_usage(self, response): 239 | usage = response.response_json.pop("usage") 240 | if usage: 241 | response.set_usage( 242 | input=usage.get("input_tokens"), output=usage.get("output_tokens") 243 | ) 244 | 245 | def __str__(self): 246 | return "Anthropic Messages: {}".format(self.model_id) 247 | 248 | 249 | class ClaudeMessages(_Shared, llm.Model): 250 | 251 | def execute(self, prompt, stream, response, conversation): 252 | client = Anthropic(api_key=self.get_key()) 253 | kwargs = self.build_kwargs(prompt, conversation) 254 | if stream: 255 | with client.messages.stream(**kwargs) as stream: 256 | for text in stream.text_stream: 257 | yield text 258 | # This records usage and other data: 259 | response.response_json = stream.get_final_message().model_dump() 260 | else: 261 | completion = client.messages.create(**kwargs) 262 | yield completion.content[0].text 263 | response.response_json = completion.model_dump() 264 | self.set_usage(response) 265 | 266 | 267 | class ClaudeMessagesLong(ClaudeMessages): 268 | class Options(ClaudeOptions): 269 | max_tokens: Optional[int] = long_field 270 | 271 | 272 | class AsyncClaudeMessages(_Shared, llm.AsyncModel): 273 | async def execute(self, prompt, stream, response, conversation): 274 | client = AsyncAnthropic(api_key=self.get_key()) 275 | kwargs = self.build_kwargs(prompt, conversation) 276 | if stream: 277 | async with client.messages.stream(**kwargs) as stream_obj: 278 | async for text in stream_obj.text_stream: 279 | yield text 280 | response.response_json = (await stream_obj.get_final_message()).model_dump() 281 | else: 282 | completion = await client.messages.create(**kwargs) 283 | yield completion.content[0].text 284 | response.response_json = completion.model_dump() 285 | self.set_usage(response) 286 | 287 | 288 | class AsyncClaudeMessagesLong(AsyncClaudeMessages): 289 | class Options(ClaudeOptions): 290 | max_tokens: Optional[int] = long_field 291 | 292 | 293 | 294 | # llm-claude-3 295 | 296 | [![PyPI](https://img.shields.io/pypi/v/llm-claude-3.svg)](https://pypi.org/project/llm-claude-3/) 297 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-claude-3?include_prereleases&label=changelog)](https://github.com/simonw/llm-claude-3/releases) 298 | [![Tests](https://github.com/simonw/llm-claude-3/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/llm-claude-3/actions/workflows/test.yml) 299 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-claude-3/blob/main/LICENSE) 300 | 301 | LLM access to Claude 3 by Anthropic 302 | 303 | ## Installation 304 | 305 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 306 | ```bash 307 | llm install llm-claude-3 308 | ``` 309 | 310 | ## Usage 311 | 312 | First, set [an API key](https://console.anthropic.com/settings/keys) for Claude 3: 313 | ```bash 314 | llm keys set claude 315 | # Paste key here 316 | ``` 317 | 318 | You can also set the key in the environment variable `ANTHROPIC_API_KEY` 319 | 320 | Run `llm models` to list the models, and `llm models --options` to include a list of their options. 321 | 322 | Run prompts like this: 323 | ```bash 324 | llm -m claude-3.5-sonnet 'Fun facts about pelicans' 325 | llm -m claude-3.5-haiku 'Fun facts about armadillos' 326 | llm -m claude-3-opus 'Fun facts about squirrels' 327 | ``` 328 | Images are supported too: 329 | ```bash 330 | llm -m claude-3.5-sonnet 'describe this image' -a https://static.simonwillison.net/static/2024/pelicans.jpg 331 | llm -m claude-3-haiku 'extract text' -a page.png 332 | ``` 333 | 334 | ## Development 335 | 336 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 337 | ```bash 338 | cd llm-claude-3 339 | python3 -m venv venv 340 | source venv/bin/activate 341 | ``` 342 | Now install the dependencies and test dependencies: 343 | ```bash 344 | llm install -e '.[test]' 345 | ``` 346 | To run the tests: 347 | ```bash 348 | pytest 349 | ``` 350 | 351 | This project uses [pytest-recording](https://github.com/kiwicom/pytest-recording) to record Anthropic API responses for the tests. 352 | 353 | If you add a new test that calls the API you can capture the API response like this: 354 | ```bash 355 | PYTEST_ANTHROPIC_API_KEY="$(llm keys get claude)" pytest --record-mode once 356 | ``` 357 | You will need to have stored a valid Anthropic API key using this command first: 358 | ```bash 359 | llm keys set claude 360 | # Paste key here 361 | ``` 362 | 363 | 364 | 365 | 366 | import click 367 | import cohere 368 | import llm 369 | from pydantic import Field 370 | import sqlite_utils 371 | import sys 372 | from typing import Optional, List 373 | 374 | 375 | @llm.hookimpl 376 | def register_commands(cli): 377 | @cli.command() 378 | @click.argument("prompt") 379 | @click.option("-s", "--system", help="System prompt to use") 380 | @click.option("model_id", "-m", "--model", help="Model to use") 381 | @click.option( 382 | "options", 383 | "-o", 384 | "--option", 385 | type=(str, str), 386 | multiple=True, 387 | help="key/value options for the model", 388 | ) 389 | @click.option("-n", "--no-log", is_flag=True, help="Don't log to database") 390 | @click.option("--key", help="API key to use") 391 | def command_r_search(prompt, system, model_id, options, no_log, key): 392 | "Prompt Command R with the web search feature" 393 | from llm.cli import logs_on, logs_db_path 394 | from llm.migrations import migrate 395 | 396 | model_id = model_id or "command-r" 397 | model = llm.get_model(model_id) 398 | if model.needs_key: 399 | model.key = llm.get_key(key, model.needs_key, model.key_env_var) 400 | validated_options = {} 401 | options = list(options) 402 | options.append(("websearch", "1")) 403 | try: 404 | validated_options = dict( 405 | (key, value) 406 | for key, value in model.Options(**dict(options)) 407 | if value is not None 408 | ) 409 | except pydantic.ValidationError as ex: 410 | raise click.ClickException(render_errors(ex.errors())) 411 | response = model.prompt(prompt, system=system, **validated_options) 412 | for chunk in response: 413 | print(chunk, end="") 414 | sys.stdout.flush() 415 | 416 | # Log to the database 417 | if (logs_on() or log) and not no_log: 418 | log_path = logs_db_path() 419 | (log_path.parent).mkdir(parents=True, exist_ok=True) 420 | db = sqlite_utils.Database(log_path) 421 | migrate(db) 422 | response.log_to_db(db) 423 | 424 | # Now output the citations 425 | documents = response.response_json.get("documents", []) 426 | if documents: 427 | print() 428 | print() 429 | print("Sources:") 430 | print() 431 | for doc in documents: 432 | print("-", doc["title"], "-", doc["url"]) 433 | 434 | 435 | @llm.hookimpl 436 | def register_models(register): 437 | # https://docs.cohere.com/docs/models 438 | register(CohereMessages("command-r"), aliases=("r",)) 439 | register(CohereMessages("command-r-plus"), aliases=("r-plus",)) 440 | 441 | 442 | class CohereMessages(llm.Model): 443 | needs_key = "cohere" 444 | key_env_var = "COHERE_API_KEY" 445 | can_stream = True 446 | 447 | class Options(llm.Options): 448 | websearch: Optional[bool] = Field( 449 | description="Use web search connector", 450 | default=False, 451 | ) 452 | 453 | def __init__(self, model_id): 454 | self.model_id = model_id 455 | 456 | def build_chat_history(self, conversation) -> List[dict]: 457 | chat_history = [] 458 | if conversation: 459 | for response in conversation.responses: 460 | chat_history.extend( 461 | [ 462 | {"role": "USER", "text": response.prompt.prompt}, 463 | {"role": "CHATBOT", "text": response.text()}, 464 | ] 465 | ) 466 | return chat_history 467 | 468 | def execute(self, prompt, stream, response, conversation): 469 | client = cohere.Client(self.get_key()) 470 | kwargs = { 471 | "message": prompt.prompt, 472 | "model": self.model_id, 473 | } 474 | if prompt.system: 475 | kwargs["preamble"] = prompt.system 476 | 477 | if conversation: 478 | kwargs["chat_history"] = self.build_chat_history(conversation) 479 | 480 | if prompt.options.websearch: 481 | kwargs["connectors"] = [{"id": "web-search"}] 482 | 483 | if stream: 484 | for event in client.chat_stream(**kwargs): 485 | if event.event_type == "text-generation": 486 | yield event.text 487 | elif event.event_type == "stream-end": 488 | response.response_json = event.response.dict() 489 | else: 490 | event = client.chat(**kwargs) 491 | answer = event.text 492 | yield answer 493 | response.response_json = event.dict() 494 | 495 | def __str__(self): 496 | return "Cohere Messages: {}".format(self.model_id) 497 | 498 | 499 | 500 | # llm-command-r 501 | 502 | [![PyPI](https://img.shields.io/pypi/v/llm-command-r.svg)](https://pypi.org/project/llm-command-r/) 503 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-command-r?include_prereleases&label=changelog)](https://github.com/simonw/llm-command-r/releases) 504 | [![Tests](https://github.com/simonw/llm-command-r/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/llm-command-r/actions/workflows/test.yml) 505 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-command-r/blob/main/LICENSE) 506 | 507 | Access the [Cohere Command R](https://docs.cohere.com/docs/command-r) family of models via the Cohere API 508 | 509 | ## Installation 510 | 511 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 512 | ```bash 513 | llm install llm-command-r 514 | ``` 515 | 516 | ## Configuration 517 | 518 | You will need a [Cohere API key](https://dashboard.cohere.com/api-keys). Configure it like this: 519 | 520 | ```bash 521 | llm keys set cohere 522 | # Paste key here 523 | ``` 524 | 525 | ## Usage 526 | 527 | This plugin adds two models. 528 | 529 | ```bash 530 | llm -m command-r 'Say hello from Command R' 531 | llm -m command-r-plus 'Say hello from Command R Plus' 532 | ``` 533 | 534 | The Command R models have the ability to search the web as part of answering a prompt. 535 | 536 | You can enable this feature using the `-o websearch 1` option to the models: 537 | 538 | ```bash 539 | llm -m command-r 'What is the LLM CLI tool?' -o websearch 1 540 | ``` 541 | Running a search costs more as it involves spending tokens including the search results in the prompt. 542 | 543 | The full search results are stored as JSON [in the LLM logs](https://llm.datasette.io/en/stable/logging.html). 544 | 545 | You can also use the `command-r-search` command provided by this plugin to see a list of documents that were used to answer your question as part of the output: 546 | 547 | ```bash 548 | llm command-r-search 'What is the LLM CLI tool by simonw?' 549 | ``` 550 | Example output: 551 | 552 | > The LLM CLI tool is a command-line utility that allows users to access large language models. It was created by Simon Willison and can be installed via pip, Homebrew or pipx. The tool supports interactions with remote APIs and models that can be locally installed and run. Users can run prompts from the command line and even build an image search engine using the CLI tool. 553 | > 554 | > Sources: 555 | > 556 | > - GitHub - simonw/llm: Access large language models from the command-line - https://github.com/simonw/llm 557 | > - llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs - https://simonwillison.net/2023/May/18/cli-tools-for-llms/ 558 | > - Sherwood Callaway on LinkedIn: GitHub - simonw/llm: Access large language models from the command-line - https://www.linkedin.com/posts/sherwoodcallaway_github-simonwllm-access-large-language-activity-7104448041041960960-2WRG 559 | > - LLM Python/CLI tool adds support for embeddings | Hacker News - https://news.ycombinator.com/item?id=37384797 560 | > - CLI tools for working with ChatGPT and other LLMs | Hacker News - https://news.ycombinator.com/item?id=35994037 561 | > - GitHub - simonw/homebrew-llm: Homebrew formulas for installing LLM and related tools - https://github.com/simonw/homebrew-llm 562 | > - LLM: A CLI utility and Python library for interacting with Large Language Models - https://llm.datasette.io/en/stable/ 563 | > - GitHub - simonw/llm-prompts: A collection of prompts for use with the LLM CLI tool - https://github.com/simonw/llm-prompts 564 | > - GitHub - simonw/llm-cmd: Use LLM to generate and execute commands in your shell - https://github.com/simonw/llm-cmd 565 | 566 | ## Development 567 | 568 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 569 | ```bash 570 | cd llm-command-r 571 | python3 -m venv venv 572 | source venv/bin/activate 573 | ``` 574 | Now install the dependencies and test dependencies: 575 | ```bash 576 | llm install -e '.[test]' 577 | ``` 578 | To run the tests: 579 | ```bash 580 | pytest 581 | ``` 582 | 583 | 584 | 585 | 586 | 587 | import click 588 | from httpx_sse import connect_sse 589 | import httpx 590 | import json 591 | import llm 592 | from pydantic import Field 593 | from typing import Optional 594 | 595 | 596 | DEFAULT_ALIASES = { 597 | "mistral/mistral-tiny": "mistral-tiny", 598 | "mistral/open-mistral-nemo": "mistral-nemo", 599 | "mistral/mistral-small": "mistral-small", 600 | "mistral/mistral-medium": "mistral-medium", 601 | "mistral/mistral-large-latest": "mistral-large", 602 | "mistral/codestral-mamba-latest": "codestral-mamba", 603 | "mistral/codestral-latest": "codestral", 604 | "mistral/ministral-3b-latest": "ministral-3b", 605 | "mistral/ministral-8b-latest": "ministral-8b", 606 | } 607 | 608 | 609 | @llm.hookimpl 610 | def register_models(register): 611 | for model_id in get_model_ids(): 612 | our_model_id = "mistral/" + model_id 613 | alias = DEFAULT_ALIASES.get(our_model_id) 614 | aliases = [alias] if alias else [] 615 | register(Mistral(our_model_id, model_id), aliases=aliases) 616 | 617 | 618 | @llm.hookimpl 619 | def register_embedding_models(register): 620 | register(MistralEmbed()) 621 | 622 | 623 | def refresh_models(): 624 | user_dir = llm.user_dir() 625 | mistral_models = user_dir / "mistral_models.json" 626 | key = llm.get_key("", "mistral", "LLM_MISTRAL_KEY") 627 | if not key: 628 | raise click.ClickException( 629 | "You must set the 'mistral' key or the LLM_MISTRAL_KEY environment variable." 630 | ) 631 | response = httpx.get( 632 | "https://api.mistral.ai/v1/models", headers={"Authorization": f"Bearer {key}"} 633 | ) 634 | response.raise_for_status() 635 | models = response.json() 636 | mistral_models.write_text(json.dumps(models, indent=2)) 637 | return models 638 | 639 | 640 | def get_model_ids(): 641 | user_dir = llm.user_dir() 642 | models = { 643 | "data": [ 644 | {"id": model_id.replace("mistral/", "")} 645 | for model_id in DEFAULT_ALIASES.keys() 646 | ] 647 | } 648 | mistral_models = user_dir / "mistral_models.json" 649 | if mistral_models.exists(): 650 | models = json.loads(mistral_models.read_text()) 651 | elif llm.get_key("", "mistral", "LLM_MISTRAL_KEY"): 652 | try: 653 | models = refresh_models() 654 | except httpx.HTTPStatusError: 655 | pass 656 | return [model["id"] for model in models["data"] if "embed" not in model["id"]] 657 | 658 | 659 | @llm.hookimpl 660 | def register_commands(cli): 661 | @cli.group() 662 | def mistral(): 663 | "Commands relating to the llm-mistral plugin" 664 | 665 | @mistral.command() 666 | def refresh(): 667 | "Refresh the list of available Mistral models" 668 | before = set(get_model_ids()) 669 | refresh_models() 670 | after = set(get_model_ids()) 671 | added = after - before 672 | removed = before - after 673 | if added: 674 | click.echo(f"Added models: {', '.join(added)}", err=True) 675 | if removed: 676 | click.echo(f"Removed models: {', '.join(removed)}", err=True) 677 | if added or removed: 678 | click.echo("New list of models:", err=True) 679 | for model_id in get_model_ids(): 680 | click.echo(model_id, err=True) 681 | else: 682 | click.echo("No changes", err=True) 683 | 684 | 685 | class Mistral(llm.Model): 686 | can_stream = True 687 | 688 | class Options(llm.Options): 689 | temperature: Optional[float] = Field( 690 | description=( 691 | "Determines the sampling temperature. Higher values like 0.8 increase randomness, " 692 | "while lower values like 0.2 make the output more focused and deterministic." 693 | ), 694 | ge=0, 695 | le=1, 696 | default=0.7, 697 | ) 698 | top_p: Optional[float] = Field( 699 | description=( 700 | "Nucleus sampling, where the model considers the tokens with top_p probability mass. " 701 | "For example, 0.1 means considering only the tokens in the top 10% probability mass." 702 | ), 703 | ge=0, 704 | le=1, 705 | default=1, 706 | ) 707 | max_tokens: Optional[int] = Field( 708 | description="The maximum number of tokens to generate in the completion.", 709 | ge=0, 710 | default=None, 711 | ) 712 | safe_mode: Optional[bool] = Field( 713 | description="Whether to inject a safety prompt before all conversations.", 714 | default=False, 715 | ) 716 | random_seed: Optional[int] = Field( 717 | description="Sets the seed for random sampling to generate deterministic results.", 718 | default=None, 719 | ) 720 | 721 | def __init__(self, our_model_id, mistral_model_id): 722 | self.model_id = our_model_id 723 | self.mistral_model_id = mistral_model_id 724 | 725 | def build_messages(self, prompt, conversation): 726 | messages = [] 727 | if not conversation: 728 | if prompt.system: 729 | messages.append({"role": "system", "content": prompt.system}) 730 | messages.append({"role": "user", "content": prompt.prompt}) 731 | return messages 732 | current_system = None 733 | for prev_response in conversation.responses: 734 | if ( 735 | prev_response.prompt.system 736 | and prev_response.prompt.system != current_system 737 | ): 738 | messages.append( 739 | {"role": "system", "content": prev_response.prompt.system} 740 | ) 741 | current_system = prev_response.prompt.system 742 | messages.append({"role": "user", "content": prev_response.prompt.prompt}) 743 | messages.append({"role": "assistant", "content": prev_response.text()}) 744 | if prompt.system and prompt.system != current_system: 745 | messages.append({"role": "system", "content": prompt.system}) 746 | messages.append({"role": "user", "content": prompt.prompt}) 747 | return messages 748 | 749 | def execute(self, prompt, stream, response, conversation): 750 | key = llm.get_key("", "mistral", "LLM_MISTRAL_KEY") or getattr( 751 | self, "key", None 752 | ) 753 | messages = self.build_messages(prompt, conversation) 754 | response._prompt_json = {"messages": messages} 755 | body = { 756 | "model": self.mistral_model_id, 757 | "messages": messages, 758 | } 759 | if prompt.options.temperature: 760 | body["temperature"] = prompt.options.temperature 761 | if prompt.options.top_p: 762 | body["top_p"] = prompt.options.top_p 763 | if prompt.options.max_tokens: 764 | body["max_tokens"] = prompt.options.max_tokens 765 | if prompt.options.safe_mode: 766 | body["safe_mode"] = prompt.options.safe_mode 767 | if prompt.options.random_seed: 768 | body["random_seed"] = prompt.options.random_seed 769 | if stream: 770 | body["stream"] = True 771 | with httpx.Client() as client: 772 | with connect_sse( 773 | client, 774 | "POST", 775 | "https://api.mistral.ai/v1/chat/completions", 776 | headers={ 777 | "Content-Type": "application/json", 778 | "Accept": "application/json", 779 | "Authorization": f"Bearer {key}", 780 | }, 781 | json=body, 782 | timeout=None, 783 | ) as event_source: 784 | # In case of unauthorized: 785 | event_source.response.raise_for_status() 786 | for sse in event_source.iter_sse(): 787 | if sse.data != "[DONE]": 788 | try: 789 | yield sse.json()["choices"][0]["delta"]["content"] 790 | except KeyError: 791 | pass 792 | else: 793 | with httpx.Client() as client: 794 | api_response = client.post( 795 | "https://api.mistral.ai/v1/chat/completions", 796 | headers={ 797 | "Content-Type": "application/json", 798 | "Accept": "application/json", 799 | "Authorization": f"Bearer {key}", 800 | }, 801 | json=body, 802 | timeout=None, 803 | ) 804 | api_response.raise_for_status() 805 | yield api_response.json()["choices"][0]["message"]["content"] 806 | response.response_json = api_response.json() 807 | 808 | 809 | class MistralEmbed(llm.EmbeddingModel): 810 | model_id = "mistral-embed" 811 | batch_size = 10 812 | 813 | def embed_batch(self, texts): 814 | key = llm.get_key("", "mistral", "LLM_MISTRAL_KEY") 815 | with httpx.Client() as client: 816 | api_response = client.post( 817 | "https://api.mistral.ai/v1/embeddings", 818 | headers={ 819 | "Content-Type": "application/json", 820 | "Accept": "application/json", 821 | "Authorization": f"Bearer {key}", 822 | }, 823 | json={ 824 | "model": "mistral-embed", 825 | "input": list(texts), 826 | "encoding_format": "float", 827 | }, 828 | timeout=None, 829 | ) 830 | api_response.raise_for_status() 831 | return [item["embedding"] for item in api_response.json()["data"]] 832 | 833 | 834 | 835 | # llm-mistral 836 | 837 | [![PyPI](https://img.shields.io/pypi/v/llm-mistral.svg)](https://pypi.org/project/llm-mistral/) 838 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-mistral?include_prereleases&label=changelog)](https://github.com/simonw/llm-mistral/releases) 839 | [![Tests](https://github.com/simonw/llm-mistral/workflows/Test/badge.svg)](https://github.com/simonw/llm-mistral/actions?query=workflow%3ATest) 840 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-mistral/blob/main/LICENSE) 841 | 842 | [LLM](https://llm.datasette.io/) plugin providing access to [Mistral](https://mistral.ai) models using the Mistral API 843 | 844 | ## Installation 845 | 846 | Install this plugin in the same environment as LLM: 847 | ```bash 848 | llm install llm-mistral 849 | ``` 850 | ## Usage 851 | 852 | First, obtain an API key for [the Mistral API](https://console.mistral.ai/). 853 | 854 | Configure the key using the `llm keys set mistral` command: 855 | ```bash 856 | llm keys set mistral 857 | ``` 858 | ``` 859 | <paste key here> 860 | ``` 861 | You can now access the Mistral hosted models. Run `llm models` for a list. 862 | 863 | To run a prompt through `mistral-tiny`: 864 | 865 | ```bash 866 | llm -m mistral-tiny 'A sassy name for a pet sasquatch' 867 | ``` 868 | To start an interactive chat session with `mistral-small`: 869 | ```bash 870 | llm chat -m mistral-small 871 | ``` 872 | ``` 873 | Chatting with mistral-small 874 | Type 'exit' or 'quit' to exit 875 | Type '!multi' to enter multiple lines, then '!end' to finish 876 | > three proud names for a pet walrus 877 | 1. "Nanuq," the Inuit word for walrus, which symbolizes strength and resilience. 878 | 2. "Sir Tuskalot," a playful and regal name that highlights the walrus' distinctive tusks. 879 | 3. "Glacier," a name that reflects the walrus' icy Arctic habitat and majestic presence. 880 | ``` 881 | To use a system prompt with `mistral-medium` to explain some code: 882 | ```bash 883 | cat example.py | llm -m mistral-medium -s 'explain this code' 884 | ``` 885 | ## Model options 886 | 887 | All three models accept the following options, using `-o name value` syntax: 888 | 889 | - `-o temperature 0.7`: The sampling temperature, between 0 and 1. Higher increases randomness, lower values are more focused and deterministic. 890 | - `-o top_p 0.1`: 0.1 means consider only tokens in the top 10% probability mass. Use this or temperature but not both. 891 | - `-o max_tokens 20`: Maximum number of tokens to generate in the completion. 892 | - `-o safe_mode 1`: Turns on [safe mode](https://docs.mistral.ai/platform/guardrailing/), which adds a system prompt to add guardrails to the model output. 893 | - `-o random_seed 123`: Set an integer random seed to generate deterministic results. 894 | 895 | ## Refreshing the model list 896 | 897 | Mistral sometimes release new models. 898 | 899 | To make those models available to an existing installation of `llm-mistral` run this command: 900 | ```bash 901 | llm mistral refresh 902 | ``` 903 | This will fetch and cache the latest list of available models. They should then become available in the output of the `llm models` command. 904 | 905 | ## Embeddings 906 | 907 | The Mistral [Embeddings API](https://docs.mistral.ai/platform/client#embeddings) can be used to generate 1,024 dimensional embeddings for any text. 908 | 909 | To embed a single string: 910 | 911 | ```bash 912 | llm embed -m mistral-embed -c 'this is text' 913 | ``` 914 | This will return a JSON array of 1,024 floating point numbers. 915 | 916 | The [LLM documentation](https://llm.datasette.io/en/stable/embeddings/index.html) has more, including how to embed in bulk and store the results in a SQLite database. 917 | 918 | See [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) and [Embeddings: What they are and why they matter](https://simonwillison.net/2023/Oct/23/embeddings/) for more about embeddings. 919 | 920 | ## Development 921 | 922 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 923 | ```bash 924 | cd llm-mistral 925 | python3 -m venv venv 926 | source venv/bin/activate 927 | ``` 928 | Now install the dependencies and test dependencies: 929 | ```bash 930 | llm install -e '.[test]' 931 | ``` 932 | To run the tests: 933 | ```bash 934 | pytest 935 | ``` 936 | 937 | 938 | 939 | 940 | 941 | import httpx 942 | import ijson 943 | import llm 944 | from pydantic import Field 945 | from typing import Optional 946 | 947 | SAFETY_SETTINGS = [ 948 | { 949 | "category": "HARM_CATEGORY_DANGEROUS_CONTENT", 950 | "threshold": "BLOCK_NONE", 951 | }, 952 | { 953 | "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", 954 | "threshold": "BLOCK_NONE", 955 | }, 956 | { 957 | "category": "HARM_CATEGORY_HATE_SPEECH", 958 | "threshold": "BLOCK_NONE", 959 | }, 960 | { 961 | "category": "HARM_CATEGORY_HARASSMENT", 962 | "threshold": "BLOCK_NONE", 963 | }, 964 | ] 965 | 966 | 967 | @llm.hookimpl 968 | def register_models(register): 969 | # Register both sync and async versions of each model 970 | for model_id in [ 971 | "gemini-pro", 972 | "gemini-1.5-pro-latest", 973 | "gemini-1.5-flash-latest", 974 | "gemini-1.5-pro-001", 975 | "gemini-1.5-flash-001", 976 | "gemini-1.5-pro-002", 977 | "gemini-1.5-flash-002", 978 | "gemini-1.5-flash-8b-latest", 979 | "gemini-1.5-flash-8b-001", 980 | "gemini-exp-1114", 981 | "gemini-exp-1121", 982 | "gemini-exp-1206", 983 | "gemini-2.0-flash-exp", 984 | "gemini-2.0-flash-thinking-exp-1219", 985 | ]: 986 | register(GeminiPro(model_id), AsyncGeminiPro(model_id)) 987 | 988 | 989 | def resolve_type(attachment): 990 | mime_type = attachment.resolve_type() 991 | # https://github.com/simonw/llm/issues/587#issuecomment-2439785140 992 | if mime_type == "audio/mpeg": 993 | mime_type = "audio/mp3" 994 | return mime_type 995 | 996 | 997 | class _SharedGemini: 998 | needs_key = "gemini" 999 | key_env_var = "LLM_GEMINI_KEY" 1000 | can_stream = True 1001 | 1002 | attachment_types = ( 1003 | # PDF 1004 | "application/pdf", 1005 | # Images 1006 | "image/png", 1007 | "image/jpeg", 1008 | "image/webp", 1009 | "image/heic", 1010 | "image/heif", 1011 | # Audio 1012 | "audio/wav", 1013 | "audio/mp3", 1014 | "audio/aiff", 1015 | "audio/aac", 1016 | "audio/ogg", 1017 | "audio/flac", 1018 | "audio/mpeg", # Treated as audio/mp3 1019 | # Video 1020 | "video/mp4", 1021 | "video/mpeg", 1022 | "video/mov", 1023 | "video/avi", 1024 | "video/x-flv", 1025 | "video/mpg", 1026 | "video/webm", 1027 | "video/wmv", 1028 | "video/3gpp", 1029 | "video/quicktime", 1030 | ) 1031 | 1032 | class Options(llm.Options): 1033 | code_execution: Optional[bool] = Field( 1034 | description="Enables the model to generate and run Python code", 1035 | default=None, 1036 | ) 1037 | temperature: Optional[float] = Field( 1038 | description=( 1039 | "Controls the randomness of the output. Use higher values for " 1040 | "more creative responses, and lower values for more " 1041 | "deterministic responses." 1042 | ), 1043 | default=None, 1044 | ge=0.0, 1045 | le=2.0, 1046 | ) 1047 | max_output_tokens: Optional[int] = Field( 1048 | description="Sets the maximum number of tokens to include in a candidate.", 1049 | default=None, 1050 | ) 1051 | top_p: Optional[float] = Field( 1052 | description=( 1053 | "Changes how the model selects tokens for output. Tokens are " 1054 | "selected from the most to least probable until the sum of " 1055 | "their probabilities equals the topP value." 1056 | ), 1057 | default=None, 1058 | ge=0.0, 1059 | le=1.0, 1060 | ) 1061 | top_k: Optional[int] = Field( 1062 | description=( 1063 | "Changes how the model selects tokens for output. A topK of 1 " 1064 | "means the selected token is the most probable among all the " 1065 | "tokens in the model's vocabulary, while a topK of 3 means " 1066 | "that the next token is selected from among the 3 most " 1067 | "probable using the temperature." 1068 | ), 1069 | default=None, 1070 | ge=1, 1071 | ) 1072 | json_object: Optional[bool] = Field( 1073 | description="Output a valid JSON object {...}", 1074 | default=None, 1075 | ) 1076 | 1077 | def __init__(self, model_id): 1078 | self.model_id = model_id 1079 | 1080 | def build_messages(self, prompt, conversation): 1081 | messages = [] 1082 | if conversation: 1083 | for response in conversation.responses: 1084 | parts = [] 1085 | for attachment in response.attachments: 1086 | mime_type = resolve_type(attachment) 1087 | parts.append( 1088 | { 1089 | "inlineData": { 1090 | "data": attachment.base64_content(), 1091 | "mimeType": mime_type, 1092 | } 1093 | } 1094 | ) 1095 | if response.prompt.prompt: 1096 | parts.append({"text": response.prompt.prompt}) 1097 | messages.append({"role": "user", "parts": parts}) 1098 | messages.append({"role": "model", "parts": [{"text": response.text_or_raise()}]}) 1099 | 1100 | parts = [] 1101 | if prompt.prompt: 1102 | parts.append({"text": prompt.prompt}) 1103 | for attachment in prompt.attachments: 1104 | mime_type = resolve_type(attachment) 1105 | parts.append( 1106 | { 1107 | "inlineData": { 1108 | "data": attachment.base64_content(), 1109 | "mimeType": mime_type, 1110 | } 1111 | } 1112 | ) 1113 | 1114 | messages.append({"role": "user", "parts": parts}) 1115 | return messages 1116 | 1117 | def build_request_body(self, prompt, conversation): 1118 | body = { 1119 | "contents": self.build_messages(prompt, conversation), 1120 | "safetySettings": SAFETY_SETTINGS, 1121 | } 1122 | if prompt.options and prompt.options.code_execution: 1123 | body["tools"] = [{"codeExecution": {}}] 1124 | if prompt.system: 1125 | body["systemInstruction"] = {"parts": [{"text": prompt.system}]} 1126 | 1127 | config_map = { 1128 | "temperature": "temperature", 1129 | "max_output_tokens": "maxOutputTokens", 1130 | "top_p": "topP", 1131 | "top_k": "topK", 1132 | } 1133 | if prompt.options and prompt.options.json_object: 1134 | body["generationConfig"] = {"response_mime_type": "application/json"} 1135 | 1136 | if any( 1137 | getattr(prompt.options, key, None) is not None for key in config_map.keys() 1138 | ): 1139 | generation_config = {} 1140 | for key, other_key in config_map.items(): 1141 | config_value = getattr(prompt.options, key, None) 1142 | if config_value is not None: 1143 | generation_config[other_key] = config_value 1144 | body["generationConfig"] = generation_config 1145 | 1146 | return body 1147 | 1148 | def process_part(self, part): 1149 | if "text" in part: 1150 | return part["text"] 1151 | elif "executableCode" in part: 1152 | return f'```{part["executableCode"]["language"].lower()}\n{part["executableCode"]["code"].strip()}\n```\n' 1153 | elif "codeExecutionResult" in part: 1154 | return f'```\n{part["codeExecutionResult"]["output"].strip()}\n```\n' 1155 | return "" 1156 | 1157 | def set_usage(self, response): 1158 | try: 1159 | usage = response.response_json[-1].pop("usageMetadata") 1160 | input_tokens = usage.pop("promptTokenCount", None) 1161 | output_tokens = usage.pop("candidatesTokenCount", None) 1162 | usage.pop("totalTokenCount", None) 1163 | if input_tokens is not None: 1164 | response.set_usage( 1165 | input=input_tokens, output=output_tokens, details=usage or None 1166 | ) 1167 | except (IndexError, KeyError): 1168 | pass 1169 | 1170 | 1171 | class GeminiPro(_SharedGemini, llm.Model): 1172 | def execute(self, prompt, stream, response, conversation): 1173 | key = self.get_key() 1174 | url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_id}:streamGenerateContent" 1175 | gathered = [] 1176 | body = self.build_request_body(prompt, conversation) 1177 | 1178 | with httpx.stream( 1179 | "POST", 1180 | url, 1181 | timeout=None, 1182 | headers={"x-goog-api-key": key}, 1183 | json=body, 1184 | ) as http_response: 1185 | events = ijson.sendable_list() 1186 | coro = ijson.items_coro(events, "item") 1187 | for chunk in http_response.iter_bytes(): 1188 | coro.send(chunk) 1189 | if events: 1190 | event = events[0] 1191 | if isinstance(event, dict) and "error" in event: 1192 | raise llm.ModelError(event["error"]["message"]) 1193 | try: 1194 | part = event["candidates"][0]["content"]["parts"][0] 1195 | yield self.process_part(part) 1196 | except KeyError: 1197 | yield "" 1198 | gathered.append(event) 1199 | events.clear() 1200 | response.response_json = gathered 1201 | self.set_usage(response) 1202 | 1203 | 1204 | class AsyncGeminiPro(_SharedGemini, llm.AsyncModel): 1205 | async def execute(self, prompt, stream, response, conversation): 1206 | key = self.get_key() 1207 | url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model_id}:streamGenerateContent" 1208 | gathered = [] 1209 | body = self.build_request_body(prompt, conversation) 1210 | 1211 | async with httpx.AsyncClient() as client: 1212 | async with client.stream( 1213 | "POST", 1214 | url, 1215 | timeout=None, 1216 | headers={"x-goog-api-key": key}, 1217 | json=body, 1218 | ) as http_response: 1219 | events = ijson.sendable_list() 1220 | coro = ijson.items_coro(events, "item") 1221 | async for chunk in http_response.aiter_bytes(): 1222 | coro.send(chunk) 1223 | if events: 1224 | event = events[0] 1225 | if isinstance(event, dict) and "error" in event: 1226 | raise llm.ModelError(event["error"]["message"]) 1227 | try: 1228 | part = event["candidates"][0]["content"]["parts"][0] 1229 | yield self.process_part(part) 1230 | except KeyError: 1231 | yield "" 1232 | gathered.append(event) 1233 | events.clear() 1234 | response.response_json = gathered 1235 | self.set_usage(response) 1236 | 1237 | 1238 | @llm.hookimpl 1239 | def register_embedding_models(register): 1240 | register( 1241 | GeminiEmbeddingModel("text-embedding-004", "text-embedding-004"), 1242 | ) 1243 | 1244 | 1245 | class GeminiEmbeddingModel(llm.EmbeddingModel): 1246 | needs_key = "gemini" 1247 | key_env_var = "LLM_GEMINI_KEY" 1248 | batch_size = 20 1249 | 1250 | def __init__(self, model_id, gemini_model_id): 1251 | self.model_id = model_id 1252 | self.gemini_model_id = gemini_model_id 1253 | 1254 | def embed_batch(self, items): 1255 | headers = { 1256 | "Content-Type": "application/json", 1257 | "x-goog-api-key": self.get_key(), 1258 | } 1259 | data = { 1260 | "requests": [ 1261 | { 1262 | "model": "models/" + self.gemini_model_id, 1263 | "content": {"parts": [{"text": item}]}, 1264 | } 1265 | for item in items 1266 | ] 1267 | } 1268 | 1269 | with httpx.Client() as client: 1270 | response = client.post( 1271 | f"https://generativelanguage.googleapis.com/v1beta/models/{self.gemini_model_id}:batchEmbedContents", 1272 | headers=headers, 1273 | json=data, 1274 | timeout=None, 1275 | ) 1276 | 1277 | response.raise_for_status() 1278 | return [item["values"] for item in response.json()["embeddings"]] 1279 | 1280 | 1281 | # llm-gemini 1282 | 1283 | [![PyPI](https://img.shields.io/pypi/v/llm-gemini.svg)](https://pypi.org/project/llm-gemini/) 1284 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-gemini?include_prereleases&label=changelog)](https://github.com/simonw/llm-gemini/releases) 1285 | [![Tests](https://github.com/simonw/llm-gemini/workflows/Test/badge.svg)](https://github.com/simonw/llm-gemini/actions?query=workflow%3ATest) 1286 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-gemini/blob/main/LICENSE) 1287 | 1288 | API access to Google's Gemini models 1289 | 1290 | ## Installation 1291 | 1292 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 1293 | ```bash 1294 | llm install llm-gemini 1295 | ``` 1296 | ## Usage 1297 | 1298 | Configure the model by setting a key called "gemini" to your [API key](https://aistudio.google.com/app/apikey): 1299 | ```bash 1300 | llm keys set gemini 1301 | ``` 1302 | ``` 1303 | 1304 | ``` 1305 | You can also set the API key by assigning it to the environment variable `LLM_GEMINI_KEY`. 1306 | 1307 | Now run the model using `-m gemini-1.5-pro-latest`, for example: 1308 | 1309 | ```bash 1310 | llm -m gemini-1.5-pro-latest "A joke about a pelican and a walrus" 1311 | ``` 1312 | 1313 | > A pelican walks into a seafood restaurant with a huge fish hanging out of its beak. The walrus, sitting at the bar, eyes it enviously. 1314 | > 1315 | > "Hey," the walrus says, "That looks delicious! What kind of fish is that?" 1316 | > 1317 | > The pelican taps its beak thoughtfully. "I believe," it says, "it's a billfish." 1318 | 1319 | Other models are: 1320 | 1321 | - `gemini-1.5-flash-latest` 1322 | - `gemini-1.5-flash-8b-latest` - the least expensive 1323 | - `gemini-exp-1114` - recent experimental #1 1324 | - `gemini-exp-1121` - recent experimental #2 1325 | - `gemini-exp-1206` - recent experimental #3 1326 | - `gemini-2.0-flash-exp` - [Gemini 2.0 Flash](https://blog.google/technology/google-deepmind/google-gemini-ai-update-december-2024/#gemini-2-0-flash) 1327 | 1328 | ### Images, audio and video 1329 | 1330 | Gemini models are multi-modal. You can provide images, audio or video files as input like this: 1331 | 1332 | ```bash 1333 | llm -m gemini-1.5-flash-latest 'extract text' -a image.jpg 1334 | ``` 1335 | Or with a URL: 1336 | ```bash 1337 | llm -m gemini-1.5-flash-8b-latest 'describe image' \ 1338 | -a https://static.simonwillison.net/static/2024/pelicans.jpg 1339 | ``` 1340 | Audio works too: 1341 | 1342 | ```bash 1343 | llm -m gemini-1.5-pro-latest 'transcribe audio' -a audio.mp3 1344 | ``` 1345 | 1346 | And video: 1347 | 1348 | ```bash 1349 | llm -m gemini-1.5-pro-latest 'describe what happens' -a video.mp4 1350 | ``` 1351 | The Gemini prompting guide includes [extensive advice](https://ai.google.dev/gemini-api/docs/file-prompting-strategies) on multi-modal prompting. 1352 | 1353 | ### JSON output 1354 | 1355 | Use `-o json_object 1` to force the output to be JSON: 1356 | 1357 | ```bash 1358 | llm -m gemini-1.5-flash-latest -o json_object 1 \ 1359 | '3 largest cities in California, list of {"name": "..."}' 1360 | ``` 1361 | Outputs: 1362 | ```json 1363 | {"cities": [{"name": "Los Angeles"}, {"name": "San Diego"}, {"name": "San Jose"}]} 1364 | ``` 1365 | 1366 | ### Code execution 1367 | 1368 | Gemini models can [write and execute code](https://ai.google.dev/gemini-api/docs/code-execution) - they can decide to write Python code, execute it in a secure sandbox and use the result as part of their response. 1369 | 1370 | To enable this feature, use `-o code_execution 1`: 1371 | 1372 | ```bash 1373 | llm -m gemini-1.5-pro-latest -o code_execution 1 \ 1374 | 'use python to calculate (factorial of 13) * 3' 1375 | ``` 1376 | 1377 | ### Chat 1378 | 1379 | To chat interactively with the model, run `llm chat`: 1380 | 1381 | ```bash 1382 | llm chat -m gemini-1.5-pro-latest 1383 | ``` 1384 | 1385 | ## Embeddings 1386 | 1387 | The plugin also adds support for the `text-embedding-004` embedding model. 1388 | 1389 | Run that against a single string like this: 1390 | ```bash 1391 | llm embed -m text-embedding-004 -c 'hello world' 1392 | ``` 1393 | This returns a JSON array of 768 numbers. 1394 | 1395 | This command will embed every `README.md` file in child directories of the current directory and store the results in a SQLite database called `embed.db` in a collection called `readmes`: 1396 | 1397 | ```bash 1398 | llm embed-multi readmes --files . '*/README.md' -d embed.db -m text-embedding-004 1399 | ``` 1400 | You can then run similarity searches against that collection like this: 1401 | ```bash 1402 | llm similar readmes -c 'upload csvs to stuff' -d embed.db 1403 | ``` 1404 | 1405 | See the [LLM embeddings documentation](https://llm.datasette.io/en/stable/embeddings/cli.html) for further details. 1406 | 1407 | ## Development 1408 | 1409 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 1410 | ```bash 1411 | cd llm-gemini 1412 | python3 -m venv venv 1413 | source venv/bin/activate 1414 | ``` 1415 | Now install the dependencies and test dependencies: 1416 | ```bash 1417 | llm install -e '.[test]' 1418 | ``` 1419 | To run the tests: 1420 | ```bash 1421 | pytest 1422 | ``` 1423 | 1424 | This project uses [pytest-recording](https://github.com/kiwicom/pytest-recording) to record Gemini API responses for the tests. 1425 | 1426 | If you add a new test that calls the API you can capture the API response like this: 1427 | ```bash 1428 | PYTEST_GEMINI_API_KEY="$(llm keys get gemini)" pytest --record-mode once 1429 | ``` 1430 | You will need to have stored a valid Gemini API key using this command first: 1431 | ```bash 1432 | llm keys set gemini 1433 | # Paste key here 1434 | ``` 1435 | 1436 | 1437 | -------------------------------------------------------------------------------- /llm_plugin_generator/few_shot_prompt_llm_plugin_utility_with_tests.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | # llm-plugin-generator 6 | 7 | [![PyPI](https://img.shields.io/pypi/v/llm-plugin-generator.svg)](https://pypi.org/project/llm-plugin-generator/) 8 | [![Changelog](https://img.shields.io/github/v/release/irthomasthomas/llm-plugin-generator?include_prereleases&label=changelog)](https://github.com/irthomasthomas/llm-plugin-generator/releases) 9 | [![Tests](https://github.com/irthomasthomas/llm-plugin-generator/workflows/Test/badge.svg)](https://github.com/irthomasthomas/llm-plugin-generator/actions?query=workflow%3ATest) 10 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/irthomasthomas/llm-plugin-generator/blob/main/LICENSE) 11 | 12 | LLM plugin to generate plugins for LLM 13 | 14 | ## Installation 15 | 16 | Install this plugin in the same environment as LLM: 17 | 18 | ```bash 19 | llm install llm-plugin-generator 20 | ``` 21 | 22 | ## Usage 23 | 24 | To generate a new LLM plugin, use the `generate-plugin` command: 25 | 26 | ```bash 27 | llm generate-plugin "Description of your plugin" 28 | ``` 29 | 30 | Options: 31 | 32 | - `PROMPT`: Description of your plugin (optional) 33 | - `INPUT_FILES`: Path(s) to input README or prompt file(s) (optional, multiple allowed) 34 | - `--output-dir`: Directory to save generated plugin files (default: current directory) 35 | - `--type`: Type of plugin to generate (default, model, or utility) 36 | - `--model`, `-m`: Model to use for generation 37 | 38 | ## --type 39 | --type model will use a few-shot prompt focused on llm model plugins. 40 | --type utility focuses on utilities. 41 | leaving off --type will use a default prompt that combines all off them. I suggest picking one of the focused options which should be faster. 42 | 43 | Examples: 44 | 45 | 1. Basic usage: 46 | ```bash 47 | llm generate-plugin "Create a plugin that translates text to emoji" --output-dir ./my-new-plugin --type utility --model gpt-4 48 | ``` 49 | 50 | 2. Using a prompt and input files - Generating plugin from a README.md 51 | ``` 52 | llm generate-plugin "Few-shot Prompt Generator. Call it llm-few-shot-generator" \ 53 | 'files/README.md' --output-dir plugins/Utilities/few-shot-generator \ 54 | --type utility -m claude-3.5-sonnet 55 | ``` 56 | 57 | 3. Using websites or remote files: 58 | ``` 59 | llm generate-plugin "Write an llm-cerebras plugin from these docs: $(curl -s https://raw.githubusercontent.com/irthomasthomas/llm-cerebras/refs/heads/main/.artefacts/cerebras-api-notes.txt)" \ 60 | --output-dir llm-cerebras --type model -m sonnet-3.5 61 | ``` 62 | 63 | This will generate a new LLM plugin based on the provided description and/or input files. The files will be saved in the specified output directory. 64 | 65 | ## Features 66 | # New: Requests are now logged to the llm db. 67 | - Generates fully functional LLM plugins based on descriptions or input files 68 | - Supports different plugin types: default, model, and utility 69 | - Uses few-shot learning with predefined examples for better generation 70 | - Allows specifying custom output directory 71 | - Compatible with various LLM models 72 | - Generates main plugin file, README.md, and pyproject.toml 73 | - Extracts plugin name from generated pyproject.toml for consistent naming 74 | 75 | ## Development 76 | 77 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 78 | 79 | ```bash 80 | cd llm-plugin-generator 81 | python -m venv venv 82 | source venv/bin/activate 83 | ``` 84 | 85 | Now install the dependencies and test dependencies: 86 | 87 | ```bash 88 | pip install -e '.[test]' 89 | ``` 90 | 91 | To run the tests: 92 | 93 | ```bash 94 | pytest 95 | ``` 96 | 97 | ## Contributing 98 | 99 | Contributions to llm-plugin-generator are welcome! Please refer to the [GitHub repository](https://github.com/irthomasthomas/llm-plugin-generator) for more information on how to contribute. 100 | 101 | ## License 102 | 103 | This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. 104 | 105 | 106 | 107 | import llm 108 | import click 109 | import os 110 | import pathlib 111 | import sqlite_utils 112 | import toml 113 | from importlib import resources 114 | 115 | # Update these lines to use just the filename 116 | DEFAULT_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_all.xml" 117 | MODEL_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_model.xml" 118 | UTILITY_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_utility.xml" 119 | 120 | def user_dir(): 121 | llm_user_path = os.environ.get("LLM_USER_PATH") 122 | if llm_user_path: 123 | path = pathlib.Path(llm_user_path) 124 | else: 125 | path = pathlib.Path(click.get_app_dir("io.datasette.llm")) 126 | path.mkdir(exist_ok=True, parents=True) 127 | return path 128 | 129 | def logs_db_path(): 130 | return user_dir() / "logs.db" 131 | 132 | def read_few_shot_prompt(file_name): 133 | with resources.open_text("llm_plugin_generator", file_name) as file: 134 | return file.read() 135 | 136 | def write_main_python_file(content, output_dir, filename): 137 | main_file = output_dir / filename 138 | with main_file.open("w") as f: 139 | f.write(content) 140 | click.echo(f"Main Python file written to {main_file}") 141 | 142 | def write_readme(content, output_dir): 143 | readme_file = output_dir / "README.md" 144 | with readme_file.open("w") as f: 145 | f.write(content) 146 | click.echo(f"README file written to {readme_file}") 147 | 148 | def write_pyproject_toml(content, output_dir): 149 | pyproject_file = output_dir / "pyproject.toml" 150 | with pyproject_file.open("w") as f: 151 | f.write(content) 152 | click.echo(f"pyproject.toml file written to {pyproject_file}") 153 | 154 | def extract_plugin_name(pyproject_content): 155 | try: 156 | pyproject_dict = toml.loads(pyproject_content) 157 | name = pyproject_dict['project']['name'] 158 | # Convert kebab-case to snake_case 159 | return name.replace('-', '_') 160 | except: 161 | # If parsing fails, return a default name 162 | return "plugin" 163 | 164 | @llm.hookimpl 165 | def register_commands(cli): 166 | @cli.command() 167 | @click.argument("prompt", required=False) 168 | @click.argument("input_files", nargs=-1, type=click.Path(exists=True)) 169 | @click.option("--output-dir", type=click.Path(), default=".", help="Directory to save generated plugin files") 170 | @click.option("--type", default="default", type=click.Choice(["default", "model", "utility"]), help="Type of plugin to generate") 171 | @click.option("--model", "-m", help="Model to use") 172 | def generate_plugin(prompt, input_files, output_dir, type, model): 173 | """Generate a new LLM plugin based on examples and a prompt or README file(s).""" 174 | # Select the appropriate few-shot prompt file based on the type 175 | if type == "model": 176 | few_shot_file = MODEL_FEW_SHOT_PROMPT_FILE 177 | elif type == "utility": 178 | few_shot_file = UTILITY_FEW_SHOT_PROMPT_FILE 179 | else: 180 | few_shot_file = DEFAULT_FEW_SHOT_PROMPT_FILE 181 | 182 | few_shot_prompt = read_few_shot_prompt(few_shot_file) 183 | 184 | input_content = "" 185 | for input_file in input_files: 186 | with open(input_file, "r") as f: 187 | input_content += f"""Content from {input_file}: 188 | {f.read()} 189 | 190 | """ 191 | if prompt: 192 | input_content += f"""Additional prompt: 193 | {prompt} 194 | """ 195 | 196 | if not input_content: 197 | input_content = click.prompt("Enter your plugin description or requirements") 198 | 199 | llm_model = llm.get_model(model) 200 | db = sqlite_utils.Database("") 201 | full_prompt = f"""Generate a new LLM plugin based on the following few-shot examples and the given input: 202 | Few-shot examples: 203 | {few_shot_prompt} 204 | 205 | Input: 206 | {input_content} 207 | 208 | Generate the plugin code, including the main plugin file, README.md, and pyproject.toml. 209 | Ensure the generated plugin follows best practices and is fully functional. 210 | Provide the content for each file separately, enclosed in XML tags like , , and .""" 211 | 212 | db = sqlite_utils.Database(logs_db_path()) 213 | response = llm_model.prompt(full_prompt) 214 | response.log_to_db(db) 215 | generated_plugin = response.text() 216 | 217 | output_path = pathlib.Path(output_dir) 218 | output_path.mkdir(parents=True, exist_ok=True) 219 | 220 | plugin_py_content = extract_content(generated_plugin, "plugin_py") 221 | readme_content = extract_content(generated_plugin, "readme_md") 222 | pyproject_content = extract_content(generated_plugin, "pyproject_toml") 223 | 224 | # Extract the plugin name from pyproject.toml 225 | plugin_name = extract_plugin_name(pyproject_content) 226 | 227 | # Use the extracted name for the main Python file 228 | write_main_python_file(plugin_py_content, output_path, f"{plugin_name}.py") 229 | write_readme(readme_content, output_path) 230 | write_pyproject_toml(pyproject_content, output_path) 231 | 232 | click.echo("Plugin generation completed.") 233 | 234 | def extract_content(text, tag): 235 | start_tag = f"<{tag}>" 236 | end_tag = f"" 237 | start = text.find(start_tag) + len(start_tag) 238 | end = text.find(end_tag) 239 | return text[start:end].strip() 240 | 241 | @llm.hookimpl 242 | def register_models(register): 243 | pass # No custom models to register for this plugin 244 | 245 | @llm.hookimpl 246 | def register_prompts(register): 247 | pass # No custom prompts to register for this plugin 248 | 249 | 250 | 251 | 252 | 253 | # llm-plugin-generator 254 | 255 | [![PyPI](https://img.shields.io/pypi/v/llm-plugin-generator.svg)](https://pypi.org/project/llm-plugin-generator/) 256 | [![Changelog](https://img.shields.io/github/v/release/irthomasthomas/llm-plugin-generator?include_prereleases&label=changelog)](https://github.com/irthomasthomas/llm-plugin-generator/releases) 257 | [![Tests](https://github.com/irthomasthomas/llm-plugin-generator/workflows/Test/badge.svg)](https://github.com/irthomasthomas/llm-plugin-generator/actions?query=workflow%3ATest) 258 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/irthomasthomas/llm-plugin-generator/blob/main/LICENSE) 259 | 260 | LLM plugin to generate plugins for LLM 261 | 262 | ## Installation 263 | 264 | Install this plugin in the same environment as LLM: 265 | 266 | ```bash 267 | llm install llm-plugin-generator 268 | ``` 269 | 270 | ## Usage 271 | 272 | To generate a new LLM plugin, use the `generate-plugin` command: 273 | 274 | ```bash 275 | llm generate-plugin "Description of your plugin" 276 | ``` 277 | 278 | Options: 279 | 280 | - `PROMPT`: Description of your plugin (optional) 281 | - `INPUT_FILES`: Path(s) to input README or prompt file(s) (optional, multiple allowed) 282 | - `--output-dir`: Directory to save generated plugin files (default: current directory) 283 | - `--type`: Type of plugin to generate (default, model, or utility) 284 | - `--model`, `-m`: Model to use for generation 285 | 286 | ## --type 287 | --type model will use a few-shot prompt focused on llm model plugins. 288 | --type utility focuses on utilities. 289 | leaving off --type will use a default prompt that combines all off them. I suggest picking one of the focused options which should be faster. 290 | 291 | Examples: 292 | 293 | 1. Basic usage: 294 | ```bash 295 | llm generate-plugin "Create a plugin that translates text to emoji" --output-dir ./my-new-plugin --type utility --model gpt-4 296 | ``` 297 | 298 | 2. Using a prompt and input files - Generating plugin from a README.md 299 | ``` 300 | llm generate-plugin "Few-shot Prompt Generator. Call it llm-few-shot-generator" \ 301 | 'files/README.md' --output-dir plugins/Utilities/few-shot-generator \ 302 | --type utility -m claude-3.5-sonnet 303 | ``` 304 | 305 | 3. Using websites or remote files: 306 | ``` 307 | llm generate-plugin "Write an llm-cerebras plugin from these docs: $(curl -s https://raw.githubusercontent.com/irthomasthomas/llm-cerebras/refs/heads/main/.artefacts/cerebras-api-notes.txt)" \ 308 | --output-dir llm-cerebras --type model -m sonnet-3.5 309 | ``` 310 | 311 | This will generate a new LLM plugin based on the provided description and/or input files. The files will be saved in the specified output directory. 312 | 313 | ## Features 314 | # New: Requests are now logged to the llm db. 315 | - Generates fully functional LLM plugins based on descriptions or input files 316 | - Supports different plugin types: default, model, and utility 317 | - Uses few-shot learning with predefined examples for better generation 318 | - Allows specifying custom output directory 319 | - Compatible with various LLM models 320 | - Generates main plugin file, README.md, and pyproject.toml 321 | - Extracts plugin name from generated pyproject.toml for consistent naming 322 | 323 | ## Development 324 | 325 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 326 | 327 | ```bash 328 | cd llm-plugin-generator 329 | python -m venv venv 330 | source venv/bin/activate 331 | ``` 332 | 333 | Now install the dependencies and test dependencies: 334 | 335 | ```bash 336 | pip install -e '.[test]' 337 | ``` 338 | 339 | To run the tests: 340 | 341 | ```bash 342 | pytest 343 | ``` 344 | 345 | ## Contributing 346 | 347 | Contributions to llm-plugin-generator are welcome! Please refer to the [GitHub repository](https://github.com/irthomasthomas/llm-plugin-generator) for more information on how to contribute. 348 | 349 | ## License 350 | 351 | This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. 352 | 353 | 354 | 355 | import llm 356 | import click 357 | import os 358 | import pathlib 359 | import sqlite_utils 360 | import toml 361 | from importlib import resources 362 | 363 | # Update these lines to use just the filename 364 | DEFAULT_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_all.xml" 365 | MODEL_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_model.xml" 366 | UTILITY_FEW_SHOT_PROMPT_FILE = "few_shot_prompt_llm_plugin_utility.xml" 367 | 368 | def user_dir(): 369 | llm_user_path = os.environ.get("LLM_USER_PATH") 370 | if llm_user_path: 371 | path = pathlib.Path(llm_user_path) 372 | else: 373 | path = pathlib.Path(click.get_app_dir("io.datasette.llm")) 374 | path.mkdir(exist_ok=True, parents=True) 375 | return path 376 | 377 | def logs_db_path(): 378 | return user_dir() / "logs.db" 379 | 380 | def read_few_shot_prompt(file_name): 381 | with resources.open_text("llm_plugin_generator", file_name) as file: 382 | return file.read() 383 | 384 | def write_main_python_file(content, output_dir, filename): 385 | main_file = output_dir / filename 386 | with main_file.open("w") as f: 387 | f.write(content) 388 | click.echo(f"Main Python file written to {main_file}") 389 | 390 | def write_readme(content, output_dir): 391 | readme_file = output_dir / "README.md" 392 | with readme_file.open("w") as f: 393 | f.write(content) 394 | click.echo(f"README file written to {readme_file}") 395 | 396 | def write_pyproject_toml(content, output_dir): 397 | pyproject_file = output_dir / "pyproject.toml" 398 | with pyproject_file.open("w") as f: 399 | f.write(content) 400 | click.echo(f"pyproject.toml file written to {pyproject_file}") 401 | 402 | def extract_plugin_name(pyproject_content): 403 | try: 404 | pyproject_dict = toml.loads(pyproject_content) 405 | name = pyproject_dict['project']['name'] 406 | # Convert kebab-case to snake_case 407 | return name.replace('-', '_') 408 | except: 409 | # If parsing fails, return a default name 410 | return "plugin" 411 | 412 | @llm.hookimpl 413 | def register_commands(cli): 414 | @cli.command() 415 | @click.argument("prompt", required=False) 416 | @click.argument("input_files", nargs=-1, type=click.Path(exists=True)) 417 | @click.option("--output-dir", type=click.Path(), default=".", help="Directory to save generated plugin files") 418 | @click.option("--type", default="default", type=click.Choice(["default", "model", "utility"]), help="Type of plugin to generate") 419 | @click.option("--model", "-m", help="Model to use") 420 | def generate_plugin(prompt, input_files, output_dir, type, model): 421 | """Generate a new LLM plugin based on examples and a prompt or README file(s).""" 422 | # Select the appropriate few-shot prompt file based on the type 423 | if type == "model": 424 | few_shot_file = MODEL_FEW_SHOT_PROMPT_FILE 425 | elif type == "utility": 426 | few_shot_file = UTILITY_FEW_SHOT_PROMPT_FILE 427 | else: 428 | few_shot_file = DEFAULT_FEW_SHOT_PROMPT_FILE 429 | 430 | few_shot_prompt = read_few_shot_prompt(few_shot_file) 431 | 432 | input_content = "" 433 | for input_file in input_files: 434 | with open(input_file, "r") as f: 435 | input_content += f"""Content from {input_file}: 436 | {f.read()} 437 | 438 | """ 439 | if prompt: 440 | input_content += f"""Additional prompt: 441 | {prompt} 442 | """ 443 | 444 | if not input_content: 445 | input_content = click.prompt("Enter your plugin description or requirements") 446 | 447 | llm_model = llm.get_model(model) 448 | db = sqlite_utils.Database("") 449 | full_prompt = f"""Generate a new LLM plugin based on the following few-shot examples and the given input: 450 | Few-shot examples: 451 | {few_shot_prompt} 452 | 453 | Input: 454 | {input_content} 455 | 456 | Generate the plugin code, including the main plugin file, README.md, and pyproject.toml. 457 | Ensure the generated plugin follows best practices and is fully functional. 458 | Provide the content for each file separately, enclosed in XML tags like <plugin_py>, <readme_md>, and <pyproject_toml>.""" 459 | 460 | db = sqlite_utils.Database(logs_db_path()) 461 | response = llm_model.prompt(full_prompt) 462 | response.log_to_db(db) 463 | generated_plugin = response.text() 464 | 465 | output_path = pathlib.Path(output_dir) 466 | output_path.mkdir(parents=True, exist_ok=True) 467 | 468 | plugin_py_content = extract_content(generated_plugin, "plugin_py") 469 | readme_content = extract_content(generated_plugin, "readme_md") 470 | pyproject_content = extract_content(generated_plugin, "pyproject_toml") 471 | 472 | # Extract the plugin name from pyproject.toml 473 | plugin_name = extract_plugin_name(pyproject_content) 474 | 475 | # Use the extracted name for the main Python file 476 | write_main_python_file(plugin_py_content, output_path, f"{plugin_name}.py") 477 | write_readme(readme_content, output_path) 478 | write_pyproject_toml(pyproject_content, output_path) 479 | 480 | click.echo("Plugin generation completed.") 481 | 482 | def extract_content(text, tag): 483 | start_tag = f"<{tag}>" 484 | end_tag = f"</{tag}>" 485 | start = text.find(start_tag) + len(start_tag) 486 | end = text.find(end_tag) 487 | return text[start:end].strip() 488 | 489 | @llm.hookimpl 490 | def register_models(register): 491 | pass # No custom models to register for this plugin 492 | 493 | @llm.hookimpl 494 | def register_prompts(register): 495 | pass # No custom prompts to register for this plugin 496 | 497 | 498 | 499 | 500 | 501 | # llm-cluster 502 | 503 | [![PyPI](https://img.shields.io/pypi/v/llm-cluster.svg)](https://pypi.org/project/llm-cluster/) 504 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-cluster?include_prereleases&label=changelog)](https://github.com/simonw/llm-cluster/releases) 505 | [![Tests](https://github.com/simonw/llm-cluster/workflows/Test/badge.svg)](https://github.com/simonw/llm-cluster/actions?query=workflow%3ATest) 506 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-cluster/blob/main/LICENSE) 507 | 508 | [LLM](https://llm.datasette.io/) plugin for clustering embeddings 509 | 510 | Background on this project: [Clustering with llm-cluster](https://simonwillison.net/2023/Sep/4/llm-embeddings/#llm-cluster). 511 | 512 | ## Installation 513 | 514 | Install this plugin in the same environment as LLM. 515 | ```bash 516 | llm install llm-cluster 517 | ``` 518 | 519 | ## Usage 520 | 521 | The plugin adds a new command, `llm cluster`. This command takes the name of an [embedding collection](https://llm.datasette.io/en/stable/embeddings/cli.html#storing-embeddings-in-sqlite) and the number of clusters to return. 522 | 523 | First, use [paginate-json](https://github.com/simonw/paginate-json) and [jq](https://stedolan.github.io/jq/) to populate a collection. I this case we are embedding the title and body of every issue in the [llm repository](https://github.com/simonw/llm), and storing the result in a `issues.db` database: 524 | ```bash 525 | paginate-json 'https://api.github.com/repos/simonw/llm/issues?state=all&filter=all' \ 526 | | jq '[.[] | {id: .id, title: .title}]' \ 527 | | llm embed-multi llm-issues - \ 528 | --database issues.db --store 529 | ``` 530 | The `--store` flag causes the content to be stored in the database along with the embedding vectors. 531 | 532 | Now we can cluster those embeddings into 10 groups: 533 | ```bash 534 | llm cluster llm-issues 10 \ 535 | -d issues.db 536 | ``` 537 | If you omit the `-d` option the default embeddings database will be used. 538 | 539 | The output should look something like this (truncated): 540 | ```json 541 | [ 542 | { 543 | "id": "2", 544 | "items": [ 545 | { 546 | "id": "1650662628", 547 | "content": "Initial design" 548 | }, 549 | { 550 | "id": "1650682379", 551 | "content": "Log prompts and responses to SQLite" 552 | } 553 | ] 554 | }, 555 | { 556 | "id": "4", 557 | "items": [ 558 | { 559 | "id": "1650760699", 560 | "content": "llm web command - launches a web server" 561 | }, 562 | { 563 | "id": "1759659476", 564 | "content": "`llm models` command" 565 | }, 566 | { 567 | "id": "1784156919", 568 | "content": "`llm.get_model(alias)` helper" 569 | } 570 | ] 571 | }, 572 | { 573 | "id": "7", 574 | "items": [ 575 | { 576 | "id": "1650765575", 577 | "content": "--code mode for outputting code" 578 | }, 579 | { 580 | "id": "1659086298", 581 | "content": "Accept PROMPT from --stdin" 582 | }, 583 | { 584 | "id": "1714651657", 585 | "content": "Accept input from standard in" 586 | } 587 | ] 588 | } 589 | ] 590 | ``` 591 | The content displayed is truncated to 100 characters. Pass `--truncate 0` to disable truncation, or `--truncate X` to truncate to X characters. 592 | 593 | ## Generating summaries for each cluster 594 | 595 | The `--summary` flag will cause the plugin to generate a summary for each cluster, by passing the content of the items (truncated according to the `--truncate` option) through a prompt to a Large Language Model. 596 | 597 | This feature is still experimental. You should experiment with custom prompts to improve the quality of your summaries. 598 | 599 | Since this can run a large amount of text through a LLM this can be expensive, depending on which model you are using. 600 | 601 | This feature only works for embeddings that have had their associated content stored in the database using the `--store` flag. 602 | 603 | You can use it like this: 604 | 605 | ```bash 606 | llm cluster llm-issues 10 \ 607 | -d issues.db \ 608 | --summary 609 | ``` 610 | This uses the default prompt and the default model. 611 | 612 | Partial example output: 613 | ```json 614 | [ 615 | { 616 | "id": "5", 617 | "items": [ 618 | { 619 | "id": "1650682379", 620 | "content": "Log prompts and responses to SQLite" 621 | }, 622 | { 623 | "id": "1650757081", 624 | "content": "Command for browsing captured logs" 625 | } 626 | ], 627 | "summary": "Log Management and Interactive Prompt Tracking" 628 | }, 629 | { 630 | "id": "6", 631 | "items": [ 632 | { 633 | "id": "1650771320", 634 | "content": "Mechanism for continuing an existing conversation" 635 | }, 636 | { 637 | "id": "1740090291", 638 | "content": "-c option for continuing a chat (using new chat_id column)" 639 | }, 640 | { 641 | "id": "1784122278", 642 | "content": "Figure out truncation strategy for continue conversation mode" 643 | } 644 | ], 645 | "summary": "Continuing Conversation Mechanism and Management" 646 | } 647 | ] 648 | ``` 649 | 650 | To use a different model, e.g. GPT-4, pass the `--model` option: 651 | ```bash 652 | llm cluster llm-issues 10 \ 653 | -d issues.db \ 654 | --summary \ 655 | --model gpt-4 656 | ``` 657 | The default prompt used is: 658 | 659 | > Short, concise title for this cluster of related documents. 660 | 661 | To use a custom prompt, pass `--prompt`: 662 | 663 | ```bash 664 | llm cluster llm-issues 10 \ 665 | -d issues.db \ 666 | --summary \ 667 | --model gpt-4 \ 668 | --prompt 'Summarize this in a short line in the style of a bored, angry panda' 669 | ``` 670 | A `"summary"` key will be added to each cluster, containing the generated summary. 671 | 672 | ## Development 673 | 674 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 675 | ```bash 676 | cd llm-cluster 677 | python3 -m venv venv 678 | source venv/bin/activate 679 | ``` 680 | Now install the dependencies and test dependencies: 681 | ```bash 682 | pip install -e '.[test]' 683 | ``` 684 | To run the tests: 685 | ```bash 686 | pytest 687 | ``` 688 | 689 | 690 | 691 | import click 692 | import json 693 | import llm 694 | import numpy as np 695 | import sklearn.cluster 696 | import sqlite_utils 697 | import textwrap 698 | 699 | DEFAULT_SUMMARY_PROMPT = """ 700 | Short, concise title for this cluster of related documents. 701 | """.strip() 702 | 703 | 704 | @llm.hookimpl 705 | def register_commands(cli): 706 | @cli.command() 707 | @click.argument("collection") 708 | @click.argument("n", type=int) 709 | @click.option( 710 | "--truncate", 711 | type=int, 712 | default=100, 713 | help="Truncate content to this many characters - 0 for no truncation", 714 | ) 715 | @click.option( 716 | "-d", 717 | "--database", 718 | type=click.Path( 719 | file_okay=True, allow_dash=False, dir_okay=False, writable=True 720 | ), 721 | envvar="LLM_EMBEDDINGS_DB", 722 | help="SQLite database file containing embeddings", 723 | ) 724 | @click.option( 725 | "--summary", is_flag=True, help="Generate summary title for each cluster" 726 | ) 727 | @click.option("-m", "--model", help="LLM model to use for the summary") 728 | @click.option("--prompt", help="Custom prompt to use for the summary") 729 | def cluster(collection, n, truncate, database, summary, model, prompt): 730 | """ 731 | Generate clusters from embeddings in a collection 732 | 733 | Example usage, to create 10 clusters: 734 | 735 | \b 736 | llm cluster my_collection 10 737 | 738 | Outputs a JSON array of {"id": "cluster_id", "items": [list of items]} 739 | 740 | Pass --summary to generate a summary for each cluster, using the default 741 | language model or the model you specify with --model. 742 | """ 743 | from llm.cli import get_default_model, get_key 744 | 745 | clustering_model = sklearn.cluster.MiniBatchKMeans(n_clusters=n, n_init="auto") 746 | if database: 747 | db = sqlite_utils.Database(database) 748 | else: 749 | db = sqlite_utils.Database(llm.user_dir() / "embeddings.db") 750 | rows = [ 751 | (row[0], llm.decode(row[1]), row[2]) 752 | for row in db.execute( 753 | """ 754 | select id, embedding, content from embeddings 755 | where collection_id = ( 756 | select id from collections where name = ? 757 | ) 758 | """, 759 | [collection], 760 | ).fetchall() 761 | ] 762 | to_cluster = np.array([item[1] for item in rows]) 763 | clustering_model.fit(to_cluster) 764 | assignments = clustering_model.labels_ 765 | 766 | def truncate_text(text): 767 | if not text: 768 | return None 769 | if truncate > 0: 770 | return text[:truncate] 771 | else: 772 | return text 773 | 774 | # Each one corresponds to an ID 775 | clusters = {} 776 | for (id, _, content), cluster in zip(rows, assignments): 777 | clusters.setdefault(str(cluster), []).append( 778 | {"id": str(id), "content": truncate_text(content)} 779 | ) 780 | # Re-arrange into a list 781 | output_clusters = [{"id": k, "items": v} for k, v in clusters.items()] 782 | 783 | # Do we need to generate summaries? 784 | if summary: 785 | model = llm.get_model(model or get_default_model()) 786 | if model.needs_key: 787 | model.key = get_key("", model.needs_key, model.key_env_var) 788 | prompt = prompt or DEFAULT_SUMMARY_PROMPT 789 | click.echo("[") 790 | for cluster, is_last in zip( 791 | output_clusters, [False] * (len(output_clusters) - 1) + [True] 792 | ): 793 | click.echo(" {") 794 | click.echo(' "id": {},'.format(json.dumps(cluster["id"]))) 795 | click.echo( 796 | ' "items": ' 797 | + textwrap.indent( 798 | json.dumps(cluster["items"], indent=2), " " 799 | ).lstrip() 800 | + "," 801 | ) 802 | prompt_content = "\n".join( 803 | [item["content"] for item in cluster["items"] if item["content"]] 804 | ) 805 | if prompt_content.strip(): 806 | summary = model.prompt( 807 | prompt_content, 808 | system=prompt, 809 | ).text() 810 | else: 811 | summary = None 812 | click.echo(' "summary": {}'.format(json.dumps(summary))) 813 | click.echo(" }" + ("," if not is_last else "")) 814 | click.echo("]") 815 | else: 816 | click.echo(json.dumps(output_clusters, indent=4)) 817 | 818 | 819 | 820 | 821 | 822 | # llm-cmd 823 | 824 | [![PyPI](https://img.shields.io/pypi/v/llm-cmd.svg)](https://pypi.org/project/llm-cmd/) 825 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-cmd?include_prereleases&label=changelog)](https://github.com/simonw/llm-cmd/releases) 826 | [![Tests](https://github.com/simonw/llm-cmd/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/llm-cmd/actions/workflows/test.yml) 827 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-cmd/blob/main/LICENSE) 828 | 829 | Use LLM to generate and execute commands in your shell 830 | 831 | ## Installation 832 | 833 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 834 | ```bash 835 | llm install llm-cmd 836 | ``` 837 | ## Usage 838 | 839 | This command could be **very dangerous**. Do not use this unless you are confident you understand what it does and are sure you could spot if it is likely to do something dangerous. 840 | 841 | Run `llm cmd` like this: 842 | 843 | ```bash 844 | llm cmd undo last git commit 845 | ``` 846 | It will use your [default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) to generate the corresponding shell command. 847 | 848 | This will then be displayed in your terminal ready for you to edit it, or hit `<enter>` to execute the prompt. 849 | 850 | If the command doesnt't look right, hit `Ctrl+C` to cancel. 851 | 852 | ## The system prompt 853 | 854 | This is the prompt used by this tool: 855 | 856 | > Return only the command to be executed as a raw string, no string delimiters 857 | wrapping it, no yapping, no markdown, no fenced code blocks, what you return 858 | will be passed to subprocess.check_output() directly. 859 | > 860 | > For example, if the user asks: undo last git commit 861 | > 862 | > You return only: git reset --soft HEAD~1 863 | 864 | ## Development 865 | 866 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 867 | ```bash 868 | cd llm-cmd 869 | python3 -m venv venv 870 | source venv/bin/activate 871 | ``` 872 | Now install the dependencies and test dependencies: 873 | ```bash 874 | llm install -e '.[test]' 875 | ``` 876 | To run the tests: 877 | ```bash 878 | pytest 879 | ``` 880 | 881 | 882 | 883 | import click 884 | import llm 885 | import subprocess 886 | from prompt_toolkit import PromptSession 887 | from prompt_toolkit.lexers import PygmentsLexer 888 | from prompt_toolkit.patch_stdout import patch_stdout 889 | from pygments.lexers.shell import BashLexer 890 | 891 | SYSTEM_PROMPT = """ 892 | Return only the command to be executed as a raw string, no string delimiters 893 | wrapping it, no yapping, no markdown, no fenced code blocks, what you return 894 | will be passed to subprocess.check_output() directly. 895 | For example, if the user asks: undo last git commit 896 | You return only: git reset --soft HEAD~1 897 | """.strip() 898 | 899 | @llm.hookimpl 900 | def register_commands(cli): 901 | @cli.command() 902 | @click.argument("args", nargs=-1) 903 | @click.option("-m", "--model", default=None, help="Specify the model to use") 904 | @click.option("-s", "--system", help="Custom system prompt") 905 | @click.option("--key", help="API key to use") 906 | def cmd(args, model, system, key): 907 | """Generate and execute commands in your shell""" 908 | from llm.cli import get_default_model 909 | prompt = " ".join(args) 910 | model_id = model or get_default_model() 911 | model_obj = llm.get_model(model_id) 912 | if model_obj.needs_key: 913 | model_obj.key = llm.get_key(key, model_obj.needs_key, model_obj.key_env_var) 914 | result = model_obj.prompt(prompt, system=system or SYSTEM_PROMPT) 915 | interactive_exec(str(result)) 916 | 917 | def interactive_exec(command): 918 | session = PromptSession(lexer=PygmentsLexer(BashLexer)) 919 | with patch_stdout(): 920 | if '\n' in command: 921 | print("Multiline command - Meta-Enter or Esc Enter to execute") 922 | edited_command = session.prompt("> ", default=command, multiline=True) 923 | else: 924 | edited_command = session.prompt("> ", default=command) 925 | try: 926 | output = subprocess.check_output( 927 | edited_command, shell=True, stderr=subprocess.STDOUT 928 | ) 929 | print(output.decode()) 930 | except subprocess.CalledProcessError as e: 931 | print(f"Command failed with error (exit status {e.returncode}): {e.output.decode()}") 932 | 933 | 934 | 935 | 936 | # llm-jq 937 | 938 | [![PyPI](https://img.shields.io/pypi/v/llm-jq.svg)](https://pypi.org/project/llm-jq/) 939 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-jq?include_prereleases&label=changelog)](https://github.com/simonw/llm-jq/releases) 940 | [![Tests](https://github.com/simonw/llm-jq/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/llm-jq/actions/workflows/test.yml) 941 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-jq/blob/main/LICENSE) 942 | 943 | Write and execute jq programs with the help of LLM 944 | 945 | See [Run a prompt to generate and execute jq programs using llm-jq](https://simonwillison.net/2024/Oct/27/llm-jq/) for background on this project. 946 | 947 | ## Installation 948 | 949 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 950 | ```bash 951 | llm install llm-jq 952 | ``` 953 | ## Usage 954 | 955 | Pipe JSON directly into `llm jq` and describe the result you would like: 956 | 957 | ```bash 958 | curl -s https://api.github.com/repos/simonw/datasette/issues | \ 959 | llm jq 'count by user.login, top 3' 960 | ``` 961 | Output: 962 | ```json 963 | [ 964 | { 965 | "login": "simonw", 966 | "count": 11 967 | }, 968 | { 969 | "login": "king7532", 970 | "count": 5 971 | }, 972 | { 973 | "login": "dependabot[bot]", 974 | "count": 2 975 | } 976 | ] 977 | ``` 978 | ``` 979 | group_by(.user.login) | map({login: .[0].user.login, count: length}) | sort_by(-.count) | .[0:3] 980 | ``` 981 | The JSON is printed to standard output, the jq program is printed to standard error. 982 | 983 | Options: 984 | 985 | - `-s/--silent`: Do not print the jq program to standard error 986 | - `-o/--output`: Output just the jq program, do not run it 987 | - `-v/--verbose`: Show the prompt sent to the model and the response 988 | - `-m/--model X`: Use a model other than the configured LLM default model 989 | - `-l/--length X`: Use a length of the input other than 1024 as the example 990 | 991 | By default, the first 1024 bytes of JSON will be sent to the model as an example along with your description. You can use `-l` to send more or less example data. 992 | 993 | ## Development 994 | 995 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 996 | ```bash 997 | cd llm-jq 998 | python -m venv venv 999 | source venv/bin/activate 1000 | ``` 1001 | Now install the dependencies and test dependencies: 1002 | ```bash 1003 | llm install -e '.[test]' 1004 | ``` 1005 | To run the tests: 1006 | ```bash 1007 | python -m pytest 1008 | ``` 1009 | 1010 | 1011 | 1012 | import click 1013 | import llm 1014 | import subprocess 1015 | import sys 1016 | import os 1017 | 1018 | 1019 | SYSTEM_PROMPT = """ 1020 | Based on the example JSON snippet and the desired query, write a jq program 1021 | 1022 | Return only the jq program to be executed as a raw string, no string delimiters 1023 | wrapping it, no yapping, no markdown, no fenced code blocks, what you return 1024 | will be passed to subprocess.check_output('jq', [...]) directly. 1025 | For example, if the user asks: extract the name of the first person 1026 | You return only: .people[0].name 1027 | """.strip() 1028 | 1029 | 1030 | @llm.hookimpl 1031 | def register_commands(cli): 1032 | @cli.command() 1033 | @click.argument("description") 1034 | @click.option("model_id", "-m", "--model", help="Model to use") 1035 | @click.option("-l", "--length", help="Example length to use", default=1024) 1036 | @click.option("-o", "--output", help="Just show the jq program", is_flag=True) 1037 | @click.option("-s", "--silent", help="Don't output jq program", is_flag=True) 1038 | @click.option( 1039 | "-v", "--verbose", help="Verbose output of prompt and response", is_flag=True 1040 | ) 1041 | def jq(description, model_id, length, output, silent, verbose): 1042 | """ 1043 | Pipe JSON data into this tool and provide a description of a 1044 | jq program you want to run against that data. 1045 | 1046 | Example usage: 1047 | 1048 | \b 1049 | cat data.json | llm jq "Just the first and last names" 1050 | """ 1051 | model = llm.get_model(model_id) 1052 | 1053 | is_pipe = not sys.stdin.isatty() 1054 | if is_pipe: 1055 | example = sys.stdin.buffer.read(length) 1056 | else: 1057 | example = "" 1058 | 1059 | prompt = description 1060 | if example: 1061 | prompt += "\n\nExample JSON snippet:\n" + example.decode() 1062 | 1063 | if verbose: 1064 | click.echo( 1065 | click.style(f"System:\n{SYSTEM_PROMPT}", fg="yellow", bold=True), 1066 | err=True, 1067 | ) 1068 | click.echo( 1069 | click.style(f"Prompt:\n{prompt}", fg="green", bold=True), 1070 | err=True, 1071 | ) 1072 | 1073 | program = ( 1074 | model.prompt( 1075 | prompt, 1076 | system=SYSTEM_PROMPT, 1077 | ) 1078 | .text() 1079 | .strip() 1080 | ) 1081 | 1082 | if verbose: 1083 | click.echo( 1084 | click.style(f"Response:\n{program}", fg="green", bold=True), 1085 | err=True, 1086 | ) 1087 | 1088 | if output or not is_pipe: 1089 | click.echo(program) 1090 | return 1091 | 1092 | # Run jq 1093 | process = subprocess.Popen( 1094 | ["jq", program], 1095 | stdin=subprocess.PIPE, 1096 | stdout=subprocess.PIPE, 1097 | stderr=subprocess.PIPE, 1098 | ) 1099 | 1100 | try: 1101 | if example: 1102 | process.stdin.write(example) 1103 | 1104 | # Stream the rest of stdin to jq, 8k at a time 1105 | if is_pipe: 1106 | while True: 1107 | chunk = sys.stdin.buffer.read(8192) 1108 | if not chunk: 1109 | break 1110 | process.stdin.write(chunk) 1111 | process.stdin.close() 1112 | 1113 | # Stream stdout 1114 | while True: 1115 | chunk = process.stdout.read(8192) 1116 | if not chunk: 1117 | break 1118 | sys.stdout.buffer.write(chunk) 1119 | sys.stdout.buffer.flush() 1120 | 1121 | # After stdout is done, read and forward any stderr 1122 | stderr_data = process.stderr.read() 1123 | if stderr_data: 1124 | sys.stderr.buffer.write(stderr_data) 1125 | sys.stderr.buffer.flush() 1126 | 1127 | # Wait for process to complete and get exit code 1128 | return_code = process.wait() 1129 | 1130 | # Output the program at the end 1131 | if not silent and not verbose: 1132 | click.echo( 1133 | click.style(f"{program}", fg="blue", bold=True), 1134 | err=True, 1135 | ) 1136 | 1137 | sys.exit(return_code) 1138 | 1139 | except BrokenPipeError: 1140 | # Handle case where output pipe is closed 1141 | devnull = os.open(os.devnull, os.O_WRONLY) 1142 | os.dup2(devnull, sys.stdout.fileno()) 1143 | sys.exit(1) 1144 | except KeyboardInterrupt: 1145 | # Handle Ctrl+C gracefully 1146 | process.terminate() 1147 | process.wait() 1148 | sys.exit(130) 1149 | finally: 1150 | # Ensure process resources are cleaned up 1151 | process.stdout.close() 1152 | process.stderr.close() 1153 | 1154 | 1155 | 1156 | 1157 | 1158 | # llm-whisper-api 1159 | 1160 | [![PyPI](https://img.shields.io/pypi/v/llm-whisper-api.svg)](https://pypi.org/project/llm-whisper-api/) 1161 | [![Changelog](https://img.shields.io/github/v/release/simonw/llm-whisper-api?include_prereleases&label=changelog)](https://github.com/simonw/llm-whisper-api/releases) 1162 | [![Tests](https://github.com/simonw/llm-whisper-api/actions/workflows/test.yml/badge.svg)](https://github.com/simonw/llm-whisper-api/actions/workflows/test.yml) 1163 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm-whisper-api/blob/main/LICENSE) 1164 | 1165 | Run transcriptions using the OpenAI Whisper API 1166 | 1167 | ## Installation 1168 | 1169 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 1170 | ```bash 1171 | llm install llm-whisper-api 1172 | ``` 1173 | ## Usage 1174 | 1175 | The plugin adds a new command, `llm whisper-api`. Use it like this: 1176 | 1177 | ```bash 1178 | llm whisper-api audio.mp3 1179 | ``` 1180 | The transcribed audio will be output directly to standard output as plain text. 1181 | 1182 | The plugin will use the OpenAI API key you have already configured using: 1183 | ```bash 1184 | llm keys set openai 1185 | # Paste key here 1186 | ``` 1187 | You can also pass an explicit API key using `--key` like this: 1188 | 1189 | ```bash 1190 | llm whisper-api audio.mp3 --key $OPENAI_API_KEY 1191 | ``` 1192 | 1193 | You can pipe data to the tool if you specify `-` as a filename: 1194 | 1195 | ```bash 1196 | curl -s 'https://static.simonwillison.net/static/2024/russian-pelican-in-spanish.mp3' \ 1197 | | llm whisper-api - 1198 | ``` 1199 | 1200 | ## Development 1201 | 1202 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 1203 | ```bash 1204 | cd llm-whisper-api 1205 | python -m venv venv 1206 | source venv/bin/activate 1207 | ``` 1208 | Now install the dependencies and test dependencies: 1209 | ```bash 1210 | llm install -e '.[test]' 1211 | ``` 1212 | To run the tests: 1213 | ```bash 1214 | python -m pytest 1215 | ``` 1216 | 1217 | 1218 | 1219 | import click 1220 | import httpx 1221 | import io 1222 | import llm 1223 | 1224 | 1225 | @llm.hookimpl 1226 | def register_commands(cli): 1227 | @cli.command() 1228 | @click.argument("audio_file", type=click.File("rb")) 1229 | @click.option("api_key", "--key", help="API key to use") 1230 | def whisper_api(audio_file, api_key): 1231 | """ 1232 | Run transcriptions using the OpenAI Whisper API 1233 | 1234 | Usage: 1235 | 1236 | \b 1237 | llm whisper-api audio.mp3 > output.txt 1238 | cat audio.mp3 | llm whisper-api - > output.txt 1239 | """ 1240 | # Read the entire content into memory first 1241 | audio_content = audio_file.read() 1242 | audio_file.close() 1243 | 1244 | key = llm.get_key(api_key, "openai") 1245 | if not key: 1246 | raise click.ClickException("OpenAI API key is required") 1247 | try: 1248 | click.echo(transcribe(audio_content, key)) 1249 | except httpx.HTTPError as ex: 1250 | raise click.ClickException(str(ex)) 1251 | 1252 | 1253 | def transcribe(audio_content: bytes, api_key: str) -> str: 1254 | """ 1255 | Transcribe audio content using OpenAI's Whisper API. 1256 | 1257 | Args: 1258 | audio_content (bytes): The audio content as bytes 1259 | api_key (str): OpenAI API key 1260 | 1261 | Returns: 1262 | str: The transcribed text 1263 | 1264 | Raises: 1265 | httpx.RequestError: If the API request fails 1266 | """ 1267 | url = "https://api.openai.com/v1/audio/transcriptions" 1268 | headers = {"Authorization": f"Bearer {api_key}"} 1269 | 1270 | audio_file = io.BytesIO(audio_content) 1271 | audio_file.name = "audio.mp3" # OpenAI API requires a filename, or 400 error 1272 | 1273 | files = {"file": audio_file} 1274 | data = {"model": "whisper-1", "response_format": "text"} 1275 | 1276 | with httpx.Client() as client: 1277 | response = client.post(url, headers=headers, files=files, data=data) 1278 | response.raise_for_status() 1279 | return response.text.strip() 1280 | 1281 | 1282 | 1283 | 1284 | 1285 | # llm-jina 1286 | 1287 | [![PyPI](https://img.shields.io/pypi/v/llm-jina.svg)](https://pypi.org/project/llm-jina/) 1288 | [![Changelog](https://img.shields.io/github/v/release/yourusername/llm-jina?include_prereleases&label=changelog)](https://github.com/yourusername/llm-jina/releases) 1289 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/yourusername/llm-jina/blob/main/LICENSE) 1290 | 1291 | LLM plugin for interacting with Jina AI APIs 1292 | 1293 | ## Table of Contents 1294 | - [llm-jina](#llm-jina) 1295 | - [Table of Contents](#table-of-contents) 1296 | - [Installation](#installation) 1297 | - [Configuration](#configuration) 1298 | - [Usage](#usage) 1299 | - [Embedding](#embedding) 1300 | - [Reranking](#reranking) 1301 | - [URL Reading](#url-reading) 1302 | - [Web Search](#web-search) 1303 | - [Fact Checking](#fact-checking) 1304 | - [Text Segmentation](#text-segmentation) 1305 | - [Classification](#classification) 1306 | - [Code Generation](#code-generation) 1307 | - [Metaprompt](#metaprompt) 1308 | - [Development](#development) 1309 | - [Contributing](#contributing) 1310 | 1311 | ## Installation 1312 | 1313 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 1314 | 1315 | ```bash 1316 | llm install llm-jina 1317 | ``` 1318 | 1319 | ## Configuration 1320 | 1321 | You need to set the `JINA_API_KEY` environment variable with your Jina AI API key. You can get a free API key from [https://jina.ai/?sui=apikey](https://jina.ai/?sui=apikey). 1322 | 1323 | ```bash 1324 | export JINA_API_KEY=your_api_key_here 1325 | ``` 1326 | 1327 | ## Usage 1328 | 1329 | This plugin adds several commands to interact with Jina AI APIs: 1330 | 1331 | ### Embedding 1332 | 1333 | Generate embeddings for given texts: 1334 | 1335 | ```bash 1336 | llm jina embed "The quick brown fox jumps over the lazy dog." 1337 | ``` 1338 | 1339 | You can specify a different model using the `--model` option: 1340 | 1341 | ```bash 1342 | llm jina embed "To be, or not to be, that is the question." --model jina-embeddings-v2-base-en 1343 | ``` 1344 | 1345 | ### Reranking 1346 | 1347 | Rerank documents based on a query: 1348 | 1349 | ```bash 1350 | llm jina rerank "Best sci-fi movies" "Star Wars: A New Hope" "The Matrix" "Blade Runner" "Interstellar" "2001: A Space Odyssey" 1351 | ``` 1352 | 1353 | You can specify a different model using the `--model` option: 1354 | 1355 | ```bash 1356 | llm jina rerank "Healthy eating tips" "Eat more fruits and vegetables" "Limit processed foods" "Stay hydrated" --model jina-reranker-v2-base-en 1357 | ``` 1358 | 1359 | ### URL Reading 1360 | 1361 | Read and parse content from a URL: 1362 | 1363 | ```bash 1364 | llm jina read https://en.wikipedia.org/wiki/Artificial_intelligence 1365 | ``` 1366 | 1367 | You can include link and image summaries: 1368 | 1369 | ```bash 1370 | llm jina read https://www.nasa.gov/topics/moon-to-mars --with-links --with-images 1371 | ``` 1372 | 1373 | ### Web Search 1374 | 1375 | Search the web for information: 1376 | 1377 | ```bash 1378 | llm jina search "History of the internet" 1379 | ``` 1380 | 1381 | You can limit the search to a specific domain: 1382 | 1383 | ```bash 1384 | llm jina search "Python programming tutorials" --site docs.python.org 1385 | ``` 1386 | 1387 | Example with multiple options: 1388 | 1389 | ```bash 1390 | llm jina search "Climate change impacts" --site nasa.gov --with-links --with-images 1391 | ``` 1392 | 1393 | ### Fact Checking 1394 | 1395 | Verify the factual accuracy of a statement: 1396 | 1397 | ```bash 1398 | llm jina ground "The Mona Lisa was painted by Leonardo da Vinci." 1399 | ``` 1400 | 1401 | You can provide specific sites for grounding: 1402 | 1403 | ```bash 1404 | llm jina ground "Jina AI offers state-of-the-art AI models." --sites https://jina.ai,https://docs.jina.ai 1405 | ``` 1406 | 1407 | ### Text Segmentation 1408 | 1409 | Segment text into tokens or chunks: 1410 | 1411 | ```bash 1412 | llm jina segment --content "Space: the final frontier. These are the voyages of the starship Enterprise. Its five-year mission: to explore strange new worlds. To seek out new life and new civilizations. To boldly go where no man has gone before 1413 | In the beginning God created the heaven and the earth. And the earth was without form, and void; and darkness was upon the face of the deep." --tokenizer cl100k_base --return-chunks 1414 | ``` 1415 | 1416 | Example response: 1417 | ```json 1418 | { 1419 | "chunks": [ 1420 | "Space: the final frontier. These are the voyages of the starship Enterprise. Its five-year mission: to explore strange new worlds. To seek out new life and new civilizations. To boldly go where no man has gone before\n", 1421 | "In the beginning God created the heaven and the earth. And the earth was without form, and void; and darkness was upon the face of the deep." 1422 | ] 1423 | } 1424 | ``` 1425 | 1426 | ### Classification 1427 | 1428 | Classify inputs into given labels: 1429 | 1430 | ```bash 1431 | llm jina classify "The movie was amazing! I loved every minute of it." "The acting was terrible and the plot made no sense." --labels positive negative neutral 1432 | ``` 1433 | 1434 | For image classification: 1435 | 1436 | ```bash 1437 | llm jina classify path/to/cat.jpg path/to/dog.jpg path/to/bird.jpg --labels feline canine avian --image 1438 | ``` 1439 | 1440 | ### Code Generation 1441 | 1442 | Generate Jina API code based on a prompt: 1443 | 1444 | ```bash 1445 | llm jina generate-code "Create a function that searches Wikipedia for information about famous scientists and reranks the results based on relevance to the query." 1446 | ``` 1447 | 1448 | ```bash 1449 | llm jina generate-code "Create a function that searches for information about AI and reranks the results based on relevance" 1450 | ``` 1451 | 1452 | ### Metaprompt 1453 | 1454 | Display the Jina metaprompt used for generating code: 1455 | 1456 | ```bash 1457 | llm jina metaprompt 1458 | ``` 1459 | 1460 | ## Development 1461 | 1462 | To set up this plugin locally, first checkout the code. Then create a new virtual environment: 1463 | 1464 | ```bash 1465 | cd llm-jina 1466 | python3 -m venv venv 1467 | source venv/bin/activate 1468 | ``` 1469 | 1470 | Now install the dependencies and test dependencies: 1471 | 1472 | ```bash 1473 | pip install -e '.[test]' 1474 | ``` 1475 | 1476 | To run the tests: 1477 | 1478 | ```bash 1479 | pytest 1480 | ``` 1481 | 1482 | ## Contributing 1483 | 1484 | Contributions to this plugin are welcome! Please refer to the [LLM plugin development documentation](https://llm.datasette.io/en/stable/plugins/index.html) for more information on how to get started. 1485 | 1486 | 1487 | import click 1488 | import json 1489 | import llm 1490 | import os 1491 | from typing import List, Dict, Any, Union 1492 | import httpx 1493 | import base64 1494 | 1495 | # Get your Jina AI API key for free: https://jina.ai/?sui=apikey 1496 | JINA_API_KEY = os.environ.get("JINA_API_KEY") 1497 | 1498 | @llm.hookimpl 1499 | def register_commands(cli): 1500 | @cli.group() 1501 | def jina(): 1502 | """Commands for interacting with Jina AI Search Foundation APIs""" 1503 | pass 1504 | 1505 | @jina.command() 1506 | @click.argument("query", type=str) 1507 | @click.option("--site", help="Limit search to a specific domain") 1508 | @click.option("--with-links", is_flag=True, help="Include links summary") 1509 | @click.option("--with-images", is_flag=True, help="Include images summary") 1510 | def search(query: str, site: str, with_links: bool, with_images: bool): 1511 | """Search the web using Jina AI Search API""" 1512 | results = jina_search(query, site, with_links, with_images) 1513 | click.echo(json.dumps(results, indent=2)) 1514 | 1515 | @jina.command() 1516 | @click.option("--content", required=True, help="The text content to segment") 1517 | @click.option("--tokenizer", default="cl100k_base", help="Tokenizer to use") 1518 | @click.option("--return-tokens", is_flag=True, help="Return tokens in the response") 1519 | @click.option("--return-chunks", is_flag=True, help="Return chunks in the response") 1520 | @click.option("--max-chunk-length", type=int, default=1000, help="Maximum characters per chunk") 1521 | def segment(content, tokenizer, return_tokens, return_chunks, max_chunk_length): 1522 | """Segment text into tokens or chunks""" 1523 | try: 1524 | result = segment_text(content, tokenizer, return_tokens, return_chunks, max_chunk_length) 1525 | click.echo(json.dumps(result, indent=2)) 1526 | except click.ClickException as e: 1527 | click.echo(str(e), err=True) 1528 | except Exception as e: 1529 | click.echo(f"An unexpected error occurred: {str(e)}", err=True) 1530 | 1531 | @jina.command() 1532 | @click.argument("url", type=str) 1533 | @click.option("--with-links", is_flag=True, help="Include links summary") 1534 | @click.option("--with-images", is_flag=True, help="Include images summary") 1535 | def read(url: str, with_links: bool, with_images: bool): 1536 | """Read and parse content from a URL using Jina AI Reader API""" 1537 | content = jina_read(url, with_links, with_images) 1538 | click.echo(json.dumps(content, indent=2)) 1539 | 1540 | @jina.command() 1541 | @click.argument("statement", type=str) 1542 | @click.option("--sites", help="Comma-separated list of URLs to use as grounding references") 1543 | def ground(statement: str, sites: str): 1544 | """Verify the factual accuracy of a statement using Jina AI Grounding API""" 1545 | result = jina_ground(statement, sites.split(",") if sites else None) 1546 | click.echo(json.dumps(result, indent=2)) 1547 | 1548 | @jina.command() 1549 | @click.argument("text", type=str) 1550 | @click.option("--model", type=str, default="jina-embeddings-v3", help="Model to use for embedding") 1551 | def embed(text: str, model: str): 1552 | """Generate embeddings for text using Jina AI Embeddings API""" 1553 | embedding = jina_embed(text, model) 1554 | click.echo(json.dumps(embedding, indent=2)) 1555 | 1556 | @jina.command() 1557 | @click.argument("query", type=str) 1558 | @click.argument("documents", nargs=-1, required=True) 1559 | @click.option("--model", default="jina-reranker-v2-base-multilingual", help="Reranking model to use") 1560 | def rerank(query: str, documents: List[str], model: str): 1561 | """Rerank a list of documents based on their relevance to a query""" 1562 | try: 1563 | result = rerank_documents(query, list(documents), model) 1564 | click.echo(json.dumps(result, indent=2)) 1565 | except click.ClickException as e: 1566 | click.echo(str(e), err=True) 1567 | except Exception as e: 1568 | click.echo(f"An unexpected error occurred: {str(e)}", err=True) 1569 | 1570 | @jina.command() 1571 | @click.argument("prompt") 1572 | def generate_code(prompt): 1573 | """Generate Jina API code based on the given prompt""" 1574 | try: 1575 | metaprompt = jina_metaprompt() 1576 | full_prompt = f"""Based on the following Jina AI API documentation and guidelines, please generate production-ready Python code for the following task: 1577 | 1578 | {metaprompt} 1579 | 1580 | Task: {prompt} 1581 | 1582 | Please provide the complete Python code implementation that follows the above guidelines and best practices. Include error handling, proper API response parsing, and any necessary setup instructions. 1583 | 1584 | Remember to: 1585 | 1. Use environment variable JINA_API_KEY for authentication 1586 | 2. Include proper error handling 1587 | 3. Follow the integration guidelines 1588 | 4. Parse API responses correctly 1589 | 5. Include any necessary imports 1590 | 6. Add setup/usage instructions as comments 1591 | 1592 | Provide the code in a format ready to be saved to a .py file and executed.""" 1593 | 1594 | response = llm.get_model().prompt(full_prompt) 1595 | result = response.text() 1596 | 1597 | click.echo("=== Generated Jina AI Code ===") 1598 | click.echo(result) 1599 | click.echo("Note: Make sure to set your JINA_API_KEY environment variable before running the code.") 1600 | click.echo("Get your API key at: https://jina.ai/?sui=apikey") 1601 | 1602 | except Exception as e: 1603 | raise click.ClickException(f"Error generating code: {str(e)}") 1604 | 1605 | @jina.command() 1606 | def metaprompt(): 1607 | """Display the Jina metaprompt""" 1608 | click.echo(jina_metaprompt()) 1609 | 1610 | @jina.command() 1611 | @click.argument("input_text", nargs=-1, required=True) 1612 | @click.option("--labels", required=True, help="Comma-separated list of labels for classification") 1613 | @click.option("--model", default="jina-embeddings-v3", help="Model to use for classification (jina-embeddings-v3 for text, jina-clip-v1 for images)") 1614 | @click.option("--image", is_flag=True, help="Treat input as image file paths") 1615 | def classify(input_text: List[str], labels: str, model: str, image: bool) -> None: 1616 | """Classify text or images using Jina AI Classifier API""" 1617 | labels_list = [label.strip() for label in labels.split(",")] 1618 | input_data = [] 1619 | 1620 | if image: 1621 | model = "jina-clip-v1" 1622 | for img_path in input_text: 1623 | try: 1624 | with open(img_path, "rb") as img_file: 1625 | img_base64 = base64.b64encode(img_file.read()).decode("utf-8") 1626 | input_data.append({"image": img_base64}) 1627 | except IOError as e: 1628 | click.echo(f"Error reading image file {img_path}: {str(e)}", err=True) 1629 | return 1630 | else: 1631 | model = "jina-embeddings-v3" 1632 | input_data = list(input_text) 1633 | 1634 | try: 1635 | result = jina_classify(input_data, labels_list, model) 1636 | click.echo(json.dumps(result, indent=2)) 1637 | except Exception as e: 1638 | click.echo(f"Error occurred while classifying: {str(e)}", err=True) 1639 | 1640 | def read_url(url: str, options: str = "Default") -> Dict[str, Any]: 1641 | api_url = "https://r.jina.ai/" 1642 | data = { 1643 | "url": url, 1644 | "options": options 1645 | } 1646 | headers = { 1647 | "X-With-Links-Summary": "true", 1648 | "X-With-Images-Summary": "true" 1649 | } 1650 | return jina_request(api_url, data, headers) 1651 | 1652 | 1653 | def fetch_metaprompt() -> str: 1654 | url = "https://docs.jina.ai" 1655 | try: 1656 | with httpx.Client(timeout=3) as client: 1657 | response = client.get(url) 1658 | response.raise_for_status() 1659 | return response.text 1660 | except (httpx.RequestError, httpx.TimeoutException) as e: 1661 | click.echo(f"Warning: Failed to fetch metaprompt from {url}: {str(e)}") 1662 | return None 1663 | 1664 | def jina_metaprompt() -> str: 1665 | metaprompt_content = fetch_metaprompt() 1666 | 1667 | if metaprompt_content is None: 1668 | try: 1669 | with open("jina-metaprompt.md", "r") as file: 1670 | return file.read() 1671 | except FileNotFoundError: 1672 | raise click.ClickException("jina-metaprompt.md file not found") 1673 | except IOError as e: 1674 | raise click.ClickException(f"Error reading jina-metaprompt.md: {str(e)}") 1675 | else: 1676 | try: 1677 | with open("jina-metaprompt.md", "w") as file: 1678 | file.write(metaprompt_content) 1679 | except IOError as e: 1680 | click.echo(f"Warning: Failed to update jina-metaprompt.md: {str(e)}") 1681 | 1682 | return metaprompt_content 1683 | 1684 | def rerank_documents(query: str, documents: List[str], model: str = "jina-reranker-v2-base-multilingual") -> List[Dict[str, Any]]: 1685 | """ 1686 | Rerank a list of documents based on their relevance to a given query. 1687 | 1688 | Args: 1689 | query (str): The query string to compare documents against. 1690 | documents (List[str]): A list of document strings to be reranked. 1691 | model (str, optional): The reranking model to use. Defaults to "jina-reranker-v2-base-multilingual". 1692 | 1693 | Returns: 1694 | List[Dict[str, Any]]: A list of dictionaries containing reranked documents and their scores. 1695 | Each dictionary includes 'text' (the document), 'index' (original position), and 'score' (relevance score). 1696 | 1697 | Raises: 1698 | click.ClickException: If there's an error in the API call. 1699 | """ 1700 | url = "https://api.jina.ai/v1/rerank" 1701 | data = { 1702 | "model": model, 1703 | "query": query, 1704 | "documents": documents 1705 | } 1706 | response = jina_request(url, data) 1707 | return response["results"] 1708 | 1709 | def segment_text(content: str, tokenizer: str = "cl100k_base", return_tokens: bool = False, return_chunks: bool = True, max_chunk_length: int = 1000) -> Dict[str, Any]: 1710 | """ 1711 | Segment text into tokens or chunks using the Jina AI Segmenter API. 1712 | 1713 | Args: 1714 | content (str): The text content to segment. 1715 | tokenizer (str): The tokenizer to use. Default is "cl100k_base". 1716 | return_tokens (bool): Whether to return tokens in the response. Default is False. 1717 | return_chunks (bool): Whether to return chunks in the response. Default is True. 1718 | max_chunk_length (int): Maximum characters per chunk. Only effective if 'return_chunks' is True. Default is 1000. 1719 | 1720 | Returns: 1721 | Dict[str, Any]: The response from the Jina AI Segmenter API. 1722 | 1723 | Raises: 1724 | click.ClickException: If there's an error in the API call or response. 1725 | """ 1726 | url = "https://segment.jina.ai/" 1727 | data = { 1728 | "content": content, 1729 | "tokenizer": tokenizer, 1730 | "return_tokens": return_tokens, 1731 | "return_chunks": return_chunks, 1732 | "max_chunk_length": max_chunk_length 1733 | } 1734 | return jina_request(url, data) 1735 | 1736 | 1737 | def jina_request(url: str, data: Dict[str, Any], headers: Dict[str, str] = None) -> Dict[str, Any]: 1738 | if not JINA_API_KEY: 1739 | raise ValueError("JINA_API_KEY environment variable is not set") 1740 | 1741 | default_headers = { 1742 | "Authorization": f"Bearer {JINA_API_KEY}", 1743 | "Content-Type": "application/json", 1744 | "Accept": "application/json" 1745 | } 1746 | if headers: 1747 | default_headers.update(headers) 1748 | 1749 | try: 1750 | with httpx.Client() as client: 1751 | response = client.post(url, json=data, headers=default_headers) 1752 | response.raise_for_status() 1753 | return response.json() 1754 | except httpx.HTTPError as e: 1755 | raise click.ClickException(f"Error calling Jina AI API: {str(e)}") 1756 | 1757 | def jina_search(query: str, site: str = None, with_links: bool = False, with_images: bool = False) -> Dict[str, Any]: 1758 | url = "https://s.jina.ai/" 1759 | headers = {} 1760 | 1761 | if site: 1762 | headers["X-Site"] = site 1763 | if with_links: 1764 | headers["X-With-Links-Summary"] = "true" 1765 | if with_images: 1766 | headers["X-With-Images-Summary"] = "true" 1767 | 1768 | data = { 1769 | "q": query, 1770 | "options": "Default" 1771 | } 1772 | 1773 | return jina_request(url, data, headers) 1774 | 1775 | def jina_read(url: str, with_links: bool = False, with_images: bool = False) -> Dict[str, Any]: 1776 | api_url = "https://r.jina.ai/" 1777 | headers = {} 1778 | 1779 | if with_links: 1780 | headers["X-With-Links-Summary"] = "true" 1781 | if with_images: 1782 | headers["X-With-Images-Summary"] = "true" 1783 | 1784 | data = { 1785 | "url": url, 1786 | "options": "Default" 1787 | } 1788 | 1789 | return jina_request(api_url, data, headers) 1790 | 1791 | def jina_ground(statement: str, sites: List[str] = None) -> Dict[str, Any]: 1792 | url = "https://g.jina.ai/" 1793 | headers = {} 1794 | 1795 | if sites: 1796 | headers["X-Site"] = ",".join(sites) 1797 | 1798 | data = { 1799 | "statement": statement 1800 | } 1801 | 1802 | return jina_request(url, data, headers) 1803 | 1804 | def jina_embed(text: str, model: str = "jina-embeddings-v3") -> Dict[str, Any]: 1805 | url = "https://api.jina.ai/v1/embeddings" 1806 | data = { 1807 | "input": [text], 1808 | "model": model 1809 | } 1810 | 1811 | return jina_request(url, data) 1812 | 1813 | def jina_classify(input_data: List[Union[str, Dict[str, str]]], labels: List[str], model: str) -> Dict[str, Any]: 1814 | url = "https://api.jina.ai/v1/classify" 1815 | data = { 1816 | "model": model, 1817 | "input": input_data, 1818 | "labels": labels 1819 | } 1820 | 1821 | try: 1822 | return jina_request(url, data) 1823 | except click.ClickException as e: 1824 | raise click.ClickException(f"Error occurred while classifying: {str(e)}. Please check your input data and try again.") 1825 | 1826 | 1827 | 1828 | 1829 | # llm-classifier 1830 | 1831 | [![PyPI](https://img.shields.io/pypi/v/llm-classifier.svg)](https://pypi.org/project/llm-classifier/) 1832 | [![Changelog](https://img.shields.io/github/v/release/yourusername/llm-classifier?include_prereleases&label=changelog)](https://github.com/yourusername/llm-classifier/releases) 1833 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/yourusername/llm-classifier/blob/main/LICENSE) 1834 | 1835 | LLM plugin for content classification using various language models 1836 | 1837 | ## Installation 1838 | 1839 | 1840 | Install this plugin in the same environment as [LLM](https://llm.datasette.io/). 1841 | 1842 | ```bash 1843 | llm install llm-classifier 1844 | ``` 1845 | 1846 | ## Usage 1847 | 1848 | This plugin adds a new `classify` command to the LLM CLI. You can use it to classify content into predefined categories using various language models. 1849 | 1850 | Basic usage: 1851 | 1852 | ```bash 1853 | llm classify "This is a happy message" -c positive -c negative -c neutral -m gpt-3.5-turbo 1854 | ``` 1855 | 1856 | Options: 1857 | 1858 | - `content`: The content(s) to classify. You can provide multiple items. 1859 | - `-c, --classes`: Class options for classification (at least two required). 1860 | - `-m, --model`: LLM model to use (default: gpt-3.5-turbo). 1861 | - `-t, --temperature`: Temperature for API call (default: 0). 1862 | - `-e, --examples`: Examples in the format 'content:class' (can be used multiple times). 1863 | - `-p, --prompt`: Custom prompt template. 1864 | - `--no-content`: Exclude content from the output. 1865 | 1866 | You can also pipe content to classify: 1867 | 1868 | ```bash 1869 | echo "This is exciting news!" | llm classify -c positive -c negative -c neutral 1870 | ``` 1871 | 1872 | ## Examples 1873 | 1874 | 1. Basic classification using a custom model and temperature: 1875 | 1876 | ```bash 1877 | llm classify "The weather is nice today" -c good -c bad -c neutral -m gpt-4 -t 0.7 1878 | ``` 1879 | Output: 1880 | ```json 1881 | [ 1882 | { 1883 | "class": "good", 1884 | "score": 0.998019085206617, 1885 | "content": "The weather is nice today" 1886 | } 1887 | ] 1888 | ``` 1889 | 1890 | 2. Basic classification multi-processing with default model: 1891 | ```bash 1892 | llm classify "I love this product" "This is terrible" -c positive -c negative -c neutral 1893 | ``` 1894 | Output: 1895 | ```json 1896 | [ 1897 | { 1898 | "class": "positive", 1899 | "score": 0.9985889762314736, 1900 | "content": "I love this product" 1901 | }, 1902 | { 1903 | "class": "negative", 1904 | "score": 0.9970504305526415, 1905 | "content": "This is terrible" 1906 | } 1907 | ] 1908 | ``` 1909 | 1910 | 1911 | 3. Providing examples for few-shot learning: 1912 | 1913 | ```bash 1914 | llm classify "The stock market crashed" -c economic -c political -c environmental \ 1915 | -e "New trade deal signed:economic" -e "President gives speech:political" \ 1916 | -e "Forest fires in California:environmental" 1917 | ``` 1918 | 1919 | 4. Using a custom prompt: 1920 | 1921 | ```bash 1922 | llm classify "Breaking news: Earthquake in Japan" -c urgent -c not-urgent \ 1923 | -p "Classify the following news headline as either urgent or not-urgent:" 1924 | ``` 1925 | 1926 | 1927 | 5. Multiple classification with Examples: 1928 | ```bash 1929 | llm classify 'news.ycombinator.com' 'facebook.com' 'ai.meta.com' \ 1930 | -c 'signal' -c 'noise' -c 'neutral' \ 1931 | -e "github.com:signal" -e "arxiv.org:signal" -e "instagram.com:noise" \ 1932 | -e "pinterest.com:noise" -e "anthropic.ai:signal" -e "twitter.com:noise" 1933 | --model openrouter/openai/gpt-4-0314 1934 | ``` 1935 | ```json 1936 | [ 1937 | { 1938 | "class": "signal", 1939 | "score": 0.9994780818067087, 1940 | "content": "news.ycombinator.com" 1941 | }, 1942 | { 1943 | "class": "noise", 1944 | "score": 0.9999876476902904, 1945 | "content": "facebook.com" 1946 | }, 1947 | { 1948 | "class": "signal", 1949 | "score": 0.9999895549275502, 1950 | "content": "ai.meta.com" 1951 | } 1952 | ] 1953 | ``` 1954 | 1955 | 6. Terminal commands classification: 1956 | ```bash 1957 | llm classify 'df -h' 'chown -R user:user /' -c 'safe' -c 'danger' -c 'neutral' -e "ls:safe" -e "rm:danger" -e "echo:neutral" --model gpt-4o-mini 1958 | ``` 1959 | ```json 1960 | [ 1961 | { 1962 | "class": "neutral", 1963 | "score": 0.9995317830277939, 1964 | "content": "df -h" 1965 | }, 1966 | { 1967 | "class": "danger", 1968 | "score": 0.9964036839906633, 1969 | "content": "chown -R user:user /" 1970 | } 1971 | ] 1972 | ``` 1973 | 1974 | 7. Classify a tweet 1975 | ```shell 1976 | llm classify $tweet -c 'AI' -c 'ASI' -c 'AGI' --model gpt-4o-mini 1977 | ``` 1978 | ```json 1979 | [ 1980 | { 1981 | "class": "asi", 1982 | "score": 0.9999984951481323, 1983 | "content": "Superintelligence is within reach.\\n\\nBuilding safe superintelligence (SSI) is the most important technical problem of our time.\\n\\nWe've started the world\u2019s first straight-shot SSI lab, with one goal and one product: a safe superintelligence." 1984 | } 1985 | ] 1986 | ``` 1987 | 1988 | 8. Verify facts 1989 | ```shell 1990 | llm classify "$(curl -s docs.jina.ai)Jina ai has an image generation api" -c True -c False --model gpt-4o --no-content 1991 | ``` 1992 | ```json 1993 | [ 1994 | { 1995 | "class": "false", 1996 | "score": 0.99997334352929 1997 | } 1998 | ] 1999 | ``` 2000 | 2001 | ## Advanced Examples 2002 | 2003 | 9. **Acting on the classification result in a shell script:** 2004 | ```bash 2005 | class-tweet() { 2006 | local tweet="$1" 2007 | local threshold=0.6 2008 | local class="machine-learning" 2009 | 2010 | result=$(llm classify "$tweet" -c 'PROGRAMMING' -c 'MACHINE-LEARNING' \ 2011 | --model openrouter/openai/gpt-4o-mini \ 2012 | | jq -r --arg class "$class" --argjson threshold "$threshold" \ 2013 | '.[0] | select(.class == $class and .score > $threshold) | .class') 2014 | 2015 | if [ -n "$result" ]; then 2016 | echo "Tweet classified as $class with high confidence. Executing demo..." 2017 | echo "Demo: This is a highly relevant tweet about $class" 2018 | else 2019 | echo "Tweet does not meet classification criteria." 2020 | fi 2021 | } 2022 | ``` 2023 | ``` 2024 | Tweet classified as machine-learning with high confidence. Executing demo... 2025 | Demo: This is a highly relevant tweet about machine-learning 2026 | ``` 2027 | 2028 | 10. **Piping multiple lines using heredoc:** 2029 | ```bash 2030 | cat < 2108 | 1: 2155 | raise click.ClickException("Temperature must be between 0 and 1") 2156 | 2157 | # Handle piped input if no content arguments provided 2158 | if not content and not sys.stdin.isatty(): 2159 | content = [line.strip() for line in sys.stdin.readlines()] 2160 | elif not content: 2161 | raise click.ClickException("No content provided. Either pipe content or provide it as an argument.") 2162 | 2163 | examples_list = None 2164 | if examples: 2165 | examples_list = [] 2166 | for example in examples: 2167 | try: 2168 | example_content, class_ = example.rsplit(':', 1) 2169 | examples_list.append({"content": example_content.strip(), "class": class_.strip()}) 2170 | except ValueError: 2171 | click.echo(f"Warning: Skipping invalid example format: {example}", err=True) 2172 | continue 2173 | 2174 | results = classify_content( 2175 | list(content), list(classes), model, temperature, examples_list, prompt, no_content 2176 | ) 2177 | click.echo(json.dumps(results, indent=2)) 2178 | 2179 | def classify_content( 2180 | content: List[str], 2181 | classes: List[str], 2182 | model: str, 2183 | temperature: float, 2184 | examples: Optional[List[Dict[str, str]]] = None, 2185 | custom_prompt: Optional[str] = None, 2186 | no_content: bool = False 2187 | ) -> List[Dict[str, Optional[str]]]: 2188 | results = [] 2189 | for item in content: 2190 | winner, probability = get_class_probability( 2191 | item, classes, model, temperature, examples, custom_prompt 2192 | ) 2193 | result = {"class": winner, "score": probability} 2194 | if not no_content: 2195 | result["content"] = item 2196 | results.append(result) 2197 | return results 2198 | 2199 | def get_class_probability( 2200 | content: str, 2201 | classes: List[str], 2202 | model: str, 2203 | temperature: float, 2204 | examples: Optional[List[Dict[str, str]]] = None, 2205 | custom_prompt: Optional[str] = None 2206 | ) -> Tuple[str, float]: 2207 | llm_model = llm.get_model(model) 2208 | 2209 | if custom_prompt: 2210 | prompt = custom_prompt 2211 | else: 2212 | prompt = f"""You are a highly efficient content classification system. Your task is to classify the given content into a single, most appropriate category from a provided list. 2213 | 2214 | 1. Read and understand the content thoroughly. 2215 | 2. Consider each category and how well it fits the content. 2216 | 3. Choose the single most appropriate category that best describes the main theme or purpose of the content. 2217 | 4. If multiple categories seem applicable, select the one that is most central or relevant to the overall message. 2218 | 2219 | Here are the categories you can choose from: 2220 | 2221 | {'\n'.join(classes)} 2222 | 2223 | 2224 | 2225 | """ 2226 | 2227 | if examples: 2228 | prompt += "Examples:" 2229 | for example in examples: 2230 | prompt += f""" 2231 | Content: {example['content']} 2232 | Class: {example['class']}""" 2233 | 2234 | prompt += f""" 2235 | Content: {content} 2236 | Class: """ 2237 | 2238 | max_retries = 3 2239 | for attempt in range(max_retries): 2240 | try: 2241 | response = llm_model.prompt(prompt, temperature=temperature, logprobs=len(classes)) 2242 | 2243 | db = sqlite_utils.Database(logs_db_path()) 2244 | response.log_to_db(db) 2245 | 2246 | 2247 | generated_text = response.text().strip().lower() 2248 | total_logprob = 0.0 2249 | logprobs = response.response_json.get('logprobs', {}) 2250 | 2251 | for token_info in logprobs['content']: 2252 | total_logprob += token_info.logprob 2253 | 2254 | probability = math.exp(total_logprob) 2255 | 2256 | # Ensure generated text matches one of the classes 2257 | found_class = None 2258 | for class_ in classes: 2259 | if class_.lower() == generated_text: 2260 | found_class = generated_text 2261 | break 2262 | 2263 | if found_class is None: 2264 | return generated_text, 0.0 2265 | 2266 | return found_class, probability 2267 | 2268 | except Exception as e: 2269 | if attempt < max_retries - 1: 2270 | click.echo(f"An error occurred: {e}. Retrying...", err=True) 2271 | time.sleep(2 ** attempt) 2272 | else: 2273 | click.echo(f"Max retries reached. An error occurred: {e}", err=True) 2274 | return "Error", 0 2275 | 2276 | @llm.hookimpl 2277 | def register_models(register): 2278 | pass # No custom models to register for this plugin 2279 | 2280 | @llm.hookimpl 2281 | def register_prompts(register): 2282 | pass # No custom prompts to register for this plugin 2283 | 2284 | 2285 | import os 2286 | import pathlib 2287 | import pytest 2288 | import click 2289 | from unittest.mock import Mock, patch 2290 | from click.testing import CliRunner 2291 | from llm_classify import logs_db_path, user_dir, classify_content, get_class_probability 2292 | from llm_classify import register_commands 2293 | 2294 | # FILE: llm_classify/test___init__.py 2295 | 2296 | 2297 | def test_logs_db_path_env_var_set(monkeypatch): 2298 | # Set the environment variable 2299 | monkeypatch.setenv("LLM_USER_PATH", "/tmp/test_llm_user_path") 2300 | 2301 | # Expected path 2302 | expected_path = pathlib.Path("/tmp/test_llm_user_path/logs.db") 2303 | 2304 | # Check if the logs_db_path function returns the correct path 2305 | assert logs_db_path() == expected_path 2306 | 2307 | def test_logs_db_path_env_var_not_set(monkeypatch): 2308 | # Unset the environment variable 2309 | monkeypatch.delenv("LLM_USER_PATH", raising=False) 2310 | 2311 | # Mock the click.get_app_dir function to return a test directory 2312 | monkeypatch.setattr("click.get_app_dir", lambda x: "/tmp/test_app_dir") 2313 | 2314 | # Expected path 2315 | expected_path = pathlib.Path("/tmp/test_app_dir/logs.db") 2316 | 2317 | # Check if the logs_db_path function returns the correct path 2318 | assert logs_db_path() == expected_path 2319 | 2320 | def test_user_dir_creates_directory(monkeypatch, tmp_path): 2321 | # Unset the environment variable 2322 | monkeypatch.delenv("LLM_USER_PATH", raising=False) 2323 | 2324 | # Mock the click.get_app_dir function to return a temporary directory 2325 | monkeypatch.setattr("click.get_app_dir", lambda x: str(tmp_path)) 2326 | 2327 | # Call the user_dir function 2328 | path = user_dir() 2329 | 2330 | # Check if the directory was created 2331 | assert path.exists() 2332 | assert path.is_dir() 2333 | 2334 | # New tests for classification functionality 2335 | def test_classify_content(): 2336 | content = ["This is a happy message"] 2337 | classes = ["positive", "negative", "neutral"] 2338 | model = "gpt-3.5-turbo" 2339 | temperature = 0 2340 | 2341 | with patch('llm_classify.get_class_probability') as mock_get_prob: 2342 | mock_get_prob.return_value = ("positive", 0.95) 2343 | results = classify_content(content, classes, model, temperature) 2344 | 2345 | assert len(results) == 1 2346 | assert results[0]["class"] == "positive" 2347 | assert results[0]["score"] == 0.95 2348 | assert results[0]["content"] == content[0] 2349 | 2350 | def test_classify_content_no_content_flag(): 2351 | content = ["Test message"] 2352 | classes = ["class1", "class2"] 2353 | 2354 | with patch('llm_classify.get_class_probability') as mock_get_prob: 2355 | mock_get_prob.return_value = ("class1", 0.8) 2356 | results = classify_content(content, classes, "test-model", 0, no_content=True) 2357 | 2358 | assert "content" not in results[0] 2359 | assert results[0]["class"] == "class1" 2360 | assert results[0]["score"] == 0.8 2361 | 2362 | def test_get_class_probability(): 2363 | content = "Test content" 2364 | classes = ["class1", "class2"] 2365 | model = "test-model" 2366 | 2367 | mock_model = Mock() 2368 | mock_response = Mock() 2369 | mock_response.text = lambda: "class1" # Use lambda to ensure consistent return 2370 | mock_response.log_to_db = Mock() 2371 | mock_response.response_json = { 2372 | 'logprobs': { 2373 | 'content': [ 2374 | Mock(logprob=-0.1) # Changed from dict to Mock object 2375 | ] 2376 | } 2377 | } 2378 | mock_model.prompt.return_value = mock_response 2379 | 2380 | with patch('llm.get_model', return_value=mock_model), \ 2381 | patch('sqlite_utils.Database'): 2382 | result_class, probability = get_class_probability( 2383 | content, classes, model, 0 2384 | ) 2385 | 2386 | assert result_class == "class1" 2387 | assert isinstance(probability, float) 2388 | mock_response.log_to_db.assert_called_once() 2389 | 2390 | @pytest.fixture 2391 | def cli(): 2392 | """Create a CLI group and register commands.""" 2393 | cli_group = click.Group() 2394 | register_commands(cli_group) 2395 | return cli_group 2396 | 2397 | def test_classify_command(cli): 2398 | runner = CliRunner() 2399 | with patch('llm_classify.classify_content') as mock_classify: 2400 | mock_classify.return_value = [{"class": "positive", "score": 0.9, "content": "test"}] 2401 | 2402 | result = runner.invoke(cli.commands['classify'], [ 2403 | "test", 2404 | "-c", "positive", 2405 | "-c", "negative", 2406 | "-m", "gpt-3.5-turbo" 2407 | ]) 2408 | 2409 | assert result.exit_code == 0 2410 | assert "positive" in result.output 2411 | 2412 | def test_classify_invalid_temperature(cli): 2413 | runner = CliRunner() 2414 | result = runner.invoke(cli.commands['classify'], [ 2415 | "test", 2416 | "-c", "a", 2417 | "-c", "b", 2418 | "-t", "1.5" 2419 | ]) 2420 | assert result.exit_code != 0 2421 | assert "Temperature must be between 0 and 1" in result.output 2422 | 2423 | def test_classify_insufficient_classes(cli): 2424 | runner = CliRunner() 2425 | result = runner.invoke(cli.commands['classify'], [ 2426 | "test", 2427 | "-c", "a" 2428 | ]) 2429 | assert result.exit_code != 0 2430 | assert "At least two classes must be provided" in result.output 2431 | 2432 | def test_examples_processing(): 2433 | content = ["Test content"] 2434 | classes = ["class1", "class2"] 2435 | examples = ["example1:class1", "example2:class2"] 2436 | 2437 | with patch('llm_classify.get_class_probability') as mock_get_prob: 2438 | mock_get_prob.return_value = ("class1", 0.9) 2439 | results = classify_content( 2440 | content, 2441 | classes, 2442 | "test-model", 2443 | 0, 2444 | [{"content": "example1", "class": "class1"}, 2445 | {"content": "example2", "class": "class2"}] 2446 | ) 2447 | 2448 | assert len(results) == 1 2449 | assert results[0]["class"] == "class1" 2450 | assert results[0]["score"] == 0.9 2451 | 2452 | 2453 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "llm-plugin-generator" 3 | version = "0.6" 4 | description = "LLM plugin to generate few-shot prompts for plugin creation" 5 | readme = "README.md" 6 | authors = [{name = "llm-plugin-generator"}] 7 | license = {text = "Apache-2.0"} 8 | classifiers = [ 9 | "License :: OSI Approved :: Apache Software License" 10 | ] 11 | dependencies = [ 12 | "llm", 13 | "click", 14 | "gitpython", 15 | ] 16 | 17 | [tool.setuptools.package-data] 18 | llm_plugin_generator = ["**/*.xml"] 19 | 20 | [project.urls] 21 | Homepage = "https://github.com/irthomasthomas/llm-plugin-generator" 22 | Changelog = "https://github.com/irthomasthomas/llm-plugin-generator/releases" 23 | Issues = "https://github.com/irthomasthomas/llm-plugin-generator/issues" 24 | 25 | [project.entry-points.llm] 26 | plugin-generator = "llm_plugin_generator" 27 | 28 | [project.optional-dependencies] 29 | test = ["pytest"] 30 | 31 | [tool.pytest.ini_options] 32 | addopts = "--assert=plain" 33 | pythonpath = "." --------------------------------------------------------------------------------