├── tests ├── __init__.py ├── conftest.py ├── test_config.py ├── test_fetchers.py └── test_cli.py ├── images ├── banner.jpg ├── trace_id.png ├── project_id.png ├── thread_id.png └── usage-example.jpg ├── src └── langsmith_cli │ ├── __init__.py │ ├── __main__.py │ ├── config.py │ ├── formatters.py │ ├── fetchers.py │ └── cli.py ├── .gitignore ├── .github └── workflows │ └── test.yml ├── pyproject.toml └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for langsmith-fetch CLI.""" 2 | -------------------------------------------------------------------------------- /images/banner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/langsmith-fetch/HEAD/images/banner.jpg -------------------------------------------------------------------------------- /images/trace_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/langsmith-fetch/HEAD/images/trace_id.png -------------------------------------------------------------------------------- /images/project_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/langsmith-fetch/HEAD/images/project_id.png -------------------------------------------------------------------------------- /images/thread_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/langsmith-fetch/HEAD/images/thread_id.png -------------------------------------------------------------------------------- /images/usage-example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/langsmith-fetch/HEAD/images/usage-example.jpg -------------------------------------------------------------------------------- /src/langsmith_cli/__init__.py: -------------------------------------------------------------------------------- 1 | """LangSmith Fetch - Minimal CLI for fetching LangSmith threads and traces.""" 2 | 3 | __version__ = "0.1.0" 4 | -------------------------------------------------------------------------------- /src/langsmith_cli/__main__.py: -------------------------------------------------------------------------------- 1 | """Entry point for python -m langsmith_cli.""" 2 | 3 | from .cli import main 4 | 5 | if __name__ == "__main__": 6 | main() 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Virtual environments 24 | venv/ 25 | ENV/ 26 | env/ 27 | .venv 28 | 29 | # IDEs 30 | .vscode/ 31 | .idea/ 32 | *.swp 33 | *.swo 34 | *~ 35 | 36 | # OS 37 | .DS_Store 38 | Thumbs.db 39 | 40 | # Config (local) 41 | .env 42 | 43 | # Example/testing code 44 | example/ 45 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: ["3.10", "3.11", "3.12", "3.13"] 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Install uv 20 | uses: astral-sh/setup-uv@v3 21 | with: 22 | enable-cache: true 23 | 24 | - name: Set up Python ${{ matrix.python-version }} 25 | run: uv python install ${{ matrix.python-version }} 26 | 27 | - name: Install dependencies 28 | run: uv sync --extra test 29 | 30 | - name: Run tests 31 | run: uv run pytest tests/ -v 32 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "langsmith-fetch" 7 | version = "0.3.1" 8 | description = "LangSmith Fetch - Minimal CLI for fetching LangSmith threads and traces" 9 | readme = "README.md" 10 | requires-python = ">=3.10" 11 | dependencies = [ 12 | "click>=8.0", 13 | "rich>=13.0", 14 | "pyyaml>=6.0", 15 | "requests>=2.31", 16 | "python-dotenv>=1.0", 17 | "langsmith>=0.1.0", 18 | ] 19 | 20 | [project.optional-dependencies] 21 | test = [ 22 | "pytest>=7.0", 23 | "pytest-mock>=3.12", 24 | "responses>=0.24", 25 | ] 26 | dev = [ 27 | "ruff>=0.1.0", 28 | ] 29 | 30 | [project.scripts] 31 | langsmith-fetch = "langsmith_cli.cli:main" 32 | 33 | [tool.hatch.build.targets.wheel] 34 | packages = ["src/langsmith_cli"] 35 | 36 | [tool.ruff] 37 | line-length = 88 38 | target-version = "py310" 39 | 40 | [tool.ruff.lint] 41 | select = [ 42 | "E", # pycodestyle errors 43 | "W", # pycodestyle warnings 44 | "F", # pyflakes 45 | "I", # isort 46 | "B", # flake8-bugbear 47 | "C4", # flake8-comprehensions 48 | "UP", # pyupgrade 49 | ] 50 | ignore = [ 51 | "E501", # line too long (handled by formatter) 52 | ] 53 | 54 | [tool.ruff.format] 55 | quote-style = "double" 56 | indent-style = "space" 57 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Pytest configuration and fixtures.""" 2 | 3 | import shutil 4 | import tempfile 5 | from pathlib import Path 6 | from unittest.mock import patch 7 | 8 | import pytest 9 | 10 | # Test IDs from examples 11 | TEST_TRACE_ID = "3b0b15fe-1e3a-4aef-afa8-48df15879cfe" 12 | TEST_THREAD_ID = "test-email-agent-thread" 13 | TEST_PROJECT_UUID = "80f1ecb3-a16b-411e-97ae-1c89adbb5c49" 14 | TEST_API_KEY = "lsv2_test_key_123" 15 | TEST_BASE_URL = "https://api.smith.langchain.com" 16 | 17 | 18 | @pytest.fixture 19 | def sample_trace_response(): 20 | """Sample trace API response.""" 21 | return { 22 | "outputs": { 23 | "messages": [ 24 | { 25 | "content": "\n**Subject**: Quick question about next week\n**From**: jane@example.com\n**To**: lance@langchain.dev\n\nHi Lance,\n\nCan we meet next Tuesday at 2pm to discuss the project roadmap?\n\nBest,\nJane\n\n---\n", 26 | "additional_kwargs": {}, 27 | "response_metadata": {}, 28 | "type": "human", 29 | "id": "964d69c7-10e2-4de2-89c9-4361c9ea5da7", 30 | }, 31 | { 32 | "content": [ 33 | { 34 | "id": "toolu_014c7iukFSFTMFTAJwGhqK8U", 35 | "input": { 36 | "reasoning": "Meeting request", 37 | "classification": "respond", 38 | }, 39 | "name": "triage_email", 40 | "type": "tool_use", 41 | } 42 | ], 43 | "additional_kwargs": {}, 44 | "response_metadata": {}, 45 | "type": "ai", 46 | "id": "msg-123", 47 | }, 48 | { 49 | "content": "Classification Decision: respond. Reasoning: This is a meeting request.", 50 | "additional_kwargs": {}, 51 | "type": "tool", 52 | "id": "tool-123", 53 | }, 54 | ] 55 | } 56 | } 57 | 58 | 59 | @pytest.fixture 60 | def sample_thread_response(): 61 | """Sample thread API response.""" 62 | return { 63 | "previews": { 64 | "all_messages": """{"role": "user", "id": "964d69c7-10e2-4de2-89c9-4361c9ea5da7", "content": "\\n**Subject**: Quick question about next week\\n**From**: jane@example.com\\n**To**: lance@langchain.dev\\n\\nHi Lance,\\n\\nCan we meet next Tuesday at 2pm to discuss the project roadmap?\\n\\nBest,\\nJane\\n\\n---\\n"} 65 | 66 | {"role": "assistant", "tool_calls": [{"id": "toolu_014c7iukFSFTMFTAJwGhqK8U", "type": "tool_use", "name": "triage_email"}]} 67 | 68 | {"role": "tool", "content": "Classification Decision: respond."}""" 69 | } 70 | } 71 | 72 | 73 | @pytest.fixture 74 | def temp_config_dir(): 75 | """Create a temporary config directory.""" 76 | temp_dir = tempfile.mkdtemp() 77 | with patch("langsmith_cli.config.CONFIG_DIR", Path(temp_dir)): 78 | with patch("langsmith_cli.config.CONFIG_FILE", Path(temp_dir) / "config.yaml"): 79 | yield Path(temp_dir) 80 | shutil.rmtree(temp_dir) 81 | 82 | 83 | @pytest.fixture 84 | def mock_env_api_key(monkeypatch): 85 | """Mock LANGSMITH_API_KEY environment variable.""" 86 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 87 | 88 | 89 | @pytest.fixture(autouse=True) 90 | def mock_base_url(monkeypatch): 91 | """Mock get_base_url to return TEST_BASE_URL.""" 92 | from langsmith_cli import config 93 | 94 | monkeypatch.setattr(config, "get_base_url", lambda: TEST_BASE_URL) 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LangSmith Fetch 2 | 3 | LangSmith Fetch is CLI for fetching threads or traces from LangSmith projects. It is designed to be easily used by humans or code agents to programmatically fetch LangSmith data for testing and debugging. 4 | 5 | ![LangSmith Fetch Banner](images/banner.jpg) 6 | 7 | ## 🚀 Quickstart 8 | 9 | ```bash 10 | pip install langsmith-fetch 11 | ``` 12 | 13 | Set your LangSmith API key and project name: 14 | 15 | ```bash 16 | export LANGSMITH_API_KEY=lsv2_... 17 | export LANGSMITH_PROJECT=your-project-name 18 | export LANGSMITH_ENDPOINT=your-langsmith-api-url # setting this if you use self-host langsmith 19 | ``` 20 | 21 | That's it! The CLI will automatically fetch traces or threads in `LANGSMITH_PROJECT`. 22 | 23 | **Fetch recent traces to directory (recommended):** 24 | ```bash 25 | langsmith-fetch traces ./my-traces --limit 10 26 | ``` 27 | 28 | **Fetch specific trace by ID:** 29 | ```bash 30 | langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe 31 | ``` 32 | 33 | **Same commands work for threads:** 34 | ```bash 35 | langsmith-fetch threads ./my-threads --limit 10 36 | langsmith-fetch thread my-thread-id 37 | ``` 38 | 39 | ![Usage Example](images/usage-example.jpg) 40 | 41 | **Include metadata and feedback:** 42 | ```bash 43 | langsmith-fetch traces ./my-traces --limit 10 --include-metadata --include-feedback 44 | ``` 45 | 46 | **For code agents:** 47 | ``` 48 | Use langsmith-fetch to fetch recent LangSmith traces. Run langsmith-fetch --help for usage details. 49 | ``` 50 | 51 | ## Commands 52 | 53 | | Command | What it fetches | Output | 54 | |---------|----------------|--------| 55 | | `trace ` | Specific **trace** by ID | stdout or file | 56 | | `thread ` | Specific **thread** by ID | stdout or file | 57 | | `traces [dir]` | Recent **traces** (bulk) | Multiple JSON files in directory (RECOMMENDED) or stdout | 58 | | `threads [dir]` | Recent **threads** (bulk) | Multiple JSON files in directory (RECOMMENDED) or stdout | 59 | 60 | ## Flags 61 | 62 | | Flag | Applies To | Description | Default | 63 | |------|-----------|-------------|---------| 64 | | `--project-uuid ` | `thread`, `threads`, `traces` | LangSmith project UUID (overrides config) | From config or env | 65 | | `-n, --limit ` | `traces`, `threads` | Maximum number to fetch | 1 | 66 | | `--last-n-minutes ` | `traces`, `threads` | Only fetch from last N minutes | None | 67 | | `--since ` | `traces`, `threads` | Only fetch since ISO timestamp | None | 68 | | `--filename-pattern ` | `traces`, `threads` | Filename pattern (use `{trace_id}`, `{thread_id}`, `{index}`) | `{trace_id}.json` or `{thread_id}.json` | 69 | | `--format ` | All commands | Output format: `pretty`, `json`, or `raw` | `pretty` | 70 | | `--file ` | `trace`, `thread` | Save to file instead of stdout | stdout | 71 | | `--include-metadata` | `traces` | Include run metadata (status, timing, tokens, costs) | Not included | 72 | | `--include-feedback` | `traces` | Include feedback data (requires extra API call) | Not included | 73 | | `--max-concurrent ` | `traces`, `threads` | Concurrent fetches (max 10 recommended) | 5 | 74 | | `--no-progress` | `traces`, `threads` | Disable progress bar | Progress shown | 75 | 76 | ### Output Formats 77 | 78 | - **`pretty`** (default): Human-readable Rich panels with color and formatting 79 | - **`json`**: Pretty-printed JSON with syntax highlighting 80 | - **`raw`**: Compact single-line JSON for piping to tools like `jq` 81 | 82 | ## Concepts 83 | 84 | LangSmith organizes data [into three levels](https://docs.langchain.com/langsmith/threads): 85 | - **Runs**: Individual LLM calls or tool executions 86 | - **Traces**: A collection of runs representing a single execution path (one trace contains multiple runs) 87 | - **Threads**: A collection of traces representing a conversation or session (one thread contains multiple traces) 88 | 89 | ## Configuration 90 | 91 | `langsmith-fetch` requires only `LANGSMITH_PROJECT` env var. It automatically looks up the Project UUID and saves both to `~/.langsmith-cli/config.yaml`. 92 | 93 | **Finding IDs in LangSmith UI:** 94 | 95 | **Project UUID** (automatic lookup via `LANGSMITH_PROJECT`): 96 | ![Project ID location](images/project_id.png) 97 | 98 | **Trace ID** (for fetching specific traces): 99 | ![Trace ID location](images/trace_id.png) 100 | 101 | **Thread ID** (for fetching specific threads): 102 | ![Thread ID location](images/thread_id.png) 103 | 104 | ## Tests 105 | 106 | Run the test suite: 107 | 108 | ```bash 109 | # Install with test dependencies 110 | pip install -e ".[test]" 111 | 112 | # Or with uv 113 | uv sync --extra test 114 | 115 | # Run all tests 116 | pytest tests/ 117 | 118 | # Run with verbose output 119 | pytest tests/ -v 120 | 121 | # Run with coverage 122 | pytest tests/ --cov=langsmith_cli 123 | ``` 124 | 125 | The test suite includes 71 tests covering: 126 | - All CLI commands (traces, trace, thread, threads, config) 127 | - All output formats (pretty, json, raw) 128 | - Config management and storage 129 | - Project UUID lookup and caching 130 | - API fetching and error handling 131 | - Time filtering and SDK integration 132 | - Edge cases and validation 133 | 134 | ## License 135 | 136 | MIT 137 | -------------------------------------------------------------------------------- /src/langsmith_cli/config.py: -------------------------------------------------------------------------------- 1 | """Configuration file management for LangSmith Fetch.""" 2 | 3 | import os 4 | from pathlib import Path 5 | from typing import Any 6 | 7 | import yaml 8 | 9 | CONFIG_DIR = Path.home() / ".langsmith-cli" 10 | CONFIG_FILE = CONFIG_DIR / "config.yaml" 11 | 12 | # Cache for project UUID lookups (avoids redundant API calls per session) 13 | _project_uuid_cache: dict[str, str | None] = {} 14 | 15 | 16 | def _ensure_config_dir(): 17 | """Ensure the config directory exists.""" 18 | CONFIG_DIR.mkdir(parents=True, exist_ok=True) 19 | 20 | 21 | def load_config() -> dict[str, Any]: 22 | """ 23 | Load configuration from file. 24 | 25 | Returns: 26 | Dictionary of configuration values, empty dict if file doesn't exist 27 | """ 28 | if not CONFIG_FILE.exists(): 29 | return {} 30 | 31 | with open(CONFIG_FILE) as f: 32 | return yaml.safe_load(f) or {} 33 | 34 | 35 | def save_config(config: dict[str, Any]): 36 | """ 37 | Save configuration to file. 38 | 39 | Args: 40 | config: Dictionary of configuration values to save 41 | """ 42 | _ensure_config_dir() 43 | 44 | with open(CONFIG_FILE, "w") as f: 45 | yaml.dump(config, f, default_flow_style=False) 46 | 47 | 48 | def get_config_value(key: str) -> str | None: 49 | """ 50 | Get a configuration value by key. 51 | 52 | Args: 53 | key: Configuration key to retrieve (supports both hyphen and underscore) 54 | 55 | Returns: 56 | Configuration value or None if not found 57 | """ 58 | config = load_config() 59 | # Try both hyphenated and underscored versions 60 | value = config.get(key) 61 | if value is None: 62 | # Try alternative format (hyphen <-> underscore) 63 | alt_key = key.replace("-", "_") if "-" in key else key.replace("_", "-") 64 | value = config.get(alt_key) 65 | return value 66 | 67 | 68 | def set_config_value(key: str, value: str): 69 | """ 70 | Set a configuration value. 71 | 72 | Args: 73 | key: Configuration key to set (will be normalized to hyphen format) 74 | value: Value to set 75 | """ 76 | config = load_config() 77 | 78 | # Normalize key to hyphen format 79 | normalized_key = key.replace("_", "-") 80 | config[normalized_key] = value 81 | 82 | # Clean up old underscore format if different from normalized 83 | if key != normalized_key and key in config: 84 | del config[key] 85 | 86 | save_config(config) 87 | 88 | # If manually setting project-uuid, clear in-memory cache 89 | # to force re-validation on next lookup 90 | if normalized_key == "project-uuid": 91 | _project_uuid_cache.clear() 92 | 93 | 94 | def _update_project_config(project_name: str, project_uuid: str): 95 | """ 96 | Update config file with both project name and UUID atomically. 97 | 98 | Args: 99 | project_name: Project name to store 100 | project_uuid: Project UUID to store 101 | """ 102 | # Load, update both fields, and save atomically 103 | config = load_config() 104 | config["project-name"] = project_name 105 | config["project-uuid"] = project_uuid 106 | 107 | # Clean up old underscore format if it exists 108 | if "project_uuid" in config: 109 | del config["project_uuid"] 110 | if "project_name" in config: 111 | del config["project_name"] 112 | 113 | save_config(config) 114 | 115 | 116 | def _lookup_project_uuid_by_name( 117 | project_name: str, 118 | api_key: str, 119 | base_url: str | None = None 120 | ) -> str: 121 | """ 122 | Look up project UUID by name using LangSmith API. 123 | 124 | Args: 125 | project_name: Project name to search for 126 | api_key: LangSmith API key 127 | base_url: Optional base URL override 128 | 129 | Returns: 130 | Project UUID string 131 | 132 | Raises: 133 | ValueError: If project not found or lookup fails 134 | """ 135 | from langsmith import Client 136 | 137 | # Initialize client 138 | client = Client(api_key=api_key, api_url=base_url) 139 | 140 | # Try direct lookup by project name 141 | try: 142 | project = client.read_project(project_name=project_name) 143 | return str(project.id) 144 | except Exception as e: 145 | raise ValueError( 146 | f"Project '{project_name}' not found: {e}\n" 147 | f"Use 'langsmith-fetch config set project-uuid ' to set explicitly, " 148 | f"or set LANGSMITH_PROJECT_UUID env var." 149 | ) 150 | 151 | 152 | def get_api_key() -> str | None: 153 | """ 154 | Get API key from config or environment variable. 155 | 156 | Returns: 157 | API key from config file, or LANGSMITH_API_KEY env var, or None 158 | """ 159 | # Try config file first 160 | api_key = get_config_value("api_key") 161 | if api_key: 162 | return api_key 163 | 164 | # Fall back to environment variable 165 | return os.environ.get("LANGSMITH_API_KEY") 166 | 167 | 168 | def get_base_url() -> str | None: 169 | """ 170 | Get base URL from config. 171 | 172 | Returns: 173 | Base URL from config file, or LANGSMITH_ENDPOINT env var, or None 174 | """ 175 | if base_url := get_config_value("base_url"): 176 | return base_url 177 | return os.environ.get("LANGSMITH_ENDPOINT") or "https://api.smith.langchain.com" 178 | 179 | 180 | def get_project_uuid() -> str | None: 181 | """ 182 | Get project UUID with automatic sync detection. 183 | 184 | Priority order: 185 | 1. LANGSMITH_PROJECT_UUID env var (explicit UUID override) 186 | 2. LANGSMITH_PROJECT env var → check if config matches → fetch if stale 187 | 3. Config file as fallback (when no env var set) 188 | 189 | Returns: 190 | Project UUID or None 191 | """ 192 | import sys 193 | 194 | # Priority 1: Explicit UUID override (bypasses all config logic) 195 | env_uuid = os.environ.get("LANGSMITH_PROJECT_UUID") 196 | if env_uuid: 197 | return env_uuid 198 | 199 | # Get current project name from env var 200 | env_project_name = os.environ.get("LANGSMITH_PROJECT") 201 | 202 | # Load config values (use hyphen format as canonical) 203 | config_project_uuid = get_config_value("project-uuid") 204 | config_project_name = get_config_value("project-name") 205 | 206 | # Case 1: No env var set - use config as default 207 | if not env_project_name: 208 | if config_project_uuid: 209 | return config_project_uuid 210 | return None 211 | 212 | # Case 2: Env var IS set - check if it matches config 213 | 214 | # Check in-memory cache first (keyed by project name) 215 | if env_project_name in _project_uuid_cache: 216 | cached_uuid = _project_uuid_cache[env_project_name] 217 | # If config is out of sync, update it 218 | if cached_uuid and config_project_name != env_project_name: 219 | _update_project_config(env_project_name, cached_uuid) 220 | return cached_uuid 221 | 222 | # Config matches env var - use cached UUID 223 | if config_project_name == env_project_name and config_project_uuid: 224 | # Add to in-memory cache 225 | _project_uuid_cache[env_project_name] = config_project_uuid 226 | return config_project_uuid 227 | 228 | # Config doesn't match (or doesn't exist) - need to fetch 229 | print(f"Project name changed to '{env_project_name}', fetching UUID...", file=sys.stderr) 230 | 231 | # Validate we have API key before attempting lookup 232 | api_key = get_api_key() 233 | if not api_key: 234 | print( 235 | "Warning: LANGSMITH_PROJECT set but no API key found. " 236 | "Set LANGSMITH_API_KEY to enable project lookup.", 237 | file=sys.stderr 238 | ) 239 | return None 240 | 241 | base_url = get_base_url() 242 | 243 | # Fetch UUID via API 244 | try: 245 | uuid = _lookup_project_uuid_by_name(env_project_name, api_key, base_url) 246 | 247 | # Update in-memory cache 248 | _project_uuid_cache[env_project_name] = uuid 249 | 250 | # Update config with BOTH name and UUID 251 | _update_project_config(env_project_name, uuid) 252 | 253 | print(f"Found project '{env_project_name}' (UUID: {uuid})", file=sys.stderr) 254 | 255 | return uuid 256 | 257 | except ValueError as e: 258 | print(f"Error: {e}", file=sys.stderr) 259 | return None 260 | except Exception as e: 261 | print( 262 | f"Warning: Failed to lookup project '{env_project_name}': {e}", 263 | file=sys.stderr 264 | ) 265 | return None 266 | 267 | 268 | def get_default_format() -> str: 269 | """ 270 | Get default output format from config. 271 | 272 | Returns: 273 | Output format ('raw', 'json', or 'pretty'), defaults to 'pretty' 274 | """ 275 | return get_config_value("default_format") or "pretty" 276 | -------------------------------------------------------------------------------- /src/langsmith_cli/formatters.py: -------------------------------------------------------------------------------- 1 | """Output formatting utilities for messages.""" 2 | 3 | import json 4 | from typing import Any 5 | 6 | from rich.console import Console 7 | from rich.panel import Panel 8 | from rich.syntax import Syntax 9 | 10 | console = Console() 11 | 12 | 13 | def format_messages(messages: list[dict[str, Any]], format_type: str) -> str: 14 | """ 15 | Format messages according to the specified format. 16 | 17 | Args: 18 | messages: List of message dictionaries 19 | format_type: Output format ('raw', 'json', or 'pretty') 20 | 21 | Returns: 22 | Formatted string representation of messages 23 | """ 24 | if format_type == "raw": 25 | return _format_raw(messages) 26 | elif format_type == "json": 27 | return _format_json(messages) 28 | elif format_type == "pretty": 29 | return _format_pretty(messages) 30 | else: 31 | raise ValueError(f"Unknown format type: {format_type}") 32 | 33 | 34 | def _format_raw(messages: list[dict[str, Any]]) -> str: 35 | """Format as raw JSON (compact).""" 36 | return json.dumps(messages) 37 | 38 | 39 | def _format_json(messages: list[dict[str, Any]]) -> str: 40 | """Format as pretty-printed JSON.""" 41 | return json.dumps(messages, indent=2) 42 | 43 | 44 | def _format_pretty(messages: list[dict[str, Any]]) -> str: 45 | """Format as human-readable structured text with Rich.""" 46 | output_parts = [] 47 | 48 | for i, msg in enumerate(messages, 1): 49 | msg_type = msg.get("type") or msg.get("role", "unknown") 50 | 51 | # Create header 52 | header = f"Message {i}: {msg_type}" 53 | output_parts.append("=" * 60) 54 | output_parts.append(header) 55 | output_parts.append("-" * 60) 56 | 57 | # Format content based on message type 58 | content = msg.get("content", "") 59 | 60 | if isinstance(content, str): 61 | output_parts.append(content) 62 | elif isinstance(content, list): 63 | # Handle structured content (tool calls, etc.) 64 | for item in content: 65 | if isinstance(item, dict): 66 | if "text" in item: 67 | output_parts.append(item["text"]) 68 | elif "type" in item and item["type"] == "tool_use": 69 | output_parts.append( 70 | f"\nTool Call: {item.get('name', 'unknown')}" 71 | ) 72 | if "input" in item: 73 | output_parts.append( 74 | f"Input: {json.dumps(item['input'], indent=2)}" 75 | ) 76 | else: 77 | output_parts.append(str(item)) 78 | else: 79 | output_parts.append(str(content)) 80 | 81 | # Handle tool calls (different format) 82 | if "tool_calls" in msg: 83 | for tool_call in msg["tool_calls"]: 84 | if isinstance(tool_call, dict): 85 | func = tool_call.get("function", {}) 86 | output_parts.append(f"\nTool Call: {func.get('name', 'unknown')}") 87 | if "arguments" in func: 88 | output_parts.append(f"Arguments: {func['arguments']}") 89 | 90 | # Handle tool responses 91 | if msg_type == "tool" or msg.get("name"): 92 | tool_name = msg.get("name", "unknown") 93 | output_parts.append(f"Tool: {tool_name}") 94 | 95 | output_parts.append("") # Empty line between messages 96 | 97 | return "\n".join(output_parts) 98 | 99 | 100 | def print_formatted( 101 | messages: list[dict[str, Any]], format_type: str, output_file: str = None 102 | ): 103 | """ 104 | Print formatted messages directly to console with Rich formatting, or save to file. 105 | 106 | Args: 107 | messages: List of message dictionaries 108 | format_type: Output format ('raw', 'json', or 'pretty') 109 | output_file: Optional file path to save output instead of printing 110 | """ 111 | # If output_file is specified, save to file 112 | if output_file: 113 | content = format_messages(messages, format_type) 114 | with open(output_file, "w") as f: 115 | f.write(content) 116 | return 117 | 118 | # Otherwise, print to console with Rich formatting 119 | if format_type == "json": 120 | # Use Rich's syntax highlighting for JSON 121 | json_str = _format_json(messages) 122 | syntax = Syntax(json_str, "json", theme="monokai", line_numbers=False) 123 | console.print(syntax) 124 | elif format_type == "pretty": 125 | # Use Rich formatting for pretty output 126 | for i, msg in enumerate(messages, 1): 127 | msg_type = msg.get("type") or msg.get("role", "unknown") 128 | title = f"Message {i}: {msg_type.upper()}" 129 | 130 | # Format content 131 | content = msg.get("content", "") 132 | if isinstance(content, str): 133 | panel_content = content 134 | elif isinstance(content, list): 135 | parts = [] 136 | for item in content: 137 | if isinstance(item, dict): 138 | if "text" in item: 139 | parts.append(item["text"]) 140 | elif "type" in item and item["type"] == "tool_use": 141 | parts.append( 142 | f"[bold]Tool:[/bold] {item.get('name', 'unknown')}" 143 | ) 144 | else: 145 | parts.append(str(item)) 146 | panel_content = "\n".join(parts) 147 | else: 148 | panel_content = str(content) 149 | 150 | # Handle tool calls 151 | if "tool_calls" in msg: 152 | tool_parts = [] 153 | for tool_call in msg["tool_calls"]: 154 | if isinstance(tool_call, dict): 155 | func = tool_call.get("function", {}) 156 | tool_parts.append( 157 | f"[bold]Tool:[/bold] {func.get('name', 'unknown')}" 158 | ) 159 | if tool_parts: 160 | panel_content += "\n" + "\n".join(tool_parts) 161 | 162 | # Create panel 163 | panel = Panel(panel_content, title=title, border_style="blue") 164 | console.print(panel) 165 | else: 166 | # Raw format - just print 167 | console.print(_format_raw(messages)) 168 | 169 | 170 | # ============================================================================ 171 | # New Formatters for Trace Data with Metadata and Feedback 172 | # ============================================================================ 173 | 174 | 175 | def format_trace_data(data: dict[str, Any] | list[dict[str, Any]], format_type: str) -> str: 176 | """Format trace data with optional metadata and feedback. 177 | 178 | Args: 179 | data: Either a list of messages (old format) or dict with keys: 180 | {trace_id/thread_id, messages, metadata, feedback} 181 | format_type: Output format ('raw', 'json', or 'pretty') 182 | 183 | Returns: 184 | Formatted string representation 185 | """ 186 | # Backward compatibility: if data is a list, treat as messages-only 187 | if isinstance(data, list): 188 | return format_messages(data, format_type) 189 | 190 | # New format with metadata 191 | if format_type == "raw": 192 | return json.dumps(data, default=str) 193 | elif format_type == "json": 194 | return json.dumps(data, indent=2, default=str) 195 | elif format_type == "pretty": 196 | return _format_pretty_with_metadata(data) 197 | else: 198 | raise ValueError(f"Unknown format type: {format_type}") 199 | 200 | 201 | def _format_pretty_with_metadata(data: dict[str, Any]) -> str: 202 | """Format trace data with metadata as human-readable output.""" 203 | parts = [] 204 | 205 | # Metadata section (if present and non-empty) 206 | metadata = data.get("metadata", {}) 207 | if metadata: 208 | parts.append(_format_metadata_section(metadata)) 209 | 210 | # Feedback section (if present and non-empty) 211 | feedback = data.get("feedback", []) 212 | if feedback: 213 | parts.append(_format_feedback_section(feedback)) 214 | 215 | # Messages section 216 | if parts: # Only add separator if we have metadata/feedback 217 | parts.append("=" * 60) 218 | parts.append("MESSAGES") 219 | parts.append("=" * 60) 220 | 221 | messages = data.get("messages", []) 222 | parts.append(_format_pretty(messages)) 223 | 224 | return "\n\n".join(parts) 225 | 226 | 227 | def _format_metadata_section(metadata: dict[str, Any]) -> str: 228 | """Format metadata section for pretty output.""" 229 | lines = ["=" * 60, "RUN METADATA", "=" * 60] 230 | 231 | # Status and timing 232 | if metadata.get("status"): 233 | lines.append(f"Status: {metadata['status']}") 234 | if metadata.get("start_time"): 235 | lines.append(f"Start Time: {metadata['start_time']}") 236 | if metadata.get("end_time"): 237 | lines.append(f"End Time: {metadata['end_time']}") 238 | if metadata.get("duration_ms"): 239 | lines.append(f"Duration: {metadata['duration_ms']}ms") 240 | 241 | # Token usage 242 | token_usage = metadata.get("token_usage", {}) 243 | if any(v is not None for v in token_usage.values()): 244 | lines.append("\nToken Usage:") 245 | if token_usage.get("prompt_tokens") is not None: 246 | lines.append(f" Prompt: {token_usage['prompt_tokens']}") 247 | if token_usage.get("completion_tokens") is not None: 248 | lines.append(f" Completion: {token_usage['completion_tokens']}") 249 | if token_usage.get("total_tokens") is not None: 250 | lines.append(f" Total: {token_usage['total_tokens']}") 251 | 252 | # Costs 253 | costs = metadata.get("costs", {}) 254 | if any(v is not None for v in costs.values()): 255 | lines.append("\nCosts:") 256 | if costs.get("total_cost") is not None: 257 | lines.append(f" Total: ${costs['total_cost']:.5f}") 258 | if costs.get("prompt_cost") is not None: 259 | lines.append(f" Prompt: ${costs['prompt_cost']:.5f}") 260 | if costs.get("completion_cost") is not None: 261 | lines.append(f" Completion: ${costs['completion_cost']:.5f}") 262 | 263 | # Custom metadata 264 | custom = metadata.get("custom_metadata", {}) 265 | if custom: 266 | lines.append("\nCustom Metadata:") 267 | for key, value in custom.items(): 268 | # Pretty print the value 269 | if isinstance(value, dict): 270 | lines.append(f" {key}: {json.dumps(value, indent=4)}") 271 | else: 272 | lines.append(f" {key}: {value}") 273 | 274 | # Feedback stats 275 | feedback_stats = metadata.get("feedback_stats", {}) 276 | if feedback_stats: 277 | lines.append("\nFeedback Stats:") 278 | for key, count in feedback_stats.items(): 279 | lines.append(f" {key}: {count}") 280 | 281 | return "\n".join(lines) 282 | 283 | 284 | def _format_feedback_section(feedback: list[dict[str, Any]]) -> str: 285 | """Format feedback section for pretty output.""" 286 | lines = ["=" * 60, "FEEDBACK", "=" * 60] 287 | 288 | for i, fb in enumerate(feedback, 1): 289 | lines.append(f"\nFeedback {i}:") 290 | lines.append(f" Key: {fb['key']}") 291 | if fb.get("score") is not None: 292 | lines.append(f" Score: {fb['score']}") 293 | if fb.get("value") is not None: 294 | lines.append(f" Value: {fb['value']}") 295 | if fb.get("comment"): 296 | lines.append(f" Comment: {fb['comment']}") 297 | if fb.get("correction"): 298 | correction = fb['correction'] 299 | if isinstance(correction, str): 300 | lines.append(f" Correction: {correction}") 301 | else: 302 | lines.append(f" Correction: {json.dumps(correction, indent=4)}") 303 | if fb.get("created_at"): 304 | lines.append(f" Created: {fb['created_at']}") 305 | 306 | return "\n".join(lines) 307 | 308 | 309 | def print_formatted_trace( 310 | data: dict[str, Any] | list[dict[str, Any]], 311 | format_type: str, 312 | output_file: str | None = None, 313 | ): 314 | """Print formatted trace data with metadata and feedback. 315 | 316 | Args: 317 | data: Either list of messages (old format) or dict with trace data 318 | format_type: Output format ('raw', 'json', or 'pretty') 319 | output_file: Optional file path to save output instead of printing 320 | """ 321 | # If output_file is specified, save to file 322 | if output_file: 323 | content = format_trace_data(data, format_type) 324 | with open(output_file, "w") as f: 325 | f.write(content) 326 | return 327 | 328 | # Otherwise, print to console 329 | if format_type == "json": 330 | # Use Rich's syntax highlighting for JSON 331 | json_str = format_trace_data(data, "json") 332 | syntax = Syntax(json_str, "json", theme="monokai", line_numbers=False) 333 | console.print(syntax) 334 | elif format_type == "pretty": 335 | # For messages-only (list), use Rich panels 336 | if isinstance(data, list): 337 | print_formatted(data, format_type, None) 338 | # For trace data with metadata (dict), use plain text format 339 | else: 340 | formatted = format_trace_data(data, "pretty") 341 | console.print(formatted) 342 | else: 343 | # Raw format 344 | console.print(format_trace_data(data, "raw")) 345 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | """Tests for config commands.""" 2 | 3 | from unittest.mock import patch 4 | 5 | from click.testing import CliRunner 6 | 7 | from langsmith_cli.cli import main 8 | from tests.conftest import TEST_API_KEY, TEST_PROJECT_UUID 9 | 10 | 11 | class TestConfigShow: 12 | """Tests for config show command.""" 13 | 14 | def test_show_empty_config(self, temp_config_dir): 15 | """Test showing config when empty.""" 16 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 17 | with patch( 18 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 19 | ): 20 | runner = CliRunner() 21 | result = runner.invoke(main, ["config", "show"]) 22 | 23 | assert result.exit_code == 0 24 | assert "No configuration found" in result.output 25 | 26 | def test_show_with_project_uuid(self, temp_config_dir): 27 | """Test showing config with project UUID.""" 28 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 29 | with patch( 30 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 31 | ): 32 | # Set config 33 | from langsmith_cli.config import set_config_value 34 | 35 | set_config_value("project-uuid", TEST_PROJECT_UUID) 36 | 37 | runner = CliRunner() 38 | result = runner.invoke(main, ["config", "show"]) 39 | 40 | assert result.exit_code == 0 41 | assert "Current configuration:" in result.output 42 | assert TEST_PROJECT_UUID in result.output 43 | 44 | def test_show_with_api_key_masked(self, temp_config_dir): 45 | """Test showing config with API key (should be masked).""" 46 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 47 | with patch( 48 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 49 | ): 50 | # Set config 51 | from langsmith_cli.config import set_config_value 52 | 53 | set_config_value("api-key", TEST_API_KEY) 54 | 55 | runner = CliRunner() 56 | result = runner.invoke(main, ["config", "show"]) 57 | 58 | assert result.exit_code == 0 59 | # Should show only first 10 chars 60 | assert TEST_API_KEY[:10] in result.output 61 | assert "..." in result.output 62 | # Should not show full key 63 | assert TEST_API_KEY not in result.output 64 | 65 | def test_show_all_config_options(self, temp_config_dir): 66 | """Test showing config with all options set.""" 67 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 68 | with patch( 69 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 70 | ): 71 | # Set all config options 72 | from langsmith_cli.config import set_config_value 73 | 74 | set_config_value("project-uuid", TEST_PROJECT_UUID) 75 | set_config_value("api-key", TEST_API_KEY) 76 | set_config_value("default-format", "json") 77 | 78 | runner = CliRunner() 79 | result = runner.invoke(main, ["config", "show"]) 80 | 81 | assert result.exit_code == 0 82 | assert "Current configuration:" in result.output 83 | assert TEST_PROJECT_UUID in result.output 84 | assert TEST_API_KEY[:10] in result.output 85 | assert "json" in result.output 86 | 87 | 88 | class TestConfigFunctions: 89 | """Tests for config module functions.""" 90 | 91 | def test_get_api_key_from_config(self, temp_config_dir, monkeypatch): 92 | """Test getting API key from config.""" 93 | monkeypatch.delenv("LANGSMITH_API_KEY", raising=False) 94 | 95 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 96 | with patch( 97 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 98 | ): 99 | from langsmith_cli.config import get_api_key, set_config_value 100 | 101 | set_config_value("api-key", TEST_API_KEY) 102 | 103 | assert get_api_key() == TEST_API_KEY 104 | 105 | def test_get_api_key_from_env(self, temp_config_dir, monkeypatch): 106 | """Test getting API key from environment variable.""" 107 | monkeypatch.setenv("LANGSMITH_API_KEY", "env_api_key") 108 | 109 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 110 | with patch( 111 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 112 | ): 113 | from langsmith_cli.config import get_api_key 114 | 115 | # Env var should take precedence over config 116 | assert get_api_key() == "env_api_key" 117 | 118 | def test_get_project_uuid(self, temp_config_dir, monkeypatch): 119 | """Test getting project UUID from config when no env var set.""" 120 | # Clear env vars to test config fallback behavior 121 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 122 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 123 | 124 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 125 | with patch( 126 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 127 | ): 128 | from langsmith_cli.config import get_project_uuid, set_config_value 129 | 130 | set_config_value("project-uuid", TEST_PROJECT_UUID) 131 | 132 | assert get_project_uuid() == TEST_PROJECT_UUID 133 | 134 | def test_get_default_format(self, temp_config_dir): 135 | """Test getting default format from config.""" 136 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 137 | with patch( 138 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 139 | ): 140 | from langsmith_cli.config import get_default_format, set_config_value 141 | 142 | # Default should be 'pretty' 143 | assert get_default_format() == "pretty" 144 | 145 | # Set to 'json' 146 | set_config_value("default-format", "json") 147 | assert get_default_format() == "json" 148 | 149 | def test_config_key_with_hyphen_and_underscore(self, temp_config_dir): 150 | """Test that config keys work with both hyphens and underscores.""" 151 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 152 | with patch( 153 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 154 | ): 155 | from langsmith_cli.config import get_config_value, set_config_value 156 | 157 | # Set with hyphen 158 | set_config_value("project-uuid", TEST_PROJECT_UUID) 159 | 160 | # Get with underscore should also work 161 | assert get_config_value("project_uuid") == TEST_PROJECT_UUID 162 | # Get with hyphen should work 163 | assert get_config_value("project-uuid") == TEST_PROJECT_UUID 164 | 165 | 166 | class TestProjectLookup: 167 | """Tests for automatic project UUID lookup from LANGSMITH_PROJECT.""" 168 | 169 | def test_get_project_uuid_priority_explicit_uuid_wins(self, temp_config_dir, monkeypatch): 170 | """Test that LANGSMITH_PROJECT_UUID env var takes highest priority.""" 171 | monkeypatch.setenv("LANGSMITH_PROJECT", "my-project") 172 | monkeypatch.setenv("LANGSMITH_PROJECT_UUID", "env-uuid") 173 | 174 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 175 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 176 | from langsmith_cli.config import get_project_uuid, set_config_value 177 | 178 | set_config_value("project-uuid", "config-uuid") 179 | set_config_value("project-name", "old-project") 180 | 181 | # LANGSMITH_PROJECT_UUID should always win (highest priority) 182 | assert get_project_uuid() == "env-uuid" 183 | 184 | def test_get_project_uuid_priority_env_uuid_no_lookup(self, temp_config_dir, monkeypatch): 185 | """Test that LANGSMITH_PROJECT_UUID env var bypasses API lookup.""" 186 | monkeypatch.setenv("LANGSMITH_PROJECT", "my-project") 187 | monkeypatch.setenv("LANGSMITH_PROJECT_UUID", "env-uuid") 188 | 189 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 190 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 191 | from langsmith_cli.config import get_project_uuid 192 | 193 | # LANGSMITH_PROJECT_UUID should be used without API lookup 194 | assert get_project_uuid() == "env-uuid" 195 | 196 | def test_lookup_project_uuid_success(self, temp_config_dir, monkeypatch): 197 | """Test successful project lookup via API.""" 198 | from unittest.mock import Mock, MagicMock 199 | 200 | monkeypatch.setenv("LANGSMITH_PROJECT", "test-project") 201 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 202 | 203 | # Mock LangSmith Client 204 | mock_project = Mock() 205 | mock_project.id = "looked-up-uuid" 206 | mock_project.name = "test-project" 207 | 208 | mock_client = MagicMock() 209 | mock_client.read_project.return_value = mock_project 210 | 211 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 212 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 213 | with patch("langsmith.Client", return_value=mock_client): 214 | from langsmith_cli.config import get_project_uuid 215 | 216 | result = get_project_uuid() 217 | assert result == "looked-up-uuid" 218 | mock_client.read_project.assert_called_once_with(project_name="test-project") 219 | 220 | def test_lookup_project_uuid_no_match(self, temp_config_dir, monkeypatch): 221 | """Test error handling when project not found.""" 222 | from unittest.mock import MagicMock 223 | 224 | monkeypatch.setenv("LANGSMITH_PROJECT", "nonexistent") 225 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 226 | 227 | mock_client = MagicMock() 228 | mock_client.read_project.side_effect = Exception("Project not found") 229 | 230 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 231 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 232 | with patch("langsmith.Client", return_value=mock_client): 233 | from langsmith_cli.config import get_project_uuid 234 | 235 | # Should return None and print error to stderr 236 | result = get_project_uuid() 237 | assert result is None 238 | 239 | def test_lookup_caching(self, temp_config_dir, monkeypatch): 240 | """Test that lookup result is cached for session.""" 241 | from unittest.mock import Mock, MagicMock 242 | 243 | monkeypatch.setenv("LANGSMITH_PROJECT", "cached-project") 244 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 245 | 246 | mock_project = Mock() 247 | mock_project.id = "cached-uuid" 248 | mock_project.name = "cached-project" 249 | 250 | mock_client = MagicMock() 251 | mock_client.read_project.return_value = mock_project 252 | 253 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 254 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 255 | with patch("langsmith.Client", return_value=mock_client): 256 | from langsmith_cli.config import get_project_uuid, _project_uuid_cache 257 | 258 | # Clear cache first 259 | _project_uuid_cache.clear() 260 | 261 | # First call should hit API 262 | result1 = get_project_uuid() 263 | assert result1 == "cached-uuid" 264 | assert mock_client.read_project.call_count == 1 265 | 266 | # Second call should use cache 267 | result2 = get_project_uuid() 268 | assert result2 == "cached-uuid" 269 | assert mock_client.read_project.call_count == 1 # Still 1 270 | 271 | def test_lookup_no_api_key(self, temp_config_dir, monkeypatch): 272 | """Test graceful handling when API key is missing.""" 273 | monkeypatch.setenv("LANGSMITH_PROJECT", "test-project") 274 | monkeypatch.delenv("LANGSMITH_API_KEY", raising=False) 275 | 276 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 277 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 278 | from langsmith_cli.config import get_project_uuid 279 | 280 | # Should return None with warning 281 | result = get_project_uuid() 282 | assert result is None 283 | 284 | def test_project_name_change_triggers_refetch(self, temp_config_dir, monkeypatch): 285 | """Test that changing project name triggers UUID re-fetch.""" 286 | from unittest.mock import Mock, MagicMock 287 | 288 | monkeypatch.setenv("LANGSMITH_PROJECT", "new-project") 289 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 290 | 291 | mock_project = Mock() 292 | mock_project.id = "new-uuid" 293 | mock_project.name = "new-project" 294 | 295 | mock_client = MagicMock() 296 | mock_client.read_project.return_value = mock_project 297 | 298 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 299 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 300 | with patch("langsmith.Client", return_value=mock_client): 301 | from langsmith_cli.config import get_project_uuid, set_config_value, get_config_value, _project_uuid_cache 302 | 303 | # Clear cache 304 | _project_uuid_cache.clear() 305 | 306 | # Set old config 307 | set_config_value("project-name", "old-project") 308 | set_config_value("project-uuid", "old-uuid") 309 | 310 | # Should detect mismatch and fetch new UUID 311 | result = get_project_uuid() 312 | assert result == "new-uuid" 313 | assert mock_client.read_project.call_count == 1 314 | 315 | # Verify config was updated with both fields 316 | assert get_config_value("project-name") == "new-project" 317 | assert get_config_value("project-uuid") == "new-uuid" 318 | 319 | def test_project_name_match_uses_cache(self, temp_config_dir, monkeypatch): 320 | """Test that matching project name uses cached UUID without API call.""" 321 | from unittest.mock import MagicMock 322 | 323 | monkeypatch.setenv("LANGSMITH_PROJECT", "test-project") 324 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 325 | 326 | mock_client = MagicMock() 327 | 328 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 329 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 330 | with patch("langsmith.Client", return_value=mock_client): 331 | from langsmith_cli.config import get_project_uuid, set_config_value, _project_uuid_cache 332 | 333 | # Clear cache 334 | _project_uuid_cache.clear() 335 | 336 | # Set matching config 337 | set_config_value("project-name", "test-project") 338 | set_config_value("project-uuid", "test-uuid") 339 | 340 | # Should use cached UUID without API call 341 | result = get_project_uuid() 342 | assert result == "test-uuid" 343 | assert mock_client.read_project.call_count == 0 344 | 345 | def test_legacy_config_migration(self, temp_config_dir, monkeypatch): 346 | """Test that legacy config (only project_uuid) triggers re-fetch and migration.""" 347 | from unittest.mock import Mock, MagicMock 348 | 349 | monkeypatch.setenv("LANGSMITH_PROJECT", "test-project") 350 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 351 | 352 | mock_project = Mock() 353 | mock_project.id = "fetched-uuid" 354 | mock_project.name = "test-project" 355 | 356 | mock_client = MagicMock() 357 | mock_client.read_project.return_value = mock_project 358 | 359 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 360 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 361 | with patch("langsmith.Client", return_value=mock_client): 362 | from langsmith_cli.config import get_project_uuid, set_config_value, get_config_value, _project_uuid_cache 363 | 364 | # Clear cache 365 | _project_uuid_cache.clear() 366 | 367 | # Set legacy config (only UUID, no name) 368 | set_config_value("project-uuid", "old-uuid") 369 | 370 | # Should detect missing project_name and fetch new UUID 371 | result = get_project_uuid() 372 | assert result == "fetched-uuid" 373 | 374 | # Verify config was updated with both fields 375 | assert get_config_value("project-name") == "test-project" 376 | assert get_config_value("project-uuid") == "fetched-uuid" 377 | 378 | def test_no_env_var_uses_config_default(self, temp_config_dir, monkeypatch): 379 | """Test that no env var uses config as default.""" 380 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 381 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 382 | 383 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 384 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 385 | from langsmith_cli.config import get_project_uuid, set_config_value 386 | 387 | # Set config 388 | set_config_value("project-name", "default-project") 389 | set_config_value("project-uuid", "default-uuid") 390 | 391 | # Should use config UUID without env var 392 | result = get_project_uuid() 393 | assert result == "default-uuid" 394 | 395 | def test_explicit_uuid_override(self, temp_config_dir, monkeypatch): 396 | """Test that LANGSMITH_PROJECT_UUID overrides everything.""" 397 | monkeypatch.setenv("LANGSMITH_PROJECT", "test-project") 398 | monkeypatch.setenv("LANGSMITH_PROJECT_UUID", "override-uuid") 399 | 400 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 401 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 402 | from langsmith_cli.config import get_project_uuid, set_config_value 403 | 404 | # Set config 405 | set_config_value("project-name", "config-project") 406 | set_config_value("project-uuid", "config-uuid") 407 | 408 | # LANGSMITH_PROJECT_UUID should override everything 409 | result = get_project_uuid() 410 | assert result == "override-uuid" 411 | 412 | def test_api_failure_handling(self, temp_config_dir, monkeypatch): 413 | """Test that API failure is handled gracefully.""" 414 | from unittest.mock import MagicMock 415 | 416 | monkeypatch.setenv("LANGSMITH_PROJECT", "nonexistent") 417 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 418 | 419 | mock_client = MagicMock() 420 | mock_client.read_project.side_effect = Exception("Project not found") 421 | 422 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 423 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 424 | with patch("langsmith.Client", return_value=mock_client): 425 | from langsmith_cli.config import get_project_uuid, get_config_value, set_config_value, _project_uuid_cache 426 | 427 | # Clear cache 428 | _project_uuid_cache.clear() 429 | 430 | # Set old config 431 | set_config_value("project-name", "old-project") 432 | set_config_value("project-uuid", "old-uuid") 433 | 434 | # Should return None on API failure 435 | result = get_project_uuid() 436 | assert result is None 437 | 438 | # Verify config was NOT updated (preserves last known good state) 439 | assert get_config_value("project-name") == "old-project" 440 | assert get_config_value("project-uuid") == "old-uuid" 441 | 442 | def test_cache_clears_on_manual_update(self, temp_config_dir): 443 | """Test that in-memory cache clears when project_uuid is manually set.""" 444 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 445 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 446 | from langsmith_cli.config import set_config_value, _project_uuid_cache 447 | 448 | # Populate cache 449 | _project_uuid_cache["test-project"] = "cached-uuid" 450 | 451 | # Manually set project_uuid 452 | set_config_value("project-uuid", "new-uuid") 453 | 454 | # Cache should be cleared 455 | assert len(_project_uuid_cache) == 0 456 | 457 | def test_in_memory_cache_updates_config(self, temp_config_dir, monkeypatch): 458 | """Test that in-memory cache updates config when out of sync.""" 459 | monkeypatch.setenv("LANGSMITH_PROJECT", "cached-project") 460 | 461 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 462 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 463 | from langsmith_cli.config import get_project_uuid, set_config_value, get_config_value, _project_uuid_cache 464 | 465 | # Set old config 466 | set_config_value("project-name", "old-project") 467 | set_config_value("project-uuid", "old-uuid") 468 | 469 | # Populate in-memory cache with different project 470 | _project_uuid_cache["cached-project"] = "cached-uuid" 471 | 472 | # Should use cache and update config 473 | result = get_project_uuid() 474 | assert result == "cached-uuid" 475 | 476 | # Verify config was updated 477 | assert get_config_value("project-name") == "cached-project" 478 | assert get_config_value("project-uuid") == "cached-uuid" 479 | 480 | def test_empty_project_name_handling(self, temp_config_dir, monkeypatch): 481 | """Test graceful handling of empty project name.""" 482 | monkeypatch.setenv("LANGSMITH_PROJECT", "") 483 | 484 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 485 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 486 | from langsmith_cli.config import get_project_uuid, set_config_value 487 | 488 | # Set config 489 | set_config_value("project-uuid", "config-uuid") 490 | 491 | # Empty string should be treated as no env var 492 | result = get_project_uuid() 493 | assert result == "config-uuid" 494 | 495 | def test_project_uuid_persists_after_lookup(self, temp_config_dir, monkeypatch): 496 | """Test that both project_name and project_uuid persist after lookup.""" 497 | from unittest.mock import Mock, MagicMock 498 | 499 | monkeypatch.setenv("LANGSMITH_PROJECT", "persist-project") 500 | monkeypatch.setenv("LANGSMITH_API_KEY", TEST_API_KEY) 501 | 502 | mock_project = Mock() 503 | mock_project.id = "persist-uuid" 504 | mock_project.name = "persist-project" 505 | 506 | mock_client = MagicMock() 507 | mock_client.read_project.return_value = mock_project 508 | 509 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 510 | with patch("langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml"): 511 | with patch("langsmith.Client", return_value=mock_client): 512 | from langsmith_cli.config import get_project_uuid, get_config_value, _project_uuid_cache 513 | 514 | # Clear cache 515 | _project_uuid_cache.clear() 516 | 517 | # First call should fetch and persist 518 | result = get_project_uuid() 519 | assert result == "persist-uuid" 520 | 521 | # Verify both fields were persisted 522 | assert get_config_value("project-name") == "persist-project" 523 | assert get_config_value("project-uuid") == "persist-uuid" 524 | -------------------------------------------------------------------------------- /tests/test_fetchers.py: -------------------------------------------------------------------------------- 1 | """Tests for fetchers module.""" 2 | 3 | import json 4 | from datetime import datetime 5 | from unittest.mock import Mock, patch 6 | 7 | import pytest 8 | import requests 9 | import responses 10 | 11 | from langsmith_cli import fetchers 12 | from tests.conftest import ( 13 | TEST_API_KEY, 14 | TEST_BASE_URL, 15 | TEST_PROJECT_UUID, 16 | TEST_THREAD_ID, 17 | TEST_TRACE_ID, 18 | ) 19 | 20 | 21 | class TestFetchTrace: 22 | """Tests for fetch_trace function.""" 23 | 24 | @responses.activate 25 | def test_fetch_trace_success(self, sample_trace_response): 26 | """Test successful trace fetching.""" 27 | responses.add( 28 | responses.GET, 29 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 30 | json=sample_trace_response, 31 | status=200, 32 | ) 33 | 34 | messages = fetchers.fetch_trace( 35 | TEST_TRACE_ID, base_url=TEST_BASE_URL, api_key=TEST_API_KEY 36 | ) 37 | 38 | assert isinstance(messages, list) 39 | assert len(messages) == 3 40 | assert messages[0]["type"] == "human" 41 | assert "jane@example.com" in messages[0]["content"] 42 | 43 | @responses.activate 44 | def test_fetch_trace_not_found(self): 45 | """Test fetch_trace with 404 error.""" 46 | responses.add( 47 | responses.GET, 48 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 49 | json={"error": "Not found"}, 50 | status=404, 51 | ) 52 | 53 | with pytest.raises(requests.HTTPError): 54 | fetchers.fetch_trace( 55 | TEST_TRACE_ID, base_url=TEST_BASE_URL, api_key=TEST_API_KEY 56 | ) 57 | 58 | @responses.activate 59 | def test_fetch_trace_api_key_sent(self, sample_trace_response): 60 | """Test that API key is sent in headers.""" 61 | responses.add( 62 | responses.GET, 63 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 64 | json=sample_trace_response, 65 | status=200, 66 | ) 67 | 68 | fetchers.fetch_trace( 69 | TEST_TRACE_ID, base_url=TEST_BASE_URL, api_key=TEST_API_KEY 70 | ) 71 | 72 | # Check that the request was made with correct headers 73 | assert len(responses.calls) == 1 74 | assert responses.calls[0].request.headers["X-API-Key"] == TEST_API_KEY 75 | 76 | 77 | class TestFetchThread: 78 | """Tests for fetch_thread function.""" 79 | 80 | @responses.activate 81 | def test_fetch_thread_success(self, sample_thread_response): 82 | """Test successful thread fetching.""" 83 | responses.add( 84 | responses.GET, 85 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 86 | json=sample_thread_response, 87 | status=200, 88 | ) 89 | 90 | messages = fetchers.fetch_thread( 91 | TEST_THREAD_ID, 92 | TEST_PROJECT_UUID, 93 | base_url=TEST_BASE_URL, 94 | api_key=TEST_API_KEY, 95 | ) 96 | 97 | assert isinstance(messages, list) 98 | assert len(messages) == 3 99 | assert messages[0]["role"] == "user" 100 | assert "jane@example.com" in messages[0]["content"] 101 | 102 | @responses.activate 103 | def test_fetch_thread_params_sent(self, sample_thread_response): 104 | """Test that correct params are sent in thread request.""" 105 | responses.add( 106 | responses.GET, 107 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 108 | json=sample_thread_response, 109 | status=200, 110 | ) 111 | 112 | fetchers.fetch_thread( 113 | TEST_THREAD_ID, 114 | TEST_PROJECT_UUID, 115 | base_url=TEST_BASE_URL, 116 | api_key=TEST_API_KEY, 117 | ) 118 | 119 | # Check that the request was made with correct params 120 | assert len(responses.calls) == 1 121 | request = responses.calls[0].request 122 | assert request.headers["X-API-Key"] == TEST_API_KEY 123 | # Check query params 124 | assert "select=all_messages" in request.url 125 | assert f"session_id={TEST_PROJECT_UUID}" in request.url 126 | 127 | @responses.activate 128 | def test_fetch_thread_not_found(self): 129 | """Test fetch_thread with 404 error.""" 130 | responses.add( 131 | responses.GET, 132 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 133 | json={"error": "Not found"}, 134 | status=404, 135 | ) 136 | 137 | with pytest.raises(requests.HTTPError): 138 | fetchers.fetch_thread( 139 | TEST_THREAD_ID, 140 | TEST_PROJECT_UUID, 141 | base_url=TEST_BASE_URL, 142 | api_key=TEST_API_KEY, 143 | ) 144 | 145 | @responses.activate 146 | def test_fetch_thread_parses_multiline_json(self, sample_thread_response): 147 | """Test that thread fetcher correctly parses newline-separated JSON.""" 148 | responses.add( 149 | responses.GET, 150 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 151 | json=sample_thread_response, 152 | status=200, 153 | ) 154 | 155 | messages = fetchers.fetch_thread( 156 | TEST_THREAD_ID, 157 | TEST_PROJECT_UUID, 158 | base_url=TEST_BASE_URL, 159 | api_key=TEST_API_KEY, 160 | ) 161 | 162 | # Should have parsed all messages from newline-separated format 163 | assert len(messages) == 3 164 | # Each message should be a valid dict 165 | for msg in messages: 166 | assert isinstance(msg, dict) 167 | assert "role" in msg or "type" in msg 168 | 169 | 170 | class TestFetchLatestTrace: 171 | """Tests for fetch_latest_trace function.""" 172 | 173 | @responses.activate 174 | @patch("langsmith.Client") 175 | def test_fetch_latest_trace_success(self, mock_client_class, sample_trace_response): 176 | """Test successful latest trace fetching.""" 177 | # Mock the Client and its list_runs method 178 | mock_client = Mock() 179 | mock_run = Mock() 180 | mock_run.id = TEST_TRACE_ID 181 | mock_client.list_runs.return_value = [mock_run] 182 | mock_client_class.return_value = mock_client 183 | 184 | # Mock the REST API call for fetch_trace 185 | responses.add( 186 | responses.GET, 187 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 188 | json=sample_trace_response, 189 | status=200, 190 | ) 191 | 192 | messages = fetchers.fetch_latest_trace( 193 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL 194 | ) 195 | 196 | # Verify Client was instantiated with correct API key 197 | mock_client_class.assert_called_once_with(api_key=TEST_API_KEY) 198 | 199 | # Verify list_runs was called with correct parameters 200 | mock_client.list_runs.assert_called_once() 201 | call_kwargs = mock_client.list_runs.call_args[1] 202 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 203 | assert call_kwargs["limit"] == 1 204 | 205 | # Verify the messages were fetched correctly 206 | assert isinstance(messages, list) 207 | assert len(messages) == 3 208 | 209 | @patch("langsmith.Client") 210 | def test_fetch_latest_trace_no_traces_found(self, mock_client_class): 211 | """Test fetch_latest_trace when no traces are found.""" 212 | # Mock empty list_runs result 213 | mock_client = Mock() 214 | mock_client.list_runs.return_value = [] 215 | mock_client_class.return_value = mock_client 216 | 217 | with pytest.raises(ValueError, match="No traces found matching criteria"): 218 | fetchers.fetch_latest_trace(api_key=TEST_API_KEY, base_url=TEST_BASE_URL) 219 | 220 | @responses.activate 221 | @patch("langsmith.Client") 222 | def test_fetch_latest_trace_with_project_uuid( 223 | self, mock_client_class, sample_trace_response 224 | ): 225 | """Test latest trace fetching with project UUID filter.""" 226 | # Mock the Client 227 | mock_client = Mock() 228 | mock_run = Mock() 229 | mock_run.id = TEST_TRACE_ID 230 | mock_client.list_runs.return_value = [mock_run] 231 | mock_client_class.return_value = mock_client 232 | 233 | # Mock the REST API call 234 | responses.add( 235 | responses.GET, 236 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 237 | json=sample_trace_response, 238 | status=200, 239 | ) 240 | 241 | messages = fetchers.fetch_latest_trace( 242 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, project_uuid=TEST_PROJECT_UUID 243 | ) 244 | 245 | # Verify list_runs was called with project_id 246 | call_kwargs = mock_client.list_runs.call_args[1] 247 | assert call_kwargs["project_id"] == TEST_PROJECT_UUID 248 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 249 | assert call_kwargs["limit"] == 1 250 | 251 | assert isinstance(messages, list) 252 | 253 | @responses.activate 254 | @patch("langsmith.Client") 255 | def test_fetch_latest_trace_with_time_window( 256 | self, mock_client_class, sample_trace_response 257 | ): 258 | """Test latest trace fetching with last_n_minutes filter.""" 259 | # Mock the Client 260 | mock_client = Mock() 261 | mock_run = Mock() 262 | mock_run.id = TEST_TRACE_ID 263 | mock_client.list_runs.return_value = [mock_run] 264 | mock_client_class.return_value = mock_client 265 | 266 | # Mock the REST API call 267 | responses.add( 268 | responses.GET, 269 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 270 | json=sample_trace_response, 271 | status=200, 272 | ) 273 | 274 | messages = fetchers.fetch_latest_trace( 275 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, last_n_minutes=30 276 | ) 277 | 278 | # Verify list_runs was called with start_time 279 | call_kwargs = mock_client.list_runs.call_args[1] 280 | assert "start_time" in call_kwargs 281 | assert isinstance(call_kwargs["start_time"], datetime) 282 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 283 | 284 | assert isinstance(messages, list) 285 | 286 | @responses.activate 287 | @patch("langsmith.Client") 288 | def test_fetch_latest_trace_with_since_timestamp( 289 | self, mock_client_class, sample_trace_response 290 | ): 291 | """Test latest trace fetching with since timestamp filter.""" 292 | # Mock the Client 293 | mock_client = Mock() 294 | mock_run = Mock() 295 | mock_run.id = TEST_TRACE_ID 296 | mock_client.list_runs.return_value = [mock_run] 297 | mock_client_class.return_value = mock_client 298 | 299 | # Mock the REST API call 300 | responses.add( 301 | responses.GET, 302 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 303 | json=sample_trace_response, 304 | status=200, 305 | ) 306 | 307 | since_timestamp = "2025-12-09T10:00:00Z" 308 | messages = fetchers.fetch_latest_trace( 309 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, since=since_timestamp 310 | ) 311 | 312 | # Verify list_runs was called with start_time 313 | call_kwargs = mock_client.list_runs.call_args[1] 314 | assert "start_time" in call_kwargs 315 | assert isinstance(call_kwargs["start_time"], datetime) 316 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 317 | 318 | assert isinstance(messages, list) 319 | 320 | @responses.activate 321 | @patch("langsmith.Client") 322 | def test_fetch_latest_trace_without_project_uuid( 323 | self, mock_client_class, sample_trace_response 324 | ): 325 | """Test latest trace searches all projects when project_uuid is None.""" 326 | # Mock the Client 327 | mock_client = Mock() 328 | mock_run = Mock() 329 | mock_run.id = TEST_TRACE_ID 330 | mock_client.list_runs.return_value = [mock_run] 331 | mock_client_class.return_value = mock_client 332 | 333 | # Mock the REST API call 334 | responses.add( 335 | responses.GET, 336 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 337 | json=sample_trace_response, 338 | status=200, 339 | ) 340 | 341 | messages = fetchers.fetch_latest_trace( 342 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, project_uuid=None 343 | ) 344 | 345 | # Verify list_runs was called WITHOUT project_id parameter 346 | call_kwargs = mock_client.list_runs.call_args[1] 347 | assert "project_id" not in call_kwargs 348 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 349 | assert call_kwargs["limit"] == 1 350 | 351 | assert isinstance(messages, list) 352 | 353 | 354 | class TestFetchRecentTraces: 355 | """Tests for fetch_recent_traces function.""" 356 | 357 | @responses.activate 358 | @patch("langsmith.Client") 359 | def test_fetch_recent_traces_success( 360 | self, mock_client_class, sample_trace_response 361 | ): 362 | """Test successful recent traces fetching.""" 363 | # Mock the Client and its list_runs method 364 | mock_client = Mock() 365 | mock_run1 = Mock() 366 | mock_run1.id = "trace-id-1" 367 | mock_run1.feedback_stats = {} # Empty dict, not a Mock 368 | mock_run1.start_time = None 369 | mock_run1.end_time = None 370 | mock_run1.extra = {} 371 | mock_run2 = Mock() 372 | mock_run2.id = "trace-id-2" 373 | mock_run2.feedback_stats = {} # Empty dict, not a Mock 374 | mock_run2.start_time = None 375 | mock_run2.end_time = None 376 | mock_run2.extra = {} 377 | mock_client.list_runs.return_value = [mock_run1, mock_run2] 378 | mock_client_class.return_value = mock_client 379 | 380 | # Mock the REST API calls for fetch_trace 381 | responses.add( 382 | responses.GET, 383 | "https://api.smith.langchain.com/runs/trace-id-1", 384 | json=sample_trace_response, 385 | status=200, 386 | ) 387 | responses.add( 388 | responses.GET, 389 | "https://api.smith.langchain.com/runs/trace-id-2", 390 | json=sample_trace_response, 391 | status=200, 392 | ) 393 | 394 | traces_data = fetchers.fetch_recent_traces( 395 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, limit=2, 396 | include_metadata=False, include_feedback=False 397 | ) 398 | 399 | # Verify Client was instantiated with correct API key 400 | mock_client_class.assert_called_once_with(api_key=TEST_API_KEY) 401 | 402 | # Verify list_runs was called with correct parameters 403 | mock_client.list_runs.assert_called_once() 404 | call_kwargs = mock_client.list_runs.call_args[1] 405 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 406 | assert call_kwargs["limit"] == 2 407 | 408 | # Verify the traces were fetched correctly 409 | assert isinstance(traces_data, list) 410 | assert len(traces_data) == 2 411 | # Order doesn't matter with concurrent fetching, just check both IDs present 412 | trace_ids = {trace_id for trace_id, _ in traces_data} 413 | assert trace_ids == {"trace-id-1", "trace-id-2"} 414 | assert all(isinstance(messages, list) for _, messages in traces_data) 415 | 416 | @patch("langsmith.Client") 417 | def test_fetch_recent_traces_no_traces_found(self, mock_client_class): 418 | """Test fetch_recent_traces when no traces are found.""" 419 | # Mock empty list_runs result 420 | mock_client = Mock() 421 | mock_client.list_runs.return_value = [] 422 | mock_client_class.return_value = mock_client 423 | 424 | with pytest.raises(ValueError, match="No traces found matching criteria"): 425 | fetchers.fetch_recent_traces(api_key=TEST_API_KEY, base_url=TEST_BASE_URL) 426 | 427 | @responses.activate 428 | @patch("langsmith.Client") 429 | def test_fetch_recent_traces_with_project_uuid( 430 | self, mock_client_class, sample_trace_response 431 | ): 432 | """Test recent traces fetching with project UUID filter.""" 433 | # Mock the Client 434 | mock_client = Mock() 435 | mock_run = Mock() 436 | mock_run.id = TEST_TRACE_ID 437 | mock_run.feedback_stats = {} 438 | mock_run.start_time = None 439 | mock_run.end_time = None 440 | mock_run.extra = {} 441 | mock_client.list_runs.return_value = [mock_run] 442 | mock_client_class.return_value = mock_client 443 | 444 | # Mock the REST API call 445 | responses.add( 446 | responses.GET, 447 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 448 | json=sample_trace_response, 449 | status=200, 450 | ) 451 | 452 | traces_data = fetchers.fetch_recent_traces( 453 | api_key=TEST_API_KEY, 454 | base_url=TEST_BASE_URL, 455 | limit=1, 456 | project_uuid=TEST_PROJECT_UUID, 457 | include_metadata=False, 458 | include_feedback=False, 459 | ) 460 | 461 | # Verify list_runs was called with project_id 462 | call_kwargs = mock_client.list_runs.call_args[1] 463 | assert call_kwargs["project_id"] == TEST_PROJECT_UUID 464 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 465 | assert call_kwargs["limit"] == 1 466 | 467 | assert isinstance(traces_data, list) 468 | assert len(traces_data) == 1 469 | 470 | @responses.activate 471 | @patch("langsmith.Client") 472 | def test_fetch_recent_traces_with_time_window( 473 | self, mock_client_class, sample_trace_response 474 | ): 475 | """Test recent traces fetching with last_n_minutes filter.""" 476 | # Mock the Client 477 | mock_client = Mock() 478 | mock_run = Mock() 479 | mock_run.id = TEST_TRACE_ID 480 | mock_run.feedback_stats = {} 481 | mock_run.start_time = None 482 | mock_run.end_time = None 483 | mock_run.extra = {} 484 | mock_client.list_runs.return_value = [mock_run] 485 | mock_client_class.return_value = mock_client 486 | 487 | # Mock the REST API call 488 | responses.add( 489 | responses.GET, 490 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 491 | json=sample_trace_response, 492 | status=200, 493 | ) 494 | 495 | traces_data = fetchers.fetch_recent_traces( 496 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, last_n_minutes=30, 497 | include_metadata=False, include_feedback=False 498 | ) 499 | 500 | # Verify list_runs was called with start_time 501 | call_kwargs = mock_client.list_runs.call_args[1] 502 | assert "start_time" in call_kwargs 503 | assert isinstance(call_kwargs["start_time"], datetime) 504 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 505 | 506 | assert isinstance(traces_data, list) 507 | 508 | @responses.activate 509 | @patch("langsmith.Client") 510 | def test_fetch_recent_traces_with_since_timestamp( 511 | self, mock_client_class, sample_trace_response 512 | ): 513 | """Test recent traces fetching with since timestamp filter.""" 514 | # Mock the Client 515 | mock_client = Mock() 516 | mock_run = Mock() 517 | mock_run.id = TEST_TRACE_ID 518 | mock_run.feedback_stats = {} 519 | mock_run.start_time = None 520 | mock_run.end_time = None 521 | mock_run.extra = {} 522 | mock_client.list_runs.return_value = [mock_run] 523 | mock_client_class.return_value = mock_client 524 | 525 | # Mock the REST API call 526 | responses.add( 527 | responses.GET, 528 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 529 | json=sample_trace_response, 530 | status=200, 531 | ) 532 | 533 | since_timestamp = "2025-12-09T10:00:00Z" 534 | traces_data = fetchers.fetch_recent_traces( 535 | api_key=TEST_API_KEY, base_url=TEST_BASE_URL, since=since_timestamp, 536 | include_metadata=False, include_feedback=False 537 | ) 538 | 539 | # Verify list_runs was called with start_time 540 | call_kwargs = mock_client.list_runs.call_args[1] 541 | assert "start_time" in call_kwargs 542 | assert isinstance(call_kwargs["start_time"], datetime) 543 | assert call_kwargs["filter"] == 'and(eq(is_root, true), neq(status, "pending"))' 544 | 545 | assert isinstance(traces_data, list) 546 | 547 | 548 | class TestFetchRecentThreads: 549 | """Tests for fetch_recent_threads function.""" 550 | 551 | @responses.activate 552 | def test_fetch_recent_threads_success(self, sample_thread_response): 553 | """Test successful recent threads fetching.""" 554 | # Mock runs query 555 | responses.add( 556 | responses.POST, 557 | f"{TEST_BASE_URL}/runs/query", 558 | json={ 559 | "runs": [ 560 | { 561 | "id": "run-1", 562 | "start_time": "2024-01-02T00:00:00Z", 563 | "extra": {"metadata": {"thread_id": "thread-1"}}, 564 | }, 565 | { 566 | "id": "run-2", 567 | "start_time": "2024-01-01T00:00:00Z", 568 | "extra": {"metadata": {"thread_id": "thread-2"}}, 569 | }, 570 | ] 571 | }, 572 | status=200, 573 | ) 574 | 575 | # Mock thread fetches 576 | responses.add( 577 | responses.GET, 578 | f"{TEST_BASE_URL}/runs/threads/thread-1", 579 | json=sample_thread_response, 580 | status=200, 581 | ) 582 | responses.add( 583 | responses.GET, 584 | f"{TEST_BASE_URL}/runs/threads/thread-2", 585 | json=sample_thread_response, 586 | status=200, 587 | ) 588 | 589 | results = fetchers.fetch_recent_threads( 590 | TEST_PROJECT_UUID, TEST_BASE_URL, TEST_API_KEY, limit=10 591 | ) 592 | 593 | assert isinstance(results, list) 594 | assert len(results) == 2 595 | assert results[0][0] == "thread-1" 596 | assert results[1][0] == "thread-2" 597 | assert len(results[0][1]) == 3 # 3 messages per thread 598 | assert len(results[1][1]) == 3 599 | 600 | @responses.activate 601 | def test_fetch_recent_threads_respects_limit(self, sample_thread_response): 602 | """Test that fetch_recent_threads respects the limit parameter.""" 603 | # Mock runs query with 3 threads 604 | responses.add( 605 | responses.POST, 606 | f"{TEST_BASE_URL}/runs/query", 607 | json={ 608 | "runs": [ 609 | { 610 | "id": "run-1", 611 | "start_time": "2024-01-03T00:00:00Z", 612 | "extra": {"metadata": {"thread_id": "thread-1"}}, 613 | }, 614 | { 615 | "id": "run-2", 616 | "start_time": "2024-01-02T00:00:00Z", 617 | "extra": {"metadata": {"thread_id": "thread-2"}}, 618 | }, 619 | { 620 | "id": "run-3", 621 | "start_time": "2024-01-01T00:00:00Z", 622 | "extra": {"metadata": {"thread_id": "thread-3"}}, 623 | }, 624 | ] 625 | }, 626 | status=200, 627 | ) 628 | 629 | # Mock only 2 thread fetches (because limit=2) 630 | for i in [1, 2]: 631 | responses.add( 632 | responses.GET, 633 | f"{TEST_BASE_URL}/runs/threads/thread-{i}", 634 | json=sample_thread_response, 635 | status=200, 636 | ) 637 | 638 | results = fetchers.fetch_recent_threads( 639 | TEST_PROJECT_UUID, TEST_BASE_URL, TEST_API_KEY, limit=2 640 | ) 641 | 642 | assert len(results) == 2 643 | assert results[0][0] == "thread-1" 644 | assert results[1][0] == "thread-2" 645 | 646 | @responses.activate 647 | def test_fetch_recent_threads_handles_missing_thread_id( 648 | self, sample_thread_response 649 | ): 650 | """Test that runs without thread_id are skipped.""" 651 | responses.add( 652 | responses.POST, 653 | f"{TEST_BASE_URL}/runs/query", 654 | json={ 655 | "runs": [ 656 | { 657 | "id": "run-1", 658 | "start_time": "2024-01-02T00:00:00Z", 659 | "extra": {"metadata": {"thread_id": "thread-1"}}, 660 | }, 661 | { 662 | "id": "run-2", 663 | "start_time": "2024-01-01T00:00:00Z", 664 | "extra": {"metadata": {}}, # No thread_id 665 | }, 666 | ] 667 | }, 668 | status=200, 669 | ) 670 | 671 | responses.add( 672 | responses.GET, 673 | f"{TEST_BASE_URL}/runs/threads/thread-1", 674 | json=sample_thread_response, 675 | status=200, 676 | ) 677 | 678 | results = fetchers.fetch_recent_threads( 679 | TEST_PROJECT_UUID, TEST_BASE_URL, TEST_API_KEY, limit=10 680 | ) 681 | 682 | assert len(results) == 1 683 | assert results[0][0] == "thread-1" 684 | 685 | @responses.activate 686 | def test_fetch_recent_threads_deduplicates(self, sample_thread_response): 687 | """Test that duplicate thread_ids are deduplicated.""" 688 | responses.add( 689 | responses.POST, 690 | f"{TEST_BASE_URL}/runs/query", 691 | json={ 692 | "runs": [ 693 | { 694 | "id": "run-1", 695 | "start_time": "2024-01-02T00:00:00Z", 696 | "extra": {"metadata": {"thread_id": "thread-1"}}, 697 | }, 698 | { 699 | "id": "run-2", 700 | "start_time": "2024-01-01T00:00:00Z", 701 | "extra": {"metadata": {"thread_id": "thread-1"}}, # Duplicate 702 | }, 703 | ] 704 | }, 705 | status=200, 706 | ) 707 | 708 | responses.add( 709 | responses.GET, 710 | f"{TEST_BASE_URL}/runs/threads/thread-1", 711 | json=sample_thread_response, 712 | status=200, 713 | ) 714 | 715 | results = fetchers.fetch_recent_threads( 716 | TEST_PROJECT_UUID, TEST_BASE_URL, TEST_API_KEY, limit=10 717 | ) 718 | 719 | # Should only have one result even though thread-1 appeared twice 720 | assert len(results) == 1 721 | assert results[0][0] == "thread-1" 722 | 723 | @responses.activate 724 | def test_fetch_recent_threads_with_last_n_minutes(self, sample_thread_response): 725 | """Test that temporal filter last_n_minutes is passed to API.""" 726 | responses.add( 727 | responses.POST, 728 | f"{TEST_BASE_URL}/runs/query", 729 | json={ 730 | "runs": [ 731 | { 732 | "id": "run-1", 733 | "start_time": "2024-01-02T00:00:00Z", 734 | "extra": {"metadata": {"thread_id": "thread-1"}}, 735 | } 736 | ] 737 | }, 738 | status=200, 739 | ) 740 | 741 | responses.add( 742 | responses.GET, 743 | f"{TEST_BASE_URL}/runs/threads/thread-1", 744 | json=sample_thread_response, 745 | status=200, 746 | ) 747 | 748 | results = fetchers.fetch_recent_threads( 749 | TEST_PROJECT_UUID, TEST_BASE_URL, TEST_API_KEY, limit=10, last_n_minutes=30 750 | ) 751 | 752 | # Verify the request was made with start_time in body 753 | assert len(responses.calls) == 2 754 | request_body = json.loads(responses.calls[0].request.body) 755 | assert "start_time" in request_body 756 | assert len(results) == 1 757 | 758 | @responses.activate 759 | def test_fetch_recent_threads_with_since(self, sample_thread_response): 760 | """Test that temporal filter since is passed to API.""" 761 | responses.add( 762 | responses.POST, 763 | f"{TEST_BASE_URL}/runs/query", 764 | json={ 765 | "runs": [ 766 | { 767 | "id": "run-1", 768 | "start_time": "2024-01-02T00:00:00Z", 769 | "extra": {"metadata": {"thread_id": "thread-1"}}, 770 | } 771 | ] 772 | }, 773 | status=200, 774 | ) 775 | 776 | responses.add( 777 | responses.GET, 778 | f"{TEST_BASE_URL}/runs/threads/thread-1", 779 | json=sample_thread_response, 780 | status=200, 781 | ) 782 | 783 | results = fetchers.fetch_recent_threads( 784 | TEST_PROJECT_UUID, 785 | TEST_BASE_URL, 786 | TEST_API_KEY, 787 | limit=10, 788 | since="2025-12-09T10:00:00Z", 789 | ) 790 | 791 | # Verify the request was made with start_time in body 792 | assert len(responses.calls) == 2 793 | request_body = json.loads(responses.calls[0].request.body) 794 | assert "start_time" in request_body 795 | assert len(results) == 1 796 | -------------------------------------------------------------------------------- /src/langsmith_cli/fetchers.py: -------------------------------------------------------------------------------- 1 | """Core fetching logic for LangSmith threads and traces.""" 2 | 3 | import json 4 | import sys 5 | from collections import OrderedDict 6 | from concurrent.futures import ThreadPoolExecutor, as_completed 7 | from time import perf_counter 8 | from typing import Any 9 | 10 | import requests 11 | from rich.console import Console 12 | from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn 13 | 14 | try: 15 | from langsmith import Client # noqa: F401 16 | 17 | HAS_LANGSMITH = True 18 | except ImportError: 19 | HAS_LANGSMITH = False 20 | 21 | 22 | def fetch_thread( 23 | thread_id: str, project_uuid: str, *, base_url: str, api_key: str 24 | ) -> list[dict[str, Any]]: 25 | """ 26 | Fetch messages for a LangGraph thread by thread_id. 27 | 28 | Args: 29 | thread_id: LangGraph thread_id (e.g., 'test-email-agent-thread') 30 | project_uuid: LangSmith project UUID (session_id) 31 | base_url: LangSmith base URL 32 | api_key: LangSmith API key 33 | 34 | Returns: 35 | List of message dictionaries 36 | 37 | Raises: 38 | requests.HTTPError: If the API request fails 39 | """ 40 | headers = {"X-API-Key": api_key, "Content-Type": "application/json"} 41 | 42 | url = f"{base_url}/runs/threads/{thread_id}" 43 | params = {"select": "all_messages", "session_id": project_uuid} 44 | 45 | response = requests.get(url, headers=headers, params=params) 46 | response.raise_for_status() 47 | 48 | data = response.json() 49 | messages_text = data["previews"]["all_messages"] 50 | 51 | # Parse the JSON messages (newline-separated JSON objects) 52 | messages = [] 53 | for line in messages_text.strip().split("\n\n"): 54 | if line.strip(): 55 | messages.append(json.loads(line)) 56 | 57 | return messages 58 | 59 | 60 | def fetch_trace(trace_id: str, *, base_url: str, api_key: str) -> list[dict[str, Any]]: 61 | """ 62 | Fetch messages for a single trace by trace ID. 63 | 64 | Args: 65 | trace_id: LangSmith trace UUID 66 | base_url: LangSmith base URL 67 | api_key: LangSmith API key 68 | 69 | Returns: 70 | List of message dictionaries with structured content 71 | 72 | Raises: 73 | requests.HTTPError: If the API request fails 74 | """ 75 | headers = {"X-API-Key": api_key, "Content-Type": "application/json"} 76 | 77 | url = f"{base_url}/runs/{trace_id}?include_messages=true" 78 | 79 | response = requests.get(url, headers=headers) 80 | response.raise_for_status() 81 | 82 | data = response.json() 83 | 84 | # Extract messages from outputs 85 | messages = data.get("messages") 86 | output_messages = (data.get("outputs") or {}).get("messages") 87 | return messages or output_messages or [] 88 | 89 | 90 | def fetch_recent_threads( 91 | project_uuid: str, 92 | base_url: str, 93 | api_key: str, 94 | limit: int = 10, 95 | last_n_minutes: int | None = None, 96 | since: str | None = None, 97 | max_workers: int = 5, 98 | show_progress: bool = True, 99 | ) -> list[tuple[str, list[dict[str, Any]]]]: 100 | """ 101 | Fetch recent threads for a project with concurrent fetching. 102 | 103 | Args: 104 | project_uuid: LangSmith project UUID (session_id) 105 | base_url: LangSmith base URL 106 | api_key: LangSmith API key 107 | limit: Maximum number of threads to return (default: 10) 108 | last_n_minutes: Optional time window to limit search. Only returns threads 109 | from the last N minutes. Mutually exclusive with `since`. 110 | since: Optional ISO timestamp string (e.g., "2025-12-09T10:00:00Z"). 111 | Only returns threads since this time. Mutually exclusive with `last_n_minutes`. 112 | max_workers: Maximum concurrent thread fetches (default: 5) 113 | show_progress: Whether to show progress bar (default: True) 114 | 115 | Returns: 116 | List of tuples (thread_id, messages) for each thread 117 | 118 | Raises: 119 | requests.HTTPError: If the API request fails 120 | """ 121 | from datetime import datetime, timedelta, timezone 122 | 123 | headers = {"X-API-Key": api_key, "Content-Type": "application/json"} 124 | 125 | # Query for root runs in the project 126 | url = f"{base_url}/runs/query" 127 | body = {"session": [project_uuid], "is_root": True} 128 | 129 | # Add time filtering if specified 130 | if last_n_minutes is not None: 131 | start_time = datetime.now(timezone.utc) - timedelta(minutes=last_n_minutes) 132 | body["start_time"] = start_time.isoformat() 133 | elif since is not None: 134 | # Parse ISO timestamp (handle both 'Z' and explicit timezone) 135 | since_clean = since.replace("Z", "+00:00") 136 | start_time = datetime.fromisoformat(since_clean) 137 | body["start_time"] = start_time.isoformat() 138 | 139 | response = requests.post(url, headers=headers, data=json.dumps(body)) 140 | 141 | # Add better error handling 142 | try: 143 | response.raise_for_status() 144 | except requests.HTTPError: 145 | # Print response content for debugging 146 | print(f"API Error Response ({response.status_code}): {response.text}") 147 | print(f"Request body was: {json.dumps(body, indent=2)}") 148 | raise 149 | 150 | data = response.json() 151 | 152 | # The response should have a 'runs' key 153 | runs = data.get("runs", []) 154 | 155 | # Extract unique thread_ids with their most recent timestamp 156 | thread_info = OrderedDict() # Maintains insertion order (most recent first) 157 | 158 | for run in runs: 159 | # Check if run has thread_id in metadata 160 | extra = run.get("extra", {}) 161 | metadata = extra.get("metadata", {}) 162 | thread_id = metadata.get("thread_id") 163 | 164 | if thread_id and thread_id not in thread_info: 165 | thread_info[thread_id] = run.get("start_time") 166 | 167 | # Stop if we've found enough unique threads 168 | if len(thread_info) >= limit: 169 | break 170 | 171 | # Fetch messages for each thread with concurrent fetching and progress bar 172 | from concurrent.futures import ThreadPoolExecutor, as_completed 173 | from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn 174 | import sys 175 | 176 | def _fetch_thread_safe(thread_id: str) -> tuple[str, list[dict[str, Any]] | None]: 177 | """Safe wrapper for fetch_thread that returns (thread_id, messages or None).""" 178 | try: 179 | messages = fetch_thread( 180 | thread_id, project_uuid, base_url=base_url, api_key=api_key 181 | ) 182 | return (thread_id, messages) 183 | except Exception as e: 184 | print(f"Warning: Failed to fetch thread {thread_id}: {e}", file=sys.stderr) 185 | return (thread_id, None) 186 | 187 | results = [] 188 | 189 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 190 | # Submit all fetch tasks 191 | future_to_thread = { 192 | executor.submit(_fetch_thread_safe, thread_id): thread_id 193 | for thread_id in thread_info.keys() 194 | } 195 | 196 | # Use progress bar if requested 197 | if show_progress: 198 | with Progress( 199 | SpinnerColumn(), 200 | TextColumn("[bold blue]Fetching {task.completed}/{task.total} threads..."), 201 | BarColumn(), 202 | TaskProgressColumn(), 203 | ) as progress: 204 | task = progress.add_task("fetch", total=len(future_to_thread)) 205 | 206 | for future in as_completed(future_to_thread): 207 | thread_id, messages = future.result() 208 | if messages is not None: 209 | results.append((thread_id, messages)) 210 | progress.update(task, advance=1) 211 | else: 212 | # No progress bar - just collect results 213 | for future in as_completed(future_to_thread): 214 | thread_id, messages = future.result() 215 | if messages is not None: 216 | results.append((thread_id, messages)) 217 | 218 | # Sort results to match original chronological order from thread_info 219 | thread_id_order = list(thread_info.keys()) 220 | results.sort(key=lambda x: thread_id_order.index(x[0])) 221 | 222 | return results 223 | 224 | 225 | def fetch_latest_trace( 226 | api_key: str, 227 | base_url: str, 228 | project_uuid: str | None = None, 229 | last_n_minutes: int | None = None, 230 | since: str | None = None, 231 | ) -> list[dict[str, Any]]: 232 | """ 233 | Fetch the most recent root trace from LangSmith. 234 | 235 | Uses the LangSmith SDK to list runs and find the latest trace, then 236 | fetches the full messages using the existing fetch_trace function. 237 | 238 | Args: 239 | api_key: LangSmith API key 240 | base_url: LangSmith base URL 241 | project_uuid: Optional project UUID to filter traces (if None, searches all projects) 242 | last_n_minutes: Optional time window in minutes to limit search 243 | since: Optional ISO timestamp string to limit search (e.g., '2025-12-09T10:00:00Z') 244 | 245 | Returns: 246 | List of message dictionaries from the latest trace 247 | 248 | Raises: 249 | ValueError: If no traces found matching criteria 250 | Exception: If API request fails 251 | """ 252 | from datetime import datetime, timedelta, timezone 253 | 254 | from langsmith import Client 255 | 256 | # Initialize langsmith client 257 | client = Client(api_key=api_key) 258 | 259 | # Build filter parameters 260 | filter_params = { 261 | "filter": 'and(eq(is_root, true), neq(status, "pending"))', 262 | "limit": 1, 263 | } 264 | 265 | # Add project filter if provided 266 | if project_uuid is not None: 267 | filter_params["project_id"] = project_uuid 268 | 269 | # Add time filtering if specified 270 | if last_n_minutes is not None: 271 | start_time = datetime.now(timezone.utc) - timedelta(minutes=last_n_minutes) 272 | filter_params["start_time"] = start_time 273 | elif since is not None: 274 | # Parse ISO timestamp 275 | start_time = datetime.fromisoformat(since.replace("Z", "+00:00")) 276 | filter_params["start_time"] = start_time 277 | 278 | # Fetch latest run 279 | runs = list(client.list_runs(**filter_params)) 280 | 281 | if not runs: 282 | raise ValueError("No traces found matching criteria") 283 | 284 | latest_run = runs[0] 285 | trace_id = str(latest_run.id) 286 | 287 | # Reuse existing fetch_trace to get full messages 288 | return fetch_trace(trace_id, base_url=base_url, api_key=api_key) 289 | 290 | 291 | def _fetch_trace_safe( 292 | trace_id: str, base_url: str, api_key: str 293 | ) -> tuple[str, list[dict[str, Any]] | None, Exception | None]: 294 | """Fetch a single trace with error handling. 295 | 296 | Returns: 297 | Tuple of (trace_id, messages or None, error or None) 298 | """ 299 | try: 300 | messages = fetch_trace(trace_id, base_url=base_url, api_key=api_key) 301 | return (trace_id, messages, None) 302 | except Exception as e: 303 | return (trace_id, None, e) 304 | 305 | 306 | def _fetch_traces_concurrent( 307 | runs: list, 308 | base_url: str, 309 | api_key: str, 310 | max_workers: int = 5, 311 | show_progress: bool = True, 312 | include_metadata: bool = False, 313 | include_feedback: bool = False, 314 | ) -> tuple[list[tuple[str, list[dict[str, Any]] | dict[str, Any]]], dict[str, float]]: 315 | """Fetch multiple traces concurrently with optional progress display. 316 | 317 | Args: 318 | runs: List of run objects from client.list_runs() 319 | base_url: LangSmith base URL 320 | api_key: LangSmith API key 321 | max_workers: Maximum number of concurrent requests (default: 5) 322 | show_progress: Whether to show progress bar (default: True) 323 | include_metadata: Whether to include metadata in results (default: False) 324 | include_feedback: Whether to fetch full feedback objects (default: False) 325 | 326 | Returns: 327 | Tuple of (results list, timing_info dict) 328 | If include_metadata=False: results are (trace_id, messages) tuples (backward compatible) 329 | If include_metadata=True: results are (trace_id, trace_data_dict) tuples 330 | """ 331 | results = [] 332 | timing_info = { 333 | "fetch_start": perf_counter(), 334 | "traces_attempted": len(runs), 335 | "traces_succeeded": 0, 336 | "traces_failed": 0, 337 | } 338 | 339 | # Extract metadata from Run objects we already have (no extra API calls!) 340 | run_metadata_map = {} 341 | runs_with_feedback = [] 342 | if include_metadata: 343 | for run in runs: 344 | trace_id = str(run.id) 345 | run_metadata_map[trace_id] = _extract_run_metadata_from_sdk_run(run) 346 | if include_feedback and _sdk_run_has_feedback(run): 347 | runs_with_feedback.append(trace_id) 348 | 349 | # Concurrent fetching with progress (for all traces, including single) 350 | individual_timings = [] 351 | 352 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 353 | # Submit all fetch tasks 354 | future_to_trace = { 355 | executor.submit(_fetch_trace_safe, str(run.id), base_url, api_key): str(run.id) 356 | for run in runs 357 | } 358 | 359 | # Setup progress bar if requested 360 | if show_progress: 361 | progress = Progress( 362 | SpinnerColumn(), 363 | TextColumn("[progress.description]{task.description}"), 364 | BarColumn(), 365 | TaskProgressColumn(), 366 | console=Console(stderr=True), 367 | ) 368 | progress.start() 369 | task = progress.add_task( 370 | f"[cyan]Fetching {len(runs)} traces...", 371 | total=len(runs), 372 | ) 373 | 374 | # Collect results as they complete 375 | for future in as_completed(future_to_trace): 376 | trace_id, messages, error = future.result() 377 | 378 | if error: 379 | msg = f"Warning: Failed to fetch trace {trace_id}: {error}" 380 | if show_progress: 381 | progress.console.print(f"[yellow]{msg}[/yellow]") 382 | else: 383 | print(msg, file=sys.stderr) 384 | timing_info["traces_failed"] += 1 385 | else: 386 | if include_metadata: 387 | trace_data = { 388 | "trace_id": trace_id, 389 | "messages": messages, 390 | "metadata": run_metadata_map.get(trace_id, {}), 391 | "feedback": [], 392 | } 393 | results.append((trace_id, trace_data)) 394 | else: 395 | results.append((trace_id, messages)) 396 | 397 | timing_info["traces_succeeded"] += 1 398 | 399 | if show_progress: 400 | progress.update(task, advance=1) 401 | 402 | if show_progress: 403 | progress.stop() 404 | 405 | timing_info["fetch_duration"] = perf_counter() - timing_info["fetch_start"] 406 | timing_info["individual_timings"] = individual_timings 407 | if timing_info["traces_succeeded"] > 0: 408 | timing_info["avg_per_trace"] = ( 409 | timing_info["fetch_duration"] / timing_info["traces_succeeded"] 410 | ) 411 | 412 | # Batch fetch feedback for all runs that have it 413 | if include_metadata and include_feedback and runs_with_feedback: 414 | feedback_start = perf_counter() 415 | feedback_map = _fetch_feedback_batch(runs_with_feedback, api_key, max_workers) 416 | timing_info["feedback_duration"] = perf_counter() - feedback_start 417 | 418 | # Add feedback to corresponding traces 419 | for idx, (trace_id, trace_data) in enumerate(results): 420 | if trace_id in feedback_map: 421 | trace_data["feedback"] = feedback_map[trace_id] 422 | 423 | return results, timing_info 424 | 425 | 426 | def fetch_recent_traces( 427 | api_key: str, 428 | base_url: str, 429 | limit: int = 1, 430 | project_uuid: str | None = None, 431 | last_n_minutes: int | None = None, 432 | since: str | None = None, 433 | max_workers: int = 5, 434 | show_progress: bool = True, 435 | return_timing: bool = False, 436 | include_metadata: bool = False, 437 | include_feedback: bool = False, 438 | ) -> list[tuple[str, list[dict[str, Any]] | dict[str, Any]]] | tuple[list[tuple[str, list[dict[str, Any]] | dict[str, Any]]], dict]: 439 | """Fetch multiple recent traces from LangSmith with concurrent fetching. 440 | 441 | Searches for recent root traces by chronological timestamp and returns 442 | their messages with metadata and feedback. Uses concurrent fetching for 443 | improved performance when fetching multiple traces. 444 | 445 | Args: 446 | api_key: LangSmith API key for authentication 447 | base_url: LangSmith base URL (e.g., https://api.smith.langchain.com) 448 | limit: Maximum number of traces to fetch (default: 1) 449 | project_uuid: Optional project UUID to filter traces to a specific project. 450 | If not provided, searches across all projects. 451 | last_n_minutes: Optional time window to limit search. Only returns traces 452 | from the last N minutes. Mutually exclusive with `since`. 453 | since: Optional ISO timestamp string (e.g., "2025-12-09T10:00:00Z"). 454 | Only returns traces since this time. Mutually exclusive with `last_n_minutes`. 455 | max_workers: Maximum number of concurrent fetch requests (default: 5) 456 | show_progress: Whether to show progress bar during fetching (default: True) 457 | return_timing: Whether to return timing information along with results (default: False) 458 | include_metadata: Whether to include metadata in results (default: True) 459 | include_feedback: Whether to fetch full feedback objects (default: True) 460 | 461 | Returns: 462 | If return_timing=False (default): 463 | If include_metadata=False: List of (trace_id, messages) tuples 464 | If include_metadata=True: List of (trace_id, trace_data_dict) tuples where 465 | trace_data_dict contains messages, metadata, and feedback 466 | If return_timing=True: 467 | Tuple of (traces list, timing_dict) where timing_dict contains performance metrics. 468 | 469 | Raises: 470 | ValueError: If no traces found matching the criteria 471 | Exception: If API request fails or langsmith package not installed 472 | 473 | Example: 474 | >>> traces = fetch_recent_traces( 475 | ... api_key="lsv2_...", 476 | ... base_url="https://api.smith.langchain.com", 477 | ... limit=5, 478 | ... project_uuid="80f1ecb3-a16b-411e-97ae-1c89adbb5c49", 479 | ... last_n_minutes=30, 480 | ... max_workers=5 481 | ... ) 482 | >>> for trace_id, trace_data in traces: 483 | ... print(f"Trace {trace_id}: {len(trace_data['messages'])} messages") 484 | ... print(f" Status: {trace_data['metadata']['status']}") 485 | ... print(f" Feedback: {len(trace_data['feedback'])} items") 486 | """ 487 | if not HAS_LANGSMITH: 488 | raise Exception( 489 | "langsmith package required for fetching multiple traces. " 490 | "Install with: pip install langsmith" 491 | ) 492 | 493 | from datetime import datetime, timedelta, timezone 494 | 495 | from langsmith import Client 496 | 497 | # Initialize client 498 | client = Client(api_key=api_key) 499 | 500 | # Build filter parameters 501 | filter_params = { 502 | "filter": 'and(eq(is_root, true), neq(status, "pending"))', 503 | "limit": limit, 504 | } 505 | 506 | if project_uuid is not None: 507 | filter_params["project_id"] = project_uuid 508 | 509 | # Add time filtering 510 | if last_n_minutes is not None: 511 | start_time = datetime.now(timezone.utc) - timedelta(minutes=last_n_minutes) 512 | filter_params["start_time"] = start_time 513 | elif since is not None: 514 | # Parse ISO timestamp (handle both 'Z' and explicit timezone) 515 | since_clean = since.replace("Z", "+00:00") 516 | start_time = datetime.fromisoformat(since_clean) 517 | filter_params["start_time"] = start_time 518 | 519 | # Fetch runs 520 | list_start = perf_counter() 521 | runs = list(client.list_runs(**filter_params)) 522 | list_duration = perf_counter() - list_start 523 | 524 | if not runs: 525 | raise ValueError("No traces found matching criteria") 526 | 527 | # Fetch messages (and metadata/feedback if requested) for each trace using concurrent fetching 528 | results, timing_info = _fetch_traces_concurrent( 529 | runs=runs, 530 | base_url=base_url, 531 | api_key=api_key, 532 | max_workers=max_workers, 533 | show_progress=show_progress, 534 | include_metadata=include_metadata, 535 | include_feedback=include_feedback, 536 | ) 537 | 538 | if not results: 539 | raise ValueError( 540 | f"Successfully queried {len(runs)} traces but failed to fetch messages for all of them" 541 | ) 542 | 543 | # Add list_runs timing to timing info 544 | timing_info["list_runs_duration"] = list_duration 545 | timing_info["total_duration"] = list_duration + timing_info["fetch_duration"] 546 | 547 | if return_timing: 548 | return results, timing_info 549 | return results 550 | 551 | 552 | # ============================================================================ 553 | # Metadata and Feedback Extraction Helpers 554 | # ============================================================================ 555 | 556 | 557 | def _extract_run_metadata(run_data: dict) -> dict[str, Any]: 558 | """Extract metadata from a Run object (REST API response). 559 | 560 | Args: 561 | run_data: Run object dict from REST API response 562 | 563 | Returns: 564 | Dictionary with extracted metadata fields 565 | """ 566 | from datetime import datetime 567 | 568 | extra = run_data.get("extra") or {} 569 | custom_metadata = extra.get("metadata") or {} 570 | 571 | # Calculate duration if we have both start and end times 572 | duration_ms = None 573 | start_time = run_data.get("start_time") 574 | end_time = run_data.get("end_time") 575 | if start_time and end_time: 576 | try: 577 | start_dt = datetime.fromisoformat(start_time.replace("Z", "+00:00")) 578 | end_dt = datetime.fromisoformat(end_time.replace("Z", "+00:00")) 579 | duration_ms = int((end_dt - start_dt).total_seconds() * 1000) 580 | except (ValueError, AttributeError): 581 | pass 582 | 583 | return { 584 | "status": run_data.get("status"), 585 | "start_time": start_time, 586 | "end_time": end_time, 587 | "duration_ms": duration_ms, 588 | "custom_metadata": custom_metadata, 589 | "token_usage": { 590 | "prompt_tokens": run_data.get("prompt_tokens"), 591 | "completion_tokens": run_data.get("completion_tokens"), 592 | "total_tokens": run_data.get("total_tokens"), 593 | }, 594 | "costs": { 595 | "prompt_cost": run_data.get("prompt_cost"), 596 | "completion_cost": run_data.get("completion_cost"), 597 | "total_cost": run_data.get("total_cost"), 598 | }, 599 | "first_token_time": run_data.get("first_token_time"), 600 | "feedback_stats": run_data.get("feedback_stats") or {}, 601 | } 602 | 603 | 604 | def _extract_run_metadata_from_sdk_run(run) -> dict[str, Any]: 605 | """Extract metadata from an SDK Run object. 606 | 607 | Args: 608 | run: Run object from langsmith SDK 609 | 610 | Returns: 611 | Dictionary with extracted metadata fields 612 | """ 613 | # Calculate duration if we have both start and end times 614 | duration_ms = None 615 | if hasattr(run, "start_time") and hasattr(run, "end_time") and run.start_time and run.end_time: 616 | try: 617 | duration_ms = int((run.end_time - run.start_time).total_seconds() * 1000) 618 | except (AttributeError, TypeError): 619 | pass 620 | 621 | # Extract custom metadata from extra field 622 | custom_metadata = {} 623 | if hasattr(run, "extra") and run.extra: 624 | custom_metadata = run.extra.get("metadata") or {} 625 | 626 | return { 627 | "status": getattr(run, "status", None), 628 | "start_time": run.start_time.isoformat() if hasattr(run, "start_time") and run.start_time else None, 629 | "end_time": run.end_time.isoformat() if hasattr(run, "end_time") and run.end_time else None, 630 | "duration_ms": duration_ms, 631 | "custom_metadata": custom_metadata, 632 | "token_usage": { 633 | "prompt_tokens": getattr(run, "prompt_tokens", None), 634 | "completion_tokens": getattr(run, "completion_tokens", None), 635 | "total_tokens": getattr(run, "total_tokens", None), 636 | }, 637 | "costs": { 638 | "prompt_cost": getattr(run, "prompt_cost", None), 639 | "completion_cost": getattr(run, "completion_cost", None), 640 | "total_cost": getattr(run, "total_cost", None), 641 | }, 642 | "first_token_time": getattr(run, "first_token_time", None), 643 | "feedback_stats": getattr(run, "feedback_stats", None) or {}, 644 | } 645 | 646 | 647 | def _has_feedback(metadata: dict) -> bool: 648 | """Check if metadata indicates feedback exists. 649 | 650 | Args: 651 | metadata: Metadata dict with feedback_stats 652 | 653 | Returns: 654 | True if feedback_stats shows any feedback exists 655 | """ 656 | feedback_stats = metadata.get("feedback_stats") or {} 657 | if not feedback_stats: 658 | return False 659 | 660 | # Check if any feedback count is positive 661 | return any( 662 | isinstance(v, (int, float)) and v > 0 663 | for v in feedback_stats.values() 664 | ) 665 | 666 | 667 | def _sdk_run_has_feedback(run) -> bool: 668 | """Check if SDK Run object has feedback. 669 | 670 | Args: 671 | run: SDK Run object 672 | 673 | Returns: 674 | True if feedback_stats shows any feedback exists 675 | """ 676 | feedback_stats = getattr(run, "feedback_stats", None) or {} 677 | if not feedback_stats: 678 | return False 679 | 680 | return any( 681 | isinstance(v, (int, float)) and v > 0 682 | for v in feedback_stats.values() 683 | ) 684 | 685 | 686 | def _serialize_feedback(fb) -> dict[str, Any]: 687 | """Convert SDK Feedback object to dictionary. 688 | 689 | Args: 690 | fb: Feedback object from langsmith SDK 691 | 692 | Returns: 693 | Dictionary with feedback fields 694 | """ 695 | return { 696 | "id": str(fb.id) if hasattr(fb, "id") else None, 697 | "key": getattr(fb, "key", None), 698 | "score": getattr(fb, "score", None), 699 | "value": getattr(fb, "value", None), 700 | "comment": getattr(fb, "comment", None), 701 | "correction": getattr(fb, "correction", None), 702 | "created_at": fb.created_at.isoformat() if hasattr(fb, "created_at") and fb.created_at else None, 703 | } 704 | 705 | 706 | def _fetch_feedback(run_id: str, *, api_key: str) -> list[dict[str, Any]]: 707 | """Fetch full feedback objects for a single run. 708 | 709 | Args: 710 | run_id: Run UUID to fetch feedback for 711 | api_key: LangSmith API key 712 | 713 | Returns: 714 | List of feedback dictionaries 715 | """ 716 | if not HAS_LANGSMITH: 717 | return [] 718 | 719 | from langsmith import Client 720 | 721 | try: 722 | client = Client(api_key=api_key) 723 | feedback_list = list(client.list_feedback(run_id=run_id)) 724 | return [_serialize_feedback(fb) for fb in feedback_list] 725 | except Exception as e: 726 | print(f"Warning: Failed to fetch feedback for run {run_id}: {e}", file=sys.stderr) 727 | return [] 728 | 729 | 730 | def _fetch_feedback_batch( 731 | run_ids: list[str], 732 | api_key: str, 733 | max_workers: int = 5, 734 | ) -> dict[str, list[dict[str, Any]]]: 735 | """Fetch feedback for multiple runs concurrently. 736 | 737 | Args: 738 | run_ids: List of run UUIDs to fetch feedback for 739 | api_key: LangSmith API key 740 | max_workers: Maximum concurrent requests (default: 5) 741 | 742 | Returns: 743 | Dictionary mapping run_id -> list of feedback dicts 744 | """ 745 | if not HAS_LANGSMITH or not run_ids: 746 | return {} 747 | 748 | def fetch_single(run_id: str) -> tuple[str, list[dict[str, Any]]]: 749 | """Fetch feedback for a single run with error handling.""" 750 | try: 751 | feedback = _fetch_feedback(run_id, api_key=api_key) 752 | return run_id, feedback 753 | except Exception: 754 | return run_id, [] 755 | 756 | results = {} 757 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 758 | futures = [executor.submit(fetch_single, rid) for rid in run_ids] 759 | for future in as_completed(futures): 760 | run_id, feedback = future.result() 761 | if feedback: 762 | results[run_id] = feedback 763 | 764 | return results 765 | 766 | 767 | # ============================================================================ 768 | # New Fetchers with Metadata and Feedback Support 769 | # ============================================================================ 770 | 771 | 772 | def fetch_trace_with_metadata( 773 | trace_id: str, 774 | *, 775 | base_url: str, 776 | api_key: str, 777 | include_feedback: bool = True, 778 | ) -> dict[str, Any]: 779 | """Fetch trace with metadata and optional feedback. 780 | 781 | Args: 782 | trace_id: LangSmith trace UUID 783 | base_url: LangSmith base URL 784 | api_key: LangSmith API key 785 | include_feedback: Whether to fetch full feedback objects (default: True) 786 | 787 | Returns: 788 | Dictionary with keys: 789 | - trace_id: Trace UUID 790 | - messages: List of message dictionaries 791 | - metadata: Metadata dict with status, timing, tokens, costs, etc. 792 | - feedback: List of feedback dicts (empty if no feedback or include_feedback=False) 793 | 794 | Raises: 795 | requests.HTTPError: If the API request fails 796 | """ 797 | headers = {"X-API-Key": api_key, "Content-Type": "application/json"} 798 | url = f"{base_url}/runs/{trace_id}?include_messages=true" 799 | 800 | response = requests.get(url, headers=headers) 801 | response.raise_for_status() 802 | 803 | data = response.json() 804 | 805 | # Extract messages 806 | messages = data.get("messages") or (data.get("outputs") or {}).get("messages") or [] 807 | 808 | # Extract metadata from the full Run object 809 | metadata = _extract_run_metadata(data) 810 | 811 | # Fetch feedback if requested and feedback exists 812 | feedback = [] 813 | if include_feedback and _has_feedback(metadata): 814 | feedback = _fetch_feedback(trace_id, api_key=api_key) 815 | 816 | return { 817 | "trace_id": trace_id, 818 | "messages": messages, 819 | "metadata": metadata, 820 | "feedback": feedback, 821 | } 822 | 823 | 824 | def fetch_thread_with_metadata( 825 | thread_id: str, 826 | project_uuid: str, 827 | *, 828 | base_url: str, 829 | api_key: str, 830 | include_feedback: bool = True, 831 | ) -> dict[str, Any]: 832 | """Fetch thread with metadata from root run and optional feedback. 833 | 834 | Args: 835 | thread_id: LangGraph thread_id 836 | project_uuid: LangSmith project UUID 837 | base_url: LangSmith base URL 838 | api_key: LangSmith API key 839 | include_feedback: Whether to fetch full feedback objects (default: True) 840 | 841 | Returns: 842 | Dictionary with keys: 843 | - thread_id: Thread ID 844 | - messages: List of message dictionaries 845 | - metadata: Metadata from most recent root run (empty dict if no run found) 846 | - feedback: List of feedback dicts 847 | 848 | Raises: 849 | requests.HTTPError: If the API request fails 850 | """ 851 | # Fetch messages using existing function 852 | messages = fetch_thread(thread_id, project_uuid, base_url=base_url, api_key=api_key) 853 | 854 | # Try to find the root run for this thread to get metadata 855 | metadata = {} 856 | feedback = [] 857 | 858 | if HAS_LANGSMITH: 859 | try: 860 | from langsmith import Client 861 | 862 | client = Client(api_key=api_key) 863 | 864 | # Query for root runs with this thread_id (most recent first) 865 | runs = list( 866 | client.list_runs( 867 | project_id=project_uuid, 868 | filter=f'and(eq(is_root, true), eq(extra.metadata.thread_id, "{thread_id}"))', 869 | limit=1, 870 | ) 871 | ) 872 | 873 | if runs: 874 | root_run = runs[0] 875 | metadata = _extract_run_metadata_from_sdk_run(root_run) 876 | 877 | # Fetch feedback if requested and feedback exists 878 | if include_feedback and _sdk_run_has_feedback(root_run): 879 | feedback = _fetch_feedback(str(root_run.id), api_key=api_key) 880 | 881 | except Exception as e: 882 | print( 883 | f"Warning: Failed to fetch metadata for thread {thread_id}: {e}", 884 | file=sys.stderr, 885 | ) 886 | 887 | return { 888 | "thread_id": thread_id, 889 | "messages": messages, 890 | "metadata": metadata, 891 | "feedback": feedback, 892 | } 893 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | """Tests for CLI commands.""" 2 | 3 | from unittest.mock import patch 4 | 5 | import responses 6 | from click.testing import CliRunner 7 | 8 | from langsmith_cli.cli import main 9 | from tests.conftest import ( 10 | TEST_BASE_URL, 11 | TEST_PROJECT_UUID, 12 | TEST_THREAD_ID, 13 | TEST_TRACE_ID, 14 | ) 15 | 16 | 17 | class TestTraceCommand: 18 | """Tests for trace command.""" 19 | 20 | @responses.activate 21 | def test_trace_default_format(self, sample_trace_response, mock_env_api_key): 22 | """Test trace command with default (pretty) format.""" 23 | responses.add( 24 | responses.GET, 25 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 26 | json=sample_trace_response, 27 | status=200, 28 | ) 29 | 30 | runner = CliRunner() 31 | result = runner.invoke(main, ["trace", TEST_TRACE_ID]) 32 | 33 | assert result.exit_code == 0 34 | # Check for Rich panel indicators 35 | assert "Message 1:" in result.output 36 | assert "human" in result.output.lower() or "user" in result.output.lower() 37 | 38 | @responses.activate 39 | def test_trace_pretty_format(self, sample_trace_response, mock_env_api_key): 40 | """Test trace command with explicit pretty format.""" 41 | responses.add( 42 | responses.GET, 43 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 44 | json=sample_trace_response, 45 | status=200, 46 | ) 47 | 48 | runner = CliRunner() 49 | result = runner.invoke(main, ["trace", TEST_TRACE_ID, "--format", "pretty"]) 50 | 51 | assert result.exit_code == 0 52 | assert "Message 1:" in result.output 53 | 54 | @responses.activate 55 | def test_trace_json_format(self, sample_trace_response, mock_env_api_key): 56 | """Test trace command with json format.""" 57 | responses.add( 58 | responses.GET, 59 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 60 | json=sample_trace_response, 61 | status=200, 62 | ) 63 | 64 | runner = CliRunner() 65 | result = runner.invoke(main, ["trace", TEST_TRACE_ID, "--format", "json"]) 66 | 67 | assert result.exit_code == 0 68 | # Output should be valid JSON with pretty formatting 69 | assert '"type": "human"' in result.output or '"type": "user"' in result.output 70 | # Check for content from the email (should be in the JSON somewhere) 71 | assert "jane" in result.output.lower() # Case-insensitive check 72 | 73 | @responses.activate 74 | def test_trace_raw_format(self, sample_trace_response, mock_env_api_key): 75 | """Test trace command with raw format.""" 76 | responses.add( 77 | responses.GET, 78 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 79 | json=sample_trace_response, 80 | status=200, 81 | ) 82 | 83 | runner = CliRunner() 84 | result = runner.invoke(main, ["trace", TEST_TRACE_ID, "--format", "raw"]) 85 | 86 | assert result.exit_code == 0 87 | # Should contain JSON array markers and message content 88 | assert "[" in result.output 89 | assert "]" in result.output 90 | assert "type" in result.output or "role" in result.output 91 | 92 | def test_trace_no_api_key(self, monkeypatch): 93 | """Test trace command fails without API key.""" 94 | monkeypatch.delenv("LANGSMITH_API_KEY", raising=False) 95 | 96 | runner = CliRunner() 97 | result = runner.invoke(main, ["trace", TEST_TRACE_ID]) 98 | 99 | assert result.exit_code == 1 100 | assert "LANGSMITH_API_KEY not found" in result.output 101 | 102 | @responses.activate 103 | def test_trace_api_error(self, mock_env_api_key): 104 | """Test trace command handles API errors.""" 105 | responses.add( 106 | responses.GET, 107 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}", 108 | json={"error": "Not found"}, 109 | status=404, 110 | ) 111 | 112 | runner = CliRunner() 113 | result = runner.invoke(main, ["trace", TEST_TRACE_ID]) 114 | 115 | assert result.exit_code == 1 116 | assert "Error fetching trace" in result.output 117 | 118 | @responses.activate 119 | def test_trace_with_metadata_flag(self, sample_trace_response, mock_env_api_key): 120 | """Test trace command with --include-metadata flag.""" 121 | responses.add( 122 | responses.GET, 123 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}?include_messages=true", 124 | json=sample_trace_response, 125 | status=200, 126 | ) 127 | 128 | runner = CliRunner() 129 | result = runner.invoke( 130 | main, ["trace", TEST_TRACE_ID, "--include-metadata", "--format", "json"] 131 | ) 132 | 133 | assert result.exit_code == 0 134 | # When metadata is included, output should contain metadata structure 135 | assert "metadata" in result.output or "trace_id" in result.output 136 | 137 | @responses.activate 138 | def test_trace_without_metadata_default( 139 | self, sample_trace_response, mock_env_api_key 140 | ): 141 | """Test trace command defaults to no metadata.""" 142 | responses.add( 143 | responses.GET, 144 | f"https://api.smith.langchain.com/runs/{TEST_TRACE_ID}?include_messages=true", 145 | json=sample_trace_response, 146 | status=200, 147 | ) 148 | 149 | runner = CliRunner() 150 | result = runner.invoke(main, ["trace", TEST_TRACE_ID, "--format", "json"]) 151 | 152 | assert result.exit_code == 0 153 | # Without flags, should just return messages array 154 | output_lower = result.output.lower() 155 | # Check that it contains message content but not metadata wrapper 156 | assert "jane" in output_lower 157 | 158 | 159 | class TestThreadCommand: 160 | """Tests for thread command.""" 161 | 162 | @responses.activate 163 | def test_thread_default_format_with_config( 164 | self, sample_thread_response, mock_env_api_key, temp_config_dir, monkeypatch 165 | ): 166 | """Test thread command with default format and config.""" 167 | # Clear env vars to test config fallback 168 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 169 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 170 | 171 | # Set up config 172 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 173 | with patch( 174 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 175 | ): 176 | from langsmith_cli.config import set_config_value 177 | 178 | set_config_value("project-uuid", TEST_PROJECT_UUID) 179 | 180 | responses.add( 181 | responses.GET, 182 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 183 | json=sample_thread_response, 184 | status=200, 185 | ) 186 | 187 | runner = CliRunner() 188 | result = runner.invoke(main, ["thread", TEST_THREAD_ID]) 189 | 190 | assert result.exit_code == 0 191 | assert "Message 1:" in result.output 192 | 193 | @responses.activate 194 | def test_thread_pretty_format( 195 | self, sample_thread_response, mock_env_api_key, temp_config_dir, monkeypatch 196 | ): 197 | """Test thread command with explicit pretty format.""" 198 | # Clear env vars to test config fallback 199 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 200 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 201 | 202 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 203 | with patch( 204 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 205 | ): 206 | from langsmith_cli.config import set_config_value 207 | 208 | set_config_value("project-uuid", TEST_PROJECT_UUID) 209 | 210 | responses.add( 211 | responses.GET, 212 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 213 | json=sample_thread_response, 214 | status=200, 215 | ) 216 | 217 | runner = CliRunner() 218 | result = runner.invoke( 219 | main, ["thread", TEST_THREAD_ID, "--format", "pretty"] 220 | ) 221 | 222 | assert result.exit_code == 0 223 | assert "Message 1:" in result.output 224 | 225 | @responses.activate 226 | def test_thread_json_format( 227 | self, sample_thread_response, mock_env_api_key, temp_config_dir, monkeypatch 228 | ): 229 | """Test thread command with json format.""" 230 | # Clear env vars to test config fallback 231 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 232 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 233 | 234 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 235 | with patch( 236 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 237 | ): 238 | from langsmith_cli.config import set_config_value 239 | 240 | set_config_value("project-uuid", TEST_PROJECT_UUID) 241 | 242 | responses.add( 243 | responses.GET, 244 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 245 | json=sample_thread_response, 246 | status=200, 247 | ) 248 | 249 | runner = CliRunner() 250 | result = runner.invoke( 251 | main, ["thread", TEST_THREAD_ID, "--format", "json"] 252 | ) 253 | 254 | assert result.exit_code == 0 255 | assert '"role":' in result.output 256 | 257 | @responses.activate 258 | def test_thread_raw_format( 259 | self, sample_thread_response, mock_env_api_key, temp_config_dir, monkeypatch 260 | ): 261 | """Test thread command with raw format.""" 262 | # Clear env vars to test config fallback 263 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 264 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 265 | 266 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 267 | with patch( 268 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 269 | ): 270 | from langsmith_cli.config import set_config_value 271 | 272 | set_config_value("project-uuid", TEST_PROJECT_UUID) 273 | 274 | responses.add( 275 | responses.GET, 276 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 277 | json=sample_thread_response, 278 | status=200, 279 | ) 280 | 281 | runner = CliRunner() 282 | result = runner.invoke( 283 | main, ["thread", TEST_THREAD_ID, "--format", "raw"] 284 | ) 285 | 286 | assert result.exit_code == 0 287 | # Should contain JSON array markers and message content 288 | assert "[" in result.output 289 | assert "]" in result.output 290 | assert "role" in result.output or "type" in result.output 291 | 292 | @responses.activate 293 | def test_thread_with_project_uuid_override( 294 | self, sample_thread_response, mock_env_api_key 295 | ): 296 | """Test thread command with --project-uuid override.""" 297 | responses.add( 298 | responses.GET, 299 | f"https://api.smith.langchain.com/runs/threads/{TEST_THREAD_ID}", 300 | json=sample_thread_response, 301 | status=200, 302 | ) 303 | 304 | runner = CliRunner() 305 | result = runner.invoke( 306 | main, ["thread", TEST_THREAD_ID, "--project-uuid", TEST_PROJECT_UUID] 307 | ) 308 | 309 | assert result.exit_code == 0 310 | 311 | def test_thread_no_project_uuid(self, mock_env_api_key, temp_config_dir): 312 | """Test thread command fails without project UUID.""" 313 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 314 | with patch( 315 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 316 | ): 317 | runner = CliRunner() 318 | result = runner.invoke(main, ["thread", TEST_THREAD_ID]) 319 | 320 | assert result.exit_code == 1 321 | assert "project-uuid required" in result.output 322 | 323 | def test_thread_no_api_key(self, monkeypatch, temp_config_dir): 324 | """Test thread command fails without API key.""" 325 | monkeypatch.delenv("LANGSMITH_API_KEY", raising=False) 326 | 327 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 328 | with patch( 329 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 330 | ): 331 | from langsmith_cli.config import set_config_value 332 | 333 | set_config_value("project-uuid", TEST_PROJECT_UUID) 334 | 335 | runner = CliRunner() 336 | result = runner.invoke(main, ["thread", TEST_THREAD_ID]) 337 | 338 | assert result.exit_code == 1 339 | assert "LANGSMITH_API_KEY not found" in result.output 340 | 341 | 342 | class TestThreadsCommand: 343 | """Tests for threads command.""" 344 | 345 | @responses.activate 346 | def test_threads_default_limit( 347 | self, sample_thread_response, mock_env_api_key, temp_config_dir, tmp_path, monkeypatch 348 | ): 349 | """Test threads command with default limit (1).""" 350 | # Clear env vars to test config fallback 351 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 352 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 353 | 354 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 355 | with patch( 356 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 357 | ): 358 | from langsmith_cli.config import set_config_value 359 | 360 | set_config_value("project-uuid", TEST_PROJECT_UUID) 361 | 362 | # Mock the runs query endpoint 363 | responses.add( 364 | responses.POST, 365 | f"{TEST_BASE_URL}/runs/query", 366 | json={ 367 | "runs": [ 368 | { 369 | "id": "run-1", 370 | "start_time": "2024-01-01T00:00:00Z", 371 | "extra": {"metadata": {"thread_id": "thread-1"}}, 372 | }, 373 | { 374 | "id": "run-2", 375 | "start_time": "2024-01-02T00:00:00Z", 376 | "extra": {"metadata": {"thread_id": "thread-2"}}, 377 | }, 378 | ] 379 | }, 380 | status=200, 381 | ) 382 | 383 | # Mock the thread fetch endpoint 384 | responses.add( 385 | responses.GET, 386 | f"{TEST_BASE_URL}/runs/threads/thread-1", 387 | json=sample_thread_response, 388 | status=200, 389 | ) 390 | 391 | runner = CliRunner() 392 | output_dir = tmp_path / "threads" 393 | result = runner.invoke(main, ["threads", str(output_dir)]) 394 | 395 | assert result.exit_code == 0 396 | assert "Found 1 thread(s)" in result.output 397 | assert "Successfully saved 1 thread(s)" in result.output 398 | 399 | # Check that only one file was created (default limit is 1) 400 | assert (output_dir / "thread-1.json").exists() 401 | assert not (output_dir / "thread-2.json").exists() 402 | 403 | @responses.activate 404 | def test_threads_custom_limit( 405 | self, sample_thread_response, mock_env_api_key, temp_config_dir, tmp_path, monkeypatch 406 | ): 407 | """Test threads command with custom limit.""" 408 | # Clear env vars to test config fallback 409 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 410 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 411 | 412 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 413 | with patch( 414 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 415 | ): 416 | from langsmith_cli.config import set_config_value 417 | 418 | set_config_value("project-uuid", TEST_PROJECT_UUID) 419 | 420 | # Mock the runs query endpoint 421 | responses.add( 422 | responses.POST, 423 | f"{TEST_BASE_URL}/runs/query", 424 | json={ 425 | "runs": [ 426 | { 427 | "id": "run-1", 428 | "start_time": "2024-01-01T00:00:00Z", 429 | "extra": {"metadata": {"thread_id": "thread-1"}}, 430 | } 431 | ] 432 | }, 433 | status=200, 434 | ) 435 | 436 | # Mock the thread fetch endpoint 437 | responses.add( 438 | responses.GET, 439 | f"{TEST_BASE_URL}/runs/threads/thread-1", 440 | json=sample_thread_response, 441 | status=200, 442 | ) 443 | 444 | runner = CliRunner() 445 | output_dir = tmp_path / "threads" 446 | result = runner.invoke( 447 | main, ["threads", str(output_dir), "--limit", "5"] 448 | ) 449 | 450 | assert result.exit_code == 0 451 | assert "thread-1" in result.output 452 | 453 | def test_threads_no_project_uuid(self, mock_env_api_key, temp_config_dir, tmp_path): 454 | """Test threads command fails without project UUID.""" 455 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 456 | with patch( 457 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 458 | ): 459 | runner = CliRunner() 460 | output_dir = tmp_path / "threads" 461 | result = runner.invoke(main, ["threads", str(output_dir)]) 462 | 463 | assert result.exit_code == 1 464 | assert "project-uuid required" in result.output 465 | 466 | @responses.activate 467 | def test_threads_custom_filename_pattern( 468 | self, sample_thread_response, mock_env_api_key, temp_config_dir, tmp_path, monkeypatch 469 | ): 470 | """Test threads command with custom filename pattern.""" 471 | # Clear env vars to test config fallback 472 | monkeypatch.delenv("LANGSMITH_PROJECT", raising=False) 473 | monkeypatch.delenv("LANGSMITH_PROJECT_UUID", raising=False) 474 | 475 | with patch("langsmith_cli.config.CONFIG_DIR", temp_config_dir): 476 | with patch( 477 | "langsmith_cli.config.CONFIG_FILE", temp_config_dir / "config.yaml" 478 | ): 479 | from langsmith_cli.config import set_config_value 480 | 481 | set_config_value("project-uuid", TEST_PROJECT_UUID) 482 | 483 | # Mock the runs query endpoint 484 | responses.add( 485 | responses.POST, 486 | f"{TEST_BASE_URL}/runs/query", 487 | json={ 488 | "runs": [ 489 | { 490 | "id": "run-1", 491 | "start_time": "2024-01-01T00:00:00Z", 492 | "extra": {"metadata": {"thread_id": "thread-1"}}, 493 | }, 494 | { 495 | "id": "run-2", 496 | "start_time": "2024-01-02T00:00:00Z", 497 | "extra": {"metadata": {"thread_id": "thread-2"}}, 498 | }, 499 | ] 500 | }, 501 | status=200, 502 | ) 503 | 504 | # Mock the thread fetch endpoints 505 | for thread_id in ["thread-1", "thread-2"]: 506 | responses.add( 507 | responses.GET, 508 | f"{TEST_BASE_URL}/runs/threads/{thread_id}", 509 | json=sample_thread_response, 510 | status=200, 511 | ) 512 | 513 | runner = CliRunner() 514 | output_dir = tmp_path / "threads" 515 | result = runner.invoke( 516 | main, 517 | [ 518 | "threads", 519 | str(output_dir), 520 | "--limit", 521 | "2", 522 | "--filename-pattern", 523 | "thread_{index:03d}.json", 524 | ], 525 | ) 526 | 527 | assert result.exit_code == 0 528 | assert "Found 2 thread(s)" in result.output 529 | 530 | # Check that files were created with custom pattern 531 | assert (output_dir / "thread_001.json").exists() 532 | assert (output_dir / "thread_002.json").exists() 533 | 534 | def test_threads_rejects_uuid_as_directory(self, mock_env_api_key): 535 | """Test threads command rejects UUID passed as directory.""" 536 | runner = CliRunner() 537 | # Pass a valid UUID instead of a directory path 538 | fake_uuid = "3a12d0b2-bda5-4500-8732-c1984f647df5" 539 | result = runner.invoke( 540 | main, ["threads", fake_uuid, "--project-uuid", TEST_PROJECT_UUID] 541 | ) 542 | 543 | assert result.exit_code == 1 544 | assert "looks like a UUID" in result.output 545 | assert "langsmith-fetch thread " in result.output 546 | assert "langsmith-fetch threads " in result.output 547 | 548 | 549 | class TestTracesCommand: 550 | """Tests for traces command.""" 551 | 552 | @responses.activate 553 | def test_traces_default_no_metadata( 554 | self, sample_trace_response, mock_env_api_key, temp_config_dir, tmp_path 555 | ): 556 | """Test traces command with directory output and default (no metadata).""" 557 | # Mock langsmith import 558 | with patch("langsmith_cli.fetchers.HAS_LANGSMITH", True): 559 | # Mock the /info endpoint (called by Client initialization) 560 | responses.add( 561 | responses.GET, 562 | f"{TEST_BASE_URL}/info", 563 | json={"version": "1.0"}, 564 | status=200, 565 | ) 566 | 567 | # Mock the runs query endpoint (called by Client.list_runs) 568 | responses.add( 569 | responses.POST, 570 | f"{TEST_BASE_URL}/runs/query", 571 | json={ 572 | "runs": [ 573 | { 574 | "id": "3b0b15fe-1e3a-4aef-afa8-48df15879cfe", 575 | "name": "test_run", 576 | "start_time": "2024-01-01T00:00:00Z", 577 | "run_type": "chain", 578 | "trace_id": "3b0b15fe-1e3a-4aef-afa8-48df15879cfe", 579 | } 580 | ] 581 | }, 582 | status=200, 583 | ) 584 | 585 | # Mock the trace fetch endpoint 586 | trace_id = "3b0b15fe-1e3a-4aef-afa8-48df15879cfe" 587 | responses.add( 588 | responses.GET, 589 | f"{TEST_BASE_URL}/runs/{trace_id}", 590 | json=sample_trace_response, 591 | status=200, 592 | ) 593 | 594 | runner = CliRunner() 595 | output_dir = tmp_path / "traces" 596 | result = runner.invoke(main, ["traces", str(output_dir), "--limit", "1"]) 597 | 598 | assert result.exit_code == 0 599 | assert "Found 1 trace(s)" in result.output 600 | assert "Successfully saved 1 trace(s)" in result.output 601 | assert "3 messages" in result.output # Should show message count 602 | 603 | # Check that file was created and contains list (not dict) 604 | import json 605 | 606 | trace_file = output_dir / f"{trace_id}.json" 607 | assert trace_file.exists() 608 | with open(trace_file) as f: 609 | data = json.load(f) 610 | assert isinstance(data, list) # Should be list when no metadata 611 | 612 | @responses.activate 613 | def test_traces_with_metadata( 614 | self, sample_trace_response, mock_env_api_key, temp_config_dir, tmp_path 615 | ): 616 | """Test traces command with --include-metadata flag.""" 617 | with patch("langsmith_cli.fetchers.HAS_LANGSMITH", True): 618 | # Mock the /info endpoint 619 | responses.add( 620 | responses.GET, 621 | f"{TEST_BASE_URL}/info", 622 | json={"version": "1.0"}, 623 | status=200, 624 | ) 625 | 626 | # Mock the runs query endpoint with metadata fields 627 | trace_id = "3b0b15fe-1e3a-4aef-afa8-48df15879cfe" 628 | responses.add( 629 | responses.POST, 630 | f"{TEST_BASE_URL}/runs/query", 631 | json={ 632 | "runs": [ 633 | { 634 | "id": trace_id, 635 | "name": "test_run", 636 | "start_time": "2024-01-01T00:00:00Z", 637 | "end_time": "2024-01-01T00:01:00Z", 638 | "run_type": "chain", 639 | "trace_id": trace_id, 640 | "status": "success", 641 | } 642 | ] 643 | }, 644 | status=200, 645 | ) 646 | 647 | responses.add( 648 | responses.GET, 649 | f"{TEST_BASE_URL}/runs/{trace_id}", 650 | json=sample_trace_response, 651 | status=200, 652 | ) 653 | 654 | runner = CliRunner() 655 | output_dir = tmp_path / "traces" 656 | result = runner.invoke( 657 | main, ["traces", str(output_dir), "--limit", "1", "--include-metadata"] 658 | ) 659 | 660 | assert result.exit_code == 0 661 | assert "Found 1 trace(s)" in result.output 662 | assert "3 messages, status:" in result.output # Should show status 663 | 664 | # Check that file contains dict with metadata 665 | import json 666 | 667 | trace_file = output_dir / f"{trace_id}.json" 668 | assert trace_file.exists() 669 | with open(trace_file) as f: 670 | data = json.load(f) 671 | assert isinstance(data, dict) 672 | assert "messages" in data 673 | assert "metadata" in data 674 | assert "feedback" in data 675 | assert len(data["messages"]) == 3 676 | 677 | @responses.activate 678 | def test_traces_custom_limit( 679 | self, sample_trace_response, mock_env_api_key, temp_config_dir, tmp_path 680 | ): 681 | """Test traces command with custom limit.""" 682 | with patch("langsmith_cli.fetchers.HAS_LANGSMITH", True): 683 | # Mock the /info endpoint 684 | responses.add( 685 | responses.GET, 686 | f"{TEST_BASE_URL}/info", 687 | json={"version": "1.0"}, 688 | status=200, 689 | ) 690 | 691 | # Mock the runs query endpoint 692 | trace_ids = [ 693 | "3b0b15fe-1e3a-4aef-afa8-48df15879cf1", 694 | "3b0b15fe-1e3a-4aef-afa8-48df15879cf2", 695 | "3b0b15fe-1e3a-4aef-afa8-48df15879cf3", 696 | ] 697 | responses.add( 698 | responses.POST, 699 | f"{TEST_BASE_URL}/runs/query", 700 | json={ 701 | "runs": [ 702 | { 703 | "id": tid, 704 | "name": f"test_run_{i}", 705 | "start_time": "2024-01-01T00:00:00Z", 706 | "run_type": "chain", 707 | "trace_id": tid, 708 | } 709 | for i, tid in enumerate(trace_ids, 1) 710 | ] 711 | }, 712 | status=200, 713 | ) 714 | 715 | # Mock trace fetch endpoints 716 | for tid in trace_ids: 717 | responses.add( 718 | responses.GET, 719 | f"{TEST_BASE_URL}/runs/{tid}", 720 | json=sample_trace_response, 721 | status=200, 722 | ) 723 | 724 | runner = CliRunner() 725 | output_dir = tmp_path / "traces" 726 | result = runner.invoke(main, ["traces", str(output_dir), "--limit", "3"]) 727 | 728 | assert result.exit_code == 0 729 | assert "Found 3 trace(s)" in result.output 730 | assert "Successfully saved 3 trace(s)" in result.output 731 | 732 | # Check that all files were created 733 | for tid in trace_ids: 734 | assert (output_dir / f"{tid}.json").exists() 735 | 736 | @responses.activate 737 | def test_traces_custom_filename_pattern( 738 | self, sample_trace_response, mock_env_api_key, temp_config_dir, tmp_path 739 | ): 740 | """Test traces command with custom filename pattern.""" 741 | with patch("langsmith_cli.fetchers.HAS_LANGSMITH", True): 742 | # Mock the /info endpoint 743 | responses.add( 744 | responses.GET, 745 | f"{TEST_BASE_URL}/info", 746 | json={"version": "1.0"}, 747 | status=200, 748 | ) 749 | 750 | # Mock the runs query endpoint 751 | trace_ids = [ 752 | "3b0b15fe-1e3a-4aef-afa8-48df15879cf1", 753 | "3b0b15fe-1e3a-4aef-afa8-48df15879cf2", 754 | ] 755 | responses.add( 756 | responses.POST, 757 | f"{TEST_BASE_URL}/runs/query", 758 | json={ 759 | "runs": [ 760 | { 761 | "id": tid, 762 | "name": f"test_run_{i}", 763 | "start_time": "2024-01-01T00:00:00Z", 764 | "run_type": "chain", 765 | "trace_id": tid, 766 | } 767 | for i, tid in enumerate(trace_ids, 1) 768 | ] 769 | }, 770 | status=200, 771 | ) 772 | 773 | # Mock trace fetch endpoints 774 | for tid in trace_ids: 775 | responses.add( 776 | responses.GET, 777 | f"{TEST_BASE_URL}/runs/{tid}", 778 | json=sample_trace_response, 779 | status=200, 780 | ) 781 | 782 | runner = CliRunner() 783 | output_dir = tmp_path / "traces" 784 | result = runner.invoke( 785 | main, 786 | [ 787 | "traces", 788 | str(output_dir), 789 | "--limit", 790 | "2", 791 | "--filename-pattern", 792 | "trace_{index:03d}.json", 793 | ], 794 | ) 795 | 796 | assert result.exit_code == 0 797 | assert "Found 2 trace(s)" in result.output 798 | 799 | # Check that files were created with custom pattern 800 | assert (output_dir / "trace_001.json").exists() 801 | assert (output_dir / "trace_002.json").exists() 802 | 803 | @responses.activate 804 | def test_traces_with_project_uuid( 805 | self, sample_trace_response, mock_env_api_key, temp_config_dir, tmp_path 806 | ): 807 | """Test traces command with --project-uuid filter.""" 808 | with patch("langsmith_cli.fetchers.HAS_LANGSMITH", True): 809 | # Mock the /info endpoint 810 | responses.add( 811 | responses.GET, 812 | f"{TEST_BASE_URL}/info", 813 | json={"version": "1.0"}, 814 | status=200, 815 | ) 816 | 817 | # Mock the runs query endpoint 818 | trace_id = "3b0b15fe-1e3a-4aef-afa8-48df15879cfe" 819 | responses.add( 820 | responses.POST, 821 | f"{TEST_BASE_URL}/runs/query", 822 | json={ 823 | "runs": [ 824 | { 825 | "id": trace_id, 826 | "name": "test_run", 827 | "start_time": "2024-01-01T00:00:00Z", 828 | "run_type": "chain", 829 | "trace_id": trace_id, 830 | } 831 | ] 832 | }, 833 | status=200, 834 | ) 835 | 836 | responses.add( 837 | responses.GET, 838 | f"{TEST_BASE_URL}/runs/{trace_id}", 839 | json=sample_trace_response, 840 | status=200, 841 | ) 842 | 843 | runner = CliRunner() 844 | output_dir = tmp_path / "traces" 845 | result = runner.invoke( 846 | main, 847 | [ 848 | "traces", 849 | str(output_dir), 850 | "--limit", 851 | "1", 852 | "--project-uuid", 853 | TEST_PROJECT_UUID, 854 | ], 855 | ) 856 | 857 | assert result.exit_code == 0 858 | assert "Found 1 trace(s)" in result.output 859 | 860 | def test_traces_rejects_uuid_as_directory(self, mock_env_api_key): 861 | """Test traces command rejects UUID passed as directory.""" 862 | runner = CliRunner() 863 | # Pass a valid UUID instead of a directory path 864 | fake_uuid = "3a12d0b2-bda5-4500-8732-c1984f647df5" 865 | result = runner.invoke(main, ["traces", fake_uuid, "--include-metadata"]) 866 | 867 | assert result.exit_code == 1 868 | assert "looks like a trace ID" in result.output 869 | assert "langsmith-fetch trace " in result.output 870 | assert "langsmith-fetch traces " in result.output 871 | -------------------------------------------------------------------------------- /src/langsmith_cli/cli.py: -------------------------------------------------------------------------------- 1 | """Main CLI interface using Click.""" 2 | 3 | import json 4 | import os 5 | import re 6 | import sys 7 | from pathlib import Path 8 | 9 | import click 10 | 11 | from . import config, fetchers, formatters 12 | 13 | 14 | def sanitize_filename(filename: str) -> str: 15 | """Sanitize a string to be used as a safe filename. 16 | 17 | Removes or replaces characters that are not safe for filenames across platforms. 18 | 19 | Args: 20 | filename: The original filename string 21 | 22 | Returns: 23 | A sanitized filename safe for all platforms 24 | """ 25 | # Remove or replace unsafe characters 26 | # Keep alphanumeric, hyphens, underscores, and dots 27 | safe_name = re.sub(r"[^\w\-.]", "_", filename) 28 | # Remove leading/trailing dots and spaces 29 | safe_name = safe_name.strip(". ") 30 | # Limit length to 255 characters (filesystem limit) 31 | if len(safe_name) > 255: 32 | safe_name = safe_name[:255] 33 | return safe_name 34 | 35 | 36 | @click.group() 37 | def main(): 38 | """LangSmith Fetch - Fetch and display LangSmith threads and traces. 39 | 40 | This CLI tool retrieves conversation messages, traces, and threads from LangSmith. 41 | 42 | REQUIREMENTS: 43 | - LANGSMITH_API_KEY environment variable or stored in config 44 | - Project UUID (required for threads, optional for traces) 45 | 46 | COMMON COMMANDS: 47 | langsmith-fetch trace # Fetch a specific trace by ID 48 | langsmith-fetch thread # Fetch a specific thread by ID 49 | langsmith-fetch traces ./dir --limit 10 # Fetch 10 traces to directory (RECOMMENDED) 50 | langsmith-fetch threads ./dir --limit 10 # Fetch 10 threads to directory (RECOMMENDED) 51 | 52 | CONFIGURATION: 53 | langsmith-fetch config show # View current configuration 54 | langsmith-fetch config set project-uuid # Configure project UUID 55 | langsmith-fetch config set api-key # Store API key in config 56 | 57 | NOTE: When using 'traces' or 'threads' commands, ALWAYS specify an output directory 58 | unless you explicitly want to print to stdout. Use directory mode for typical usage. 59 | 60 | OUTPUT FORMATS: 61 | --format pretty Human-readable with Rich panels (default) 62 | --format json Pretty-printed JSON with syntax highlighting 63 | --format raw Compact single-line JSON for piping 64 | 65 | FOR LLMs AND AUTOMATION: 66 | When fetching data programmatically, use these guidelines: 67 | 68 | FETCH BY ID (when you have a specific trace/thread ID): 69 | langsmith-fetch trace --format raw 70 | langsmith-fetch thread --format raw 71 | 72 | BULK FETCH MULTIPLE (saves to directory - RECOMMENDED DEFAULT): 73 | langsmith-fetch traces ./output-folder --limit 10 74 | langsmith-fetch threads ./output-folder --limit 10 75 | # Creates one JSON file per trace/thread in output-folder/ 76 | # USE THIS BY DEFAULT unless user explicitly asks for stdout 77 | 78 | FETCH TO STDOUT (only if user explicitly requests it): 79 | langsmith-fetch traces --project-uuid --format raw 80 | langsmith-fetch threads --project-uuid --format raw 81 | NOTE: For threads, --project-uuid is REQUIRED 82 | 83 | WORKFLOW EXAMPLES: 84 | # Bulk export to directory (RECOMMENDED - use this by default) 85 | langsmith-fetch traces ./my-traces --limit 10 86 | langsmith-fetch threads ./my-threads --limit 25 87 | 88 | # Fetch to stdout (only if user explicitly wants stdout output) 89 | langsmith-fetch traces --limit 5 --format json 90 | langsmith-fetch threads --limit 5 --format json 91 | 92 | # Quick inspection of single item 93 | langsmith-fetch trace 94 | langsmith-fetch thread 95 | """ 96 | pass 97 | 98 | 99 | @main.command() 100 | @click.argument("thread_id", metavar="THREAD_ID") 101 | @click.option( 102 | "--project-uuid", 103 | metavar="UUID", 104 | help="LangSmith project UUID (overrides config). Find in UI or via trace session_id.", 105 | ) 106 | @click.option( 107 | "--format", 108 | "format_type", 109 | type=click.Choice(["raw", "json", "pretty"]), 110 | help="Output format: raw (compact JSON), json (pretty JSON), pretty (human-readable panels)", 111 | ) 112 | @click.option( 113 | "--file", 114 | "output_file", 115 | metavar="PATH", 116 | help="Save output to file instead of printing to stdout", 117 | ) 118 | def thread(thread_id, project_uuid, format_type, output_file): 119 | """Fetch messages for a LangGraph thread by thread_id. 120 | 121 | A thread represents a conversation or session containing multiple traces. Each 122 | trace in the thread represents one turn or execution. This command retrieves 123 | all messages from all traces in the thread. 124 | 125 | \b 126 | ARGUMENTS: 127 | THREAD_ID LangGraph thread identifier (e.g., 'test-email-agent-thread') 128 | 129 | \b 130 | RETURNS: 131 | List of all messages from all traces in the thread, ordered chronologically. 132 | 133 | \b 134 | EXAMPLES: 135 | # Fetch thread with project UUID from config 136 | langsmith-fetch thread test-email-agent-thread 137 | 138 | # Fetch thread with explicit project UUID 139 | langsmith-fetch thread my-thread --project-uuid 80f1ecb3-a16b-411e-97ae-1c89adbb5c49 140 | 141 | # Fetch thread as JSON for parsing 142 | langsmith-fetch thread test-email-agent-thread --format json 143 | 144 | \b 145 | PREREQUISITES: 146 | - LANGSMITH_API_KEY environment variable must be set, or 147 | API key stored via: langsmith-fetch config set api-key 148 | - Project UUID must be set via: langsmith-fetch config set project-uuid 149 | or provided with --project-uuid option 150 | 151 | \b 152 | FINDING PROJECT UUID: 153 | The project UUID can be found in the LangSmith UI or programmatically: 154 | from langsmith import Client 155 | run = Client().read_run('') 156 | print(run.session_id) # This is your project UUID 157 | """ 158 | 159 | # Get API key 160 | base_url = config.get_base_url() 161 | api_key = config.get_api_key() 162 | if not api_key: 163 | click.echo( 164 | "Error: LANGSMITH_API_KEY not found in environment or config", err=True 165 | ) 166 | sys.exit(1) 167 | 168 | # Get project UUID (from option or config) 169 | if not project_uuid: 170 | project_uuid = config.get_project_uuid() 171 | 172 | if not project_uuid: 173 | click.echo( 174 | "Error: project-uuid required. Pass --project-uuid flag", 175 | err=True, 176 | ) 177 | sys.exit(1) 178 | 179 | # Get format (from option or config) 180 | if not format_type: 181 | format_type = config.get_default_format() 182 | 183 | try: 184 | # Fetch thread with metadata and feedback 185 | thread_data = fetchers.fetch_thread_with_metadata( 186 | thread_id, project_uuid, base_url=base_url, api_key=api_key 187 | ) 188 | 189 | # Output with metadata and feedback 190 | formatters.print_formatted_trace(thread_data, format_type, output_file) 191 | 192 | except Exception as e: 193 | click.echo(f"Error fetching thread: {e}", err=True) 194 | sys.exit(1) 195 | 196 | 197 | @main.command() 198 | @click.argument("trace_id", metavar="TRACE_ID") 199 | @click.option( 200 | "--format", 201 | "format_type", 202 | type=click.Choice(["raw", "json", "pretty"]), 203 | help="Output format: raw (compact JSON), json (pretty JSON), pretty (human-readable panels)", 204 | ) 205 | @click.option( 206 | "--file", 207 | "output_file", 208 | metavar="PATH", 209 | help="Save output to file instead of printing to stdout", 210 | ) 211 | @click.option( 212 | "--include-metadata", 213 | is_flag=True, 214 | default=False, 215 | help="Include run metadata (status, timing, tokens, costs) in output", 216 | ) 217 | @click.option( 218 | "--include-feedback", 219 | is_flag=True, 220 | default=False, 221 | help="Include feedback data in output (requires extra API call)", 222 | ) 223 | def trace(trace_id, format_type, output_file, include_metadata, include_feedback): 224 | """Fetch messages for a single trace by trace ID. 225 | 226 | A trace represents a single execution path containing multiple runs (LLM calls, 227 | tool executions). This command retrieves all messages from that trace. 228 | 229 | \b 230 | ARGUMENTS: 231 | TRACE_ID LangSmith trace UUID (e.g., 3b0b15fe-1e3a-4aef-afa8-48df15879cfe) 232 | 233 | \b 234 | RETURNS: 235 | List of messages with role, content, tool calls (default). 236 | With --include-metadata: Dictionary with messages, metadata, and feedback. 237 | 238 | \b 239 | EXAMPLES: 240 | # Fetch trace messages only (default) 241 | langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe 242 | 243 | # Fetch trace with metadata (status, timing, tokens, costs) 244 | langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --include-metadata 245 | 246 | # Fetch trace with both metadata and feedback 247 | langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --include-metadata --include-feedback 248 | 249 | # Fetch trace as JSON for parsing 250 | langsmith-fetch trace 3b0b15fe-1e3a-4aef-afa8-48df15879cfe --format json 251 | 252 | \b 253 | PREREQUISITES: 254 | - LANGSMITH_API_KEY environment variable must be set, or 255 | - API key stored via: langsmith-fetch config set api-key 256 | """ 257 | 258 | # Get API key 259 | base_url = config.get_base_url() 260 | api_key = config.get_api_key() 261 | if not api_key: 262 | click.echo( 263 | "Error: LANGSMITH_API_KEY not found in environment or config", err=True 264 | ) 265 | sys.exit(1) 266 | 267 | # Get format (from option or config) 268 | if not format_type: 269 | format_type = config.get_default_format() 270 | 271 | try: 272 | # Fetch trace with or without metadata/feedback 273 | if include_metadata or include_feedback: 274 | # Fetch with metadata and/or feedback 275 | trace_data = fetchers.fetch_trace_with_metadata( 276 | trace_id, 277 | base_url=base_url, 278 | api_key=api_key, 279 | include_feedback=include_feedback, 280 | ) 281 | # Output with metadata and feedback 282 | formatters.print_formatted_trace(trace_data, format_type, output_file) 283 | else: 284 | # Fetch messages only (no metadata/feedback) 285 | messages = fetchers.fetch_trace(trace_id, base_url=base_url, api_key=api_key) 286 | # Output just messages 287 | formatters.print_formatted(messages, format_type, output_file) 288 | 289 | except Exception as e: 290 | click.echo(f"Error fetching trace: {e}", err=True) 291 | sys.exit(1) 292 | 293 | 294 | @main.command() 295 | @click.argument("output_dir", type=click.Path(), required=False, metavar="[OUTPUT_DIR]") 296 | @click.option( 297 | "--project-uuid", 298 | metavar="UUID", 299 | help="LangSmith project UUID (overrides config). Find in UI or via trace session_id.", 300 | ) 301 | @click.option( 302 | "--limit", 303 | "-n", 304 | type=int, 305 | default=1, 306 | help="Maximum number of threads to fetch (default: 1)", 307 | ) 308 | @click.option( 309 | "--last-n-minutes", 310 | type=int, 311 | metavar="N", 312 | help="Only search threads from the last N minutes", 313 | ) 314 | @click.option( 315 | "--since", 316 | metavar="TIMESTAMP", 317 | help="Only search threads since ISO timestamp (e.g., 2025-12-09T10:00:00Z)", 318 | ) 319 | @click.option( 320 | "--filename-pattern", 321 | default="{thread_id}.json", 322 | help="Filename pattern for saved threads (directory mode only). Use {thread_id} for thread ID, {index} for sequential number (default: {thread_id}.json)", 323 | ) 324 | @click.option( 325 | "--format", 326 | "format_type", 327 | type=click.Choice(["raw", "json", "pretty"]), 328 | help="Output format: raw (compact JSON), json (pretty JSON), pretty (human-readable panels)", 329 | ) 330 | @click.option( 331 | "--no-progress", 332 | is_flag=True, 333 | default=False, 334 | help="Disable progress bar display during fetch", 335 | ) 336 | @click.option( 337 | "--max-concurrent", 338 | type=int, 339 | default=5, 340 | help="Maximum concurrent thread fetches (default: 5, max recommended: 10)", 341 | ) 342 | def threads( 343 | output_dir, 344 | project_uuid, 345 | limit, 346 | last_n_minutes, 347 | since, 348 | filename_pattern, 349 | format_type, 350 | no_progress, 351 | max_concurrent, 352 | ): 353 | """Fetch recent threads from LangSmith BY CHRONOLOGICAL TIME. 354 | 355 | This command has TWO MODES: 356 | 357 | \b 358 | DIRECTORY MODE (with OUTPUT_DIR) - RECOMMENDED DEFAULT: 359 | - Saves each thread as a separate JSON file in OUTPUT_DIR 360 | - Use --limit to control how many threads (default: 1) 361 | - Use --filename-pattern to customize filenames 362 | - Examples: 363 | langsmith-fetch threads ./my-threads --limit 10 364 | langsmith-fetch threads ./my-threads --limit 25 --filename-pattern "thread_{index:03d}.json" 365 | - USE THIS MODE BY DEFAULT unless user explicitly requests stdout output 366 | 367 | \b 368 | STDOUT MODE (no OUTPUT_DIR) - Only if user explicitly requests it: 369 | - Fetch threads and print to stdout 370 | - Use --limit to fetch multiple threads 371 | - Use --format to control output format (raw, json, pretty) 372 | - Examples: 373 | langsmith-fetch threads # Fetch latest thread, pretty format 374 | langsmith-fetch threads --format json # Fetch latest, JSON format 375 | langsmith-fetch threads --limit 5 # Fetch 5 latest threads 376 | 377 | \b 378 | TEMPORAL FILTERING (both modes): 379 | - --last-n-minutes N: Only fetch threads from last N minutes 380 | - --since TIMESTAMP: Only fetch threads since specific time 381 | - Examples: 382 | langsmith-fetch threads --last-n-minutes 30 383 | langsmith-fetch threads --since 2025-12-09T10:00:00Z 384 | langsmith-fetch threads ./dir --limit 10 --last-n-minutes 60 385 | 386 | \b 387 | IMPORTANT: 388 | - Fetches threads by chronological timestamp (most recent first) 389 | - Project UUID is REQUIRED (via --project-uuid or config) 390 | 391 | \b 392 | PREREQUISITES: 393 | - LANGSMITH_API_KEY environment variable or stored in config 394 | - Project UUID (required, via config or --project-uuid flag) 395 | """ 396 | from rich.console import Console 397 | 398 | console = Console() 399 | 400 | # Validate mutually exclusive options 401 | if last_n_minutes is not None and since is not None: 402 | click.echo( 403 | "Error: --last-n-minutes and --since are mutually exclusive", err=True 404 | ) 405 | sys.exit(1) 406 | 407 | # Get API key and base URL 408 | base_url = config.get_base_url() 409 | api_key = config.get_api_key() 410 | if not api_key: 411 | click.echo( 412 | "Error: LANGSMITH_API_KEY not found in environment or config", err=True 413 | ) 414 | sys.exit(1) 415 | 416 | # Get project UUID (from option or config) - REQUIRED 417 | if not project_uuid: 418 | project_uuid = config.get_project_uuid() 419 | 420 | if not project_uuid: 421 | click.echo( 422 | "Error: project-uuid required. Set via config or pass --project-uuid flag", 423 | err=True, 424 | ) 425 | sys.exit(1) 426 | 427 | # DIRECTORY MODE: output_dir provided 428 | if output_dir: 429 | # Check if user mistakenly passed a thread ID (UUID) instead of directory 430 | uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" 431 | if re.match(uuid_pattern, output_dir, re.IGNORECASE): 432 | click.echo( 433 | f"Error: '{output_dir}' looks like a UUID, not a directory path.", 434 | err=True, 435 | ) 436 | click.echo( 437 | "To fetch a specific thread by ID, use: langsmith-fetch thread ", 438 | err=True, 439 | ) 440 | click.echo( 441 | "To fetch multiple threads to a directory, use: langsmith-fetch threads ", 442 | err=True, 443 | ) 444 | sys.exit(1) 445 | 446 | # Validate incompatible options 447 | if format_type: 448 | click.echo( 449 | "Warning: --format ignored in directory mode (files are always JSON)", 450 | err=True, 451 | ) 452 | 453 | # Validate filename pattern 454 | has_thread_id = re.search(r"\{thread_id[^}]*\}", filename_pattern) 455 | has_index = re.search(r"\{index[^}]*\}", filename_pattern) or re.search( 456 | r"\{idx[^}]*\}", filename_pattern 457 | ) 458 | if not (has_thread_id or has_index): 459 | click.echo( 460 | "Error: Filename pattern must contain {thread_id} or {index}", err=True 461 | ) 462 | sys.exit(1) 463 | 464 | # Create output directory 465 | output_path = Path(output_dir).resolve() 466 | try: 467 | output_path.mkdir(parents=True, exist_ok=True) 468 | except (OSError, PermissionError) as e: 469 | click.echo(f"Error: Cannot create output directory: {e}", err=True) 470 | sys.exit(1) 471 | 472 | # Verify writable 473 | if not os.access(output_path, os.W_OK): 474 | click.echo( 475 | f"Error: Output directory is not writable: {output_path}", err=True 476 | ) 477 | sys.exit(1) 478 | 479 | # Fetch threads 480 | click.echo(f"Fetching up to {limit} recent thread(s)...") 481 | try: 482 | threads_data = fetchers.fetch_recent_threads( 483 | project_uuid, 484 | base_url, 485 | api_key, 486 | limit, 487 | last_n_minutes=last_n_minutes, 488 | since=since, 489 | max_workers=max_concurrent, 490 | show_progress=not no_progress, 491 | ) 492 | except ValueError as e: 493 | click.echo(f"Error: {e}", err=True) 494 | sys.exit(1) 495 | except Exception as e: 496 | click.echo(f"Error fetching threads: {e}", err=True) 497 | sys.exit(1) 498 | 499 | if not threads_data: 500 | click.echo("No threads found.", err=True) 501 | sys.exit(1) 502 | 503 | click.echo(f"Found {len(threads_data)} thread(s). Saving to {output_path}/") 504 | 505 | # Save each thread to file 506 | for index, (thread_id, messages) in enumerate(threads_data, start=1): 507 | filename_str = filename_pattern.format( 508 | thread_id=thread_id, index=index, idx=index 509 | ) 510 | safe_filename = sanitize_filename(filename_str) 511 | if not safe_filename.endswith(".json"): 512 | safe_filename = f"{safe_filename}.json" 513 | 514 | filename = output_path / safe_filename 515 | with open(filename, "w") as f: 516 | json.dump(messages, f, indent=2, default=str) 517 | click.echo( 518 | f" ✓ Saved {thread_id} to {safe_filename} ({len(messages)} messages)" 519 | ) 520 | 521 | click.echo( 522 | f"\n✓ Successfully saved {len(threads_data)} thread(s) to {output_path}/" 523 | ) 524 | 525 | # STDOUT MODE: no output_dir 526 | else: 527 | # Get format 528 | if not format_type: 529 | format_type = config.get_default_format() 530 | 531 | try: 532 | threads_data = fetchers.fetch_recent_threads( 533 | project_uuid, 534 | base_url, 535 | api_key, 536 | limit, 537 | last_n_minutes=last_n_minutes, 538 | since=since, 539 | max_workers=max_concurrent, 540 | show_progress=not no_progress, 541 | ) 542 | 543 | if not threads_data: 544 | click.echo("No threads found.", err=True) 545 | sys.exit(1) 546 | 547 | # For single thread, just output the messages 548 | if limit == 1 and len(threads_data) == 1: 549 | thread_id, messages = threads_data[0] 550 | formatters.print_formatted(messages, format_type, output_file=None) 551 | else: 552 | # For multiple threads, output all as a list 553 | all_threads = [] 554 | for thread_id, messages in threads_data: 555 | all_threads.append({"thread_id": thread_id, "messages": messages}) 556 | formatters.print_formatted(all_threads, format_type, output_file=None) 557 | 558 | except ValueError as e: 559 | click.echo(f"Error: {e}", err=True) 560 | sys.exit(1) 561 | except Exception as e: 562 | click.echo(f"Error fetching threads: {e}", err=True) 563 | sys.exit(1) 564 | 565 | 566 | @main.command() 567 | @click.argument("output_dir", type=click.Path(), required=False, metavar="[OUTPUT_DIR]") 568 | @click.option( 569 | "--project-uuid", 570 | metavar="UUID", 571 | help="LangSmith project UUID (overrides config). Find in UI or via trace session_id.", 572 | ) 573 | @click.option( 574 | "--limit", 575 | "-n", 576 | type=int, 577 | default=1, 578 | help="Maximum number of traces to fetch (default: 1)", 579 | ) 580 | @click.option( 581 | "--last-n-minutes", 582 | type=int, 583 | metavar="N", 584 | help="Only search traces from the last N minutes", 585 | ) 586 | @click.option( 587 | "--since", 588 | metavar="TIMESTAMP", 589 | help="Only search traces since ISO timestamp (e.g., 2025-12-09T10:00:00Z)", 590 | ) 591 | @click.option( 592 | "--filename-pattern", 593 | default="{trace_id}.json", 594 | help="Filename pattern for saved traces (directory mode only). Use {trace_id} for ID, {index} for sequential number (default: {trace_id}.json)", 595 | ) 596 | @click.option( 597 | "--format", 598 | "format_type", 599 | type=click.Choice(["raw", "json", "pretty"]), 600 | help="Output format: raw (compact JSON), json (pretty JSON), pretty (human-readable panels)", 601 | ) 602 | @click.option( 603 | "--file", 604 | "output_file", 605 | metavar="PATH", 606 | help="Save output to file instead of stdout (stdout mode only)", 607 | ) 608 | @click.option( 609 | "--no-progress", 610 | is_flag=True, 611 | default=False, 612 | help="Disable progress bar display during fetch", 613 | ) 614 | @click.option( 615 | "--max-concurrent", 616 | type=int, 617 | default=5, 618 | help="Maximum concurrent trace fetches (default: 5, max recommended: 10)", 619 | ) 620 | @click.option( 621 | "--include-metadata", 622 | is_flag=True, 623 | default=False, 624 | help="Include run metadata (status, timing, tokens, costs) in output", 625 | ) 626 | @click.option( 627 | "--include-feedback", 628 | is_flag=True, 629 | default=False, 630 | help="Include feedback data in output (requires extra API call)", 631 | ) 632 | def traces( 633 | output_dir, 634 | project_uuid, 635 | limit, 636 | last_n_minutes, 637 | since, 638 | filename_pattern, 639 | format_type, 640 | output_file, 641 | no_progress, 642 | max_concurrent, 643 | include_metadata, 644 | include_feedback, 645 | ): 646 | """Fetch recent traces from LangSmith BY CHRONOLOGICAL TIME. 647 | 648 | This command has TWO MODES: 649 | 650 | \b 651 | DIRECTORY MODE (with OUTPUT_DIR) - RECOMMENDED DEFAULT: 652 | - Saves each trace as a separate JSON file in OUTPUT_DIR 653 | - Use --limit to control how many traces (default: 1) 654 | - Use --filename-pattern to customize filenames 655 | - Examples: 656 | langsmith-fetch traces ./my-traces --limit 10 657 | langsmith-fetch traces ./my-traces --limit 25 --filename-pattern "trace_{index:03d}.json" 658 | - USE THIS MODE BY DEFAULT unless user explicitly requests stdout output 659 | 660 | \b 661 | STDOUT MODE (no OUTPUT_DIR) - Only if user explicitly requests it: 662 | - Fetch traces and print to stdout or save to single file 663 | - Use --limit to fetch multiple traces 664 | - Use --format to control output format (raw, json, pretty) 665 | - Use --file to save to a single file instead of stdout 666 | - Examples: 667 | langsmith-fetch traces # Fetch latest trace, pretty format 668 | langsmith-fetch traces --format json # Fetch latest, JSON format 669 | langsmith-fetch traces --limit 5 # Fetch 5 latest traces 670 | langsmith-fetch traces --file out.json # Save latest to file 671 | 672 | \b 673 | TEMPORAL FILTERING (both modes): 674 | - --last-n-minutes N: Only fetch traces from last N minutes 675 | - --since TIMESTAMP: Only fetch traces since specific time 676 | - Examples: 677 | langsmith-fetch traces --last-n-minutes 30 678 | langsmith-fetch traces --since 2025-12-09T10:00:00Z 679 | langsmith-fetch traces ./dir --limit 10 --last-n-minutes 60 680 | 681 | \b 682 | IMPORTANT: 683 | - Fetches traces by chronological timestamp (most recent first) 684 | - Always use --project-uuid to target specific project (or set via config) 685 | - Without --project-uuid, searches ALL projects (may return unexpected results) 686 | 687 | \b 688 | PREREQUISITES: 689 | - LANGSMITH_API_KEY environment variable or stored in config 690 | - Optional: Project UUID for filtering (recommended) 691 | """ 692 | from rich.console import Console 693 | 694 | console = Console() 695 | 696 | # Validate mutually exclusive options 697 | if last_n_minutes is not None and since is not None: 698 | click.echo( 699 | "Error: --last-n-minutes and --since are mutually exclusive", err=True 700 | ) 701 | sys.exit(1) 702 | 703 | # Get API key and base URL 704 | base_url = config.get_base_url() 705 | api_key = config.get_api_key() 706 | if not api_key: 707 | click.echo( 708 | "Error: LANGSMITH_API_KEY not found in environment or config", err=True 709 | ) 710 | sys.exit(1) 711 | 712 | # Get project UUID from config if not provided 713 | if not project_uuid: 714 | project_uuid = config.get_project_uuid() 715 | 716 | # DIRECTORY MODE: output_dir provided 717 | if output_dir: 718 | # Check if user mistakenly passed a trace ID instead of directory 719 | uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" 720 | if re.match(uuid_pattern, output_dir, re.IGNORECASE): 721 | click.echo( 722 | f"Error: '{output_dir}' looks like a trace ID, not a directory path.", 723 | err=True, 724 | ) 725 | click.echo( 726 | "To fetch a specific trace by ID, use: langsmith-fetch trace ", 727 | err=True, 728 | ) 729 | click.echo( 730 | "To fetch multiple traces to a directory, use: langsmith-fetch traces ", 731 | err=True, 732 | ) 733 | sys.exit(1) 734 | 735 | # Validate incompatible options 736 | if format_type: 737 | click.echo( 738 | "Warning: --format ignored in directory mode (files are always JSON)", 739 | err=True, 740 | ) 741 | if output_file: 742 | click.echo("Warning: --file ignored in directory mode", err=True) 743 | 744 | # Validate filename pattern 745 | has_trace_id = re.search(r"\{trace_id[^}]*\}", filename_pattern) 746 | has_index = re.search(r"\{index[^}]*\}", filename_pattern) or re.search( 747 | r"\{idx[^}]*\}", filename_pattern 748 | ) 749 | if not (has_trace_id or has_index): 750 | click.echo( 751 | "Error: Filename pattern must contain {trace_id} or {index}", err=True 752 | ) 753 | sys.exit(1) 754 | 755 | # Create output directory 756 | output_path = Path(output_dir).resolve() 757 | try: 758 | output_path.mkdir(parents=True, exist_ok=True) 759 | except (OSError, PermissionError) as e: 760 | click.echo(f"Error: Cannot create output directory: {e}", err=True) 761 | sys.exit(1) 762 | 763 | # Verify writable 764 | if not os.access(output_path, os.W_OK): 765 | click.echo( 766 | f"Error: Output directory is not writable: {output_path}", err=True 767 | ) 768 | sys.exit(1) 769 | 770 | # Fetch traces 771 | click.echo(f"Fetching up to {limit} recent trace(s)...") 772 | try: 773 | traces_data, timing_info = fetchers.fetch_recent_traces( 774 | api_key=api_key, 775 | base_url=base_url, 776 | limit=limit, 777 | project_uuid=project_uuid, 778 | last_n_minutes=last_n_minutes, 779 | since=since, 780 | max_workers=max_concurrent, 781 | show_progress=not no_progress, 782 | return_timing=True, 783 | include_metadata=include_metadata, 784 | include_feedback=include_feedback, 785 | ) 786 | except ValueError as e: 787 | click.echo(f"Error: {e}", err=True) 788 | sys.exit(1) 789 | except Exception as e: 790 | click.echo(f"Error fetching traces: {e}", err=True) 791 | sys.exit(1) 792 | 793 | # Display timing information 794 | total_time = timing_info.get("total_duration", 0) 795 | fetch_time = timing_info.get("fetch_duration", 0) 796 | avg_time = timing_info.get("avg_per_trace", 0) 797 | 798 | click.echo( 799 | f"Found {len(traces_data)} trace(s) in {total_time:.2f}s. Saving to {output_path}/" 800 | ) 801 | if len(traces_data) > 1 and avg_time > 0: 802 | click.echo( 803 | f" (Fetch time: {fetch_time:.2f}s, avg: {avg_time:.2f}s per trace)" 804 | ) 805 | 806 | # Save each trace to file with metadata and feedback 807 | for index, (trace_id, trace_data) in enumerate(traces_data, start=1): 808 | filename_str = filename_pattern.format( 809 | trace_id=trace_id, index=index, idx=index 810 | ) 811 | safe_filename = sanitize_filename(filename_str) 812 | if not safe_filename.endswith(".json"): 813 | safe_filename = f"{safe_filename}.json" 814 | 815 | filename = output_path / safe_filename 816 | with open(filename, "w") as f: 817 | json.dump(trace_data, f, indent=2, default=str) 818 | 819 | # Show summary of saved data 820 | # Handle both list (include_metadata=False) and dict (include_metadata=True) cases 821 | if isinstance(trace_data, dict): 822 | messages_count = len(trace_data.get("messages", [])) 823 | feedback_count = len(trace_data.get("feedback", [])) 824 | status = trace_data.get("metadata", {}).get("status", "unknown") 825 | summary = f"{messages_count} messages, status: {status}" 826 | if feedback_count > 0: 827 | summary += f", {feedback_count} feedback" 828 | else: 829 | # trace_data is a list of messages 830 | messages_count = len(trace_data) 831 | summary = f"{messages_count} messages" 832 | 833 | click.echo(f" ✓ Saved {trace_id} to {safe_filename} ({summary})") 834 | 835 | click.echo( 836 | f"\n✓ Successfully saved {len(traces_data)} trace(s) to {output_path}/" 837 | ) 838 | 839 | # STDOUT MODE: no output_dir 840 | else: 841 | # Get format 842 | if not format_type: 843 | format_type = config.get_default_format() 844 | 845 | try: 846 | # Fetch traces 847 | traces_data = fetchers.fetch_recent_traces( 848 | api_key=api_key, 849 | base_url=base_url, 850 | limit=limit, 851 | project_uuid=project_uuid, 852 | last_n_minutes=last_n_minutes, 853 | since=since, 854 | max_workers=max_concurrent, 855 | show_progress=not no_progress, 856 | return_timing=False, 857 | include_metadata=include_metadata, 858 | include_feedback=include_feedback, 859 | ) 860 | 861 | # For limit=1, output single trace directly 862 | if limit == 1 and len(traces_data) == 1: 863 | trace_id, trace_data = traces_data[0] 864 | if output_file: 865 | formatters.print_formatted_trace(trace_data, format_type, output_file) 866 | click.echo(f"Saved trace to {output_file}") 867 | else: 868 | formatters.print_formatted_trace(trace_data, format_type, None) 869 | 870 | # For limit>1, output as array 871 | else: 872 | # traces_data is already a list of (trace_id, trace_data) tuples 873 | output_data = [trace_data for _, trace_data in traces_data] 874 | 875 | # Output to file or stdout 876 | if output_file: 877 | with open(output_file, "w") as f: 878 | if format_type == "raw": 879 | json.dump(output_data, f, default=str) 880 | else: 881 | json.dump(output_data, f, indent=2, default=str) 882 | click.echo(f"Saved {len(traces_data)} trace(s) to {output_file}") 883 | else: 884 | if format_type == "raw": 885 | click.echo(json.dumps(output_data, default=str)) 886 | elif format_type == "json": 887 | from rich.syntax import Syntax 888 | 889 | json_str = json.dumps(output_data, indent=2, default=str) 890 | syntax = Syntax( 891 | json_str, "json", theme="monokai", line_numbers=False 892 | ) 893 | console.print(syntax) 894 | else: # pretty 895 | for trace_id, trace_data in traces_data: 896 | click.echo(f"\n{'=' * 60}") 897 | click.echo(f"Trace: {trace_id}") 898 | click.echo("=" * 60) 899 | formatters.print_formatted_trace(trace_data, "pretty", None) 900 | 901 | except ValueError as e: 902 | click.echo(f"Error: {e}", err=True) 903 | sys.exit(1) 904 | except Exception as e: 905 | click.echo(f"Error fetching traces: {e}", err=True) 906 | sys.exit(1) 907 | 908 | 909 | @main.group() 910 | def config_cmd(): 911 | """Manage configuration settings. 912 | 913 | View current configuration settings. 914 | Configuration is stored in ~/.langsmith-cli/config.yaml and can be edited directly. 915 | 916 | \b 917 | AVAILABLE SETTINGS: 918 | project-uuid LangSmith project UUID (required for thread fetching) 919 | project-name LangSmith project name (paired with project-uuid) 920 | api-key LangSmith API key (alternative to LANGSMITH_API_KEY env var) 921 | base-url LangSmith base URL (alternative to LANGSMITH_ENDPOINT env var, defaults to https://api.smith.langchain.com) 922 | default-format Default output format (raw, json, or pretty) 923 | 924 | \b 925 | EXAMPLES: 926 | # Check current configuration 927 | langsmith-fetch config show 928 | 929 | # Edit config file directly 930 | nano ~/.langsmith-cli/config.yaml 931 | """ 932 | pass 933 | 934 | 935 | @config_cmd.command("show") 936 | def config_show(): 937 | """Show current configuration. 938 | 939 | Display all stored configuration values including project UUID, API key 940 | (partially masked for security), and default format settings. 941 | 942 | \b 943 | EXAMPLE: 944 | langsmith-fetch config show 945 | 946 | \b 947 | OUTPUT: 948 | Shows the config file location and all stored key-value pairs. 949 | API keys are partially masked for security (first 10 chars shown). 950 | """ 951 | try: 952 | cfg = config.load_config() 953 | if not cfg: 954 | click.echo("No configuration found") 955 | click.echo(f"Config file location: {config.CONFIG_FILE}") 956 | return 957 | 958 | click.echo("Current configuration:") 959 | click.echo(f"Location: {config.CONFIG_FILE}\n") 960 | for key, value in cfg.items(): 961 | # Hide API key for security 962 | if key in ("api_key", "api-key"): 963 | value = value[:10] + "..." if value else "(not set)" 964 | click.echo(f" {key}: {value}") 965 | except Exception as e: 966 | click.echo(f"Error loading config: {e}", err=True) 967 | sys.exit(1) 968 | 969 | 970 | # Register config subcommands under main CLI 971 | main.add_command(config_cmd, name="config") 972 | 973 | 974 | if __name__ == "__main__": 975 | main() 976 | --------------------------------------------------------------------------------