srcdoc should be defined by the user
', 23 | ); 24 | }); 25 | 26 | it('should be createable with a value', () => { 27 | const state = { srcdoc: 'it is alright' }; 28 | const model = createTestModel(StitchModel, state); 29 | expect(model).toBeInstanceOf(StitchModel); 30 | expect(model.get('srcdoc')).toEqual('it is alright'); 31 | }); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /packages/python/stitch/src/__tests__/utils.ts: -------------------------------------------------------------------------------- 1 | // Copyright (c) Jupyter Development Team. 2 | // Distributed under the terms of the Modified BSD License. 3 | 4 | import * as widgets from '@jupyter-widgets/base'; 5 | import * as baseManager from '@jupyter-widgets/base-manager'; 6 | import * as services from '@jupyterlab/services'; 7 | 8 | let numComms = 0; 9 | 10 | export class MockComm implements widgets.IClassicComm { 11 | constructor() { 12 | this.comm_id = `mock-comm-id-${numComms}`; 13 | numComms += 1; 14 | } 15 | on_close(fn: ((x?: any) => void) | null): void { 16 | this._on_close = fn; 17 | } 18 | on_msg(fn: (x?: any) => void): void { 19 | this._on_msg = fn; 20 | } 21 | _process_msg(msg: services.KernelMessage.ICommMsgMsg): void | Promisesrcdoc should be defined by the user
").tag(sync=True) 29 | initial_height = Unicode("1px").tag(sync=True) 30 | initial_width = Unicode("1px").tag(sync=True) 31 | initial_border = Unicode("0").tag(sync=True) 32 | 33 | # NOTE(nopdive): Should we sync or not? There are overheads when we deal with bandwidth on real time applications. 34 | state = Unicode("").tag(sync=True) -------------------------------------------------------------------------------- /packages/python/stitch/stitch/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/packages/python/stitch/stitch/tests/__init__.py -------------------------------------------------------------------------------- /packages/python/stitch/stitch/tests/conftest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Copyright (c) Guidance Contributors. 5 | # Distributed under the terms of the Modified BSD License. 6 | 7 | import pytest 8 | 9 | from ipykernel.comm import Comm 10 | from ipywidgets import Widget 11 | 12 | class MockComm(Comm): 13 | """A mock Comm object. 14 | 15 | Can be used to inspect calls to Comm's open/send/close methods. 16 | """ 17 | comm_id = 'a-b-c-d' 18 | kernel = 'Truthy' 19 | 20 | def __init__(self, *args, **kwargs): 21 | self.log_open = [] 22 | self.log_send = [] 23 | self.log_close = [] 24 | super(MockComm, self).__init__(*args, **kwargs) 25 | 26 | def open(self, *args, **kwargs): 27 | self.log_open.append((args, kwargs)) 28 | 29 | def send(self, *args, **kwargs): 30 | self.log_send.append((args, kwargs)) 31 | 32 | def close(self, *args, **kwargs): 33 | self.log_close.append((args, kwargs)) 34 | 35 | _widget_attrs = {} 36 | undefined = object() 37 | 38 | 39 | @pytest.fixture 40 | def mock_comm(): 41 | _widget_attrs['_comm_default'] = getattr(Widget, '_comm_default', undefined) 42 | Widget._comm_default = lambda self: MockComm() 43 | _widget_attrs['_ipython_display_'] = Widget._ipython_display_ 44 | def raise_not_implemented(*args, **kwargs): 45 | raise NotImplementedError() 46 | Widget._ipython_display_ = raise_not_implemented 47 | 48 | yield MockComm() 49 | 50 | for attr, value in _widget_attrs.items(): 51 | if value is undefined: 52 | delattr(Widget, attr) 53 | else: 54 | setattr(Widget, attr, value) 55 | -------------------------------------------------------------------------------- /packages/python/stitch/stitch/tests/test_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Copyright (c) Guidance Contributors. 5 | # Distributed under the terms of the Modified BSD License. 6 | 7 | import pytest 8 | 9 | from ..stitch import StitchWidget 10 | 11 | 12 | def test_example_creation_blank(): 13 | w = StitchWidget() 14 | assert w.kernelmsg == "" 15 | assert w.clientmsg == "" 16 | assert w.srcdoc == "srcdoc should be defined by the user
" 17 | -------------------------------------------------------------------------------- /packages/python/stitch/stitch/tests/test_nbextension_path.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Copyright (c) Guidance Contributors. 5 | # Distributed under the terms of the Modified BSD License. 6 | 7 | 8 | def test_nbextension_path(): 9 | # Check that magic function can be imported from package root: 10 | from stitch import _jupyter_nbextension_paths 11 | # Ensure that it can be called without incident: 12 | path = _jupyter_nbextension_paths() 13 | # Some sanity checks: 14 | assert len(path) == 1 15 | assert isinstance(path[0], dict) 16 | -------------------------------------------------------------------------------- /packages/python/stitch/tsconfig.eslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src/**/*.ts", "src/**/*.tsx"], 4 | "exclude": [] 5 | } -------------------------------------------------------------------------------- /packages/python/stitch/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": true, 4 | "esModuleInterop":true, 5 | "lib": ["es2015", "dom"], 6 | "module": "commonjs", 7 | "moduleResolution": "node", 8 | "noEmitOnError": true, 9 | "noUnusedLocals": true, 10 | "outDir": "lib", 11 | "resolveJsonModule": true, 12 | "rootDir": "src", 13 | "skipLibCheck": true, 14 | "sourceMap": true, 15 | "strict": true, 16 | "strictPropertyInitialization": false, 17 | "target": "es2015", 18 | "types": ["jest"] 19 | }, 20 | "include": [ 21 | "src/**/*.ts", 22 | "src/**/*.tsx", 23 | ], 24 | "exclude": ["src/**/__tests__"] 25 | } 26 | -------------------------------------------------------------------------------- /packages/python/stitch/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const version = require('./package.json').version; 3 | 4 | // Custom webpack rules 5 | const rules = [ 6 | { test: /\.ts$/, loader: 'ts-loader' }, 7 | { test: /\.js$/, loader: 'source-map-loader' }, 8 | { test: /\.css$/, use: ['style-loader', 'css-loader']} 9 | ]; 10 | 11 | // Packages that shouldn't be bundled but loaded at runtime 12 | const externals = ['@jupyter-widgets/base']; 13 | 14 | const resolve = { 15 | // Add '.ts' and '.tsx' as resolvable extensions. 16 | extensions: [".webpack.js", ".web.js", ".ts", ".js"] 17 | }; 18 | 19 | module.exports = [ 20 | /** 21 | * Notebook extension 22 | * 23 | * This bundle only contains the part of the JavaScript that is run on load of 24 | * the notebook. 25 | */ 26 | { 27 | entry: './src/extension.ts', 28 | output: { 29 | filename: 'index.js', 30 | path: path.resolve(__dirname, 'stitch', 'nbextension'), 31 | libraryTarget: 'amd', 32 | publicPath: '', 33 | }, 34 | module: { 35 | rules: rules 36 | }, 37 | devtool: 'source-map', 38 | externals, 39 | resolve, 40 | }, 41 | 42 | /** 43 | * Embeddable @guidance-ai/stitch bundle 44 | * 45 | * This bundle is almost identical to the notebook extension bundle. The only 46 | * difference is in the configuration of the webpack public path for the 47 | * static assets. 48 | * 49 | * The target bundle is always `dist/index.js`, which is the path required by 50 | * the custom widget embedder. 51 | */ 52 | { 53 | entry: './src/index.ts', 54 | output: { 55 | filename: 'index.js', 56 | path: path.resolve(__dirname, 'dist'), 57 | libraryTarget: 'amd', 58 | library: "@guidance-ai/stitch", 59 | publicPath: 'https://unpkg.com/@guidance-ai/stitch@' + version + '/dist/' 60 | }, 61 | devtool: 'source-map', 62 | module: { 63 | rules: rules 64 | }, 65 | externals, 66 | resolve, 67 | }, 68 | 69 | 70 | /** 71 | * Documentation widget bundle 72 | * 73 | * This bundle is used to embed widgets in the package documentation. 74 | */ 75 | { 76 | entry: './src/index.ts', 77 | output: { 78 | filename: 'embed-bundle.js', 79 | path: path.resolve(__dirname, 'docs', 'source', '_static'), 80 | library: "stitch", 81 | libraryTarget: 'amd' 82 | }, 83 | module: { 84 | rules: rules 85 | }, 86 | devtool: 'source-map', 87 | externals, 88 | resolve, 89 | } 90 | 91 | ]; 92 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel", 5 | "pybind11>=2.10.0", 6 | ] 7 | build-backend = "setuptools.build_meta" 8 | 9 | [tool.pytest.ini_options] 10 | addopts = "--strict-markers" 11 | markers = [ 12 | "resource_intensive: Tests which readily exceed resources such as memory", 13 | "asyncio: Make async tests to run" 14 | ] 15 | 16 | [tool.black] 17 | line-length = 99 18 | target_version = ['py39', 'py310', 'py311', 'py312'] 19 | 20 | [tool.isort] 21 | profile = "black" 22 | 23 | [tool.mypy] 24 | no_implicit_optional = false # TODO: PEP484, set to true 25 | strict = false # TODO: set to true 26 | exclude = ["tests"] 27 | 28 | [[tool.mypy.overrides]] 29 | module = "pybind11.*" 30 | ignore_missing_imports = true 31 | 32 | [[tool.mypy.overrides]] 33 | module = "vertexai.*" 34 | ignore_missing_imports = true 35 | 36 | [[tool.mypy.overrides]] 37 | module = "google.*" 38 | ignore_missing_imports = true 39 | 40 | [[tool.mypy.overrides]] 41 | module = "llama_cpp.*" 42 | ignore_missing_imports = true 43 | 44 | [[tool.mypy.overrides]] 45 | module = "anthropic.*" 46 | ignore_missing_imports = true 47 | 48 | [[tool.mypy.overrides]] 49 | module = "litellm.*" 50 | ignore_missing_imports = true 51 | 52 | [[tool.mypy.overrides]] 53 | module = "transformers.*" 54 | ignore_missing_imports = true 55 | 56 | [[tool.mypy.overrides]] 57 | module = "diskcache.*" 58 | ignore_missing_imports = true 59 | 60 | [[tool.mypy.overrides]] 61 | module = "tokenizers.*" 62 | ignore_missing_imports = true -------------------------------------------------------------------------------- /tests/ReadMe.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | ## Organisation 4 | 5 | The tests are arranged into the following directories: 6 | 7 | - `unit` tests do not depend on LLMs (but may use `model.Mock`) 8 | - `model_integration` tests should run on any (fully supported) model, supplied by the `selected_model` fixture 9 | - `model_specific` tests are for isolating particular issues with individual LLMs 10 | - `need_credentials` tests are for tests which need access to various credentials (mainly `Grammarless` models for endpoints without full Guidance support) 11 | - `notebook` tests are for notebooks 12 | 13 | The `model_specific` tests should make use of the `selected_model` machinery, but skip themselves if the appropriate model is not supplied. 14 | A sample means of achieving this: 15 | 16 | ```python 17 | @pytest.fixture(scope="module") 18 | def phi3_model(selected_model, selected_model_name): 19 | if selected_model_name in ["transformers_phi3_mini_4k_instruct_cpu"]: 20 | return selected_model 21 | else: 22 | pytest.skip("Requires Phi3 model") 23 | ``` 24 | 25 | ## Selecting a model 26 | 27 | To select a particular model when running the tests, use the `--selected_model` command line option. 28 | For example: 29 | 30 | ```bash 31 | python -m pytest --selected_model transformers_gemma2_9b_cpu ./tests/model_integration/ 32 | ``` 33 | 34 | The allowed values for `--selected_model` are in the [`confest.py`](./conftest.py) file, and are defined in the `selected_model` function. 35 | Alternatively, the `GUIDANCE_SELECTED_MODEL` environment variable can be used to override the default value for `--selected_model` (which can be useful when using a debugger). 36 | 37 | ### A Note on Credentials 38 | 39 | As noted above the `need_credentials` tests are mainly for `Grammarless` models - those for remote endpoints which do not support Guidance grammars (there are a few exceptions, which is why the directory isn't simply named `grammarless`). 40 | As endpoints with Guidance grammar support come online, their tests should *not* go in there; these should go into `model_integration` and `model_specific`, but will only be run in CI builds. 41 | Similarly, some models (e.g. LLama3) require credentials in order to download their weights from Hugging Face. 42 | These should be run through the `model_integration` and `model_specific` tests, but this run will happen from the CI build, and hence have credential access. 43 | 44 | ## Testing Goal 45 | 46 | Ideally, when creating a new feature, most of the tests should go into the `unit` directory, and make use of `model.Mock` if needed. 47 | These should always be able to be run with quite a minimal Guidance installation (have to add `pytest`, obviously). 48 | These tests should be fast, and facilitate a developer experience build around running 49 | 50 | ```bash 51 | pytest tests/unit 52 | ``` 53 | very frequently. 54 | 55 | There should also be a handful of tests in `model_integration`, which should work with _any_ fully supported Guidance model. 56 | Finally, if any model quirks are noted (and _especially_ if workarounds are required in the code), tests to characterise these should go into `model_specific`. 57 | 58 | In this paradigm, no tests in `unit` or `model_integration` should be using `pytest.skip` (or its variants). 59 | Those in `model_specific` will use `pytest.skip` for when the `selected_model` fixture is not of the appropriate type. 60 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/__init__.py -------------------------------------------------------------------------------- /tests/bench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/bench/__init__.py -------------------------------------------------------------------------------- /tests/bench/test_api.py: -------------------------------------------------------------------------------- 1 | from guidance.bench._api import bench, AVAILABLE_MODELS 2 | from pathlib import Path 3 | import tempfile 4 | import pytest 5 | 6 | @pytest.mark.skip("Waiting on CI upgrades. Need access to env var LANGCHAIN_API_KEY.") 7 | def test_bench(): 8 | # TODO(nopdive): Parameterize models once CI is upgraded. 9 | with tempfile.TemporaryDirectory() as tmp_dir: 10 | db_path = Path(tmp_dir) / "bench.db" 11 | db_url = f"sqlite:///{db_path}" 12 | status_df, result_df = bench(db_url, "bench-test", models=AVAILABLE_MODELS[:1], debug_mode=True) 13 | 14 | assert len(status_df) > 0 15 | assert len(result_df) > 0 16 | assert (status_df['status'] == 'COMPLETE').all() -------------------------------------------------------------------------------- /tests/bench/test_powerlift.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import tempfile 3 | 4 | from guidance.bench._powerlift import retrieve_langchain 5 | from pathlib import Path 6 | 7 | def test_retrieve_langchain_err(monkeypatch): 8 | monkeypatch.delenv("LANGCHAIN_API_KEY", raising=False) 9 | with pytest.raises(ValueError): 10 | gen = retrieve_langchain() 11 | _ = list(gen) 12 | 13 | @pytest.mark.skip("Waiting on CI upgrades. Need access to env var LANGCHAIN_API_KEY.") 14 | def test_retrieve_langchain_basic(): 15 | with tempfile.TemporaryDirectory() as tmp_dir: 16 | # Run once 17 | first_results = list(retrieve_langchain(cache_dir=tmp_dir)) 18 | langchain_cache_path = Path(tmp_dir, "langchain") 19 | assert Path.exists(langchain_cache_path) 20 | 21 | # Run another time to trigger the cache 22 | second_results = list(retrieve_langchain(cache_dir=tmp_dir)) 23 | for first, second in zip(first_results, second_results): 24 | assert first.inputs.equals(second.inputs) -------------------------------------------------------------------------------- /tests/bench/test_utils.py: -------------------------------------------------------------------------------- 1 | from guidance.bench._utils import lib_bench_dir 2 | import tempfile 3 | from pathlib import Path 4 | 5 | def test_lib_bench_dir_basic(): 6 | expected_dir = Path.home() / ".guidance-bench" 7 | actual_dir = lib_bench_dir() 8 | 9 | assert expected_dir == actual_dir 10 | assert Path.exists(actual_dir) 11 | 12 | 13 | def test_lib_bench_dir_env_var(monkeypatch): 14 | with tempfile.TemporaryDirectory() as tmp_dir: 15 | expected_dir = Path(tmp_dir) / "guidance-bench" 16 | monkeypatch.setenv("GUIDANCE_BENCH_DIR", expected_dir) 17 | 18 | actual_dir = lib_bench_dir() 19 | assert expected_dir == actual_dir 20 | assert Path.exists(actual_dir) -------------------------------------------------------------------------------- /tests/model_integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/model_integration/__init__.py -------------------------------------------------------------------------------- /tests/model_integration/library/test_subgrammar.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numpy as np 3 | import pytest 4 | from jsonschema import validate 5 | import json 6 | 7 | import guidance 8 | from guidance import ( 9 | gen, 10 | select, 11 | optional, 12 | one_or_more, 13 | ) 14 | from guidance.library._subgrammar import subgrammar, lexeme 15 | 16 | 17 | @guidance(stateless=True) 18 | def json_string(lm): 19 | return lm + lexeme(r'"(\\(["\\\/bfnrt]|u[a-fA-F0-9]{4})|[^"\\\x00-\x1F\x7F]+)*"') 20 | 21 | 22 | @guidance(stateless=True) 23 | def json_number(lm): 24 | return lm + lexeme(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?") 25 | 26 | 27 | @guidance(stateless=True) 28 | def json_value(lm): 29 | return lm + select( 30 | [ 31 | json_string(), 32 | json_number(), 33 | json_object(), 34 | json_array(), 35 | "true", 36 | "false", 37 | "null", 38 | ] 39 | ) 40 | 41 | 42 | @guidance(stateless=True) 43 | def json_member(lm): 44 | return lm + json_string() + ":" + json_value() 45 | 46 | 47 | @guidance(stateless=True) 48 | def json_object(lm): 49 | return lm + "{" + optional(json_member() + one_or_more("," + json_member())) + "}" 50 | 51 | 52 | @guidance(stateless=True) 53 | def json_array(lm): 54 | return lm + "[" + optional(json_value() + one_or_more("," + json_value())) + "]" 55 | 56 | 57 | @guidance(stateless=True) 58 | def gen_json_object(lm, name: str, max_tokens=100000000): 59 | grm = subgrammar( 60 | body=json_object(), 61 | name=name, 62 | skip_regex=r"[\x20\x0A\x0D\x09]+", 63 | max_tokens=max_tokens 64 | ) 65 | return lm + grm 66 | 67 | 68 | def test_greedy_json_object(selected_model: guidance.models.Model): 69 | lm = selected_model 70 | lm += "John Doe's name, age, and birthday:\n" 71 | lm += gen_json_object("hacker", max_tokens=1000) 72 | lm += "\nScore: " + gen("score", regex="[1-3]") 73 | # make sure it parses as JSON 74 | obj = json.loads(lm["hacker"]) 75 | assert isinstance(obj, dict) 76 | assert lm["score"] in ["1", "2", "3"] 77 | 78 | 79 | def test_greedy_single_terminal(selected_model: guidance.models.Model): 80 | lm = selected_model 81 | lm += "A number: " 82 | lm += subgrammar(body=lexeme(r"[0-9]{3}")) 83 | assert re.search(r": [0-9]{3}$", str(lm)) 84 | -------------------------------------------------------------------------------- /tests/model_integration/library/test_substring.py: -------------------------------------------------------------------------------- 1 | from guidance import gen, models, substring 2 | 3 | 4 | def test_substring_equal_unconstrained(selected_model: models.Model): 5 | target_model = selected_model 6 | lm = target_model + "ae galera " + gen(max_tokens=10, name="test") 7 | lm2 = target_model + "ae galera " + substring(lm["test"], name="capture") 8 | assert lm2["capture"] in lm["test"] 9 | -------------------------------------------------------------------------------- /tests/model_integration/test_grammar.py: -------------------------------------------------------------------------------- 1 | from guidance import models, select 2 | 3 | 4 | def test_select_simple(selected_model: models.Model): 5 | lm = selected_model 6 | options = ["baad I think", "bad I think", "bad"] 7 | lm = lm + "Scott is quite " + select(name="bad", options=options) 8 | assert lm["bad"] in options 9 | -------------------------------------------------------------------------------- /tests/model_integration/test_tokenizers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from tests.tokenizer_common import TOKENIZER_ROUND_TRIP_STRINGS, BaseTestTransformerTokenizers 3 | 4 | # These are not _strictly_ unit tests, since they refer 5 | # to specific tokenisers. However, tokenisers are small, 6 | # so if the tokeniser can be loaded separately from the 7 | # model, then this is a good place to have them live. 8 | 9 | # The LlamaCpp tokenisers have tests in test_llamacpp.py 10 | # since those tokenisers cannot be loaded separately from 11 | # their models. 12 | 13 | # The transformer tests have an authenticated version under 14 | # need_credentials 15 | 16 | 17 | class TestUnauthenticatedTransformerTokenizers(BaseTestTransformerTokenizers): 18 | TRANSFORMER_MODELS = [ 19 | "gpt2", 20 | "microsoft/phi-2", 21 | "microsoft/Phi-3-small-8k-instruct", 22 | "microsoft/Phi-3-mini-4k-instruct", 23 | ] 24 | 25 | @pytest.mark.parametrize( 26 | "model_name", 27 | TRANSFORMER_MODELS, 28 | ) 29 | def test_smoke(self, model_name: str): 30 | self.base_smoke(model_name) 31 | 32 | @pytest.mark.parametrize("model_name", TRANSFORMER_MODELS) 33 | @pytest.mark.parametrize("target_string", TOKENIZER_ROUND_TRIP_STRINGS) 34 | def test_string_roundtrip(self, model_name: str, target_string: str): 35 | self.base_string_roundtrip(model_name, target_string) 36 | 37 | @pytest.mark.parametrize("model_name", TRANSFORMER_MODELS) 38 | def test_eos_bos_token_round_trip(self, model_name: str): 39 | self.base_eos_bos_token_round_trip(model_name) 40 | -------------------------------------------------------------------------------- /tests/model_specific/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/model_specific/__init__.py -------------------------------------------------------------------------------- /tests/model_specific/common_chat_testing.py: -------------------------------------------------------------------------------- 1 | from guidance import assistant, gen, models, system, user 2 | 3 | 4 | def smoke_chat(lm: models.Model, has_system_role: bool = True): 5 | # lm.engine.reset_metrics() 6 | if has_system_role: 7 | with system(): 8 | lm += "You are a math wiz." 9 | 10 | with user(): 11 | lm += "What is 1 + 1?" 12 | 13 | with assistant(): 14 | lm += gen(max_tokens=10, name="text", temperature=0.5) 15 | 16 | print(str(lm)) 17 | # print(f"{lm.engine.metrics=}") 18 | assert len(lm["text"]) > 0 19 | # assert lm.engine.metrics.engine_input_tokens > 2, "Expect some input tokens" 20 | # assert lm.engine.metrics.engine_output_tokens > 0, "Expect some output tokens" 21 | 22 | 23 | def longer_chat_1(lm: models.Model, has_system_role: bool = True): 24 | if has_system_role: 25 | with system(): 26 | lm += "You are a math wiz." 27 | 28 | with user(): 29 | lm += "What is 1 + 1?" 30 | 31 | with assistant(): 32 | lm += gen(max_tokens=10, name="text") 33 | 34 | print(str(lm)) 35 | assert len(lm["text"]) > 0 36 | 37 | with user(): 38 | lm += "10. Now you pick a number between 0 and 20" 39 | 40 | with assistant(): 41 | lm += gen(max_tokens=2, name="number") 42 | 43 | print(str(lm)) 44 | assert len(lm["number"]) > 0 45 | 46 | 47 | def longer_chat_2(lm: models.Model, has_system_role: bool = True): 48 | if has_system_role: 49 | with system(): 50 | lm += "You are a math wiz." 51 | 52 | with user(): 53 | lm += "What is 1 + 1?" 54 | 55 | # This is the new part compared to longer_chat_1 56 | with assistant(): 57 | lm += "2" 58 | 59 | with user(): 60 | lm += "What is 2 + 3?" 61 | 62 | # Resume the previous 63 | with assistant(): 64 | lm += gen(max_tokens=10, name="text") 65 | 66 | print(str(lm)) 67 | assert len(lm["text"]) > 0 68 | 69 | with user(): 70 | lm += "10. Now you pick a number between 0 and 20" 71 | 72 | with assistant(): 73 | lm += gen(max_tokens=2, name="number") 74 | 75 | print(str(lm)) 76 | assert len(lm["number"]) > 0 77 | -------------------------------------------------------------------------------- /tests/model_specific/llama_cpp_tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Set up for pytest -------------------------------------------------------------------------------- /tests/model_specific/llama_cpp_tests/test_chat_templates.py: -------------------------------------------------------------------------------- 1 | import jinja2 2 | import pytest 3 | 4 | import guidance 5 | 6 | from guidance.chat import CHAT_TEMPLATE_CACHE 7 | 8 | 9 | def test_chat_format_smoke(llamacpp_model: guidance.models.LlamaCpp, selected_model_name): 10 | # Retrieve the template string 11 | if ( 12 | hasattr(llamacpp_model.engine.model_obj, "metadata") 13 | and "tokenizer.chat_template" in llamacpp_model.engine.model_obj.metadata 14 | ): 15 | model_chat_template = llamacpp_model.engine.model_obj.metadata["tokenizer.chat_template"] 16 | else: 17 | pytest.skip("Chat template not available from LlamaCpp object") 18 | 19 | lm = guidance.models.Mock("") 20 | lm._interpreter.chat_template = CHAT_TEMPLATE_CACHE[model_chat_template]() 21 | 22 | messages = [ 23 | {"role": "user", "content": "Good_day_to_you!"}, 24 | {"role": "assistant", "content": "Hello!"}, 25 | ] 26 | 27 | # Note that llama-cpp-python does provide a llama_chat_apply_template function 28 | # but details about its use are thin on the ground and according to 29 | # https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template 30 | # it does its own thing internally 31 | jinja2_template = jinja2.Environment(loader=jinja2.BaseLoader()).from_string( 32 | model_chat_template 33 | ) 34 | jinja2_render = jinja2_template.render( 35 | messages=messages, 36 | bos_token=llamacpp_model.engine.tokenizer.bos_token.decode(), 37 | eos_token=llamacpp_model.engine.tokenizer.eos_token.decode(), 38 | ) 39 | 40 | with guidance.user(): 41 | lm += "Good_day_to_you!" 42 | with guidance.assistant(): 43 | lm += "Hello!" 44 | # Only check substring due to BOS/EOS tokens 45 | if selected_model_name == "llamacpp_mistral_7b_cpu": 46 | # The templates extracted via Transformers and GGUF are somewhat 47 | # different for Mistral. This is showing up in slightly 48 | # different spacing (our template is putting in a few extra spaces) 49 | # so at least make sure the 'tags' are correct 50 | assert str(lm).replace(" ", "") in jinja2_render.replace(" ", "") 51 | else: 52 | assert str(lm) in jinja2_render 53 | -------------------------------------------------------------------------------- /tests/model_specific/test_visual.py: -------------------------------------------------------------------------------- 1 | from guidance.registry import get_renderer 2 | 3 | 4 | def test_repeat_simple_model(): 5 | from guidance.models import Transformers 6 | from guidance import gen 7 | from guidance.registry import set_renderer, get_trace_handler 8 | from guidance.visual import JupyterWidgetRenderer 9 | 10 | trace_handler = get_trace_handler() 11 | original_renderer = get_renderer() 12 | for i in range(2): 13 | set_renderer(JupyterWidgetRenderer(trace_handler)) 14 | 15 | lm = Transformers('gpt2') 16 | lm += 'Hi hi hi' 17 | lm += gen(max_tokens=5) 18 | 19 | set_renderer(original_renderer) 20 | 21 | assert True 22 | 23 | 24 | def test_roles(): 25 | from guidance.models import Transformers 26 | from guidance import gen, user, system 27 | 28 | m0 = Transformers("gpt2") 29 | with system(): 30 | m1 = m0 + "You are responsible for writing an epic poem." 31 | with user(): 32 | m2 = m1 + "Roses are red and " + gen(name="suffix", regex=r'[\w\s]{20,30}', max_tokens=30) 33 | 34 | assert m2 is not None -------------------------------------------------------------------------------- /tests/need_credentials/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/need_credentials/__init__.py -------------------------------------------------------------------------------- /tests/need_credentials/test_anthropic.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import guidance 4 | from guidance import assistant, capture, gen, select, system, user 5 | 6 | from ..utils import get_model 7 | 8 | 9 | def test_anthropic_chat(): 10 | try: 11 | lm = guidance.models.Anthropic(model="claude-3-haiku-20240307") 12 | except: 13 | pytest.skip("Skipping Anthropic test because we can't load the model!") 14 | with system(): 15 | lm += "You are a math wiz." 16 | 17 | with user(): 18 | lm += "What is 1 + 1?" 19 | 20 | with assistant(): 21 | lm += gen(max_tokens=10, name="text") 22 | lm += "Pick a number: " 23 | 24 | assert len(lm["text"]) > 0 25 | 26 | 27 | def test_anthropic_select(): 28 | try: 29 | lm = guidance.models.Anthropic(model="claude-instant-1.2") 30 | except: 31 | pytest.skip("Skipping Anthropic test because we can't load the model!") 32 | 33 | # We can't meaningfully test or enforce select on this model 34 | with pytest.raises(guidance.models._model.ConstraintException): 35 | with user(): 36 | lm += "Write the next number in the list: 1,2,3,4,5,6," 37 | with assistant(): 38 | lm += select( 39 | ["harsha", "scott", "marco"], name="the number" 40 | ) 41 | 42 | 43 | def test_anthropic_chat_loop(): 44 | # tests issue #509 45 | try: 46 | model = guidance.models.Anthropic(model="claude-3-haiku-20240307") 47 | except: 48 | pytest.skip("Skipping Anthropic test because we can't load the model!") 49 | 50 | for i in range(2): 51 | 52 | with system(): 53 | lm = model + "You will just return whatever number I give you" 54 | 55 | with user(): 56 | lm += f"The number is: {i}" 57 | 58 | with assistant(): 59 | lm += gen(name="answer", max_tokens=2) 60 | 61 | # def test_direct_anthropic_api(): 62 | # import anthropic 63 | 64 | # client = anthropic.Anthropic() 65 | 66 | # with client.messages.stream( 67 | # max_tokens=10, 68 | # system="You are a counting robot. Do nothing but continue counting numbers in the same format the user presented.", 69 | # messages=[{"role": "user", "content": "1,2,3,4,5,"}], 70 | # model="claude-3-haiku-20240307", 71 | # ) as stream: 72 | # text_list = [] 73 | # for text in stream.text_stream: 74 | # print(text, end="", flush=True) 75 | # text_list.append(text) 76 | 77 | # assert len(text_list) > 0 -------------------------------------------------------------------------------- /tests/need_credentials/test_azureai_studio.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from guidance import models 4 | from guidance.models._azureai import create_azure_aifoundry_model 5 | 6 | 7 | from ..model_specific import common_chat_testing 8 | from ..utils import env_or_fail, slowdown 9 | 10 | # pytest.skip("Deployments temporarily deleted", allow_module_level=True) 11 | 12 | # How to fill out the environment variables to 13 | # set up the models 14 | # Temporarily remove mistral pending endpoint investigation 15 | # _chat_models = {"phi3": "PHI3", "llama3": "LLAMA3_CHAT"} 16 | _chat_models = {"phi4": "PHI4"} 17 | 18 | 19 | def _get_chat_model(model_name: str): 20 | env_string = _chat_models[model_name] 21 | 22 | azureai_studio_endpoint = env_or_fail(f"AZUREAI_STUDIO_{env_string}_ENDPOINT") 23 | azureai_studio_model_name = env_or_fail(f"AZUREAI_STUDIO_{env_string}_MODEL_NAME") 24 | azureai_studio_key = env_or_fail(f"AZUREAI_STUDIO_{env_string}_KEY") 25 | 26 | lm = create_azure_aifoundry_model( 27 | azure_endpoint=azureai_studio_endpoint, 28 | api_key=azureai_studio_key, 29 | # token_credential=DefaultAzureCredential(), 30 | model_name=azureai_studio_model_name, 31 | ) 32 | assert isinstance(lm, models.Model) 33 | return lm 34 | 35 | 36 | @pytest.mark.parametrize("chat_model_name", _chat_models.keys()) 37 | def test_azureai_chat_smoke(chat_model_name: str): 38 | slowdown() 39 | 40 | lm = _get_chat_model(chat_model_name) 41 | 42 | common_chat_testing.smoke_chat(lm, chat_model_name != "mistral") 43 | 44 | 45 | @pytest.mark.parametrize("chat_model_name", _chat_models.keys()) 46 | def test_azureai_chat_longer_1(chat_model_name: str): 47 | slowdown() 48 | 49 | lm = _get_chat_model(chat_model_name) 50 | common_chat_testing.longer_chat_1(lm, chat_model_name != "mistral") 51 | 52 | 53 | @pytest.mark.parametrize("chat_model_name", _chat_models.keys()) 54 | def test_azureai_chat_longer_2(chat_model_name: str): 55 | slowdown() 56 | 57 | lm = _get_chat_model(chat_model_name) 58 | common_chat_testing.longer_chat_2(lm, chat_model_name != "mistral") 59 | -------------------------------------------------------------------------------- /tests/need_credentials/test_cohere.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import guidance 4 | from guidance import assistant, capture, gen, role, select, system, user 5 | 6 | 7 | def test_lite_llm_basic(): 8 | try: 9 | lm = guidance.models.CohereCompletion("command-nightly") 10 | except: 11 | pytest.skip("Skipping Cohere test because we can't load the model!") 12 | lm += "Count to 20: 1,2,3,4," 13 | nl = "\n" 14 | lm += f"""\ 15 | 5,6,7""" 16 | lm += f"""{gen(max_tokens=1, suffix=nl)}aaaaaa""" 17 | assert str(lm)[-5:] == "aaaaa" 18 | 19 | 20 | def test_lite_llm_instruct(): 21 | try: 22 | lm = guidance.models.CohereInstruct("command-nightly") 23 | except: 24 | pytest.skip("Skipping LiteLLM test because we can't load the model!") 25 | with role("instruction"): 26 | lm += "Count to 20." 27 | lm += gen("val", max_tokens=1) 28 | assert len(lm["val"]) > 0 29 | -------------------------------------------------------------------------------- /tests/need_credentials/test_googleai.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from guidance import gen, models, select 4 | 5 | from ..utils import get_model 6 | 7 | 8 | def test_gemini_pro(): 9 | from guidance import assistant, gen, models, system, user 10 | 11 | try: 12 | vmodel = models.GoogleAI("gemini-pro") 13 | except: 14 | pytest.skip("Skipping GoogleAI test because we can't load the model!") 15 | 16 | lm = vmodel 17 | 18 | with user(): 19 | lm += "The economy is crashing!" 20 | 21 | with assistant(): 22 | lm += gen("test1", max_tokens=100) 23 | 24 | with user(): 25 | lm += "What is the best again?" 26 | 27 | with assistant(): 28 | lm += gen("test2", max_tokens=100) 29 | 30 | assert len(lm["test1"]) > 0 31 | assert len(lm["test2"]) > 0 32 | 33 | # second time to make sure cache reuse is okay 34 | lm = vmodel 35 | 36 | with user(): 37 | lm += "The economy is crashing!" 38 | 39 | with assistant(): 40 | lm += gen("test1", max_tokens=100) 41 | 42 | with user(): 43 | lm += "What is the best again?" 44 | 45 | with assistant(): 46 | lm += gen("test2", max_tokens=100) 47 | 48 | assert len(lm["test1"]) > 0 49 | assert len(lm["test2"]) > 0 50 | assert lm["test1"].find("<|im_end|>") < 0 51 | -------------------------------------------------------------------------------- /tests/need_credentials/test_lite_llm.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import guidance 4 | from guidance import assistant, capture, gen, select, system, user 5 | 6 | from ..utils import get_model 7 | 8 | 9 | def test_lite_llm_basic_openai(): 10 | try: 11 | lm = guidance.models.LiteLLMCompletion("gpt-3.5-turbo-instruct") 12 | except: 13 | pytest.skip("Skipping LiteLLM test because we can't load the model!") 14 | lm += "Count to 20: 1,2,3,4," 15 | nl = "\n" 16 | lm += f"""\ 17 | 5,6,7""" 18 | lm += f"""{gen(max_tokens=1, suffix=nl)}aaaaaa""" 19 | assert str(lm)[-5:] == "aaaaa" 20 | 21 | 22 | def test_lite_llm_basic_cohere(): 23 | try: 24 | lm = guidance.models.LiteLLMCompletion("command-nightly") 25 | except: 26 | pytest.skip("Skipping LiteLLM test because we can't load the model!") 27 | lm += "Count to 20: 1,2,3,4," 28 | nl = "\n" 29 | lm += f"""\ 30 | 5,6,7""" 31 | lm += f"""{gen(max_tokens=1, suffix=nl)}aaaaaa""" 32 | assert str(lm)[-5:] == "aaaaa" 33 | 34 | 35 | def test_lite_llm_select(): 36 | try: 37 | lm = guidance.models.LiteLLMCompletion("gpt-3.5-turbo-instruct") 38 | except: 39 | pytest.skip("Skipping LiteLLM test because we can't load the model!") 40 | lm += "Pick a number: " 41 | lm += select( 42 | ["1", "11", "111", "1111", "11111", "111111", "1111111"], name="the number" 43 | ) 44 | assert str(lm)[-1] in "123" 45 | 46 | 47 | def test_lite_llm_chat(): 48 | try: 49 | lm = guidance.models.LiteLLMChat("gpt-3.5-turbo") 50 | except: 51 | pytest.skip("Skipping LiteLLM test because we can't load the model!") 52 | with system(): 53 | lm += "You are a math wiz." 54 | 55 | with user(): 56 | lm += "What is 1 + 1?" 57 | 58 | with assistant(): 59 | lm += gen(max_tokens=10, name="text") 60 | lm += "Pick a number: " 61 | 62 | assert len(lm["text"]) > 0 63 | -------------------------------------------------------------------------------- /tests/need_credentials/test_togetherai.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import guidance 4 | from guidance import assistant, gen, select, system, user 5 | 6 | 7 | def test_togetherai_basic(): 8 | try: 9 | lm = guidance.models.TogetherAI("mistralai/Mistral-7B-v0.1") 10 | except: 11 | pytest.skip("Skipping TogetherAI test because we can't load the model!") 12 | lm += "Count to 20: 1,2,3,4," 13 | stop = "\n" 14 | lm += f"""{gen(max_tokens=1, stop=stop, name="text")}""" 15 | assert str(lm)[-1] == "5" 16 | 17 | 18 | def test_togetherai_select(): 19 | try: 20 | lm = guidance.models.TogetherAI("mistralai/Mistral-7B-v0.1") 21 | except: 22 | pytest.skip("Skipping TogetherAI test because we can't load the model!") 23 | nums = ["1", "11", "111", "1111", "11111", "111111", "1111111"] 24 | lm += "Pick a number: " 25 | lm += select(nums, name="number") 26 | assert str(lm["number"]) in nums 27 | 28 | 29 | def test_togetherai_chat(): 30 | try: 31 | lm = guidance.models.TogetherAIChat("teknium/OpenHermes-2-Mistral-7B") 32 | except: 33 | pytest.skip("Skipping TogetherAI test because we can't load the model!") 34 | with system(): 35 | lm += "You are a math wiz." 36 | 37 | with user(): 38 | lm += "What is 1 + 1?" 39 | 40 | with assistant(): 41 | lm += gen(max_tokens=10, name="text") 42 | lm += "Pick a number: " 43 | 44 | assert len(lm["text"]) > 0 45 | 46 | 47 | def test_togetherai_chat_without_roles(): 48 | try: 49 | lm = guidance.models.TogetherAIChat("teknium/OpenHermes-2-Mistral-7B") 50 | except: 51 | pytest.skip("Skipping TogetherAI test because we can't load the model!") 52 | with pytest.raises(ValueError) as error_info: 53 | lm += "You are a math wiz. What is 1+1?" + gen(max_tokens=10, name="text") 54 | 55 | 56 | def test_togetherai_chat_loop(): 57 | try: 58 | model = guidance.models.TogetherAIChat( 59 | "teknium/OpenHermes-2-Mistral-7B", echo=False 60 | ) 61 | except: 62 | pytest.skip("Skipping TogetherAI test because we can't load the model!") 63 | 64 | with system(): 65 | lm = model + "You will just return whatever number I give you" 66 | 67 | for i in range(2): 68 | with user(): 69 | lm += f"The number is: {i}" 70 | 71 | with assistant(): 72 | lm += gen(name="answer", max_tokens=10) 73 | assert len(lm["answer"]) > 0 74 | -------------------------------------------------------------------------------- /tests/need_credentials/test_tokenizers.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from guidance import models 4 | 5 | from tests.tokenizer_common import TOKENIZER_ROUND_TRIP_STRINGS, BaseTestTransformerTokenizers 6 | 7 | # Since this is under 'need_credentials' we can assume that HF_TOKEN 8 | # will be available when run 9 | 10 | 11 | class TestAuthenticatedTransformerTokenizers(BaseTestTransformerTokenizers): 12 | TRANSFORMER_MODELS = [ 13 | # "google/gemma-2-9b-it", # Works locally, fails in build 14 | "meta-llama/Meta-Llama-3-8B-Instruct", 15 | ] 16 | 17 | @pytest.mark.parametrize( 18 | "model_name", 19 | TRANSFORMER_MODELS, 20 | ) 21 | def test_smoke(self, model_name: str): 22 | try: 23 | self.base_smoke(model_name) 24 | except OSError: 25 | pytest.skip("HuggingFace raises OSError if user is not authenticated.") 26 | 27 | @pytest.mark.parametrize("model_name", TRANSFORMER_MODELS) 28 | @pytest.mark.parametrize("target_string", TOKENIZER_ROUND_TRIP_STRINGS) 29 | def test_string_roundtrip(self, model_name: str, target_string: str): 30 | try: 31 | self.base_string_roundtrip(model_name, target_string) 32 | except OSError: 33 | pytest.skip("HuggingFace raises OSError if user is not authenticated.") 34 | -------------------------------------------------------------------------------- /tests/need_credentials/test_vertexai.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from guidance import gen, role, models, select 4 | 5 | from ..utils import get_model 6 | 7 | 8 | def test_palm2_instruct(): 9 | try: 10 | vmodel = models.VertexAI("text-bison@001") 11 | except: 12 | pytest.skip("Skipping VertexAI test because we can't load the model!") 13 | 14 | with role("instruction"): 15 | lm = vmodel + "this is a test about" 16 | lm += gen("test", max_tokens=100) 17 | assert len(lm["test"]) > 0 18 | 19 | 20 | def test_palm2_chat(): 21 | from guidance import assistant, gen, models, system, user 22 | 23 | try: 24 | vmodel = models.VertexAI("chat-bison@001") 25 | except: 26 | pytest.skip("Skipping VertexAI test because we can't load the model!") 27 | 28 | with system(): 29 | lm = vmodel + "You are an always-happy agent no matter what." 30 | 31 | with user(): 32 | lm += "The economy is crashing!" 33 | 34 | with assistant(): 35 | lm += gen("test1", max_tokens=100) 36 | 37 | with user(): 38 | lm += "What is the best again?" 39 | 40 | with assistant(): 41 | lm += gen("test2", max_tokens=100) 42 | 43 | assert len(lm["test1"]) > 0 44 | assert len(lm["test2"]) > 0 45 | 46 | # second time to make sure cache reuse is okay 47 | with system(): 48 | lm = vmodel + "You are an always-happy agent no matter what." 49 | 50 | with user(): 51 | lm += "The economy is crashing!" 52 | 53 | with assistant(): 54 | lm += gen("test1", max_tokens=100) 55 | 56 | with user(): 57 | lm += "What is the best again?" 58 | 59 | with assistant(): 60 | lm += gen("test2", max_tokens=100) 61 | 62 | assert len(lm["test1"]) > 0 63 | assert len(lm["test2"]) > 0 64 | assert lm["test1"].find("<|im_end|>") < 0 65 | 66 | 67 | def test_gemini_chat(): 68 | from guidance import assistant, gen, models, system, user 69 | 70 | try: 71 | vmodel = models.VertexAI("gemini-pro") 72 | except: 73 | pytest.skip("Skipping VertexAI test because we can't load the model!") 74 | 75 | lm = vmodel 76 | 77 | with user(): 78 | lm += "The economy is crashing!" 79 | 80 | with assistant(): 81 | lm += gen("test1", max_tokens=100) 82 | 83 | with user(): 84 | lm += "What is the best again?" 85 | 86 | with assistant(): 87 | lm += gen("test2", max_tokens=100) 88 | 89 | assert len(lm["test1"]) > 0 90 | assert len(lm["test2"]) > 0 91 | 92 | # second time to make sure cache reuse is okay 93 | lm = vmodel 94 | 95 | with user(): 96 | lm += "The economy is crashing!" 97 | 98 | with assistant(): 99 | lm += gen("test1", max_tokens=100) 100 | 101 | with user(): 102 | lm += "What is the best again?" 103 | 104 | with assistant(): 105 | lm += gen("test2", max_tokens=100) 106 | 107 | assert len(lm["test1"]) > 0 108 | assert len(lm["test2"]) > 0 109 | assert lm["test1"].find("<|im_end|>") < 0 110 | -------------------------------------------------------------------------------- /tests/notebooks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/notebooks/__init__.py -------------------------------------------------------------------------------- /tests/tokenizer_common.py: -------------------------------------------------------------------------------- 1 | from guidance import models 2 | 3 | TOKENIZER_ROUND_TRIP_STRINGS = [ 4 | "", 5 | " ", 6 | "hello", 7 | " hello", 8 | "two words", 9 | " two words", 10 | " two words ", 11 | "two words ", 12 | "’", 13 | "’•¶∂ƒ˙∆£Ħ爨ൠᅘ∰፨", 14 | ] 15 | 16 | 17 | class BaseTestTransformerTokenizers: 18 | def base_smoke(self, model_name: str): 19 | my_tok = models.TransformersTokenizer.from_pretrained( 20 | model_name, trust_remote_code=True, 21 | ) 22 | assert my_tok is not None 23 | 24 | def base_string_roundtrip(self, model_name: str, target_string: str): 25 | my_tok = models.TransformersTokenizer.from_pretrained( 26 | model_name, 27 | trust_remote_code=True, 28 | ) 29 | 30 | encoded = my_tok.encode(target_string.encode()) 31 | decoded = my_tok.decode(encoded) 32 | final_string = decoded.decode() 33 | 34 | assert final_string == target_string 35 | 36 | def base_eos_bos_token_round_trip( 37 | self, model_name: str 38 | ): 39 | my_tok = models.TransformersTokenizer.from_pretrained( 40 | model_name, 41 | trust_remote_code=True, 42 | ) 43 | 44 | assert my_tok.eos_token == my_tok.decode([my_tok.eos_token_id]) 45 | assert my_tok.encode(my_tok.eos_token) == [my_tok.eos_token_id] 46 | 47 | if my_tok.bos_token is not None: 48 | assert my_tok.bos_token == my_tok.decode([my_tok.bos_token_id]) 49 | assert my_tok.encode(my_tok.bos_token) == [my_tok.bos_token_id] 50 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/library/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/unit/library/__init__.py -------------------------------------------------------------------------------- /tests/unit/library/json/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guidance-ai/guidance/e5be7b6bf41e93faeb540ae169509568a1941c9a/tests/unit/library/json/__init__.py -------------------------------------------------------------------------------- /tests/unit/library/json/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from functools import partial 3 | from json import dumps as json_dumps 4 | from json import loads as json_loads 5 | from typing import Any, Optional, Union 6 | 7 | from jsonschema import validate 8 | 9 | from guidance import json as gen_json 10 | from guidance.library._json import JSONSchema 11 | 12 | from ....utils import check_match_failure as _check_match_failure 13 | from ....utils import check_run_with_temperature 14 | from ....utils import generate_and_check as _generate_and_check 15 | 16 | 17 | def generate_and_check( 18 | target_obj: Any, 19 | schema_obj: Union[str, JSONSchema], 20 | desired_temperature: Optional[float] = None, 21 | ): 22 | if isinstance(schema_obj, str): 23 | schema_obj = json_loads(schema_obj) 24 | 25 | # Sanity check what we're being asked 26 | validate(instance=target_obj, schema=schema_obj) 27 | prepared_json = json_dumps(target_obj) 28 | assert json.loads(prepared_json) == target_obj 29 | 30 | # Now test that the grammar can recognize and generate prepared_json 31 | # We partial in the grammar_callable 32 | if desired_temperature is not None: 33 | grammar_callable = partial(gen_json, schema=schema_obj, temperature=desired_temperature) 34 | else: 35 | grammar_callable = partial(gen_json, schema=schema_obj) 36 | 37 | lm = _generate_and_check( 38 | grammar_callable, 39 | test_string=prepared_json, 40 | ) 41 | check_run_with_temperature(lm, desired_temperature) 42 | 43 | 44 | def check_match_failure( 45 | *, 46 | bad_string: str, 47 | good_bytes: Optional[bytes] = None, 48 | failure_byte: Optional[bytes] = None, 49 | allowed_bytes: Optional[set[bytes]] = None, 50 | schema_obj: Union[str, JSONSchema], 51 | ): 52 | grammar = gen_json(schema=schema_obj) 53 | 54 | _check_match_failure( 55 | bad_string=bad_string, 56 | good_bytes=good_bytes, 57 | failure_byte=failure_byte, 58 | allowed_bytes=allowed_bytes, 59 | grammar=grammar, 60 | ) 61 | -------------------------------------------------------------------------------- /tests/unit/library/test_block.py: -------------------------------------------------------------------------------- 1 | from guidance import regex, block, models 2 | import pytest 3 | 4 | 5 | def test_text_opener(): 6 | model = models.Mock("